From c87f74e544c5222d8b74794a55e1d106c7d0a109 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:54:22 +0300 Subject: [PATCH 001/110] feat: add shared product analytics contract --- apps/desktop/package.json | 1 + apps/web/package.json | 1 + packages/analytics/package.json | 13 + packages/analytics/src/index.test.ts | 241 ++++++++++++++ packages/analytics/src/index.ts | 448 +++++++++++++++++++++++++++ packages/analytics/tsconfig.json | 9 + packages/web-backend/package.json | 1 + pnpm-lock.yaml | 29 +- 8 files changed, 736 insertions(+), 7 deletions(-) create mode 100644 packages/analytics/package.json create mode 100644 packages/analytics/src/index.test.ts create mode 100644 packages/analytics/src/index.ts create mode 100644 packages/analytics/tsconfig.json diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 3eb2586b91e..5b58754d170 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -15,6 +15,7 @@ }, "dependencies": { "@aerofoil/rive-solid-canvas": "^2.1.4", + "@cap/analytics": "workspace:*", "@cap/database": "workspace:*", "@cap/ui-solid": "workspace:*", "@cap/utils": "workspace:*", diff --git a/apps/web/package.json b/apps/web/package.json index 4aa4d2d1f78..4de17b939df 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -21,6 +21,7 @@ "@aws-sdk/cloudfront-signer": "^3.821.0", "@aws-sdk/s3-presigned-post": "^3.485.0", "@aws-sdk/s3-request-presigner": "^3.485.0", + "@cap/analytics": "workspace:*", "@cap/database": "workspace:*", "@cap/env": "workspace:*", "@cap/recorder-core": "workspace:*", diff --git a/packages/analytics/package.json b/packages/analytics/package.json new file mode 100644 index 00000000000..ef81deb5796 --- /dev/null +++ b/packages/analytics/package.json @@ -0,0 +1,13 @@ +{ + "name": "@cap/analytics", + "private": true, + "main": "./src/index.ts", + "types": "./src/index.ts", + "type": "module", + "scripts": { + "test": "vitest run" + }, + "devDependencies": { + "vitest": "^3.2.0" + } +} diff --git a/packages/analytics/src/index.test.ts b/packages/analytics/src/index.test.ts new file mode 100644 index 00000000000..f4c753bb469 --- /dev/null +++ b/packages/analytics/src/index.test.ts @@ -0,0 +1,241 @@ +import { describe, expect, it } from "vitest"; +import { + CORE_EVENT_NAMES, + createProductEventRows, + isCoreEventName, + isServerOnlyEventName, + normalizeProductEventInput, + normalizeProductEventProperties, + PRODUCT_ANALYTICS_LIMITS, +} from "./index"; + +describe("product analytics contract", () => { + it("keeps the catalog unique and consistently named", () => { + expect(new Set(CORE_EVENT_NAMES).size).toBe(CORE_EVENT_NAMES.length); + for (const name of CORE_EVENT_NAMES) { + expect(name).toMatch(/^[a-z][a-z0-9_]*$/); + } + }); + + it("rejects noisy events outside the core catalog", () => { + expect(isCoreEventName("recording_started")).toBe(true); + expect(isCoreEventName("mouse_moved")).toBe(false); + expect(isCoreEventName("$autocapture")).toBe(false); + }); + + it("marks revenue and lifecycle events as server authoritative", () => { + expect(isServerOnlyEventName("purchase_completed")).toBe(true); + expect(isServerOnlyEventName("user_signed_up")).toBe(true); + expect(isServerOnlyEventName("page_view")).toBe(false); + expect(isServerOnlyEventName("recording_completed")).toBe(false); + }); +}); + +describe("normalizeProductEventProperties", () => { + it("retains only finite scalar values", () => { + expect( + normalizeProductEventProperties({ + valid_string: "value", + valid_number: 42, + valid_boolean: false, + valid_null: null, + not_finite: Number.POSITIVE_INFINITY, + object_value: { nested: true }, + array_value: ["nope"], + }), + ).toEqual({ + valid_string: "value", + valid_number: 42, + valid_boolean: false, + valid_null: null, + }); + }); + + it("drops invalid keys and truncates strings", () => { + const longValue = "x".repeat( + PRODUCT_ANALYTICS_LIMITS.propertyStringLength + 20, + ); + expect( + normalizeProductEventProperties({ + valid_key: longValue, + "Invalid-Key": "drop", + ["x".repeat(PRODUCT_ANALYTICS_LIMITS.propertyKeyLength + 1)]: "drop", + }), + ).toEqual({ + valid_key: "x".repeat(PRODUCT_ANALYTICS_LIMITS.propertyStringLength), + }); + }); + + it("caps the number of retained properties", () => { + const properties = Object.fromEntries( + Array.from( + { length: PRODUCT_ANALYTICS_LIMITS.propertyCount + 10 }, + (_, i) => [`property_${i}`, i], + ), + ); + const normalized = normalizeProductEventProperties(properties); + expect(Object.keys(normalized ?? {})).toHaveLength( + PRODUCT_ANALYTICS_LIMITS.propertyCount, + ); + }); + + it("caps the serialized property payload", () => { + const properties = Object.fromEntries( + Array.from({ length: PRODUCT_ANALYTICS_LIMITS.propertyCount }, (_, i) => [ + `property_${i}`, + "🙂".repeat(PRODUCT_ANALYTICS_LIMITS.propertyStringLength), + ]), + ); + const normalized = normalizeProductEventProperties(properties); + expect( + new TextEncoder().encode(JSON.stringify(normalized)).byteLength, + ).toBeLessThanOrEqual(PRODUCT_ANALYTICS_LIMITS.propertiesBytes); + }); + + it("returns undefined when nothing is safe to retain", () => { + expect(normalizeProductEventProperties()).toBeUndefined(); + expect(normalizeProductEventProperties({ nested: {} })).toBeUndefined(); + }); + + it("drops property keys that could contain customer content", () => { + expect( + normalizeProductEventProperties({ + transcript: "private", + file_path: "/Users/private/recording.cap", + error: "upload failed at /Users/private/recording.cap", + reason: "raw operating system error", + video_id: "private-recording-id", + status: "completed", + }), + ).toEqual({ status: "completed" }); + }); +}); + +describe("normalizeProductEventInput", () => { + const now = Date.parse("2026-07-12T12:00:00.000Z"); + const baseEvent = { + eventId: "event-1", + eventName: "recording_started", + occurredAt: "2026-07-12T11:59:59.000Z", + anonymousId: "anonymous-1", + sessionId: "session-1", + platform: "desktop", + }; + + it("normalizes a valid event", () => { + expect(normalizeProductEventInput(baseEvent, now)).toEqual(baseEvent); + }); + + it.each([ + ["unknown event", { ...baseEvent, eventName: "mouse_moved" }], + ["missing id", { ...baseEvent, eventId: "" }], + ["unknown platform", { ...baseEvent, platform: "mobile" }], + ["server-authored platform", { ...baseEvent, platform: "server" }], + ["invalid timestamp", { ...baseEvent, occurredAt: "not-a-date" }], + [ + "stale timestamp", + { ...baseEvent, occurredAt: "2026-07-01T11:59:59.000Z" }, + ], + [ + "future timestamp", + { ...baseEvent, occurredAt: "2026-07-12T12:06:00.000Z" }, + ], + ])("rejects %s", (_label, event) => { + expect(normalizeProductEventInput(event, now)).toBeNull(); + }); + + it("truncates bounded context and sanitizes properties", () => { + const normalized = normalizeProductEventInput( + { + ...baseEvent, + pathname: `/${"short/".repeat(PRODUCT_ANALYTICS_LIMITS.pathnameLength)}`, + properties: { + status: "completed", + content: "private", + nested: { invalid: true }, + }, + }, + now, + ); + expect(normalized?.pathname).toHaveLength( + PRODUCT_ANALYTICS_LIMITS.pathnameLength, + ); + expect(normalized?.properties).toEqual({ status: "completed" }); + }); + + it("removes query strings and high-cardinality path segments", () => { + const normalized = normalizeProductEventInput( + { + ...baseEvent, + pathname: + "https://cap.so/s/019f1ad7-2deb-7730-8d27-916abc9cd4d8?token=private", + }, + now, + ); + expect(normalized?.pathname).toBe("/s/:id"); + }); + + it.each([ + "/screen-recorder-windows", + "/loom-alternative", + "/blog/how-to-record-your-screen-with-audio", + ])("preserves static acquisition route %s", (pathname) => { + expect( + normalizeProductEventInput({ ...baseEvent, pathname }, now)?.pathname, + ).toBe(pathname); + }); + + it("normalizes Cap IDs only on dynamic route segments", () => { + expect( + normalizeProductEventInput( + { ...baseEvent, pathname: "/dashboard/spaces/01abcdefghjkmnp" }, + now, + )?.pathname, + ).toBe("/dashboard/spaces/:id"); + }); + + it("keeps only the referrer hostname", () => { + const normalized = normalizeProductEventInput( + { + ...baseEvent, + referrer: "https://www.google.com/search?q=private", + }, + now, + ); + expect(normalized?.referrer).toBe("www.google.com"); + }); +}); + +describe("createProductEventRows", () => { + it("adds trusted server context without accepting client identity", () => { + const [row] = createProductEventRows( + [ + { + eventId: "event-1", + eventName: "purchase_completed", + occurredAt: "2026-07-12T12:00:00.000Z", + anonymousId: "guest-checkout", + platform: "server", + properties: { plan: "monthly" }, + }, + ], + { + receivedAt: "2026-07-12T12:00:01.000Z", + source: "server", + userId: "user-1", + organizationId: "org-1", + country: "CY", + }, + ); + + expect(row).toMatchObject({ + event_id: "event-1", + event_name: "purchase_completed", + source: "server", + user_id: "user-1", + organization_id: "org-1", + country: "CY", + properties: '{"plan":"monthly"}', + }); + }); +}); diff --git a/packages/analytics/src/index.ts b/packages/analytics/src/index.ts new file mode 100644 index 00000000000..0ee0de947d2 --- /dev/null +++ b/packages/analytics/src/index.ts @@ -0,0 +1,448 @@ +export const CORE_EVENT_NAMES = [ + "page_view", + "download_cta_clicked", + "pricing_cta_clicked", + "cli_install_command_copied", + "auth_started", + "auth_email_sent", + "user_signed_up", + "recording_started", + "recording_completed", + "multipart_upload_complete", + "multipart_upload_failed", + "export_button_clicked", + "create_shareable_link_clicked", + "checkout_started", + "guest_checkout_started", + "purchase_completed", + "organization_invite_sent", + "organization_member_joined", + "seat_quantity_changed", + "loom_import_started", + "loom_import_completed", + "loom_import_failed", + "first_view_received", + "recording_recovery_failed", +] as const; + +export const SERVER_ONLY_EVENT_NAMES = [ + "user_signed_up", + "checkout_started", + "guest_checkout_started", + "purchase_completed", + "organization_invite_sent", + "organization_member_joined", + "seat_quantity_changed", + "first_view_received", +] as const satisfies readonly CoreEventName[]; + +export type CoreEventName = (typeof CORE_EVENT_NAMES)[number]; +export type ProductEventPlatform = "web" | "desktop" | "server"; +export type ProductEventProperty = string | number | boolean | null; +export type ProductEventProperties = Record; + +export const PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE = + "cap_analytics_anonymous_id"; + +export interface ProductEventInput { + eventId: string; + eventName: CoreEventName; + occurredAt: string; + anonymousId: string; + sessionId?: string; + platform: ProductEventPlatform; + appVersion?: string; + pathname?: string; + referrer?: string; + properties?: ProductEventProperties; +} + +export interface ProductEventContext { + receivedAt: string; + source: "client" | "server"; + userId?: string; + organizationId?: string; + country?: string; + region?: string; + city?: string; +} + +export interface ProductEventRow { + event_id: string; + occurred_at: string; + received_at: string; + event_name: CoreEventName; + schema_version: 1; + source: "client" | "server"; + platform: ProductEventPlatform; + anonymous_id: string; + session_id: string; + user_id: string; + organization_id: string; + app_version: string; + pathname: string; + referrer: string; + country: string; + region: string; + city: string; + properties: string; +} + +export const PRODUCT_ANALYTICS_LIMITS = { + batchSize: 20, + queueSize: 100, + requestBytes: 64 * 1024, + propertyCount: 32, + propertyKeyLength: 64, + propertyStringLength: 512, + propertiesBytes: 16 * 1024, + identifierLength: 128, + appVersionLength: 64, + pathnameLength: 2048, + referrerLength: 2048, + maxPastAgeMs: 7 * 24 * 60 * 60 * 1000, + maxFutureAgeMs: 5 * 60 * 1000, +} as const; + +export class ProductAnalyticsError extends Error { + readonly _tag = "ProductAnalyticsError"; + readonly retryable: boolean; + readonly status?: number; + + constructor(options: { + cause: unknown; + retryable: boolean; + status?: number; + }) { + super("Product analytics request failed", { cause: options.cause }); + this.name = "ProductAnalyticsError"; + this.retryable = options.retryable; + this.status = options.status; + } +} + +interface ProductAnalyticsTransportOptions { + host: string; + token: string; + rows: readonly ProductEventRow[]; + wait?: boolean; + maxAttempts?: number; + fetchImpl?: typeof fetch; +} + +export async function sendProductAnalyticsRows({ + host, + token, + rows, + wait = false, + maxAttempts = 2, + fetchImpl = fetch, +}: ProductAnalyticsTransportOptions) { + if (rows.length === 0) return; + + const url = new URL("/v0/events", host); + url.searchParams.set("name", "product_events_v1"); + url.searchParams.set("format", "ndjson"); + if (wait) url.searchParams.set("wait", "true"); + + const body = rows.map((row) => JSON.stringify(row)).join("\n"); + let lastError: ProductAnalyticsError | undefined; + + for (let attempt = 0; attempt < maxAttempts; attempt += 1) { + try { + const response = await fetchImpl(url, { + method: "POST", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/x-ndjson", + }, + body, + signal: AbortSignal.timeout(wait ? 10_000 : 2_000), + }); + + if (response.ok) return; + + const retryable = response.status === 429 || response.status >= 500; + lastError = new ProductAnalyticsError({ + cause: await response.text(), + retryable, + status: response.status, + }); + if (!retryable) throw lastError; + } catch (cause) { + if (cause instanceof ProductAnalyticsError) throw cause; + lastError = new ProductAnalyticsError({ cause, retryable: true }); + } + + if (attempt + 1 < maxAttempts) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + } + + throw ( + lastError ?? + new ProductAnalyticsError({ + cause: "Product analytics request failed", + retryable: false, + }) + ); +} + +const CORE_EVENT_NAME_SET = new Set(CORE_EVENT_NAMES); +const SERVER_ONLY_EVENT_NAME_SET = new Set(SERVER_ONLY_EVENT_NAMES); +const PROPERTY_KEY_PATTERN = /^[a-z][a-z0-9_]*$/; +const FORBIDDEN_PROPERTY_KEYS = new Set([ + "comment", + "content", + "email", + "error", + "error_message", + "file_name", + "file_path", + "raw_error", + "reason", + "recording_name", + "organization_id", + "session_id", + "subscription_id", + "title", + "transcript", + "user_email", + "user_id", + "video_id", +]); +const CLIENT_PRODUCT_EVENT_PLATFORMS = new Set([ + "web", + "desktop", +]); +const DYNAMIC_ID_PARENT_SEGMENTS = new Set([ + "apps", + "c", + "dev", + "embed", + "folder", + "invite", + "messenger", + "s", + "spaces", + "videos", +]); +const UUID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +const ULID_PATTERN = /^[0-9A-HJKMNP-TV-Z]{26}$/; +const CAP_NANOID_PATTERN = /^(?:[0-9abcdefghjkmnpqrstvwxyz]{15}){1,2}$/; + +export function isCoreEventName(value: string): value is CoreEventName { + return CORE_EVENT_NAME_SET.has(value); +} + +export function isServerOnlyEventName(value: CoreEventName) { + return SERVER_ONLY_EVENT_NAME_SET.has(value); +} + +export function normalizeProductEventProperties( + properties?: Record, +): ProductEventProperties | undefined { + if (!properties) return undefined; + + const normalized: ProductEventProperties = {}; + let count = 0; + + for (const [key, value] of Object.entries(properties)) { + if (count >= PRODUCT_ANALYTICS_LIMITS.propertyCount) break; + if ( + key.length > PRODUCT_ANALYTICS_LIMITS.propertyKeyLength || + !PROPERTY_KEY_PATTERN.test(key) || + FORBIDDEN_PROPERTY_KEYS.has(key) + ) { + continue; + } + + let normalizedValue: ProductEventProperty; + if (typeof value === "string") { + normalizedValue = value.slice( + 0, + PRODUCT_ANALYTICS_LIMITS.propertyStringLength, + ); + } else if (typeof value === "number" && Number.isFinite(value)) { + normalizedValue = value; + } else if (typeof value === "boolean" || value === null) { + normalizedValue = value; + } else { + continue; + } + + normalized[key] = normalizedValue; + if ( + new TextEncoder().encode(JSON.stringify(normalized)).byteLength > + PRODUCT_ANALYTICS_LIMITS.propertiesBytes + ) { + delete normalized[key]; + continue; + } + + count += 1; + } + + return count > 0 ? normalized : undefined; +} + +export function normalizeProductEventInput( + value: unknown, + now = Date.now(), +): ProductEventInput | null { + if (!isRecord(value)) return null; + + const eventId = normalizeIdentifier(value.eventId); + const anonymousId = normalizeIdentifier(value.anonymousId); + const sessionId = normalizeOptionalIdentifier(value.sessionId); + const eventName = value.eventName; + const platform = value.platform; + const occurredAt = normalizeOccurredAt(value.occurredAt, now); + + if ( + !eventId || + !anonymousId || + !occurredAt || + typeof eventName !== "string" || + !isCoreEventName(eventName) || + typeof platform !== "string" || + !CLIENT_PRODUCT_EVENT_PLATFORMS.has(platform as ProductEventPlatform) + ) { + return null; + } + + const properties = + "properties" in value && isRecord(value.properties) + ? normalizeProductEventProperties(value.properties) + : undefined; + + return { + eventId, + eventName, + occurredAt, + anonymousId, + ...(sessionId ? { sessionId } : {}), + platform: platform as ProductEventPlatform, + ...normalizeOptionalStringField( + "appVersion", + value.appVersion, + PRODUCT_ANALYTICS_LIMITS.appVersionLength, + ), + ...normalizeOptionalPathname(value.pathname), + ...normalizeOptionalReferrer(value.referrer), + ...(properties ? { properties } : {}), + }; +} + +export function createProductEventRows( + events: readonly ProductEventInput[], + context: ProductEventContext, +): ProductEventRow[] { + return events.map((event) => ({ + event_id: event.eventId, + occurred_at: event.occurredAt, + received_at: context.receivedAt, + event_name: event.eventName, + schema_version: 1, + source: context.source, + platform: event.platform, + anonymous_id: event.anonymousId, + session_id: event.sessionId ?? "", + user_id: context.userId ?? "", + organization_id: context.organizationId ?? "", + app_version: event.appVersion ?? "", + pathname: event.pathname ?? "", + referrer: event.referrer ?? "", + country: context.country ?? "", + region: context.region ?? "", + city: context.city ?? "", + properties: event.properties ? JSON.stringify(event.properties) : "{}", + })); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function normalizeIdentifier(value: unknown) { + if (typeof value !== "string") return null; + const normalized = value.trim(); + if ( + !normalized || + normalized.length > PRODUCT_ANALYTICS_LIMITS.identifierLength + ) { + return null; + } + return normalized; +} + +function normalizeOptionalIdentifier(value: unknown) { + if (value === undefined || value === null || value === "") return undefined; + return normalizeIdentifier(value) ?? undefined; +} + +function normalizeOccurredAt(value: unknown, now: number) { + if (typeof value !== "string") return null; + const timestamp = Date.parse(value); + if (!Number.isFinite(timestamp)) return null; + if (timestamp < now - PRODUCT_ANALYTICS_LIMITS.maxPastAgeMs) return null; + if (timestamp > now + PRODUCT_ANALYTICS_LIMITS.maxFutureAgeMs) return null; + return new Date(timestamp).toISOString(); +} + +function normalizeOptionalStringField( + key: Key, + value: unknown, + maxLength: number, +): Partial> { + if (typeof value !== "string") return {}; + const normalized = value.trim().slice(0, maxLength); + return normalized ? ({ [key]: normalized } as Record) : {}; +} + +function normalizeOptionalPathname(value: unknown) { + if (typeof value !== "string") return {}; + + let pathname = value.trim(); + try { + pathname = new URL(pathname).pathname; + } catch { + pathname = pathname.split(/[?#]/, 1)[0] ?? ""; + } + + const segments = pathname.split("/"); + const normalized = segments + .map((segment, index) => + isHighCardinalityPathSegment(segment, segments[index - 1]) + ? ":id" + : segment, + ) + .join("/") + .slice(0, PRODUCT_ANALYTICS_LIMITS.pathnameLength); + + return normalized ? { pathname: normalized } : {}; +} + +function normalizeOptionalReferrer(value: unknown) { + if (typeof value !== "string" || !value.trim()) return {}; + try { + return { + referrer: new URL(value).hostname.slice( + 0, + PRODUCT_ANALYTICS_LIMITS.referrerLength, + ), + }; + } catch { + return {}; + } +} + +function isHighCardinalityPathSegment(segment: string, parentSegment?: string) { + if (UUID_PATTERN.test(segment) || ULID_PATTERN.test(segment)) return true; + return Boolean( + parentSegment && + DYNAMIC_ID_PARENT_SEGMENTS.has(parentSegment) && + CAP_NANOID_PATTERN.test(segment), + ); +} diff --git a/packages/analytics/tsconfig.json b/packages/analytics/tsconfig.json new file mode 100644 index 00000000000..e8b0bc24b9c --- /dev/null +++ b/packages/analytics/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "../config/base.tsconfig.json", + "exclude": ["node_modules"], + "include": ["**/*.ts"], + "compilerOptions": { + "baseUrl": ".", + "moduleResolution": "bundler" + } +} diff --git a/packages/web-backend/package.json b/packages/web-backend/package.json index 6d644700853..90afa3f2465 100644 --- a/packages/web-backend/package.json +++ b/packages/web-backend/package.json @@ -17,6 +17,7 @@ "@aws-sdk/credential-providers": "^3.908.0", "@aws-sdk/s3-presigned-post": "^3.485.0", "@aws-sdk/s3-request-presigner": "^3.485.0", + "@cap/analytics": "workspace:*", "@cap/database": "workspace:*", "@cap/utils": "workspace:*", "@cap/web-domain": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 0e13e2503e9..ca004a5ccce 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -111,6 +111,9 @@ importers: '@aerofoil/rive-solid-canvas': specifier: ^2.1.4 version: 2.1.4 + '@cap/analytics': + specifier: workspace:* + version: link:../../packages/analytics '@cap/database': specifier: workspace:* version: link:../../packages/database @@ -300,7 +303,7 @@ importers: version: 4.3.2 '@tailwindcss/typography': specifier: ^0.5.9 - version: 0.5.20(tailwindcss@3.4.19(yaml@2.9.0)) + version: 0.5.20(tailwindcss@4.3.2) '@tauri-apps/cli': specifier: '>=2.1.0' version: 2.11.4 @@ -611,6 +614,9 @@ importers: '@aws-sdk/s3-request-presigner': specifier: ^3.485.0 version: 3.1077.0 + '@cap/analytics': + specifier: workspace:* + version: link:../../packages/analytics '@cap/database': specifier: workspace:* version: link:../../packages/database @@ -1128,6 +1134,12 @@ importers: specifier: 3.17.14 version: 3.17.14 + packages/analytics: + devDependencies: + vitest: + specifier: ^3.2.0 + version: 3.2.6(@types/debug@4.1.13)(@types/node@22.20.0)(@vitest/ui@3.2.6)(jiti@2.7.0)(jsdom@26.1.0)(lightningcss@1.32.0)(terser@5.48.0)(yaml@2.9.0) + packages/config: dependencies: '@vitejs/plugin-react': @@ -1598,6 +1610,9 @@ importers: '@aws-sdk/s3-request-presigner': specifier: ^3.485.0 version: 3.1077.0 + '@cap/analytics': + specifier: workspace:* + version: link:../analytics '@cap/database': specifier: workspace:* version: link:../database @@ -19958,7 +19973,7 @@ snapshots: semver: 7.8.5 string-width: 4.2.3 supports-color: 8.1.1 - tinyglobby: 0.2.15 + tinyglobby: 0.2.17 widest-line: 3.1.0 wordwrap: 1.0.0 wrap-ansi: 7.0.0 @@ -26505,7 +26520,7 @@ snapshots: eslint: 9.39.4(jiti@2.7.0) eslint-import-resolver-node: 0.3.10 eslint-import-resolver-typescript: 3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint@9.39.4(jiti@2.7.0)))(eslint@9.39.4(jiti@2.7.0)) - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint@9.39.4(jiti@2.7.0)))(eslint@9.39.4(jiti@2.7.0)))(eslint@9.39.4(jiti@2.7.0)) eslint-plugin-jsx-a11y: 6.10.2(eslint@9.39.4(jiti@2.7.0)) eslint-plugin-react: 7.37.5(eslint@9.39.4(jiti@2.7.0)) eslint-plugin-react-hooks: 7.1.1(eslint@9.39.4(jiti@2.7.0)) @@ -26547,7 +26562,7 @@ snapshots: tinyglobby: 0.2.17 unrs-resolver: 1.12.2 optionalDependencies: - eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)) + eslint-plugin-import: 2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint@9.39.4(jiti@2.7.0)))(eslint@9.39.4(jiti@2.7.0)))(eslint@9.39.4(jiti@2.7.0)) transitivePeerDependencies: - supports-color @@ -26617,7 +26632,7 @@ snapshots: - eslint-import-resolver-webpack - supports-color - eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1)(eslint@9.39.4(jiti@2.7.0)): + eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint-import-resolver-typescript@3.10.1(eslint-plugin-import@2.32.0(@typescript-eslint/parser@8.62.1(eslint@9.39.4(jiti@2.7.0))(typescript@5.9.3))(eslint@9.39.4(jiti@2.7.0)))(eslint@9.39.4(jiti@2.7.0)))(eslint@9.39.4(jiti@2.7.0)): dependencies: '@rtsao/scc': 1.1.0 array-includes: 3.1.9 @@ -34257,7 +34272,7 @@ snapshots: werift-common@0.0.3: dependencies: '@shinyoshiaki/jspack': 0.0.6 - debug: 4.4.0 + debug: 4.4.3(supports-color@8.1.1) transitivePeerDependencies: - supports-color @@ -34276,7 +34291,7 @@ snapshots: dependencies: '@shinyoshiaki/jspack': 0.0.6 buffer-crc32: 1.0.0 - debug: 4.4.0 + debug: 4.4.3(supports-color@8.1.1) int64-buffer: 1.1.0 ip: 2.0.1 lodash: 4.18.1 From ca358875ac9153fa83e396a238263539dbef1805 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:54:31 +0300 Subject: [PATCH 002/110] feat: add first-party web analytics --- .../unit/guest-checkout-analytics.test.ts | 79 ++++ .../unit/product-analytics-queue.test.ts | 356 +++++++++++++++++ .../unit/product-analytics-request.test.ts | 161 ++++++++ .../unit/product-analytics-scheduler.test.ts | 47 +++ .../unit/product-analytics-server.test.ts | 52 +++ .../unit/product-analytics-transport.test.ts | 134 +++++++ .../subscription-analytics-webhook.test.ts | 246 ++++++++++++ .../actions/analytics/track-user-signed-up.ts | 50 ++- apps/web/actions/organization/send-invites.ts | 26 ++ .../organization/update-seat-quantity.ts | 22 + apps/web/app/Layout/PosthogIdentify.tsx | 5 +- apps/web/app/Layout/PosthogPageView.tsx | 28 +- .../app/Layout/ProductAnalyticsPageView.tsx | 29 ++ apps/web/app/Layout/providers.tsx | 8 +- apps/web/app/api/desktop/[...route]/root.ts | 53 ++- apps/web/app/api/events/route.ts | 152 +++++++ apps/web/app/api/invite/accept/route.ts | 27 ++ .../settings/billing/guest-checkout/route.ts | 54 ++- .../api/settings/billing/subscribe/route.ts | 52 ++- apps/web/app/api/webhooks/stripe/route.ts | 180 +++++++-- apps/web/app/layout.tsx | 2 + apps/web/app/utils/analytics.ts | 16 +- apps/web/app/utils/product-analytics.ts | 378 ++++++++++++++++++ apps/web/lib/analytics/request.ts | 148 +++++++ apps/web/lib/analytics/server-event.ts | 54 +++ apps/web/lib/analytics/server.ts | 77 ++++ apps/web/lib/rate-limit.ts | 1 + apps/web/lib/server.ts | 18 +- packages/env/server.ts | 2 + .../web-backend/src/ProductAnalytics/index.ts | 113 ++++++ packages/web-backend/src/index.ts | 8 + 31 files changed, 2436 insertions(+), 142 deletions(-) create mode 100644 apps/web/__tests__/unit/guest-checkout-analytics.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-queue.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-request.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-scheduler.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-server.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-transport.test.ts create mode 100644 apps/web/__tests__/unit/subscription-analytics-webhook.test.ts create mode 100644 apps/web/app/Layout/ProductAnalyticsPageView.tsx create mode 100644 apps/web/app/api/events/route.ts create mode 100644 apps/web/app/utils/product-analytics.ts create mode 100644 apps/web/lib/analytics/request.ts create mode 100644 apps/web/lib/analytics/server-event.ts create mode 100644 apps/web/lib/analytics/server.ts create mode 100644 packages/web-backend/src/ProductAnalytics/index.ts diff --git a/apps/web/__tests__/unit/guest-checkout-analytics.test.ts b/apps/web/__tests__/unit/guest-checkout-analytics.test.ts new file mode 100644 index 00000000000..8782675c934 --- /dev/null +++ b/apps/web/__tests__/unit/guest-checkout-analytics.test.ts @@ -0,0 +1,79 @@ +import { NextRequest } from "next/server"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + createSession: vi.fn(), + product: vi.fn(), + posthog: vi.fn(), + readAnonymousId: vi.fn(), +})); + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ WEB_URL: "https://cap.so" }), +})); +vi.mock("@cap/utils", () => ({ + stripe: () => ({ + checkout: { sessions: { create: mocks.createSession } }, + }), +})); +vi.mock("@/lib/analytics/server", () => ({ + readAnalyticsAnonymousId: mocks.readAnonymousId, + scheduleLegacyPostHogEvent: mocks.posthog, + scheduleServerProductEvent: mocks.product, +})); + +describe("guest checkout analytics", () => { + let POST: typeof import("@/app/api/settings/billing/guest-checkout/route").POST; + + beforeEach(async () => { + vi.clearAllMocks(); + mocks.createSession.mockResolvedValue({ + id: "cs_guest_1", + url: "https://checkout.stripe.com/session", + }); + POST = (await import("@/app/api/settings/billing/guest-checkout/route")) + .POST; + }); + + it("uses one stable fallback identity through checkout and purchase metadata", async () => { + mocks.readAnonymousId.mockReturnValue(undefined); + const response = await POST( + new NextRequest("https://cap.so/api/settings/billing/guest-checkout", { + method: "POST", + body: JSON.stringify({ priceId: "price_team", quantity: 3 }), + }), + ); + expect(response.status).toBe(200); + + const metadata = mocks.createSession.mock.calls[0]?.[0].metadata; + expect(metadata.analyticsAnonymousId).toMatch(/^guest:/); + expect(metadata.analyticsIsFirstPurchase).toBe("true"); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: "checkout:cs_guest_1", + anonymousId: metadata.analyticsAnonymousId, + }), + ); + expect(mocks.posthog).toHaveBeenCalledWith( + expect.objectContaining({ + distinctId: metadata.analyticsAnonymousId, + properties: expect.objectContaining({ + $insert_id: "checkout:cs_guest_1", + }), + }), + ); + }); + + it("preserves an existing browser identity", async () => { + mocks.readAnonymousId.mockReturnValue("anonymous-browser-1"); + await POST( + new NextRequest("https://cap.so/api/settings/billing/guest-checkout", { + method: "POST", + body: JSON.stringify({ priceId: "price_team", quantity: 1 }), + }), + ); + expect( + mocks.createSession.mock.calls[0]?.[0].metadata.analyticsAnonymousId, + ).toBe("anonymous-browser-1"); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-queue.test.ts b/apps/web/__tests__/unit/product-analytics-queue.test.ts new file mode 100644 index 00000000000..6debe8cda01 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-queue.test.ts @@ -0,0 +1,356 @@ +import { + PRODUCT_ANALYTICS_LIMITS, + type ProductEventInput, +} from "@cap/analytics"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + createProductEventId, + getOrCreateStorageId, + ProductAnalyticsQueue, + type ProductAnalyticsTransport, + readFirstTouchAttribution, + sendBrowserProductAnalytics, + shouldCaptureProductPageView, +} from "@/app/utils/product-analytics"; + +const makeEvent = (index: number): ProductEventInput => ({ + eventId: `event-${index}`, + eventName: "page_view", + occurredAt: "2026-07-12T12:00:00.000Z", + anonymousId: "anonymous-1", + sessionId: "session-1", + platform: "web", +}); + +describe("ProductAnalyticsQueue", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => vi.useRealTimers()); + + it("does not perform network work on enqueue", () => { + const transport = vi.fn(); + const queue = new ProductAnalyticsQueue(transport); + queue.enqueue(makeEvent(1)); + expect(transport).not.toHaveBeenCalled(); + }); + + it("flushes one batch after the interval", async () => { + const transport = vi + .fn() + .mockResolvedValue("success"); + const queue = new ProductAnalyticsQueue(transport); + queue.enqueue(makeEvent(1)); + queue.enqueue(makeEvent(2)); + + await vi.advanceTimersByTimeAsync(5_000); + expect(transport).toHaveBeenCalledTimes(1); + expect(transport.mock.calls[0]?.[0]).toHaveLength(2); + }); + + it("flushes immediately when a full batch is queued", async () => { + const transport = vi + .fn() + .mockResolvedValue("success"); + const queue = new ProductAnalyticsQueue(transport); + for (let i = 0; i < PRODUCT_ANALYTICS_LIMITS.batchSize; i += 1) { + queue.enqueue(makeEvent(i)); + } + await queue.flush(); + expect(transport).toHaveBeenCalledTimes(1); + expect(transport.mock.calls[0]?.[0]).toHaveLength( + PRODUCT_ANALYTICS_LIMITS.batchSize, + ); + }); + + it("allows only one request in flight", async () => { + let resolveTransport: ((value: "success") => void) | undefined; + const transport = vi.fn().mockImplementation( + () => + new Promise((resolve) => { + resolveTransport = resolve; + }), + ); + const queue = new ProductAnalyticsQueue(transport); + queue.enqueue(makeEvent(1)); + const first = queue.flush(); + const second = queue.flush(); + expect(first).toBe(second); + expect(transport).toHaveBeenCalledTimes(1); + resolveTransport?.("success"); + await first; + }); + + it("retries a failed batch once", async () => { + const transport = vi + .fn() + .mockResolvedValueOnce("retry") + .mockResolvedValueOnce("retry"); + const queue = new ProductAnalyticsQueue(transport); + queue.enqueue(makeEvent(1)); + await queue.flush(); + await vi.advanceTimersByTimeAsync(3_000); + expect(transport).toHaveBeenCalledTimes(2); + expect(queue.size).toBe(0); + }); + + it("honors retry backoff for a full failed batch", async () => { + const transport = vi + .fn() + .mockResolvedValueOnce("retry") + .mockResolvedValueOnce("success"); + const queue = new ProductAnalyticsQueue(transport); + for (let i = 0; i < PRODUCT_ANALYTICS_LIMITS.batchSize; i += 1) { + queue.enqueue(makeEvent(i)); + } + await Promise.resolve(); + await Promise.resolve(); + + expect(transport).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1_999); + expect(transport).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + expect(transport).toHaveBeenCalledTimes(2); + }); + + it("does not retry a rejected batch", async () => { + const transport = vi + .fn() + .mockResolvedValue("drop"); + const queue = new ProductAnalyticsQueue(transport); + queue.enqueue(makeEvent(1)); + await queue.flush(); + await vi.runAllTimersAsync(); + expect(transport).toHaveBeenCalledTimes(1); + }); + + it("bounds memory and drops the oldest queued events", async () => { + let resolveFirst: ((value: "success") => void) | undefined; + const transport = vi + .fn() + .mockImplementationOnce( + () => + new Promise((resolve) => { + resolveFirst = resolve; + }), + ) + .mockResolvedValue("success"); + const queue = new ProductAnalyticsQueue(transport); + for (let i = 0; i < PRODUCT_ANALYTICS_LIMITS.queueSize + 30; i += 1) { + queue.enqueue(makeEvent(i)); + } + expect(queue.size).toBe(PRODUCT_ANALYTICS_LIMITS.queueSize); + expect(transport.mock.calls[0]?.[0][0]?.eventId).toBe("event-0"); + resolveFirst?.("success"); + await vi.waitFor(() => + expect(transport.mock.calls.length).toBeGreaterThanOrEqual(2), + ); + expect(transport.mock.calls[1]?.[0][0]?.eventId).toBe("event-30"); + }); + + it("keeps every request under the body size limit", async () => { + const requestSizes: number[] = []; + const transport = vi.fn(async (events) => { + requestSizes.push( + new TextEncoder().encode(JSON.stringify({ events })).byteLength, + ); + return "success"; + }); + const queue = new ProductAnalyticsQueue(transport); + for (let i = 0; i < 10; i += 1) { + queue.enqueue({ + ...makeEvent(i), + properties: { value: "x".repeat(20_000) }, + }); + } + + await vi.runAllTimersAsync(); + expect( + requestSizes.every( + (size) => size <= PRODUCT_ANALYTICS_LIMITS.requestBytes, + ), + ).toBe(true); + expect( + transport.mock.calls.reduce( + (count, [events]) => count + events.length, + 0, + ), + ).toBe(10); + }); + + it("drops a single event larger than the request limit", async () => { + const transport = vi.fn(); + const queue = new ProductAnalyticsQueue(transport); + queue.enqueue({ + ...makeEvent(1), + properties: { + value: "x".repeat(PRODUCT_ANALYTICS_LIMITS.requestBytes), + }, + }); + + await vi.runAllTimersAsync(); + expect(transport).not.toHaveBeenCalled(); + expect(queue.size).toBe(0); + }); +}); + +describe("browser analytics identity", () => { + it("falls back when secure UUID generation is unavailable", () => { + expect(createProductEventId(null, 1_000, 0.5)).toBe("fallback-rs-i"); + expect( + createProductEventId( + () => { + throw new Error("blocked"); + }, + 1_000, + 0.5, + ), + ).toBe("fallback-rs-i"); + }); + + it("reuses a persisted identifier", () => { + const storage = { + getItem: vi.fn(() => "existing-id"), + setItem: vi.fn(), + }; + expect(getOrCreateStorageId(storage, "key", () => "new-id")).toBe( + "existing-id", + ); + expect(storage.setItem).not.toHaveBeenCalled(); + }); + + it("creates and persists an identifier once", () => { + const storage = { getItem: vi.fn(() => null), setItem: vi.fn() }; + expect(getOrCreateStorageId(storage, "key", () => "new-id")).toBe("new-id"); + expect(storage.setItem).toHaveBeenCalledWith("key", "new-id"); + }); + + it("falls back when storage is unavailable", () => { + const storage = { + getItem: vi.fn(() => { + throw new Error("blocked"); + }), + setItem: vi.fn(), + }; + expect(getOrCreateStorageId(storage, "key", () => "memory-id")).toBe( + "memory-id", + ); + }); + + it("keeps one generated id when persistence is unavailable", () => { + const createId = vi.fn(() => "memory-id"); + const storage = { + getItem: vi.fn(() => null), + setItem: vi.fn(() => { + throw new Error("blocked"); + }), + }; + expect(getOrCreateStorageId(storage, "key", createId)).toBe("memory-id"); + expect(createId).toHaveBeenCalledOnce(); + }); +}); + +describe("first-touch attribution", () => { + it("stores only allowlisted attribution fields", () => { + const storage = { getItem: vi.fn(() => null), setItem: vi.fn() }; + const result = readFirstTouchAttribution( + "?utm_source=google&utm_campaign=launch&email=private%40example.com", + storage, + ); + expect(result).toEqual({ utm_source: "google", utm_campaign: "launch" }); + expect(storage.setItem).toHaveBeenCalledOnce(); + }); + + it("does not overwrite existing attribution", () => { + const storage = { + getItem: vi.fn(() => '{"utm_source":"original"}'), + setItem: vi.fn(), + }; + expect(readFirstTouchAttribution("?utm_source=new", storage)).toEqual({ + utm_source: "original", + }); + expect(storage.setItem).not.toHaveBeenCalled(); + }); +}); + +describe("product page views", () => { + it.each(["/", "/pricing", "/dashboard", "/dashboard/settings"])( + "captures %s", + (pathname) => { + expect(shouldCaptureProductPageView(pathname)).toBe(true); + }, + ); + + it.each(["/s/video-id", "/c/comment-id", "/embed/video-id"])( + "excludes high-volume viewer route %s", + (pathname) => { + expect(shouldCaptureProductPageView(pathname)).toBe(false); + }, + ); +}); + +describe("browser product analytics transport", () => { + it("uses a beacon during unload without also fetching", async () => { + const fetchImpl = vi.fn(); + const sendBeacon = vi.fn(() => true); + await expect( + sendBrowserProductAnalytics([makeEvent(1)], "unload", { + fetchImpl, + sendBeacon, + }), + ).resolves.toBe("success"); + expect(sendBeacon).toHaveBeenCalledOnce(); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("falls back to keepalive fetch when a beacon is rejected", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(null, { status: 202 })); + await expect( + sendBrowserProductAnalytics([makeEvent(1)], "unload", { + fetchImpl, + sendBeacon: () => false, + }), + ).resolves.toBe("success"); + expect(fetchImpl.mock.calls[0]?.[1]).toMatchObject({ keepalive: true }); + }); + + it.each([ + [429, "retry"], + [503, "retry"], + [400, "drop"], + ] as const)("maps HTTP %s to %s", async (status, result) => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(null, { status })); + await expect( + sendBrowserProductAnalytics([makeEvent(1)], "normal", { fetchImpl }), + ).resolves.toBe(result); + }); + + it("retries transport failures", async () => { + const fetchImpl = vi + .fn() + .mockRejectedValue(new Error("offline")); + await expect( + sendBrowserProductAnalytics([makeEvent(1)], "normal", { fetchImpl }), + ).resolves.toBe("retry"); + }); + + it("times out a stalled request", async () => { + vi.useFakeTimers(); + const fetchImpl = vi.fn( + (_url, init) => + new Promise((_resolve, reject) => { + init?.signal?.addEventListener("abort", () => + reject(new DOMException("Aborted", "AbortError")), + ); + }), + ); + const result = sendBrowserProductAnalytics([makeEvent(1)], "normal", { + fetchImpl, + }); + await vi.advanceTimersByTimeAsync(3_000); + await expect(result).resolves.toBe("retry"); + vi.useRealTimers(); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-request.test.ts b/apps/web/__tests__/unit/product-analytics-request.test.ts new file mode 100644 index 00000000000..561f60a1979 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-request.test.ts @@ -0,0 +1,161 @@ +import { PRODUCT_ANALYTICS_LIMITS } from "@cap/analytics"; +import { describe, expect, it } from "vitest"; +import { + getProductAnalyticsRateLimitKey, + isTrustedAnalyticsRequest, + normalizeGeoHeader, + normalizeProductEventBatch, + ProductAnalyticsRateLimiter, +} from "@/lib/analytics/request"; + +const allowedOrigins = ["https://cap.so", "tauri://localhost"]; +const event = { + eventId: "event-1", + eventName: "page_view", + occurredAt: "2026-07-12T12:00:00.000Z", + anonymousId: "anonymous-1", + sessionId: "session-1", + platform: "web", +}; +const now = Date.parse("2026-07-12T12:00:01.000Z"); + +describe("isTrustedAnalyticsRequest", () => { + it.each([ + [ + "same-origin browser", + { origin: "https://cap.so", secFetchSite: "same-origin" }, + ], + [ + "same-site browser", + { origin: "https://cap.so", secFetchSite: "same-site" }, + ], + ["Tauri", { origin: "tauri://localhost" }], + ["server client", {}], + ])("accepts %s", (_label, headers) => { + expect(isTrustedAnalyticsRequest(headers, allowedOrigins)).toBe(true); + }); + + it("rejects cross-site browser requests", () => { + expect( + isTrustedAnalyticsRequest( + { origin: "https://attacker.example", secFetchSite: "cross-site" }, + allowedOrigins, + ), + ).toBe(false); + }); + + it("rejects oversized declared bodies", () => { + expect( + isTrustedAnalyticsRequest( + { contentLength: String(PRODUCT_ANALYTICS_LIMITS.requestBytes + 1) }, + allowedOrigins, + ), + ).toBe(false); + }); + + it.each(["invalid", "-1", "1.5"])( + "rejects malformed content length %s", + (contentLength) => { + expect(isTrustedAnalyticsRequest({ contentLength }, allowedOrigins)).toBe( + false, + ); + }, + ); +}); + +describe("normalizeProductEventBatch", () => { + it("accepts a bounded valid batch", () => { + expect(normalizeProductEventBatch([event], now)).toEqual([event]); + }); + + it("rejects an empty batch", () => { + expect(normalizeProductEventBatch([], now)).toBeNull(); + }); + + it("rejects a batch above the cap", () => { + expect( + normalizeProductEventBatch( + Array.from( + { length: PRODUCT_ANALYTICS_LIMITS.batchSize + 1 }, + () => event, + ), + now, + ), + ).toBeNull(); + }); + + it("rejects the whole batch when one event is invalid", () => { + expect( + normalizeProductEventBatch( + [event, { ...event, eventName: "$autocapture" }], + now, + ), + ).toBeNull(); + }); + + it("rejects an undeclared oversized body", () => { + expect( + normalizeProductEventBatch( + [ + { + ...event, + properties: { + value: "x".repeat(PRODUCT_ANALYTICS_LIMITS.requestBytes), + }, + }, + ], + now, + ), + ).toBeNull(); + }); + + it.each([ + "user_signed_up", + "checkout_started", + "guest_checkout_started", + "purchase_completed", + ] as const)("rejects client-authored %s", (eventName) => { + expect( + normalizeProductEventBatch([{ ...event, eventName }], now), + ).toBeNull(); + }); +}); + +describe("ProductAnalyticsRateLimiter", () => { + it("enforces per-key and process-wide fallback limits", () => { + const limiter = new ProductAnalyticsRateLimiter({ + perKeyLimit: 2, + globalLimit: 4, + windowMs: 1_000, + }); + expect(limiter.isRateLimited("a", 0)).toBe(false); + expect(limiter.isRateLimited("a", 0)).toBe(false); + expect(limiter.isRateLimited("a", 0)).toBe(true); + expect(limiter.isRateLimited("b", 0)).toBe(false); + expect(limiter.isRateLimited("c", 0)).toBe(true); + expect(limiter.isRateLimited("a", 1_000)).toBe(false); + }); + + it("uses a bounded request identity", () => { + expect( + getProductAnalyticsRateLimitKey({ + xForwardedFor: "203.0.113.10, 10.0.0.1", + }), + ).toBe("203.0.113.10"); + expect(getProductAnalyticsRateLimitKey({})).toBe("unknown"); + }); +}); + +describe("normalizeGeoHeader", () => { + it("decodes and bounds a city header", () => { + expect(normalizeGeoHeader("Nicosia%20Centre", true)).toBe("Nicosia Centre"); + }); + + it("rejects malformed encoded data", () => { + expect(normalizeGeoHeader("%E0%A4%A", true)).toBeUndefined(); + }); + + it("removes unknown values", () => { + expect(normalizeGeoHeader("unknown")).toBeUndefined(); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-scheduler.test.ts b/apps/web/__tests__/unit/product-analytics-scheduler.test.ts new file mode 100644 index 00000000000..3ed5183580f --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-scheduler.test.ts @@ -0,0 +1,47 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + after: vi.fn(() => { + throw new Error("after unavailable"); + }), +})); + +vi.mock("next/server", () => ({ after: mocks.after })); +vi.mock("@cap/env", () => ({ + buildEnv: { + NEXT_PUBLIC_POSTHOG_KEY: "", + NEXT_PUBLIC_POSTHOG_HOST: "", + }, + serverEnv: () => ({ + PRODUCT_ANALYTICS_TINYBIRD_HOST: undefined, + PRODUCT_ANALYTICS_TINYBIRD_TOKEN: undefined, + }), +})); +vi.mock("posthog-node", () => ({ PostHog: vi.fn() })); + +describe("analytics scheduling", () => { + afterEach(() => vi.restoreAllMocks()); + + it("cannot make a business route fail when after is unavailable", async () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + const { scheduleLegacyPostHogEvent, scheduleServerProductEvent } = + await import("@/lib/analytics/server"); + + expect(() => + scheduleServerProductEvent({ + eventId: "checkout:cs_1", + eventName: "checkout_started", + anonymousId: "anonymous-1", + platform: "web", + }), + ).not.toThrow(); + expect(() => + scheduleLegacyPostHogEvent({ + distinctId: "anonymous-1", + eventName: "checkout_started", + }), + ).not.toThrow(); + await Promise.resolve(); + expect(mocks.after).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-server.test.ts b/apps/web/__tests__/unit/product-analytics-server.test.ts new file mode 100644 index 00000000000..f1dda108db9 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-server.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import { createServerProductEventRows } from "@/lib/analytics/server-event"; + +describe("server product analytics", () => { + it("builds a deterministic trusted server event", () => { + const [row] = createServerProductEventRows({ + eventId: "stripe:evt_123:purchase_completed", + eventName: "purchase_completed", + occurredAt: "2026-07-12T12:00:00.000Z", + anonymousId: "anonymous-1", + platform: "web", + userId: "user-1", + organizationId: "org-1", + properties: { + quantity: 3, + email: "private@example.com", + nested: { private: true }, + }, + }); + + expect(row).toMatchObject({ + event_id: "stripe:evt_123:purchase_completed", + event_name: "purchase_completed", + source: "server", + platform: "web", + anonymous_id: "anonymous-1", + user_id: "user-1", + organization_id: "org-1", + properties: '{"quantity":3}', + }); + }); + + it("uses an authenticated fallback identity", () => { + const [row] = createServerProductEventRows({ + eventId: "signup:user-1", + eventName: "user_signed_up", + platform: "server", + userId: "user-1", + }); + expect(row?.anonymous_id).toBe("user:user-1"); + }); + + it("drops an event without any stable identity", () => { + expect( + createServerProductEventRows({ + eventId: "event-1", + eventName: "page_view", + platform: "server", + }), + ).toEqual([]); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-transport.test.ts b/apps/web/__tests__/unit/product-analytics-transport.test.ts new file mode 100644 index 00000000000..2fa585cbe32 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-transport.test.ts @@ -0,0 +1,134 @@ +import { + createProductEventRows, + type ProductAnalyticsError, + sendProductAnalyticsRows, +} from "@cap/analytics"; +import { hasAnalyticsSessionCookie } from "@cap/web-backend"; +import { describe, expect, it, vi } from "vitest"; + +const rows = createProductEventRows( + [ + { + eventId: "event-1", + eventName: "page_view", + occurredAt: "2026-07-12T12:00:00.000Z", + anonymousId: "anonymous-1", + platform: "web", + }, + ], + { + receivedAt: "2026-07-12T12:00:01.000Z", + source: "client", + }, +); + +describe("Tinybird product event transport", () => { + it("skips session resolution for anonymous requests", () => { + expect(hasAnalyticsSessionCookie()).toBe(false); + expect(hasAnalyticsSessionCookie("theme=dark; visitor=123")).toBe(false); + expect( + hasAnalyticsSessionCookie( + "theme=dark; next-auth.session-token=token; visitor=123", + ), + ).toBe(true); + expect(hasAnalyticsSessionCookie("next-auth.session-token.0=chunk")).toBe( + true, + ); + }); + + it("posts NDJSON with append-only credentials", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response(null, { status: 202 })); + + await sendProductAnalyticsRows({ + host: "https://api.tinybird.co", + token: "append-token", + rows, + fetchImpl, + }); + + const [url, request] = fetchImpl.mock.calls[0] ?? []; + expect(String(url)).toBe( + "https://api.tinybird.co/v0/events?name=product_events_v1&format=ndjson", + ); + expect(request).toMatchObject({ + method: "POST", + headers: { + Authorization: "Bearer append-token", + "Content-Type": "application/x-ndjson", + }, + body: JSON.stringify(rows[0]), + }); + }); + + it("retries a transient response once", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValueOnce(new Response("busy", { status: 503 })) + .mockResolvedValueOnce(new Response(null, { status: 202 })); + + await sendProductAnalyticsRows({ + host: "https://api.tinybird.co", + token: "append-token", + rows, + fetchImpl, + }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it("does not retry a permanent response", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response("invalid", { status: 400 })); + + await expect( + sendProductAnalyticsRows({ + host: "https://api.tinybird.co", + token: "append-token", + rows, + fetchImpl, + }), + ).rejects.toMatchObject({ + _tag: "ProductAnalyticsError", + retryable: false, + status: 400, + } satisfies Partial); + expect(fetchImpl).toHaveBeenCalledOnce(); + }); + + it("supports a single-attempt collector path", async () => { + const fetchImpl = vi + .fn() + .mockRejectedValue(new Error("offline")); + await expect( + sendProductAnalyticsRows({ + host: "https://api.tinybird.co", + token: "append-token", + rows, + maxAttempts: 1, + fetchImpl, + }), + ).rejects.toMatchObject({ retryable: true }); + expect(fetchImpl).toHaveBeenCalledOnce(); + }); + + it("fails after two network attempts", async () => { + const fetchImpl = vi + .fn() + .mockRejectedValue(new Error("offline")); + + await expect( + sendProductAnalyticsRows({ + host: "https://api.tinybird.co", + token: "append-token", + rows, + fetchImpl, + }), + ).rejects.toMatchObject({ + _tag: "ProductAnalyticsError", + retryable: true, + }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts b/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts new file mode 100644 index 00000000000..c0e530ffede --- /dev/null +++ b/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts @@ -0,0 +1,246 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + product: vi.fn(), + posthog: vi.fn(), + constructEvent: vi.fn(), + retrieveCustomer: vi.fn(), + retrieveSubscription: vi.fn(), +})); + +const dbChain = { + select: vi.fn(), + from: vi.fn(), + where: vi.fn(), + limit: vi.fn(), + update: vi.fn(), + set: vi.fn(), +}; + +vi.mock("@/lib/analytics/server", () => ({ + scheduleLegacyPostHogEvent: mocks.posthog, + scheduleServerProductEvent: mocks.product, +})); +vi.mock("@/lib/developer-credits", () => ({ addCreditsToAccount: vi.fn() })); +vi.mock("@cap/database", () => ({ db: () => dbChain })); +vi.mock("@cap/database/helpers", () => ({ nanoId: () => "new-user" })); +vi.mock("@cap/database/schema", () => ({ + developerCreditTransactions: {}, + users: { + id: "id", + email: "email", + }, +})); +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ STRIPE_WEBHOOK_SECRET: "whsec_test" }), +})); +vi.mock("@cap/utils", () => ({ + stripe: () => ({ + webhooks: { constructEvent: mocks.constructEvent }, + customers: { + retrieve: mocks.retrieveCustomer, + update: vi.fn(), + }, + subscriptions: { + retrieve: mocks.retrieveSubscription, + list: vi.fn(), + }, + }), +})); +vi.mock("@cap/web-domain", () => ({ + Organisation: { OrganisationId: { make: (value: string) => value } }, + User: { UserId: { make: (value: string) => value } }, +})); +vi.mock("drizzle-orm", () => ({ + and: (...args: unknown[]) => args, + eq: (left: unknown, right: unknown) => ({ left, right }), +})); + +const dbUser = { + id: "user-1", + email: "user@example.com", + activeOrganizationId: "org-1", + stripeSubscriptionId: null, + name: "User", +}; + +const customer = { + id: "cus_1", + deleted: false, + email: "user@example.com", + metadata: { userId: "user-1" }, +}; + +const subscription = { + id: "sub_1", + status: "active", + items: { + data: [ + { + quantity: 3, + price: { + id: "price_team", + unit_amount: 900, + recurring: { interval: "month", interval_count: 1 }, + }, + }, + ], + }, +}; + +function session(overrides: Record = {}) { + return { + id: "cs_1", + customer: "cus_1", + subscription: "sub_1", + payment_status: "paid", + amount_total: 2700, + amount_subtotal: 3000, + currency: "usd", + total_details: { amount_discount: 300 }, + metadata: { + platform: "web", + analyticsAnonymousId: "anonymous-1", + analyticsIsFirstPurchase: "true", + }, + ...overrides, + }; +} + +function request() { + return new Request("https://cap.so/api/webhooks/stripe", { + method: "POST", + headers: { "Stripe-Signature": "signature" }, + body: "{}", + }); +} + +function event(type: string, checkoutSession: ReturnType) { + return { + id: `evt_${type}`, + created: 1_752_537_600, + type, + data: { object: checkoutSession }, + }; +} + +describe("Stripe subscription analytics", () => { + let POST: typeof import("@/app/api/webhooks/stripe/route").POST; + + beforeEach(async () => { + vi.clearAllMocks(); + dbChain.select.mockReturnValue(dbChain); + dbChain.from.mockReturnValue(dbChain); + dbChain.where.mockReturnValue(dbChain); + dbChain.limit.mockResolvedValue([dbUser]); + dbChain.update.mockReturnValue(dbChain); + dbChain.set.mockReturnValue(dbChain); + mocks.retrieveCustomer.mockResolvedValue(customer); + mocks.retrieveSubscription.mockResolvedValue(subscription); + POST = (await import("@/app/api/webhooks/stripe/route")).POST; + }); + + it("emits a paid purchase with revenue dimensions and deterministic IDs", async () => { + mocks.constructEvent.mockReturnValue( + event("checkout.session.completed", session()), + ); + expect((await POST(request())).status).toBe(200); + + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: "stripe:evt_checkout.session.completed:purchase_completed", + eventName: "purchase_completed", + occurredAt: "2025-07-15T00:00:00.000Z", + anonymousId: "anonymous-1", + userId: "user-1", + organizationId: "org-1", + properties: expect.objectContaining({ + payment_status: "paid", + amount_total_minor: 2700, + currency: "usd", + unit_amount_minor: 900, + billing_interval: "month", + quantity: 3, + }), + }), + ); + expect(mocks.posthog).toHaveBeenCalledWith( + expect.objectContaining({ + properties: expect.objectContaining({ + $insert_id: + "stripe:evt_checkout.session.completed:purchase_completed", + }), + }), + ); + }); + + it("keeps first-purchase attribution stable on duplicate delivery", async () => { + dbChain.limit.mockResolvedValue([ + { ...dbUser, stripeSubscriptionId: "sub_1" }, + ]); + mocks.constructEvent.mockReturnValue( + event("checkout.session.completed", session()), + ); + + expect((await POST(request())).status).toBe(200); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + properties: expect.objectContaining({ + is_first_purchase: true, + }), + }), + ); + }); + + it("does not count an unpaid checkout as a purchase", async () => { + mocks.constructEvent.mockReturnValue( + event( + "checkout.session.completed", + session({ payment_status: "unpaid" }), + ), + ); + expect((await POST(request())).status).toBe(200); + expect(mocks.product).not.toHaveBeenCalled(); + expect(mocks.posthog).not.toHaveBeenCalled(); + }); + + it("emits when an asynchronous subscription payment settles", async () => { + mocks.constructEvent.mockReturnValue( + event("checkout.session.async_payment_succeeded", session()), + ); + expect((await POST(request())).status).toBe(200); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: + "stripe:evt_checkout.session.async_payment_succeeded:purchase_completed", + userId: "user-1", + }), + ); + }); + + it("counts a no-payment trial while exposing its zero revenue", async () => { + mocks.retrieveSubscription.mockResolvedValue({ + ...subscription, + status: "trialing", + }); + mocks.constructEvent.mockReturnValue( + event( + "checkout.session.completed", + session({ + payment_status: "no_payment_required", + amount_total: 0, + }), + ), + ); + expect((await POST(request())).status).toBe(200); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + properties: expect.objectContaining({ + payment_status: "no_payment_required", + amount_total_minor: 0, + subscription_status: "trialing", + }), + }), + ); + }); +}); diff --git a/apps/web/actions/analytics/track-user-signed-up.ts b/apps/web/actions/analytics/track-user-signed-up.ts index 2ffb33c6f0a..ce0af3d7b3c 100644 --- a/apps/web/actions/analytics/track-user-signed-up.ts +++ b/apps/web/actions/analytics/track-user-signed-up.ts @@ -1,9 +1,16 @@ "use server"; +import { PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE } from "@cap/analytics"; import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; import { users } from "@cap/database/schema"; import { sql } from "drizzle-orm"; +import { cookies } from "next/headers"; +import { + captureServerProductEvent, + scheduleAfterResponse, +} from "@/lib/analytics/server"; +import { normalizeServerIdentifier } from "@/lib/analytics/server-event"; const SIGNUP_TRACKING_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; @@ -15,6 +22,7 @@ type UserPreferences = { pauseReactions: boolean; }; trackedEvents?: { + product_user_signed_up?: boolean; user_signed_up?: boolean; }; } | null; @@ -53,10 +61,9 @@ export async function checkAndMarkUserSignedUpTracked(): Promise<{ try { const prefs = currentUser.preferences as UserPreferences; const alreadyTracked = Boolean(prefs?.trackedEvents?.user_signed_up); - - if (alreadyTracked) { - return { shouldTrack: false }; - } + const productAlreadyTracked = Boolean( + prefs?.trackedEvents?.product_user_signed_up, + ); const createdAtTime = getCreatedAtTime(currentUser.created_at); @@ -67,6 +74,41 @@ export async function checkAndMarkUserSignedUpTracked(): Promise<{ return { shouldTrack: false }; } + if (!productAlreadyTracked) { + const analyticsAnonymousId = normalizeServerIdentifier( + (await cookies()).get(PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE)?.value, + ); + scheduleAfterResponse(async () => { + try { + const captured = await captureServerProductEvent({ + eventId: `signup:${currentUser.id}`, + eventName: "user_signed_up", + occurredAt: new Date(createdAtTime).toISOString(), + anonymousId: analyticsAnonymousId, + platform: "web", + userId: currentUser.id, + organizationId: currentUser.activeOrganizationId, + }); + if (!captured) return; + + await db() + .update(users) + .set({ + preferences: sql`JSON_SET(COALESCE(${users.preferences}, JSON_OBJECT()), '$.trackedEvents.product_user_signed_up', true)`, + }) + .where( + sql`(${users.id} = ${currentUser.id}) AND JSON_CONTAINS(COALESCE(${users.preferences}, JSON_OBJECT()), CAST(true AS JSON), '$.trackedEvents.product_user_signed_up') = 0`, + ); + } catch (error) { + console.error("Failed to capture user_signed_up", error); + } + }); + } + + if (alreadyTracked) { + return { shouldTrack: false }; + } + const result = await db() .update(users) .set({ diff --git a/apps/web/actions/organization/send-invites.ts b/apps/web/actions/organization/send-invites.ts index ad3642d4e3b..b46f8e5193b 100644 --- a/apps/web/actions/organization/send-invites.ts +++ b/apps/web/actions/organization/send-invites.ts @@ -15,6 +15,7 @@ import { serverEnv } from "@cap/env"; import type { Organisation } from "@cap/web-domain"; import { and, eq, inArray } from "drizzle-orm"; import { revalidatePath } from "next/cache"; +import { scheduleServerProductEvent } from "@/lib/analytics/server"; import { provisionOrganizationInvitee } from "@/lib/organization-provisioning"; import { type AssignableOrganizationRole, @@ -211,6 +212,31 @@ export async function sendOrganizationInvites( } } + const failedInviteIds = new Set(failedInvites.map((invite) => invite.id)); + const successfulInvites = inviteRecords.filter( + (invite) => !failedInviteIds.has(invite.id), + ); + const firstSuccessfulInvite = successfulInvites[0]; + if (firstSuccessfulInvite) { + scheduleServerProductEvent({ + eventId: `organization_invites:${firstSuccessfulInvite.id}`, + eventName: "organization_invite_sent", + platform: "web", + userId: user.id, + organizationId, + properties: { + invite_count: successfulInvites.length, + admin_count: successfulInvites.filter( + (invite) => invite.role === "admin", + ).length, + member_count: successfulInvites.filter( + (invite) => invite.role === "member", + ).length, + delivery: "email", + }, + }); + } + revalidatePath("/dashboard/settings/organization"); return { success: true, failedEmails }; diff --git a/apps/web/actions/organization/update-seat-quantity.ts b/apps/web/actions/organization/update-seat-quantity.ts index 942bbb0bf08..89f365161a1 100644 --- a/apps/web/actions/organization/update-seat-quantity.ts +++ b/apps/web/actions/organization/update-seat-quantity.ts @@ -11,6 +11,7 @@ import { stripe } from "@cap/utils"; import type { Organisation } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; +import { scheduleServerProductEvent } from "@/lib/analytics/server"; import { calculateProSeats } from "@/utils/organization"; async function getOwnerSubscription( @@ -205,6 +206,27 @@ export async function updateSeatQuantity( } revalidatePath("/dashboard/settings/organization"); + const latestInvoice = + typeof updatedSubscription.latest_invoice === "string" + ? updatedSubscription.latest_invoice + : updatedSubscription.latest_invoice?.id; + scheduleServerProductEvent({ + eventId: `seat_quantity:${subscription.id}:${latestInvoice ?? updatedSubscription.current_period_start}:${newQuantity}`, + eventName: "seat_quantity_changed", + platform: "web", + userId: user.id, + organizationId, + properties: { + previous_quantity: currentQuantity, + new_quantity: newQuantity, + quantity_delta: newQuantity - currentQuantity, + direction: isSeatIncrease ? "increase" : "decrease", + price_id: subscriptionItem.price.id, + unit_amount_minor: subscriptionItem.price.unit_amount, + currency: subscriptionItem.price.currency, + billing_interval: subscriptionItem.price.recurring?.interval, + }, + }); return { success: true, newQuantity }; } diff --git a/apps/web/app/Layout/PosthogIdentify.tsx b/apps/web/app/Layout/PosthogIdentify.tsx index 851143c9a4d..ef074fa5f06 100644 --- a/apps/web/app/Layout/PosthogIdentify.tsx +++ b/apps/web/app/Layout/PosthogIdentify.tsx @@ -5,7 +5,7 @@ import { checkAndMarkUserSignedUpTracked } from "@/actions/analytics/track-user- import { identifyUser, initAnonymousUser, - trackEvent, + trackExternalEvent, } from "../utils/analytics"; import { useCurrentUser } from "./AuthContext"; @@ -30,9 +30,8 @@ function Inner() { (async () => { const { shouldTrack } = await checkAndMarkUserSignedUpTracked(); if (shouldTrack) { - trackEvent("user_signed_up"); + trackExternalEvent("user_signed_up"); } - trackEvent("user_signed_in"); })(); } }, [user]); diff --git a/apps/web/app/Layout/PosthogPageView.tsx b/apps/web/app/Layout/PosthogPageView.tsx index 41996db03bb..0838ced327a 100644 --- a/apps/web/app/Layout/PosthogPageView.tsx +++ b/apps/web/app/Layout/PosthogPageView.tsx @@ -1,28 +1,23 @@ -// app/PostHogPageView.tsx "use client"; -import { usePathname, useSearchParams } from "next/navigation"; +import { usePathname } from "next/navigation"; import { usePostHog } from "posthog-js/react"; -import { Suspense, useEffect, useMemo } from "react"; +import { useEffect } from "react"; +import { shouldCaptureProductPageView } from "../utils/product-analytics"; let lastTrackedUrl: string | null = null; function PostHogPageView(): null { const pathname = usePathname(); - const searchParams = useSearchParams(); const posthog = usePostHog(); - const search = useMemo(() => searchParams?.toString() ?? "", [searchParams]); useEffect(() => { - if (!pathname || !posthog) { + if (!pathname || !posthog || !shouldCaptureProductPageView(pathname)) { return; } try { - let url = window.location.origin + pathname; - if (search) { - url = `${url}?${search}`; - } + const url = window.location.origin + pathname; if (lastTrackedUrl === url) { return; @@ -33,18 +28,9 @@ function PostHogPageView(): null { } catch (error) { console.error("Error capturing pageview:", error); } - }, [pathname, search, posthog]); + }, [pathname, posthog]); return null; } -// Wrap this in Suspense to avoid the `useSearchParams` usage above -// from de-opting the whole app into client-side rendering -// See: https://nextjs.org/docs/messages/deopted-into-client-rendering -export default function SuspendedPostHogPageView() { - return ( - - - - ); -} +export default PostHogPageView; diff --git a/apps/web/app/Layout/ProductAnalyticsPageView.tsx b/apps/web/app/Layout/ProductAnalyticsPageView.tsx new file mode 100644 index 00000000000..5bccfd96b9d --- /dev/null +++ b/apps/web/app/Layout/ProductAnalyticsPageView.tsx @@ -0,0 +1,29 @@ +"use client"; + +import { usePathname } from "next/navigation"; +import { useEffect } from "react"; +import { + captureProductPageView, + shouldCaptureProductPageView, +} from "../utils/product-analytics"; + +let lastCapturedPathname: string | undefined; + +export function ProductAnalyticsPageView() { + const pathname = usePathname(); + + useEffect(() => { + if ( + !pathname || + pathname === lastCapturedPathname || + !shouldCaptureProductPageView(pathname) + ) { + return; + } + + lastCapturedPathname = pathname; + captureProductPageView(); + }, [pathname]); + + return null; +} diff --git a/apps/web/app/Layout/providers.tsx b/apps/web/app/Layout/providers.tsx index 79ff4a0675e..8076fe1bae8 100644 --- a/apps/web/app/Layout/providers.tsx +++ b/apps/web/app/Layout/providers.tsx @@ -44,17 +44,15 @@ export function PostHogProvider({ if (!host) return undefined; const base: CapPostHogConfig = { api_host: host, + autocapture: false, capture_pageview: false, - capture_pageleave: true, + capture_pageleave: false, + disable_session_recording: true, bootstrap: initialBootstrap.current?.distinctID ? initialBootstrap.current : undefined, }; - if (process.env.NEXT_PUBLIC_POSTHOG_DISABLE_SESSION_RECORDING === "true") { - base.disable_session_recording = true; - } - return base; }, [host]); diff --git a/apps/web/app/api/desktop/[...route]/root.ts b/apps/web/app/api/desktop/[...route]/root.ts index 694119771b9..b6f4e798242 100644 --- a/apps/web/app/api/desktop/[...route]/root.ts +++ b/apps/web/app/api/desktop/[...route]/root.ts @@ -8,7 +8,7 @@ import { organizations, users, } from "@cap/database/schema"; -import { buildEnv, serverEnv } from "@cap/env"; +import { serverEnv } from "@cap/env"; import { stripe, userIsPro } from "@cap/utils"; import { OrganizationBrandingPatchBody } from "@cap/web-api-contract"; import { ImageUploads } from "@cap/web-backend"; @@ -17,9 +17,12 @@ import { zValidator } from "@hono/zod-validator"; import { and, eq, isNull } from "drizzle-orm"; import { Effect, Option } from "effect"; import { type Context, Hono } from "hono"; -import { PostHog } from "posthog-node"; import type Stripe from "stripe"; import { z } from "zod"; +import { + scheduleLegacyPostHogEvent, + scheduleServerProductEvent, +} from "@/lib/analytics/server"; import { runPromise } from "@/lib/server"; import { withAuth, withOptionalAuth } from "../../utils"; import { @@ -756,31 +759,37 @@ app.post( success_url: `${serverEnv().WEB_URL}/dashboard/caps?upgrade=true`, cancel_url: `${serverEnv().WEB_URL}/pricing`, allow_promotion_codes: true, - metadata: { platform: "desktop", dubCustomerId: user.id }, + metadata: { + platform: "desktop", + dubCustomerId: user.id, + analyticsIsFirstPurchase: user.stripeSubscriptionId ? "false" : "true", + }, }); if (checkoutSession.url) { console.log("[POST] Checkout session created successfully"); + scheduleServerProductEvent({ + eventId: `checkout:${checkoutSession.id}`, + eventName: "checkout_started", + platform: "desktop", + userId: user.id, + organizationId: user.activeOrganizationId, + properties: { + price_id: priceId, + quantity: 1, + }, + }); - try { - const ph = new PostHog(buildEnv.NEXT_PUBLIC_POSTHOG_KEY || "", { - host: buildEnv.NEXT_PUBLIC_POSTHOG_HOST || "", - }); - - ph.capture({ - distinctId: user.id, - event: "checkout_started", - properties: { - price_id: priceId, - quantity: 1, - platform: "desktop", - }, - }); - - await ph.shutdown(); - } catch (e) { - console.error("Failed to capture checkout_started in PostHog", e); - } + scheduleLegacyPostHogEvent({ + distinctId: user.id, + eventName: "checkout_started", + properties: { + $insert_id: `checkout:${checkoutSession.id}`, + price_id: priceId, + quantity: 1, + platform: "desktop", + }, + }); return c.json({ url: checkoutSession.url }); } diff --git a/apps/web/app/api/events/route.ts b/apps/web/app/api/events/route.ts new file mode 100644 index 00000000000..394a40fbae5 --- /dev/null +++ b/apps/web/app/api/events/route.ts @@ -0,0 +1,152 @@ +import { + createProductEventRows, + PRODUCT_ANALYTICS_LIMITS, +} from "@cap/analytics"; +import { + ProductAnalytics, + resolveProductAnalyticsActor, +} from "@cap/web-backend"; +import { + HttpApi, + HttpApiBuilder, + HttpApiEndpoint, + HttpApiError, + HttpApiGroup, + HttpApiSchema, + HttpServerRequest, +} from "@effect/platform"; +import { Effect, Layer, Schema } from "effect"; +import { + getProductAnalyticsRateLimitKey, + isTrustedAnalyticsRequest, + normalizeGeoHeader, + normalizeProductEventBatch, + ProductAnalyticsRateLimiter, +} from "@/lib/analytics/request"; +import { isRateLimited, RATE_LIMIT_IDS } from "@/lib/rate-limit"; +import { apiToHandler } from "@/lib/server"; +import { allowedOrigins } from "@/utils/cors"; + +class RateLimited extends Schema.TaggedError()( + "RateLimited", + {}, + HttpApiSchema.annotations({ status: 429 }), +) {} + +class Api extends HttpApi.make("ProductAnalyticsApi").add( + HttpApiGroup.make("events").add( + HttpApiEndpoint.post("capture", "/api/events") + .setPayload( + Schema.Struct({ + events: Schema.Array(Schema.Unknown).pipe( + Schema.minItems(1), + Schema.maxItems(PRODUCT_ANALYTICS_LIMITS.batchSize), + ), + }), + ) + .addSuccess(Schema.Struct({ accepted: Schema.Number })) + .addError(HttpApiError.BadRequest) + .addError(HttpApiError.ServiceUnavailable) + .addError(RateLimited), + ), +) {} + +const RequestHeaders = Schema.Struct({ + "content-length": Schema.optional(Schema.String), + "sec-fetch-site": Schema.optional(Schema.String), + origin: Schema.optional(Schema.String), + "x-vercel-ip-country": Schema.optional(Schema.String), + "x-vercel-ip-country-region": Schema.optional(Schema.String), + "x-vercel-ip-city": Schema.optional(Schema.String), + "x-forwarded-for": Schema.optional(Schema.String), + "x-real-ip": Schema.optional(Schema.String), +}); + +const fallbackRateLimiter = new ProductAnalyticsRateLimiter(); + +const ApiLive = HttpApiBuilder.api(Api).pipe( + Layer.provide( + HttpApiBuilder.group(Api, "events", (handlers) => + Effect.gen(function* () { + const analytics = yield* ProductAnalytics; + + return handlers.handle("capture", ({ payload }) => + Effect.gen(function* () { + const headers = yield* HttpServerRequest.schemaHeaders( + RequestHeaders, + ).pipe(Effect.mapError(() => new HttpApiError.BadRequest())); + if ( + !isTrustedAnalyticsRequest( + { + contentLength: headers["content-length"], + origin: headers.origin, + secFetchSite: headers["sec-fetch-site"], + }, + allowedOrigins, + ) + ) { + return yield* Effect.fail(new HttpApiError.BadRequest()); + } + + if ( + fallbackRateLimiter.isRateLimited( + getProductAnalyticsRateLimitKey({ + xForwardedFor: headers["x-forwarded-for"], + xRealIp: headers["x-real-ip"], + }), + ) + ) { + return yield* Effect.fail(new RateLimited()); + } + + if ( + yield* Effect.promise(() => + isRateLimited(RATE_LIMIT_IDS.PRODUCT_ANALYTICS_EVENTS), + ) + ) { + return yield* Effect.fail(new RateLimited()); + } + + const events = normalizeProductEventBatch(payload.events); + if (!events) { + return yield* Effect.fail(new HttpApiError.BadRequest()); + } + + const actor = yield* resolveProductAnalyticsActor; + const rows = createProductEventRows(events, { + receivedAt: new Date().toISOString(), + source: "client", + userId: actor?.userId, + organizationId: actor?.organizationId, + country: normalizeGeoHeader(headers["x-vercel-ip-country"]), + region: normalizeGeoHeader(headers["x-vercel-ip-country-region"]), + city: normalizeGeoHeader(headers["x-vercel-ip-city"], true), + }); + + yield* analytics + .append(rows) + .pipe( + Effect.catchTag("ProductAnalyticsError", (error) => + Effect.logWarning( + "Product analytics ingestion failed", + error, + ).pipe( + Effect.andThen( + Effect.fail(new HttpApiError.ServiceUnavailable()), + ), + ), + ), + ); + + return { accepted: rows.length }; + }), + ); + }), + ), + ), +); + +const handler = apiToHandler(ApiLive); + +export const POST = handler; +export const OPTIONS = handler; diff --git a/apps/web/app/api/invite/accept/route.ts b/apps/web/app/api/invite/accept/route.ts index 4f99e2022a8..1352bd25390 100644 --- a/apps/web/app/api/invite/accept/route.ts +++ b/apps/web/app/api/invite/accept/route.ts @@ -9,6 +9,10 @@ import { } from "@cap/database/schema"; import { and, eq } from "drizzle-orm"; import { type NextRequest, NextResponse } from "next/server"; +import { + readAnalyticsAnonymousId, + scheduleServerProductEvent, +} from "@/lib/analytics/server"; import { normalizeAssignableOrganizationRole } from "@/lib/permissions/roles"; import { calculateProSeats } from "@/utils/organization"; @@ -33,6 +37,10 @@ export async function POST(request: NextRequest) { } try { + let joinedMemberId: string | undefined; + let joinedOrganizationId: string | undefined; + let joinedRole: string | undefined; + let assignedProSeat = false; await db().transaction(async (tx) => { const [invite] = await tx .select() @@ -72,6 +80,9 @@ export async function POST(request: NextRequest) { role, }); memberId = newId; + joinedMemberId = newId; + joinedOrganizationId = invite.organizationId; + joinedRole = role; } const [org] = await tx @@ -111,6 +122,7 @@ export async function POST(request: NextRequest) { }); if (proSeatsRemaining > 0) { + assignedProSeat = true; await tx .update(organizationMembers) .set({ hasProSeat: true }) @@ -148,6 +160,21 @@ export async function POST(request: NextRequest) { .where(eq(organizationInvites.id, inviteId)); }); + if (joinedMemberId && joinedOrganizationId) { + scheduleServerProductEvent({ + eventId: `organization_member:${joinedMemberId}:joined`, + eventName: "organization_member_joined", + anonymousId: readAnalyticsAnonymousId(request), + platform: "web", + userId: user.id, + organizationId: joinedOrganizationId, + properties: { + role: joinedRole, + assigned_pro_seat: assignedProSeat, + }, + }); + } + return NextResponse.json({ success: true }); } catch (error) { if (error instanceof Error) { diff --git a/apps/web/app/api/settings/billing/guest-checkout/route.ts b/apps/web/app/api/settings/billing/guest-checkout/route.ts index a8e5dc63f30..775225e834a 100644 --- a/apps/web/app/api/settings/billing/guest-checkout/route.ts +++ b/apps/web/app/api/settings/billing/guest-checkout/route.ts @@ -1,11 +1,18 @@ -import { buildEnv, serverEnv } from "@cap/env"; +import { randomUUID } from "node:crypto"; +import { serverEnv } from "@cap/env"; import { stripe } from "@cap/utils"; import type { NextRequest } from "next/server"; -import { PostHog } from "posthog-node"; +import { + readAnalyticsAnonymousId, + scheduleLegacyPostHogEvent, + scheduleServerProductEvent, +} from "@/lib/analytics/server"; export async function POST(request: NextRequest) { console.log("Starting guest checkout process"); const { priceId, quantity } = await request.json(); + const analyticsAnonymousId = readAnalyticsAnonymousId(request); + const checkoutAnonymousId = analyticsAnonymousId ?? `guest:${randomUUID()}`; console.log("Received guest checkout request:", { priceId, quantity }); @@ -25,32 +32,35 @@ export async function POST(request: NextRequest) { metadata: { platform: "web", guestCheckout: "true", + analyticsIsFirstPurchase: "true", + analyticsAnonymousId: checkoutAnonymousId, }, }); if (checkoutSession.url) { console.log("Successfully created guest checkout session"); + scheduleServerProductEvent({ + eventId: `checkout:${checkoutSession.id}`, + eventName: "guest_checkout_started", + anonymousId: checkoutAnonymousId, + platform: "web", + properties: { + price_id: priceId, + quantity: quantity || 1, + }, + }); - try { - const ph = new PostHog(buildEnv.NEXT_PUBLIC_POSTHOG_KEY || "", { - host: buildEnv.NEXT_PUBLIC_POSTHOG_HOST || "", - }); - - ph.capture({ - distinctId: `guest-${checkoutSession.id}`, - event: "guest_checkout_started", - properties: { - price_id: priceId, - quantity: quantity || 1, - platform: "web", - session_id: checkoutSession.id, - }, - }); - - await ph.shutdown(); - } catch (e) { - console.error("Failed to capture guest_checkout_started in PostHog", e); - } + scheduleLegacyPostHogEvent({ + distinctId: checkoutAnonymousId, + eventName: "guest_checkout_started", + properties: { + $insert_id: `checkout:${checkoutSession.id}`, + price_id: priceId, + quantity: quantity || 1, + platform: "web", + session_id: checkoutSession.id, + }, + }); return Response.json({ url: checkoutSession.url }, { status: 200 }); } diff --git a/apps/web/app/api/settings/billing/subscribe/route.ts b/apps/web/app/api/settings/billing/subscribe/route.ts index ebb9b816bad..c702471c621 100644 --- a/apps/web/app/api/settings/billing/subscribe/route.ts +++ b/apps/web/app/api/settings/billing/subscribe/route.ts @@ -1,17 +1,22 @@ import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; import { users } from "@cap/database/schema"; -import { buildEnv, serverEnv } from "@cap/env"; +import { serverEnv } from "@cap/env"; import { stripe, userIsPro } from "@cap/utils"; import { eq } from "drizzle-orm"; import type { NextRequest } from "next/server"; -import { PostHog } from "posthog-node"; import type Stripe from "stripe"; +import { + readAnalyticsAnonymousId, + scheduleLegacyPostHogEvent, + scheduleServerProductEvent, +} from "@/lib/analytics/server"; export async function POST(request: NextRequest) { const user = await getCurrentUser(); let customerId = user?.stripeCustomerId; const { priceId, quantity, isOnBoarding } = await request.json(); + const analyticsAnonymousId = readAnalyticsAnonymousId(request); if (!priceId) { console.error("Price ID not found"); @@ -78,29 +83,36 @@ export async function POST(request: NextRequest) { platform: "web", dubCustomerId: user.id, isOnBoarding: isOnBoarding ? "true" : "false", + analyticsIsFirstPurchase: user.stripeSubscriptionId ? "false" : "true", + ...(analyticsAnonymousId ? { analyticsAnonymousId } : {}), }, }); if (checkoutSession.url) { - try { - const ph = new PostHog(buildEnv.NEXT_PUBLIC_POSTHOG_KEY || "", { - host: buildEnv.NEXT_PUBLIC_POSTHOG_HOST || "", - }); - - ph.capture({ - distinctId: user.id, - event: "checkout_started", - properties: { - price_id: priceId, - quantity: quantity, - platform: "web", - }, - }); + scheduleServerProductEvent({ + eventId: `checkout:${checkoutSession.id}`, + eventName: "checkout_started", + anonymousId: analyticsAnonymousId, + platform: "web", + userId: user.id, + organizationId: user.activeOrganizationId, + properties: { + price_id: priceId, + quantity: quantity ?? 1, + is_onboarding: Boolean(isOnBoarding), + }, + }); - await ph.shutdown(); - } catch (e) { - console.error("Failed to capture checkout_started in PostHog", e); - } + scheduleLegacyPostHogEvent({ + distinctId: user.id, + eventName: "checkout_started", + properties: { + $insert_id: `checkout:${checkoutSession.id}`, + price_id: priceId, + quantity, + platform: "web", + }, + }); return Response.json({ url: checkoutSession.url }, { status: 200 }); } diff --git a/apps/web/app/api/webhooks/stripe/route.ts b/apps/web/app/api/webhooks/stripe/route.ts index e0f3dff4f06..ce7ee372ada 100644 --- a/apps/web/app/api/webhooks/stripe/route.ts +++ b/apps/web/app/api/webhooks/stripe/route.ts @@ -1,13 +1,16 @@ import { db } from "@cap/database"; import { nanoId } from "@cap/database/helpers"; import { developerCreditTransactions, users } from "@cap/database/schema"; -import { buildEnv, serverEnv } from "@cap/env"; +import { serverEnv } from "@cap/env"; import { stripe } from "@cap/utils"; import { Organisation, User } from "@cap/web-domain"; import { and, eq } from "drizzle-orm"; import { NextResponse } from "next/server"; -import { PostHog } from "posthog-node"; import type Stripe from "stripe"; +import { + scheduleLegacyPostHogEvent, + scheduleServerProductEvent, +} from "@/lib/analytics/server"; import { addCreditsToAccount } from "@/lib/developer-credits"; const relevantEvents = new Set([ @@ -17,6 +20,98 @@ const relevantEvents = new Set([ "customer.subscription.deleted", ]); +type PurchaseAnalyticsUser = Pick< + typeof users.$inferSelect, + "id" | "activeOrganizationId" +>; + +function isSettledSubscriptionPurchase( + session: Stripe.Checkout.Session, + subscription: Stripe.Subscription, +) { + return ( + (session.payment_status === "paid" || + session.payment_status === "no_payment_required") && + (subscription.status === "active" || subscription.status === "trialing") + ); +} + +function scheduleSubscriptionPurchaseEvents({ + eventId, + occurredAt, + session, + subscription, + inviteQuota, + user, + isFirstPurchase, +}: { + eventId: string; + occurredAt: string; + session: Stripe.Checkout.Session; + subscription: Stripe.Subscription; + inviteQuota: number; + user?: PurchaseAnalyticsUser; + isFirstPurchase: boolean; +}) { + const isGuestCheckout = session.metadata?.guestCheckout === "true"; + const platform = + session.metadata?.platform === "desktop" + ? "desktop" + : session.metadata?.platform === "web" + ? "web" + : "server"; + const anonymousId = session.metadata?.analyticsAnonymousId; + const distinctId = user?.id ?? anonymousId ?? `stripe:${session.id}`; + const price = subscription.items.data[0]?.price; + const revenueProperties = { + payment_status: session.payment_status, + subscription_status: subscription.status, + amount_total_minor: session.amount_total, + amount_subtotal_minor: session.amount_subtotal, + discount_amount_minor: session.total_details?.amount_discount, + currency: session.currency, + unit_amount_minor: price?.unit_amount, + billing_interval: price?.recurring?.interval, + billing_interval_count: price?.recurring?.interval_count, + }; + + scheduleServerProductEvent({ + eventId: `stripe:${eventId}:purchase_completed`, + eventName: "purchase_completed", + occurredAt, + anonymousId, + platform, + userId: user?.id, + organizationId: user?.activeOrganizationId, + properties: { + ...revenueProperties, + invite_quota: inviteQuota, + price_id: price?.id, + quantity: inviteQuota, + is_onboarding: session.metadata?.isOnBoarding === "true", + is_first_purchase: isFirstPurchase, + is_guest_checkout: isGuestCheckout, + }, + }); + + scheduleLegacyPostHogEvent({ + distinctId, + eventName: "purchase_completed", + properties: { + $insert_id: `stripe:${eventId}:purchase_completed`, + subscription_id: subscription.id, + ...revenueProperties, + invite_quota: inviteQuota, + price_id: price?.id, + quantity: inviteQuota, + is_onboarding: session.metadata?.isOnBoarding === "true", + platform: platform === "server" ? "unknown" : platform, + is_first_purchase: isFirstPurchase, + is_guest_checkout: isGuestCheckout, + }, + }); +} + async function grantDeveloperCredits( session: Stripe.Checkout.Session, ): Promise { @@ -329,39 +424,17 @@ export const POST = async (req: Request) => { console.log("Successfully updated user in database"); - try { - const serverPostHog = new PostHog( - buildEnv.NEXT_PUBLIC_POSTHOG_KEY || "", - { host: buildEnv.NEXT_PUBLIC_POSTHOG_HOST || "" }, - ); - - const isFirstPurchase = !dbUser.stripeSubscriptionId; - const isGuestCheckout = session.metadata?.guestCheckout === "true"; - serverPostHog.capture({ - distinctId: dbUser.id, - event: "purchase_completed", - properties: { - subscription_id: subscription.id, - subscription_status: subscription.status, - invite_quota: inviteQuota, - price_id: subscription.items.data[0]?.price.id, - quantity: inviteQuota, - is_onboarding: session.metadata?.isOnBoarding === "true", - platform: - session.metadata?.platform === "desktop" - ? "desktop" - : session.metadata?.platform === "web" - ? "web" - : "unknown", - is_first_purchase: isFirstPurchase, - is_guest_checkout: isGuestCheckout, - }, + if (isSettledSubscriptionPurchase(session, subscription)) { + scheduleSubscriptionPurchaseEvents({ + eventId: event.id, + occurredAt: new Date(event.created * 1000).toISOString(), + session, + subscription, + inviteQuota, + user: dbUser, + isFirstPurchase: + session.metadata?.analyticsIsFirstPurchase === "true", }); - - await serverPostHog.shutdown(); - console.log("Successfully tracked purchase event in PostHog"); - } catch (error) { - console.error("Error tracking purchase in PostHog:", error); } } @@ -374,6 +447,45 @@ export const POST = async (req: Request) => { if (session.metadata?.type === "developer_credits") { return await grantDeveloperCredits(session); } + + if (typeof session.subscription === "string") { + const subscription = await stripe().subscriptions.retrieve( + session.subscription, + ); + if (isSettledSubscriptionPurchase(session, subscription)) { + const inviteQuota = subscription.items.data.reduce( + (total, item) => total + (item.quantity || 1), + 0, + ); + let dbUser: typeof users.$inferSelect | null = null; + if (typeof session.customer === "string") { + const customer = await stripe().customers.retrieve( + session.customer, + ); + if (!customer.deleted) { + const userId = customer.metadata.userId + ? User.UserId.make(customer.metadata.userId) + : undefined; + dbUser = await findUserWithRetry( + customer.email ?? "", + userId, + 1, + ); + } + } + + scheduleSubscriptionPurchaseEvents({ + eventId: event.id, + occurredAt: new Date(event.created * 1000).toISOString(), + session, + subscription, + inviteQuota, + ...(dbUser ? { user: dbUser } : {}), + isFirstPurchase: + session.metadata?.analyticsIsFirstPurchase === "true", + }); + } + } } if (event.type === "customer.subscription.updated") { diff --git a/apps/web/app/layout.tsx b/apps/web/app/layout.tsx index f8c3bbcff49..0a446ced264 100644 --- a/apps/web/app/layout.tsx +++ b/apps/web/app/layout.tsx @@ -3,6 +3,7 @@ import type { Metadata } from "next"; import localFont from "next/font/local"; import Script from "next/script"; import type { PropsWithChildren } from "react"; +import { ProductAnalyticsPageView } from "./Layout/ProductAnalyticsPageView"; const defaultFont = localFont({ src: [ @@ -84,6 +85,7 @@ export default function RootLayout({ children }: PropsWithChildren) { ', + "https://preview.vercel.app", + ), + ["https://preview.vercel.app/_next/static/a.js"], + ); +}); + +test("bundle budgets require absolute and live-baseline gates", () => { + assert.deepEqual( + evaluateBundleBudget({ + baselineBytes: 1_000_000, + measuredBytes: 1_040_000, + absoluteMaximumBytes: 5_000_000, + regressionFactor: 1.05, + regressionFloorBytes: 25_000, + }), + { + absoluteMaximumBytes: 5_000_000, + regressionLimitBytes: 1_050_000, + deltaBytes: 40_000, + regressionRatio: 1.04, + passed: true, + }, + ); + assert.equal( + evaluateBundleBudget({ + baselineBytes: 1_000_000, + measuredBytes: 1_200_000, + absoluteMaximumBytes: 5_000_000, + regressionFactor: 1.05, + regressionFloorBytes: 25_000, + }).passed, + false, + ); +}); + +test("latency budgets require both absolute and measured-baseline gates", () => { + const baseline = latencySummary([80, 90, 100, 110, 120]); + assert.deepEqual( + evaluateLatencyBudget({ + baseline, + measured: latencySummary([90, 100, 110, 120, 130]), + absoluteP95Ms: 2_500, + regressionFactor: 2.5, + regressionFloorMs: 250, + }), + { + absoluteP95Ms: 2_500, + regressionLimitMs: 370, + regressionRatio: 130 / 120, + passed: true, + }, + ); + assert.equal( + evaluateLatencyBudget({ + baseline, + measured: latencySummary([500]), + absoluteP95Ms: 2_500, + regressionFactor: 2.5, + regressionFloorMs: 250, + }).passed, + false, + ); +}); + test("CI assertion normalization proves decision deduplication and conflict quarantine", () => { const assertions = normalizeCiAssertions({ data: [ From 178e2f83f04f9cb65c66f29ec1c2d8eec1f0f69a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:13:43 +0100 Subject: [PATCH 046/110] test: type analytics performance scheduler --- apps/web/__tests__/unit/product-analytics-queue.test.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/apps/web/__tests__/unit/product-analytics-queue.test.ts b/apps/web/__tests__/unit/product-analytics-queue.test.ts index c2b889ae5cd..133dc9777f1 100644 --- a/apps/web/__tests__/unit/product-analytics-queue.test.ts +++ b/apps/web/__tests__/unit/product-analytics-queue.test.ts @@ -210,7 +210,10 @@ describe("ProductAnalyticsQueue", () => { for (let sample = 0; sample < 10; sample += 1) { const queue = new ProductAnalyticsQueue( () => new Promise(() => {}), - () => 0 as unknown as ReturnType, + (() => + 0 as unknown as ReturnType< + typeof setTimeout + >) as unknown as typeof setTimeout, () => {}, ); const startedAt = performance.now(); From 6b2a9790736184fb91b8e0185bb1b4148e9fcc22 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Fri, 31 Jul 2026 20:15:54 +0100 Subject: [PATCH 047/110] chore: format analytics recovery tests --- apps/mobile/src/analytics/product-analytics-storage.test.ts | 3 ++- scripts/analytics/tests/tooling.test.js | 4 +--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/mobile/src/analytics/product-analytics-storage.test.ts b/apps/mobile/src/analytics/product-analytics-storage.test.ts index f350eebf96b..68be0361d31 100644 --- a/apps/mobile/src/analytics/product-analytics-storage.test.ts +++ b/apps/mobile/src/analytics/product-analytics-storage.test.ts @@ -39,7 +39,8 @@ const createHarness = () => { }), getInfoAsync: vi.fn(async (uri: string) => ({ exists: files.has(uri) })), moveAsync: vi.fn(async ({ from, to }: { from: string; to: string }) => { - if (failFinalMove && from.endsWith(".next")) throw new Error("interrupted"); + if (failFinalMove && from.endsWith(".next")) + throw new Error("interrupted"); const contents = files.get(from); if (contents === undefined) throw new Error("missing"); files.set(to, contents); diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index 2a76115a7f9..58a99f59f4b 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -126,9 +126,7 @@ test("local fixtures use bounded current dates", () => { assert.match(fixture, /2026-07-31/); assert.ok(rows.every((row) => row.event_id.endsWith("_20260731"))); assert.ok(rows.every((row) => /^[0-9a-f]{32}$/.test(row.payload_hash))); - const engagement = rows.find( - (row) => row.event_name === "page_engagement", - ); + const engagement = rows.find((row) => row.event_name === "page_engagement"); assert.match(JSON.parse(engagement.properties).page_view_id, /_20260731$/); }); From 298f9d9c7dbf296a239f165291a961f15f6a95f1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:32:07 +0100 Subject: [PATCH 048/110] test: isolate Tinybird local fixtures --- scripts/analytics/tests/tooling.test.js | 39 ++++++++++++++++++++----- scripts/analytics/tooling.js | 37 ++++++++++++++++++++--- 2 files changed, 65 insertions(+), 11 deletions(-) diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index 58a99f59f4b..bdcdc063f05 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -59,7 +59,8 @@ test("cloud deploy checks before deploying and waits for completion", () => { }); test("local setup builds, verifies copied endpoints and writes its deterministic environment", () => { - const commands = operationPlan("local") + const steps = operationPlan("local"); + const commands = steps .filter((step) => step.command) .map((step) => step.args.join(" ")); assert.ok(commands.some((command) => command.endsWith("--local build"))); @@ -70,15 +71,20 @@ test("local setup builds, verifies copied endpoints and writes its deterministic ), ), ); - assert.ok( - operationPlan("local").some((step) => step.type === "verify-local"), - ); - assert.ok( - operationPlan("local").some((step) => step.type === "write-local-env"), + const appendIndex = steps.findIndex((step) => + step.args?.join(" ").includes("datasource append product_events_v1"), ); + const pauseIndexes = steps + .map((step, index) => ({ index, command: step.args?.join(" ") ?? "" })) + .filter(({ command }) => command.includes("--local copy pause")) + .map(({ index }) => index); + assert.equal(pauseIndexes.length, 7); + assert.ok(pauseIndexes.every((index) => index < appendIndex)); + assert.ok(steps.some((step) => step.type === "verify-local")); + assert.ok(steps.some((step) => step.type === "write-local-env")); assert.ok( commands.some((command) => - command.includes("up -d --wait --wait-timeout 60 tinybird-local"), + command.includes("up -d --wait --wait-timeout 120 tinybird-local"), ), ); const first = localEnvironment({}); @@ -125,6 +131,25 @@ test("local fixtures use bounded current dates", () => { assert.doesNotMatch(fixture, /2099-01-/); assert.match(fixture, /2026-07-31/); assert.ok(rows.every((row) => row.event_id.endsWith("_20260731"))); + assert.ok( + rows.every( + (row) => !row.anonymous_id || row.anonymous_id.endsWith("_20260731"), + ), + ); + assert.ok( + rows.every( + (row) => !row.session_id || row.session_id.endsWith("_20260731"), + ), + ); + assert.ok( + rows.every((row) => !row.user_id || row.user_id.endsWith("_20260731")), + ); + assert.ok( + rows.every( + (row) => + !row.organization_id || row.organization_id.endsWith("_20260731"), + ), + ); assert.ok(rows.every((row) => /^[0-9a-f]{32}$/.test(row.payload_hash))); const engagement = rows.find((row) => row.event_name === "page_engagement"); assert.match(JSON.parse(engagement.properties).page_view_id, /_20260731$/); diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index b1cdf0c3a15..fa1ec5c2f8f 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -153,12 +153,15 @@ const operationPlan = (operation) => { "-d", "--wait", "--wait-timeout", - "60", + "120", "tinybird-local", ), localAuth: true, }, localCliStep("--local", "build"), + ...PRODUCT_COPY_PIPES.map((name) => + localCliStep("--local", "copy", "pause", name), + ), localCliStep( "--local", "datasource", @@ -186,11 +189,14 @@ const operationPlan = (operation) => { "-d", "--wait", "--wait-timeout", - "60", + "120", "tinybird-local", ), localAuth: true, }, + ...PRODUCT_COPY_PIPES.map((name) => + localCliStep("--local", "copy", "pause", name), + ), localCliStep( "--local", "datasource", @@ -730,6 +736,20 @@ const prepareLocalFixture = (now = new Date()) => { `${row.event_id}_${fixtureSuffix}`, ]), ); + const identityFields = [ + "anonymous_id", + "session_id", + "user_id", + "organization_id", + ]; + const identityIds = new Map( + templateRows.flatMap((row) => + identityFields + .map((field) => row[field]) + .filter(Boolean) + .map((value) => [value, `${value}_${fixtureSuffix}`]), + ), + ); const fixtureRows = templateRows.map((templateRow) => { const row = structuredClone(templateRow); for (const [template, replacement] of Object.entries(dates)) { @@ -737,9 +757,18 @@ const prepareLocalFixture = (now = new Date()) => { row.received_at = row.received_at.replace(template, replacement); } row.event_id = eventIds.get(row.event_id); + for (const field of identityFields) { + if (identityIds.has(row[field])) { + row[field] = identityIds.get(row[field]); + } + } const properties = JSON.parse(row.properties); - if (eventIds.has(properties.page_view_id)) { - properties.page_view_id = eventIds.get(properties.page_view_id); + for (const [key, value] of Object.entries(properties)) { + if (typeof value === "string" && eventIds.has(value)) { + properties[key] = eventIds.get(value); + } else if (typeof value === "string" && identityIds.has(value)) { + properties[key] = identityIds.get(value); + } } row.properties = JSON.stringify(properties); const hashPayload = { ...row }; From 42e7fef4efdbcf084a7e89a35bcc3eb04a0e69a3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:56:55 +0100 Subject: [PATCH 049/110] fix: preserve analytics delivery recovery --- .../src-tauri/src/product_analytics.rs | 108 ++++++++++++++++-- .../unit/product-analytics-request.test.ts | 9 +- apps/web/app/api/analytics/track/route.ts | 6 + apps/web/app/api/events/route.ts | 3 + apps/web/lib/analytics/request.ts | 10 +- 5 files changed, 118 insertions(+), 18 deletions(-) diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index 30f01b93f1d..7696cf6b641 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -426,19 +426,12 @@ fn should_retry_product_status(status: u16) -> bool { } fn fallback_outbox_encryption_key(app: &AppHandle) -> Result<[u8; 32], String> { + if let Some(existing) = existing_fallback_outbox_encryption_key(app)? { + return Ok(existing); + } let store = app .store("store") .map_err(|_| "fallback_key_store_unavailable".to_string())?; - if let Some(encoded) = store - .get(PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY) - .and_then(|value| value.as_str().map(str::to_owned)) - { - return BASE64 - .decode(encoded) - .map_err(|_| "invalid_fallback_key".to_string())? - .try_into() - .map_err(|_| "invalid_fallback_key".to_string()); - } let mut generated = [0_u8; 32]; SystemRandom::new() .fill(&mut generated) @@ -453,6 +446,24 @@ fn fallback_outbox_encryption_key(app: &AppHandle) -> Result<[u8; 32], String> { Ok(generated) } +fn existing_fallback_outbox_encryption_key(app: &AppHandle) -> Result, String> { + let store = app + .store("store") + .map_err(|_| "fallback_key_store_unavailable".to_string())?; + let Some(encoded) = store + .get(PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY) + .and_then(|value| value.as_str().map(str::to_owned)) + else { + return Ok(None); + }; + let key = BASE64 + .decode(encoded) + .map_err(|_| "invalid_fallback_key".to_string())? + .try_into() + .map_err(|_| "invalid_fallback_key".to_string())?; + Ok(Some(key)) +} + fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { if let Some(key) = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.get() { return Ok(key); @@ -521,7 +532,45 @@ fn encrypt_outbox_with_key( } fn decrypt_outbox(app: &AppHandle, value: &str) -> Result { - decrypt_outbox_with_key(value, outbox_encryption_key(app)?) + if let Some(key) = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.get() + && let Ok(outbox) = decrypt_outbox_with_key(value, key) + { + return Ok(outbox); + } + + let mut candidates = Vec::new(); + if let Ok(entry) = keyring::Entry::new( + PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE, + PRODUCT_EVENT_OUTBOX_KEYRING_USER, + ) && let Ok(encoded) = entry.get_password() + && let Ok(decoded) = BASE64.decode(encoded) + && let Ok(key) = decoded.try_into() + { + candidates.push(key); + } + if let Some(key) = existing_fallback_outbox_encryption_key(app)? + && !candidates.contains(&key) + { + candidates.push(key); + } + let (outbox, key) = decrypt_outbox_with_keys(value, &candidates)?; + let _ = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.set(key); + Ok(outbox) +} + +fn decrypt_outbox_with_keys( + value: &str, + candidates: &[[u8; 32]], +) -> Result<(ProductEventOutbox, [u8; 32]), String> { + if candidates.is_empty() { + return Err("outbox_key_unavailable".to_string()); + } + for key in candidates { + if let Ok(outbox) = decrypt_outbox_with_key(value, key) { + return Ok((outbox, *key)); + } + } + Err("outbox_decryption_failed".to_string()) } fn decrypt_outbox_with_key( @@ -554,6 +603,9 @@ fn decrypt_outbox_with_key( } fn persist_outbox(app: &AppHandle, outbox: &ProductEventOutbox) -> Result<(), String> { + ensure_outbox_persistence_allowed( + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.load(Ordering::Acquire), + )?; let encrypted = encrypt_outbox(app, outbox)?; let store = app .store("store") @@ -562,6 +614,14 @@ fn persist_outbox(app: &AppHandle, outbox: &ProductEventOutbox) -> Result<(), St store.save().map_err(|_| "store_write_failed".to_string()) } +fn ensure_outbox_persistence_allowed(restore_blocked: bool) -> Result<(), String> { + if restore_blocked { + Err("outbox_restore_blocked".to_string()) + } else { + Ok(()) + } +} + fn load_outbox(app: &AppHandle) -> Result { let store = app .store("store") @@ -750,6 +810,7 @@ pub fn init_product_session(app: &AppHandle) { match load_outbox(app) { Ok(loaded) => { + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(false, Ordering::Release); let has_pending = !loaded.pending.is_empty(); *outbox_guard() = loaded; if has_pending { @@ -757,6 +818,7 @@ pub fn init_product_session(app: &AppHandle) { } } Err(failure_class) => { + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( failure_class, @@ -771,6 +833,7 @@ static PRODUCT_EVENT_SESSION_ID: OnceLock = OnceLock::new(); static PRODUCT_EVENT_SENDER: OnceLock> = OnceLock::new(); static PRODUCT_EVENT_OUTBOX: OnceLock> = OnceLock::new(); static PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY: OnceLock<[u8; 32]> = OnceLock::new(); +static PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED: AtomicBool = AtomicBool::new(false); static PRODUCT_EVENTS_DROPPED: AtomicU64 = AtomicU64::new(0); static PRODUCT_EVENT_RETRIES: AtomicU64 = AtomicU64::new(0); static PRODUCT_EVENT_DEAD_LETTERS: AtomicU64 = AtomicU64::new(0); @@ -984,6 +1047,29 @@ mod tests { assert_eq!(decrypted.pending[0].event_id, event_id); } + #[test] + fn encrypted_outbox_tries_every_persisted_key_without_replacing_it() { + let data = event_data(recording_started()); + let event = product_event(&data, "install-id".to_string()).unwrap(); + let event_id = event.event_id.clone(); + let outbox = ProductEventOutbox { + pending: vec![event], + dead_letters: Vec::new(), + }; + let valid_key = [7_u8; 32]; + let encrypted = encrypt_outbox_with_key(&outbox, &valid_key).unwrap(); + + let (decrypted, selected_key) = + decrypt_outbox_with_keys(&encrypted, &[[8_u8; 32], valid_key]).unwrap(); + + assert_eq!(selected_key, valid_key); + assert_eq!(decrypted.pending[0].event_id, event_id); + assert_eq!( + ensure_outbox_persistence_allowed(true), + Err("outbox_restore_blocked".to_string()) + ); + } + #[test] fn product_status_retry_policy_matches_the_browser() { assert!(!should_retry_product_status(400)); diff --git a/apps/web/__tests__/unit/product-analytics-request.test.ts b/apps/web/__tests__/unit/product-analytics-request.test.ts index f22fdd2b388..3d1b96b0d8e 100644 --- a/apps/web/__tests__/unit/product-analytics-request.test.ts +++ b/apps/web/__tests__/unit/product-analytics-request.test.ts @@ -290,9 +290,12 @@ describe("ProductAnalyticsRateLimiter", () => { fallbackIdentity: "browser-2", }), ); - expect(getProductAnalyticsRateLimitKey({ trustedVercelProxy: true })).toBe( - "vercel-unknown", - ); + expect( + getProductAnalyticsRateLimitKey({ trustedVercelProxy: true }), + ).toBeNull(); + expect( + getProductAnalyticsRateLimitKey({ trustedVercelProxy: false }), + ).toBeNull(); }); }); diff --git a/apps/web/app/api/analytics/track/route.ts b/apps/web/app/api/analytics/track/route.ts index b4cd843d58e..6c4d1fe0d3b 100644 --- a/apps/web/app/api/analytics/track/route.ts +++ b/apps/web/app/api/analytics/track/route.ts @@ -84,6 +84,12 @@ export async function POST(request: NextRequest) { fallbackIdentity: sessionId ?? request.headers.get("user-agent") ?? undefined, }); + if (!rateLimitKey) { + return Response.json( + { error: "Missing request identity" }, + { status: 400 }, + ); + } if ( fallbackRateLimiter.isRateLimited(rateLimitKey) || (await isRateLimited(RATE_LIMIT_IDS.ANALYTICS_TRACK, { diff --git a/apps/web/app/api/events/route.ts b/apps/web/app/api/events/route.ts index f81e785281c..8a302734124 100644 --- a/apps/web/app/api/events/route.ts +++ b/apps/web/app/api/events/route.ts @@ -139,6 +139,9 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( (browserClaims ? browserClaims.anonymousId : undefined) ?? headers.authorization, }); + if (!rateLimitKey) { + return yield* Effect.fail(new HttpApiError.BadRequest()); + } if (fallbackRateLimiter.isRateLimited(rateLimitKey)) { return yield* Effect.fail(new RateLimited()); } diff --git a/apps/web/lib/analytics/request.ts b/apps/web/lib/analytics/request.ts index 810af0025e7..9c07d6fd04d 100644 --- a/apps/web/lib/analytics/request.ts +++ b/apps/web/lib/analytics/request.ts @@ -171,12 +171,14 @@ export function getProductAnalyticsRateLimitKey(headers: { fallbackIdentity?: string; }) { if (!headers.trustedVercelProxy) { - const identity = headers.fallbackIdentity?.trim() || "unknown"; + const identity = headers.fallbackIdentity?.trim(); + if (!identity) return null; return `self-hosted:${createHash("sha256").update(identity).digest("hex")}`; } - return ( - headers.xVercelForwardedFor?.split(",")[0]?.trim() || "vercel-unknown" - ).slice(0, PRODUCT_ANALYTICS_LIMITS.identifierLength); + const identity = headers.xVercelForwardedFor?.split(",")[0]?.trim(); + return identity + ? identity.slice(0, PRODUCT_ANALYTICS_LIMITS.identifierLength) + : null; } export function normalizeGeoHeader(value?: string, decode = false) { From a1895a162ac127ff787bb33f36b68128348ee8ca Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:56:55 +0100 Subject: [PATCH 050/110] ci: retire legacy staging analytics resources --- .github/workflows/analytics.yml | 4 ++-- scripts/analytics/staging-ci-lib.js | 2 +- scripts/analytics/tests/staging-ci.test.js | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 7c39c6b5c91..0f78c56a711 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -133,7 +133,7 @@ jobs: TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: >- docker compose --file packages/local-docker/docker-compose.yml --profile analytics - run --rm tinybird-cloud-cli --cloud deployment create --check + run --rm tinybird-cloud-cli --cloud deployment create --allow-destructive-operations --check - name: Create isolated Tinybird staging deployment id: create-deployment env: @@ -143,7 +143,7 @@ jobs: TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: >- docker compose --file packages/local-docker/docker-compose.yml --profile analytics - run --rm tinybird-cloud-cli --cloud deployment create --wait + run --rm tinybird-cloud-cli --cloud deployment create --allow-destructive-operations --wait - name: Resolve only the deployment created by this run id: tinybird env: diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 29da835fe63..0143737be14 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -499,7 +499,7 @@ export const assertWorkflowSafety = (workflow) => { String(FEATURE_PULL_REQUEST), "cancel-in-progress: true", "pull_request.head.sha", - "deployment create --check", + "deployment create --allow-destructive-operations --check", "deployment promote", "deployment discard", "environment: staging", diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 52b5f742e41..e13bd76c329 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -368,4 +368,10 @@ test("the analytics workflow is statically restricted to staging", () => { "utf8", ); assert.doesNotThrow(() => assertWorkflowSafety(workflow)); + assert.equal( + workflow.match( + /deployment create --allow-destructive-operations --(?:check|wait)/g, + )?.length, + 2, + ); }); From 56d0e8b529c8e700255957f0be95e5e0c392a294 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:22:33 +0100 Subject: [PATCH 051/110] fix: keep analytics recovery durable --- .../src-tauri/src/product_analytics.rs | 158 +++++++++++++++--- apps/web/app/api/analytics/track/route.ts | 3 +- apps/web/app/s/[videoId]/Share.tsx | 20 ++- 3 files changed, 149 insertions(+), 32 deletions(-) diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index 7696cf6b641..fec19bed098 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -30,6 +30,7 @@ const PRODUCT_EVENT_RETRY_DELAY: Duration = Duration::from_millis(500); const PRODUCT_EVENT_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); const PRODUCT_EVENT_SESSION_STORE_KEY: &str = "product_analytics_session_id"; const PRODUCT_EVENT_OUTBOX_STORE_KEY: &str = "product_analytics_outbox_v1"; +const PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY: &str = "product_analytics_outbox_recovery_v1"; const PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY: &str = "product_analytics_outbox_fallback_key_v1"; const PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE: &str = "so.cap.desktop"; @@ -482,7 +483,7 @@ fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { .try_into() .map_err(|_| "invalid_keyring_value".to_string())? } - Err(_) => { + Err(keyring::Error::NoEntry) => { let mut generated = [0_u8; 32]; SystemRandom::new() .fill(&mut generated) @@ -493,6 +494,7 @@ fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { fallback_outbox_encryption_key(app)? } } + Err(_) => fallback_outbox_encryption_key(app)?, }, Err(_) => fallback_outbox_encryption_key(app)?, }; @@ -602,31 +604,32 @@ fn decrypt_outbox_with_key( serde_json::from_slice(plaintext).map_err(|_| "outbox_deserialization_failed".to_string()) } -fn persist_outbox(app: &AppHandle, outbox: &ProductEventOutbox) -> Result<(), String> { - ensure_outbox_persistence_allowed( - PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.load(Ordering::Acquire), - )?; +fn write_outbox( + app: &AppHandle, + store_key: &str, + outbox: &ProductEventOutbox, +) -> Result<(), String> { let encrypted = encrypt_outbox(app, outbox)?; let store = app .store("store") .map_err(|_| "store_unavailable".to_string())?; - store.set(PRODUCT_EVENT_OUTBOX_STORE_KEY, encrypted); + store.set(store_key, encrypted); store.save().map_err(|_| "store_write_failed".to_string()) } -fn ensure_outbox_persistence_allowed(restore_blocked: bool) -> Result<(), String> { - if restore_blocked { - Err("outbox_restore_blocked".to_string()) - } else { - Ok(()) - } +fn delete_stored_outbox(app: &AppHandle, store_key: &str) -> Result<(), String> { + let store = app + .store("store") + .map_err(|_| "store_unavailable".to_string())?; + store.delete(store_key); + store.save().map_err(|_| "store_write_failed".to_string()) } -fn load_outbox(app: &AppHandle) -> Result { +fn load_stored_outbox(app: &AppHandle, store_key: &str) -> Result { let store = app .store("store") .map_err(|_| "store_unavailable".to_string())?; - let Some(value) = store.get(PRODUCT_EVENT_OUTBOX_STORE_KEY) else { + let Some(value) = store.get(store_key) else { return Ok(ProductEventOutbox::default()); }; let Some(encrypted) = value.as_str() else { @@ -635,6 +638,48 @@ fn load_outbox(app: &AppHandle) -> Result { decrypt_outbox(app, encrypted) } +fn merge_outbox(target: &mut ProductEventOutbox, source: ProductEventOutbox) { + let mut known_event_ids = target + .pending + .iter() + .map(|event| event.event_id.clone()) + .chain( + target + .dead_letters + .iter() + .map(|entry| entry.event.event_id.clone()), + ) + .collect::>(); + for event in source.pending { + if known_event_ids.insert(event.event_id.clone()) { + target.pending.push(event); + } + } + for entry in source.dead_letters { + if known_event_ids.insert(entry.event.event_id.clone()) { + target.dead_letters.push(entry); + } + } +} + +fn persist_outbox(app: &AppHandle, outbox: &mut ProductEventOutbox) -> Result<(), String> { + if !PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.load(Ordering::Acquire) { + return write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, outbox); + } + + let Ok(mut restored) = load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY) else { + return write_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY, outbox); + }; + merge_outbox(&mut restored, outbox.clone()); + let recovery = load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY)?; + merge_outbox(&mut restored, recovery); + write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, &restored)?; + delete_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY)?; + *outbox = restored; + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(false, Ordering::Release); + Ok(()) +} + fn outbox_guard() -> std::sync::MutexGuard<'static, ProductEventOutbox> { PRODUCT_EVENT_OUTBOX .get_or_init(|| Mutex::new(ProductEventOutbox::default())) @@ -643,8 +688,8 @@ fn outbox_guard() -> std::sync::MutexGuard<'static, ProductEventOutbox> { } fn persist_current_outbox(app: &AppHandle) { - let outbox = outbox_guard(); - if let Err(failure_class) = persist_outbox(app, &outbox) { + let mut outbox = outbox_guard(); + if let Err(failure_class) = persist_outbox(app, &mut outbox) { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( failure_class, @@ -662,7 +707,7 @@ fn remove_delivered_events(app: &AppHandle, delivered: &[ProductEvent]) { outbox .pending .retain(|event| !delivered_ids.contains(event.event_id.as_str())); - if let Err(failure_class) = persist_outbox(app, &outbox) { + if let Err(failure_class) = persist_outbox(app, &mut outbox) { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( failure_class, @@ -693,7 +738,7 @@ fn move_to_dead_letters(app: &AppHandle, events: &[ProductEvent], error: &Delive }); } PRODUCT_EVENT_DEAD_LETTERS.fetch_add(events.len() as u64, Ordering::Relaxed); - if let Err(failure_class) = persist_outbox(app, &outbox) { + if let Err(failure_class) = persist_outbox(app, &mut outbox) { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( failure_class, @@ -808,8 +853,37 @@ pub fn init_product_session(app: &AppHandle) { Err(err) => warn!("Failed to access store for product analytics session: {err}"), } - match load_outbox(app) { - Ok(loaded) => { + match load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY) { + Ok(mut loaded) => { + match load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) { + Ok(recovery) => { + merge_outbox(&mut loaded, recovery); + if let Err(failure_class) = + write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, &loaded) + { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class, + "Failed to consolidate product analytics recovery outbox" + ); + } else if let Err(failure_class) = + delete_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) + { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class, + "Failed to remove consolidated product analytics recovery outbox" + ); + } + } + Err(failure_class) => { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class, + "Failed to restore product analytics recovery outbox" + ); + } + } PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(false, Ordering::Release); let has_pending = !loaded.pending.is_empty(); *outbox_guard() = loaded; @@ -820,10 +894,23 @@ pub fn init_product_session(app: &AppHandle) { Err(failure_class) => { PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + match load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) { + Ok(recovery) => *outbox_guard() = recovery, + Err(recovery_failure_class) => { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class = recovery_failure_class, + "Failed to restore product analytics recovery outbox" + ); + } + } warn!( failure_class, "Failed to restore encrypted product analytics outbox" ); + if !outbox_guard().pending.is_empty() { + let _ = product_event_sender(app).try_send(()); + } } } } @@ -1064,10 +1151,35 @@ mod tests { assert_eq!(selected_key, valid_key); assert_eq!(decrypted.pending[0].event_id, event_id); - assert_eq!( - ensure_outbox_persistence_allowed(true), - Err("outbox_restore_blocked".to_string()) + } + + #[test] + fn recovery_outbox_merge_preserves_unique_events() { + let first = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + let mut duplicate = first.clone(); + duplicate.event_name = "recording_completed".to_string(); + let second = product_event( + &event_data(recording_started()), + "second-install-id".to_string(), + ) + .unwrap(); + let mut target = ProductEventOutbox { + pending: vec![first.clone()], + dead_letters: Vec::new(), + }; + + merge_outbox( + &mut target, + ProductEventOutbox { + pending: vec![duplicate, second.clone()], + dead_letters: Vec::new(), + }, ); + + assert_eq!(target.pending.len(), 2); + assert_eq!(target.pending[0].event_name, first.event_name); + assert_eq!(target.pending[1].event_id, second.event_id); } #[test] diff --git a/apps/web/app/api/analytics/track/route.ts b/apps/web/app/api/analytics/track/route.ts index 6c4d1fe0d3b..719d4ff75f7 100644 --- a/apps/web/app/api/analytics/track/route.ts +++ b/apps/web/app/api/analytics/track/route.ts @@ -81,8 +81,7 @@ export async function POST(request: NextRequest) { trustedVercelProxy: process.env.VERCEL === "1", xVercelForwardedFor: request.headers.get("x-vercel-forwarded-for") ?? undefined, - fallbackIdentity: - sessionId ?? request.headers.get("user-agent") ?? undefined, + fallbackIdentity: sessionId ?? undefined, }); if (!rateLimitKey) { return Response.json( diff --git a/apps/web/app/s/[videoId]/Share.tsx b/apps/web/app/s/[videoId]/Share.tsx index 1811f0ec2f2..fab166c4728 100644 --- a/apps/web/app/s/[videoId]/Share.tsx +++ b/apps/web/app/s/[videoId]/Share.tsx @@ -41,20 +41,26 @@ export type CommentType = typeof commentsSchema.$inferSelect & { const SESSION_STORAGE_KEY = "cap_tb_session_id"; const SESSION_TTL_MS = 30 * 60 * 1000; // 30 minutes +let volatileAnalyticsSessionId: string | undefined; const ensureAnalyticsSessionId = () => { if (typeof window === "undefined") return "anonymous"; + const now = Date.now(); + const newId = + volatileAnalyticsSessionId ?? + (typeof crypto !== "undefined" && "randomUUID" in crypto + ? crypto.randomUUID() + : `${now}-${Math.random().toString(36).slice(2)}`); + volatileAnalyticsSessionId = newId; try { const raw = window.localStorage.getItem(SESSION_STORAGE_KEY); - const now = Date.now(); if (raw) { const parsed = JSON.parse(raw) as { value: string; expiry: number }; - if (parsed?.value && parsed.expiry > now) return parsed.value; + if (parsed?.value && parsed.expiry > now) { + volatileAnalyticsSessionId = parsed.value; + return parsed.value; + } } - const newId = - typeof crypto !== "undefined" && "randomUUID" in crypto - ? crypto.randomUUID() - : Math.random().toString(36).slice(2); window.localStorage.setItem( SESSION_STORAGE_KEY, JSON.stringify({ value: newId, expiry: now + SESSION_TTL_MS }), @@ -62,7 +68,7 @@ const ensureAnalyticsSessionId = () => { return newId; } catch (error) { console.warn("Failed to persist analytics session id", error); - return "anonymous"; + return newId; } }; From 0054d58de234655251c48d2f0d22483f48bd6355 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:22:36 +0100 Subject: [PATCH 052/110] fix: evolve exact analytics aggregates safely --- scripts/analytics/tests/datafiles.test.js | 2 +- ...e => product_events_health_hourly_exact.datasource} | 0 .../tinybird/pipes/product_analytics_freshness.pipe | 2 +- .../tinybird/pipes/product_events_health.pipe | 2 +- .../pipes/snapshot_product_events_health_hourly.pipe | 2 +- .../snapshot_product_traffic_pages_daily_exact.pipe | 4 ++-- scripts/analytics/tooling.js | 10 ++++++---- 7 files changed, 12 insertions(+), 10 deletions(-) rename scripts/analytics/tinybird/datasources/{product_events_health_hourly.datasource => product_events_health_hourly_exact.datasource} (100%) diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index 1c3cae1c75b..25cd3c18e58 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -222,7 +222,7 @@ test("health queries use stable hourly aggregates and a bounded window", () => { contents, /toStartOfHour\(toDateTime64\(\{\{DateTime64\(start_time\)\}\}, 3\)\)/, ); - assert.match(contents, /FROM product_events_health_hourly/); + assert.match(contents, /FROM product_events_health_hourly_exact/); assert.match(contents, /uniqExactMerge\(unique_events\)/); assert.match(contents, /uniqExactMerge\(unique_payloads\)/); assert.match(contents, /payload_conflicts/); diff --git a/scripts/analytics/tinybird/datasources/product_events_health_hourly.datasource b/scripts/analytics/tinybird/datasources/product_events_health_hourly_exact.datasource similarity index 100% rename from scripts/analytics/tinybird/datasources/product_events_health_hourly.datasource rename to scripts/analytics/tinybird/datasources/product_events_health_hourly_exact.datasource diff --git a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe index 303d4d1be69..4cb69f0003f 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe @@ -11,6 +11,6 @@ SQL > (SELECT max(calculated_at) FROM product_events_daily_exact) AS product_calculated_at, (SELECT max(calculated_at) FROM product_traffic_daily_exact) AS traffic_calculated_at, (SELECT max(calculated_at) FROM product_creator_retention_exact) AS retention_calculated_at - FROM product_events_health_hourly + FROM product_events_health_hourly_exact TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_events_health.pipe b/scripts/analytics/tinybird/pipes/product_events_health.pipe index fefa92d8302..39c4077a685 100644 --- a/scripts/analytics/tinybird/pipes/product_events_health.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_health.pipe @@ -22,7 +22,7 @@ SQL > sum(late_events) AS late_events, sum(missing_identity_events) AS missing_identity_events, quantilesTDigestMerge(0.5, 0.95, 0.99)(ingestion_lag_ms) AS ingestion_lag_ms - FROM product_events_health_hourly + FROM product_events_health_hourly_exact WHERE hour >= toStartOfHour(toDateTime64({{DateTime64(start_time)}}, 3)) AND hour <= toDateTime64({{DateTime64(end_time)}}, 3) {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe index 10b8ed8a17d..3c2ff894d12 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe @@ -18,6 +18,6 @@ SQL > GROUP BY hour, platform, app_version TYPE COPY -TARGET_DATASOURCE product_events_health_hourly +TARGET_DATASOURCE product_events_health_hourly_exact COPY_MODE replace COPY_SCHEDULE 1-59/8 * * * * diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe index 2197c1fd37c..1db2c7742cd 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe @@ -52,14 +52,14 @@ SQL > os, channel, uniqExactState(visitor_id) AS visitors, - uniqExactState(session_id) AS visits, + uniqExactState(page_views.session_id) AS visits, toUInt64(count()) AS pageviews, toUInt64(countIf(event_id = landing_event_id)) AS landings, toUInt64(countIf(event_id = exit_event_id)) AS exits, toUInt64(sum(coalesce(engaged_ms, 0))) AS engaged_ms, toFloat64(sum(coalesce(scroll_depth, 0))) AS scroll_depth_sum FROM page_views - INNER JOIN sessions USING session_id + INNER JOIN sessions ON page_views.session_id = sessions.session_id LEFT JOIN engagements ON page_views.event_id = engagements.page_view_id GROUP BY date, hostname, pathname, country, device, browser, os, channel diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index fa1ec5c2f8f..6d0f8a3b716 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -172,7 +172,7 @@ const operationPlan = (operation) => { ), ...PRODUCT_COPY_PIPES.map((name) => ({ ...localCliStep("--local", "copy", "run", name, "--wait"), - attempts: 5, + attempts: 8, retryPattern: /CANNOT_SCHEDULE_TASK|no free thread/i, })), { type: "verify-local" }, @@ -207,7 +207,7 @@ const operationPlan = (operation) => { ), ...PRODUCT_COPY_PIPES.map((name) => ({ ...localCliStep("--local", "copy", "run", name, "--wait"), - attempts: 5, + attempts: 8, retryPattern: /CANNOT_SCHEDULE_TASK|no free thread/i, })), { type: "verify-local" }, @@ -423,10 +423,12 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { } } const healthHourly = project.datasources.find( - (datasource) => datasource.name === "product_events_health_hourly", + (datasource) => datasource.name === "product_events_health_hourly_exact", ); if (!healthHourly || healthHourly.engine !== "AggregatingMergeTree") { - issues.push("Missing product_events_health_hourly aggregate datasource"); + issues.push( + "Missing product_events_health_hourly_exact aggregate datasource", + ); } else if (hasToken(healthHourly, "product_events_agent_read", "READ")) { issues.push("Product event health states must be endpoint-only"); } From 0b74725671f4821fad3b1d54307fb4966aa8d836 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:34:22 +0100 Subject: [PATCH 053/110] fix: close analytics recovery abuse gaps --- ...st-party-analytics-production-readiness.md | 2 + .../src-tauri/src/product_analytics.rs | 114 +++++++++++++----- apps/web/app/api/analytics/track/route.ts | 22 +++- 3 files changed, 102 insertions(+), 36 deletions(-) diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index fdaa9724352..657f3783c3c 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -30,6 +30,8 @@ The collector rejects unregistered events, unknown properties, raw error fields, Known bots, crawlers, previews, internal IP hashes, and synthetic runs are visible to health monitoring but excluded from decision snapshots. Anonymous write tokens are rate-limited by both source IP and anonymous ID. Replay and automation can be made expensive and observable, but public traffic analytics cannot cryptographically guarantee a human browser. Signup, organization, billing, and other business outcomes therefore remain server-authoritative. +Vercel request-IP buckets use only the platform-owned forwarding header. A self-hosted deployment does not trust forwarding headers by default and instead applies a shared 600-request-per-minute process cap to legacy public view tracking. `CAP_ANALYTICS_TRUST_PROXY=1` switches self-hosted tracking to the first `x-forwarded-for` address and is safe only behind a reverse proxy that overwrites, rather than appends or passes through, that header. Do not set it on Vercel. + Account and organization deletion writes its durable pending marker or tombstone before erasure begins. Delivery and reconciliation reject marked identities, then deletion waits longer than the bounded in-flight delivery attempt before removing matching raw rows with a dedicated erasure token and rebuilding every replace-mode canonical, traffic, activation, retention, and health snapshot. This closes the race where a workflow that passed its identity check could otherwise append after a completed erasure. Deletion fails closed if the erasure credential or any rebuild is unavailable. The staging suite deletes a synthetic user and organization, proves their raw-health and decision state is gone, and proves an out-of-scope row sharing the anonymous ID remains until final test cleanup. ## Data quality and performance gates diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index fec19bed098..6ae79e163f4 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -35,6 +35,7 @@ const PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY: &str = "product_analytics_outbox_fallback_key_v1"; const PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE: &str = "so.cap.desktop"; const PRODUCT_EVENT_OUTBOX_KEYRING_USER: &str = "product-analytics-outbox-v1"; +const PRODUCT_EVENT_OUTBOX_KEY_HISTORY_CAPACITY: usize = 4; const PRODUCT_EVENT_OUTBOX_CAPACITY: usize = 500; const PRODUCT_EVENT_DEAD_LETTER_CAPACITY: usize = 100; const PRODUCT_EVENT_MAX_RETRY_DELAY: Duration = Duration::from_secs(5 * 60); @@ -427,42 +428,75 @@ fn should_retry_product_status(status: u16) -> bool { } fn fallback_outbox_encryption_key(app: &AppHandle) -> Result<[u8; 32], String> { - if let Some(existing) = existing_fallback_outbox_encryption_key(app)? { - return Ok(existing); + if let Some(existing) = persisted_outbox_encryption_keys(app)?.first() { + return Ok(*existing); } - let store = app - .store("store") - .map_err(|_| "fallback_key_store_unavailable".to_string())?; let mut generated = [0_u8; 32]; SystemRandom::new() .fill(&mut generated) .map_err(|_| "key_generation_failed".to_string())?; - store.set( - PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY, - BASE64.encode(generated), - ); - store - .save() - .map_err(|_| "fallback_key_store_write_failed".to_string())?; + persist_outbox_encryption_key(app, generated)?; Ok(generated) } -fn existing_fallback_outbox_encryption_key(app: &AppHandle) -> Result, String> { +fn persisted_outbox_encryption_keys(app: &AppHandle) -> Result, String> { let store = app .store("store") .map_err(|_| "fallback_key_store_unavailable".to_string())?; - let Some(encoded) = store - .get(PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY) - .and_then(|value| value.as_str().map(str::to_owned)) - else { - return Ok(None); + let Some(value) = store.get(PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY) else { + return Ok(Vec::new()); }; - let key = BASE64 - .decode(encoded) - .map_err(|_| "invalid_fallback_key".to_string())? - .try_into() - .map_err(|_| "invalid_fallback_key".to_string())?; - Ok(Some(key)) + decode_persisted_outbox_encryption_keys(&value) +} + +fn decode_persisted_outbox_encryption_keys( + value: &serde_json::Value, +) -> Result, String> { + let encoded_keys = if let Some(encoded) = value.as_str() { + vec![encoded] + } else if let Some(values) = value.as_array() { + values + .iter() + .map(|value| { + value + .as_str() + .ok_or_else(|| "invalid_fallback_key".to_string()) + }) + .collect::, _>>()? + } else { + return Err("invalid_fallback_key".to_string()); + }; + encoded_keys + .into_iter() + .map(|encoded| { + BASE64 + .decode(encoded) + .map_err(|_| "invalid_fallback_key".to_string())? + .try_into() + .map_err(|_| "invalid_fallback_key".to_string()) + }) + .collect() +} + +fn persist_outbox_encryption_key(app: &AppHandle, key: [u8; 32]) -> Result<(), String> { + let mut keys = persisted_outbox_encryption_keys(app)?; + keys.retain(|candidate| candidate != &key); + keys.insert(0, key); + keys.truncate(PRODUCT_EVENT_OUTBOX_KEY_HISTORY_CAPACITY); + let encoded_keys = keys + .into_iter() + .map(|candidate| serde_json::Value::String(BASE64.encode(candidate))) + .collect(); + let store = app + .store("store") + .map_err(|_| "fallback_key_store_unavailable".to_string())?; + store.set( + PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY, + serde_json::Value::Array(encoded_keys), + ); + store + .save() + .map_err(|_| "fallback_key_store_write_failed".to_string()) } fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { @@ -479,9 +513,11 @@ fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { let decoded = BASE64 .decode(encoded) .map_err(|_| "invalid_keyring_value".to_string())?; - decoded + let key = decoded .try_into() - .map_err(|_| "invalid_keyring_value".to_string())? + .map_err(|_| "invalid_keyring_value".to_string())?; + persist_outbox_encryption_key(app, key)?; + key } Err(keyring::Error::NoEntry) => { let mut generated = [0_u8; 32]; @@ -489,6 +525,7 @@ fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { .fill(&mut generated) .map_err(|_| "key_generation_failed".to_string())?; if entry.set_password(&BASE64.encode(generated)).is_ok() { + persist_outbox_encryption_key(app, generated)?; generated } else { fallback_outbox_encryption_key(app)? @@ -550,10 +587,10 @@ fn decrypt_outbox(app: &AppHandle, value: &str) -> Result { const trimmed = value?.trim(); @@ -77,22 +81,28 @@ export async function POST(request: NextRequest) { return Response.json({ error: "Invalid origin" }, { status: 403 }); } } + const isVercel = process.env.VERCEL === "1"; + const trustedNetworkProxy = + isVercel || process.env.CAP_ANALYTICS_TRUST_PROXY === "1"; const rateLimitKey = getProductAnalyticsRateLimitKey({ - trustedVercelProxy: process.env.VERCEL === "1", + trustedVercelProxy: trustedNetworkProxy, xVercelForwardedFor: - request.headers.get("x-vercel-forwarded-for") ?? undefined, - fallbackIdentity: sessionId ?? undefined, + request.headers.get( + isVercel ? "x-vercel-forwarded-for" : "x-forwarded-for", + ) ?? undefined, }); - if (!rateLimitKey) { + if (!rateLimitKey && trustedNetworkProxy) { return Response.json( { error: "Missing request identity" }, { status: 400 }, ); } if ( - fallbackRateLimiter.isRateLimited(rateLimitKey) || + (rateLimitKey + ? fallbackRateLimiter.isRateLimited(rateLimitKey) + : untrustedProxyRateLimiter.isRateLimited("self-hosted")) || (await isRateLimited(RATE_LIMIT_IDS.ANALYTICS_TRACK, { - key: rateLimitKey, + key: rateLimitKey ?? "self-hosted", headers: request.headers, })) ) { From 7b1f97ad4c3ea60cfc8a045fa5a94fc7ab6f39fe Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:34:27 +0100 Subject: [PATCH 054/110] test: bound Tinybird local copy threads --- scripts/analytics/tests/datafiles.test.js | 1 + scripts/analytics/tests/tooling.test.js | 9 +++++++++ ...apshot_product_activation_daily_exact.pipe | 4 ++++ ...pshot_product_creator_retention_exact.pipe | 4 ++++ .../snapshot_product_events_canonical_v1.pipe | 4 ++++ .../snapshot_product_events_daily_exact.pipe | 4 ++++ ...snapshot_product_events_health_hourly.pipe | 4 ++++ .../snapshot_product_traffic_daily_exact.pipe | 4 ++++ ...hot_product_traffic_pages_daily_exact.pipe | 4 ++++ scripts/analytics/tooling.js | 20 +++++++++++++++++-- 10 files changed, 56 insertions(+), 2 deletions(-) diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index 25cd3c18e58..800751fcd1f 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -207,6 +207,7 @@ test("copy schedules serialize canonical and derived rebuilds", () => { "utf8", ); assert.ok(contents.split("\n").includes(`COPY_SCHEDULE ${schedule}`)); + assert.match(contents, /\{\{max_threads\(Int32\(copy_max_threads\)\)\}\}/); } assert.equal(new Set(schedules.values()).size, schedules.size); }); diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index bdcdc063f05..829a8c18286 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -80,6 +80,15 @@ test("local setup builds, verifies copied endpoints and writes its deterministic .map(({ index }) => index); assert.equal(pauseIndexes.length, 7); assert.ok(pauseIndexes.every((index) => index < appendIndex)); + const copyCommands = commands.filter((command) => + command.includes("--local copy run"), + ); + assert.equal(copyCommands.length, 7); + assert.ok( + copyCommands.every((command) => + command.includes("--param copy_max_threads=1 --wait"), + ), + ); assert.ok(steps.some((step) => step.type === "verify-local")); assert.ok(steps.some((step) => step.type === "write-local-env")); assert.ok( diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe index 763b5377345..b9000ce8e9b 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe @@ -3,6 +3,10 @@ DESCRIPTION > NODE snapshot_product_activation_daily_exact_node SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} WITH signups AS ( SELECT user_id, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe index c098a0c11be..91761580850 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe @@ -3,6 +3,10 @@ DESCRIPTION > NODE snapshot_product_creator_retention_exact_node SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} WITH activity AS ( SELECT DISTINCT user_id, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe index a4a209bec7e..7d8d138f662 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe @@ -3,6 +3,10 @@ DESCRIPTION > NODE snapshot_product_events_canonical_v1_node SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} SELECT event_id, any(raw_payload_hash) AS payload_hash, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe index f22f40027fe..987af0a2789 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe @@ -3,6 +3,10 @@ DESCRIPTION > NODE snapshot_product_events_daily_exact_node SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} SELECT now64(3) AS calculated_at, toDate(occurred_at, 'UTC') AS date, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe index 3c2ff894d12..ece93f8febe 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe @@ -3,6 +3,10 @@ DESCRIPTION > NODE snapshot_product_events_health_hourly_node SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} SELECT toStartOfHour(received_at) AS hour, platform, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe index a18c5367ee3..7d0b5726fcc 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe @@ -3,6 +3,10 @@ DESCRIPTION > NODE snapshot_product_traffic_daily_exact_node SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} WITH session_engagement AS ( SELECT session_id, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe index 1db2c7742cd..dd1dfb82b85 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe @@ -3,6 +3,10 @@ DESCRIPTION > NODE snapshot_product_traffic_pages_daily_exact_node SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} WITH sessions AS ( SELECT session_id, diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index 6d0f8a3b716..fea17c8e1a8 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -171,7 +171,15 @@ const operationPlan = (operation) => { "fixtures/product_events_v1.local.ndjson", ), ...PRODUCT_COPY_PIPES.map((name) => ({ - ...localCliStep("--local", "copy", "run", name, "--wait"), + ...localCliStep( + "--local", + "copy", + "run", + name, + "--param", + "copy_max_threads=1", + "--wait", + ), attempts: 8, retryPattern: /CANNOT_SCHEDULE_TASK|no free thread/i, })), @@ -206,7 +214,15 @@ const operationPlan = (operation) => { "fixtures/product_events_v1.local.ndjson", ), ...PRODUCT_COPY_PIPES.map((name) => ({ - ...localCliStep("--local", "copy", "run", name, "--wait"), + ...localCliStep( + "--local", + "copy", + "run", + name, + "--param", + "copy_max_threads=1", + "--wait", + ), attempts: 8, retryPattern: /CANNOT_SCHEDULE_TASK|no free thread/i, })), From ad881ea882004e619bb7b14499c1d8a8bc6499ed Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:43:40 +0100 Subject: [PATCH 055/110] fix: keep analytics keys in the OS keyring --- ...st-party-analytics-production-readiness.md | 2 +- .../src-tauri/src/product_analytics.rs | 187 ++++++------------ apps/web/app/api/analytics/track/route.ts | 18 +- 3 files changed, 71 insertions(+), 136 deletions(-) diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index 657f3783c3c..7393e9ab50f 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -30,7 +30,7 @@ The collector rejects unregistered events, unknown properties, raw error fields, Known bots, crawlers, previews, internal IP hashes, and synthetic runs are visible to health monitoring but excluded from decision snapshots. Anonymous write tokens are rate-limited by both source IP and anonymous ID. Replay and automation can be made expensive and observable, but public traffic analytics cannot cryptographically guarantee a human browser. Signup, organization, billing, and other business outcomes therefore remain server-authoritative. -Vercel request-IP buckets use only the platform-owned forwarding header. A self-hosted deployment does not trust forwarding headers by default and instead applies a shared 600-request-per-minute process cap to legacy public view tracking. `CAP_ANALYTICS_TRUST_PROXY=1` switches self-hosted tracking to the first `x-forwarded-for` address and is safe only behind a reverse proxy that overwrites, rather than appends or passes through, that header. Do not set it on Vercel. +Vercel request-IP buckets use only the platform-owned forwarding header. A self-hosted deployment does not trust forwarding headers by default, so legacy public view tracking returns `503` until `CAP_ANALYTICS_TRUST_PROXY=1` is configured. That setting uses the first `x-forwarded-for` address and is safe only behind a reverse proxy that overwrites, rather than appends or passes through, that header. Do not set it on Vercel. Account and organization deletion writes its durable pending marker or tombstone before erasure begins. Delivery and reconciliation reject marked identities, then deletion waits longer than the bounded in-flight delivery attempt before removing matching raw rows with a dedicated erasure token and rebuilding every replace-mode canonical, traffic, activation, retention, and health snapshot. This closes the race where a workflow that passed its identity check could otherwise append after a completed erasure. Deletion fails closed if the erasure credential or any rebuild is unavailable. The staging suite deletes a synthetic user and organization, proves their raw-health and decision state is gone, and proves an out-of-scope row sharing the anonymous ID remains until final test cleanup. diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index 6ae79e163f4..bbfd49b0148 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -31,11 +31,9 @@ const PRODUCT_EVENT_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); const PRODUCT_EVENT_SESSION_STORE_KEY: &str = "product_analytics_session_id"; const PRODUCT_EVENT_OUTBOX_STORE_KEY: &str = "product_analytics_outbox_v1"; const PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY: &str = "product_analytics_outbox_recovery_v1"; -const PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY: &str = - "product_analytics_outbox_fallback_key_v1"; const PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE: &str = "so.cap.desktop"; const PRODUCT_EVENT_OUTBOX_KEYRING_USER: &str = "product-analytics-outbox-v1"; -const PRODUCT_EVENT_OUTBOX_KEY_HISTORY_CAPACITY: usize = 4; +const PRODUCT_EVENT_OUTBOX_KEYRING_BACKUP_USER: &str = "product-analytics-outbox-v1-backup"; const PRODUCT_EVENT_OUTBOX_CAPACITY: usize = 500; const PRODUCT_EVENT_DEAD_LETTER_CAPACITY: usize = 100; const PRODUCT_EVENT_MAX_RETRY_DELAY: Duration = Duration::from_secs(5 * 60); @@ -427,114 +425,71 @@ fn should_retry_product_status(status: u16) -> bool { status == 429 || status >= 500 } -fn fallback_outbox_encryption_key(app: &AppHandle) -> Result<[u8; 32], String> { - if let Some(existing) = persisted_outbox_encryption_keys(app)?.first() { - return Ok(*existing); - } - let mut generated = [0_u8; 32]; - SystemRandom::new() - .fill(&mut generated) - .map_err(|_| "key_generation_failed".to_string())?; - persist_outbox_encryption_key(app, generated)?; - Ok(generated) +fn decode_outbox_encryption_key(encoded: &str) -> Result<[u8; 32], String> { + BASE64 + .decode(encoded) + .map_err(|_| "invalid_keyring_value".to_string())? + .try_into() + .map_err(|_| "invalid_keyring_value".to_string()) } -fn persisted_outbox_encryption_keys(app: &AppHandle) -> Result, String> { - let store = app - .store("store") - .map_err(|_| "fallback_key_store_unavailable".to_string())?; - let Some(value) = store.get(PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY) else { - return Ok(Vec::new()); - }; - decode_persisted_outbox_encryption_keys(&value) +fn read_keyring_outbox_key(user: &str) -> Result, String> { + let entry = keyring::Entry::new(PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE, user) + .map_err(|_| "outbox_keyring_unavailable".to_string())?; + match entry.get_password() { + Ok(encoded) => decode_outbox_encryption_key(&encoded).map(Some), + Err(keyring::Error::NoEntry) => Ok(None), + Err(_) => Err("outbox_keyring_unavailable".to_string()), + } } -fn decode_persisted_outbox_encryption_keys( - value: &serde_json::Value, -) -> Result, String> { - let encoded_keys = if let Some(encoded) = value.as_str() { - vec![encoded] - } else if let Some(values) = value.as_array() { - values - .iter() - .map(|value| { - value - .as_str() - .ok_or_else(|| "invalid_fallback_key".to_string()) - }) - .collect::, _>>()? - } else { - return Err("invalid_fallback_key".to_string()); - }; - encoded_keys - .into_iter() - .map(|encoded| { - BASE64 - .decode(encoded) - .map_err(|_| "invalid_fallback_key".to_string())? - .try_into() - .map_err(|_| "invalid_fallback_key".to_string()) - }) - .collect() +fn keyring_outbox_encryption_keys() -> Result, String> { + let mut keys = Vec::new(); + let mut failure = None; + for user in [ + PRODUCT_EVENT_OUTBOX_KEYRING_USER, + PRODUCT_EVENT_OUTBOX_KEYRING_BACKUP_USER, + ] { + match read_keyring_outbox_key(user) { + Ok(Some(key)) if !keys.contains(&key) => keys.push(key), + Ok(_) => {} + Err(error) => failure = Some(error), + } + } + if keys.is_empty() + && let Some(error) = failure + { + return Err(error); + } + Ok(keys) } -fn persist_outbox_encryption_key(app: &AppHandle, key: [u8; 32]) -> Result<(), String> { - let mut keys = persisted_outbox_encryption_keys(app)?; - keys.retain(|candidate| candidate != &key); - keys.insert(0, key); - keys.truncate(PRODUCT_EVENT_OUTBOX_KEY_HISTORY_CAPACITY); - let encoded_keys = keys - .into_iter() - .map(|candidate| serde_json::Value::String(BASE64.encode(candidate))) - .collect(); - let store = app - .store("store") - .map_err(|_| "fallback_key_store_unavailable".to_string())?; - store.set( - PRODUCT_EVENT_OUTBOX_FALLBACK_KEY_STORE_KEY, - serde_json::Value::Array(encoded_keys), - ); - store - .save() - .map_err(|_| "fallback_key_store_write_failed".to_string()) +fn persist_keyring_outbox_key(user: &str, key: [u8; 32]) -> Result<(), String> { + let entry = keyring::Entry::new(PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE, user) + .map_err(|_| "outbox_keyring_unavailable".to_string())?; + entry + .set_password(&BASE64.encode(key)) + .map_err(|_| "outbox_keyring_write_failed".to_string()) } -fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { +fn outbox_encryption_key(_app: &AppHandle) -> Result<&'static [u8; 32], String> { if let Some(key) = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.get() { return Ok(key); } - let key = match keyring::Entry::new( - PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE, - PRODUCT_EVENT_OUTBOX_KEYRING_USER, - ) { - Ok(entry) => match entry.get_password() { - Ok(encoded) => { - let decoded = BASE64 - .decode(encoded) - .map_err(|_| "invalid_keyring_value".to_string())?; - let key = decoded - .try_into() - .map_err(|_| "invalid_keyring_value".to_string())?; - persist_outbox_encryption_key(app, key)?; - key - } - Err(keyring::Error::NoEntry) => { - let mut generated = [0_u8; 32]; - SystemRandom::new() - .fill(&mut generated) - .map_err(|_| "key_generation_failed".to_string())?; - if entry.set_password(&BASE64.encode(generated)).is_ok() { - persist_outbox_encryption_key(app, generated)?; - generated - } else { - fallback_outbox_encryption_key(app)? - } - } - Err(_) => fallback_outbox_encryption_key(app)?, - }, - Err(_) => fallback_outbox_encryption_key(app)?, + let keys = keyring_outbox_encryption_keys()?; + let key = if let Some(key) = keys.first() { + *key + } else { + let mut generated = [0_u8; 32]; + SystemRandom::new() + .fill(&mut generated) + .map_err(|_| "key_generation_failed".to_string())?; + persist_keyring_outbox_key(PRODUCT_EVENT_OUTBOX_KEYRING_USER, generated)?; + generated }; + let _ = persist_keyring_outbox_key(PRODUCT_EVENT_OUTBOX_KEYRING_USER, key); + let _ = persist_keyring_outbox_key(PRODUCT_EVENT_OUTBOX_KEYRING_BACKUP_USER, key); let _ = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.set(key); PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY .get() @@ -570,28 +525,14 @@ fn encrypt_outbox_with_key( Ok(BASE64.encode(stored)) } -fn decrypt_outbox(app: &AppHandle, value: &str) -> Result { +fn decrypt_outbox(value: &str) -> Result { if let Some(key) = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.get() && let Ok(outbox) = decrypt_outbox_with_key(value, key) { return Ok(outbox); } - let mut candidates = Vec::new(); - if let Ok(entry) = keyring::Entry::new( - PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE, - PRODUCT_EVENT_OUTBOX_KEYRING_USER, - ) && let Ok(encoded) = entry.get_password() - && let Ok(decoded) = BASE64.decode(encoded) - && let Ok(key) = decoded.try_into() - { - candidates.push(key); - } - for key in persisted_outbox_encryption_keys(app)? { - if !candidates.contains(&key) { - candidates.push(key); - } - } + let candidates = keyring_outbox_encryption_keys()?; let (outbox, key) = decrypt_outbox_with_keys(value, &candidates)?; let _ = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.set(key); Ok(outbox) @@ -672,7 +613,7 @@ fn load_stored_outbox(app: &AppHandle, store_key: &str) -> Result { const trimmed = value?.trim(); @@ -84,6 +80,12 @@ export async function POST(request: NextRequest) { const isVercel = process.env.VERCEL === "1"; const trustedNetworkProxy = isVercel || process.env.CAP_ANALYTICS_TRUST_PROXY === "1"; + if (!trustedNetworkProxy) { + return Response.json( + { error: "Analytics proxy identity is not configured" }, + { status: 503 }, + ); + } const rateLimitKey = getProductAnalyticsRateLimitKey({ trustedVercelProxy: trustedNetworkProxy, xVercelForwardedFor: @@ -91,18 +93,16 @@ export async function POST(request: NextRequest) { isVercel ? "x-vercel-forwarded-for" : "x-forwarded-for", ) ?? undefined, }); - if (!rateLimitKey && trustedNetworkProxy) { + if (!rateLimitKey) { return Response.json( { error: "Missing request identity" }, { status: 400 }, ); } if ( - (rateLimitKey - ? fallbackRateLimiter.isRateLimited(rateLimitKey) - : untrustedProxyRateLimiter.isRateLimited("self-hosted")) || + fallbackRateLimiter.isRateLimited(rateLimitKey) || (await isRateLimited(RATE_LIMIT_IDS.ANALYTICS_TRACK, { - key: rateLimitKey ?? "self-hosted", + key: rateLimitKey, headers: request.headers, })) ) { From 2f915b4cd620405d6c0ef55025d8f25a694f97df Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 01:43:40 +0100 Subject: [PATCH 056/110] ci: install dependencies for staging assertions --- .github/workflows/analytics.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 0f78c56a711..adbfb963a15 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -104,9 +104,7 @@ jobs: with: ref: ${{ github.event.pull_request.head.sha || github.sha }} fetch-depth: 1 - - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 - with: - node-version: 20 + - uses: ./.github/actions/setup-js - name: Refuse an out-of-scope event, branch, PR, or checkout run: node scripts/analytics/staging-ci.js verify-scope --actual-sha "$(git rev-parse HEAD)" - name: Refuse missing, non-staging, or wrong-workspace credentials From c5a464fd898e601f16bb4c78f38cbd1168b35505 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:04:57 +0100 Subject: [PATCH 057/110] fix: migrate durable analytics outbox keys --- .../src-tauri/src/product_analytics.rs | 234 +++++++++++++++--- 1 file changed, 205 insertions(+), 29 deletions(-) diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index bbfd49b0148..ae3167a2903 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -5,14 +5,19 @@ use ring::{ }; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +#[cfg(unix)] +use std::os::unix::fs::PermissionsExt; use std::{ + fs, + io::Write, + path::{Path, PathBuf}, sync::{ Mutex, OnceLock, atomic::{AtomicBool, AtomicU64, Ordering}, }, time::Duration, }; -use tauri::AppHandle; +use tauri::{AppHandle, Manager}; use tauri_plugin_store::StoreExt; use tokio::sync::mpsc; use tracing::{error, warn}; @@ -31,9 +36,11 @@ const PRODUCT_EVENT_REQUEST_TIMEOUT: Duration = Duration::from_secs(3); const PRODUCT_EVENT_SESSION_STORE_KEY: &str = "product_analytics_session_id"; const PRODUCT_EVENT_OUTBOX_STORE_KEY: &str = "product_analytics_outbox_v1"; const PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY: &str = "product_analytics_outbox_recovery_v1"; +const PRODUCT_EVENT_OUTBOX_LEGACY_KEY_STORE_KEY: &str = "product_analytics_outbox_fallback_key_v1"; +const PRODUCT_EVENT_OUTBOX_KEY_FILE: &str = "product-analytics-outbox-key-v1"; const PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE: &str = "so.cap.desktop"; const PRODUCT_EVENT_OUTBOX_KEYRING_USER: &str = "product-analytics-outbox-v1"; -const PRODUCT_EVENT_OUTBOX_KEYRING_BACKUP_USER: &str = "product-analytics-outbox-v1-backup"; +const PRODUCT_EVENT_OUTBOX_LEGACY_KEYRING_USER: &str = "product-analytics-outbox-v1-backup"; const PRODUCT_EVENT_OUTBOX_CAPACITY: usize = 500; const PRODUCT_EVENT_DEAD_LETTER_CAPACITY: usize = 100; const PRODUCT_EVENT_MAX_RETRY_DELAY: Duration = Duration::from_secs(5 * 60); @@ -427,10 +434,10 @@ fn should_retry_product_status(status: u16) -> bool { fn decode_outbox_encryption_key(encoded: &str) -> Result<[u8; 32], String> { BASE64 - .decode(encoded) - .map_err(|_| "invalid_keyring_value".to_string())? + .decode(encoded.trim()) + .map_err(|_| "invalid_outbox_key".to_string())? .try_into() - .map_err(|_| "invalid_keyring_value".to_string()) + .map_err(|_| "invalid_outbox_key".to_string()) } fn read_keyring_outbox_key(user: &str) -> Result, String> { @@ -448,7 +455,7 @@ fn keyring_outbox_encryption_keys() -> Result, String> { let mut failure = None; for user in [ PRODUCT_EVENT_OUTBOX_KEYRING_USER, - PRODUCT_EVENT_OUTBOX_KEYRING_BACKUP_USER, + PRODUCT_EVENT_OUTBOX_LEGACY_KEYRING_USER, ] { match read_keyring_outbox_key(user) { Ok(Some(key)) if !keys.contains(&key) => keys.push(key), @@ -472,24 +479,160 @@ fn persist_keyring_outbox_key(user: &str, key: [u8; 32]) -> Result<(), String> { .map_err(|_| "outbox_keyring_write_failed".to_string()) } -fn outbox_encryption_key(_app: &AppHandle) -> Result<&'static [u8; 32], String> { +fn file_outbox_key_path(app: &AppHandle) -> Result { + app.path() + .app_local_data_dir() + .map(|directory| directory.join(PRODUCT_EVENT_OUTBOX_KEY_FILE)) + .map_err(|_| "outbox_key_directory_unavailable".to_string()) +} + +fn read_file_outbox_key(path: &Path) -> Result, String> { + match fs::read_to_string(path) { + Ok(encoded) => decode_outbox_encryption_key(&encoded).map(Some), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(_) => Err("outbox_key_file_unavailable".to_string()), + } +} + +fn persist_file_outbox_key(path: &Path, key: [u8; 32]) -> Result<(), String> { + if let Some(existing) = read_file_outbox_key(path)? { + return if existing == key { + Ok(()) + } else { + Err("outbox_key_file_conflict".to_string()) + }; + } + let parent = path + .parent() + .ok_or_else(|| "outbox_key_directory_unavailable".to_string())?; + fs::create_dir_all(parent).map_err(|_| "outbox_key_directory_unavailable".to_string())?; + let mut temporary = tempfile::NamedTempFile::new_in(parent) + .map_err(|_| "outbox_key_file_write_failed".to_string())?; + #[cfg(unix)] + temporary + .as_file() + .set_permissions(fs::Permissions::from_mode(0o600)) + .map_err(|_| "outbox_key_file_write_failed".to_string())?; + temporary + .write_all(BASE64.encode(key).as_bytes()) + .and_then(|()| temporary.as_file().sync_all()) + .map_err(|_| "outbox_key_file_write_failed".to_string())?; + match temporary.persist_noclobber(path) { + Ok(file) => { + file.sync_all() + .map_err(|_| "outbox_key_file_write_failed".to_string())?; + #[cfg(unix)] + fs::File::open(parent) + .and_then(|directory| directory.sync_all()) + .map_err(|_| "outbox_key_file_write_failed".to_string())?; + Ok(()) + } + Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { + if read_file_outbox_key(path)? == Some(key) { + Ok(()) + } else { + Err("outbox_key_file_conflict".to_string()) + } + } + Err(_) => Err("outbox_key_file_write_failed".to_string()), + } +} + +fn decode_legacy_outbox_encryption_keys(value: &Value) -> Result, String> { + if let Some(encoded) = value.as_str() { + return decode_outbox_encryption_key(encoded).map(|key| vec![key]); + } + value + .as_array() + .ok_or_else(|| "invalid_legacy_outbox_key".to_string())? + .iter() + .map(|value| { + value + .as_str() + .ok_or_else(|| "invalid_legacy_outbox_key".to_string()) + .and_then(decode_outbox_encryption_key) + }) + .collect() +} + +fn legacy_store_outbox_encryption_keys(app: &AppHandle) -> Result, String> { + let store = app + .store("store") + .map_err(|_| "legacy_outbox_key_store_unavailable".to_string())?; + match store.get(PRODUCT_EVENT_OUTBOX_LEGACY_KEY_STORE_KEY) { + Some(value) => decode_legacy_outbox_encryption_keys(&value), + None => Ok(Vec::new()), + } +} + +fn delete_legacy_store_outbox_encryption_keys(app: &AppHandle) -> Result<(), String> { + let store = app + .store("store") + .map_err(|_| "legacy_outbox_key_store_unavailable".to_string())?; + store.delete(PRODUCT_EVENT_OUTBOX_LEGACY_KEY_STORE_KEY); + store + .save() + .map_err(|_| "legacy_outbox_key_store_write_failed".to_string()) +} + +fn outbox_encryption_keys(app: &AppHandle) -> Result, String> { + let mut keys = Vec::new(); + let mut failure = None; + if let Some(key) = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.get() { + keys.push(*key); + } + match file_outbox_key_path(app).and_then(|path| read_file_outbox_key(&path)) { + Ok(Some(key)) if !keys.contains(&key) => keys.push(key), + Ok(_) => {} + Err(error) => failure = Some(error), + } + match keyring_outbox_encryption_keys() { + Ok(keyring_keys) => { + for key in keyring_keys { + if !keys.contains(&key) { + keys.push(key); + } + } + } + Err(error) => failure = Some(error), + } + match legacy_store_outbox_encryption_keys(app) { + Ok(legacy_keys) => { + for key in legacy_keys { + if !keys.contains(&key) { + keys.push(key); + } + } + } + Err(error) => failure = Some(error), + } + if keys.is_empty() + && let Some(error) = failure + { + return Err(error); + } + Ok(keys) +} + +fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { if let Some(key) = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.get() { return Ok(key); } - let keys = keyring_outbox_encryption_keys()?; - let key = if let Some(key) = keys.first() { + let key_path = file_outbox_key_path(app)?; + let key = if let Some(key) = read_file_outbox_key(&key_path)? { + key + } else if let Some(key) = keyring_outbox_encryption_keys().unwrap_or_default().first() { *key } else { let mut generated = [0_u8; 32]; SystemRandom::new() .fill(&mut generated) .map_err(|_| "key_generation_failed".to_string())?; - persist_keyring_outbox_key(PRODUCT_EVENT_OUTBOX_KEYRING_USER, generated)?; generated }; + persist_file_outbox_key(&key_path, key)?; let _ = persist_keyring_outbox_key(PRODUCT_EVENT_OUTBOX_KEYRING_USER, key); - let _ = persist_keyring_outbox_key(PRODUCT_EVENT_OUTBOX_KEYRING_BACKUP_USER, key); let _ = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.set(key); PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY .get() @@ -525,17 +668,15 @@ fn encrypt_outbox_with_key( Ok(BASE64.encode(stored)) } -fn decrypt_outbox(value: &str) -> Result { +fn decrypt_outbox(app: &AppHandle, value: &str) -> Result { if let Some(key) = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.get() && let Ok(outbox) = decrypt_outbox_with_key(value, key) { return Ok(outbox); } - let candidates = keyring_outbox_encryption_keys()?; - let (outbox, key) = decrypt_outbox_with_keys(value, &candidates)?; - let _ = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.set(key); - Ok(outbox) + let candidates = outbox_encryption_keys(app)?; + decrypt_outbox_with_keys(value, &candidates).map(|(outbox, _)| outbox) } fn decrypt_outbox_with_keys( @@ -613,7 +754,7 @@ fn load_stored_outbox(app: &AppHandle, store_key: &str) -> Result Result<() merge_outbox(&mut restored, recovery); write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, &restored)?; delete_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY)?; + delete_legacy_store_outbox_encryption_keys(app)?; *outbox = restored; PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(false, Ordering::Release); Ok(()) @@ -836,25 +978,24 @@ pub fn init_product_session(app: &AppHandle) { match load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) { Ok(recovery) => { merge_outbox(&mut loaded, recovery); - if let Err(failure_class) = - write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, &loaded) - { + let consolidation = write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, &loaded) + .and_then(|()| { + delete_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) + }) + .and_then(|()| delete_legacy_store_outbox_encryption_keys(app)); + if let Err(failure_class) = consolidation { + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( failure_class, "Failed to consolidate product analytics recovery outbox" ); - } else if let Err(failure_class) = - delete_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) - { - PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); - warn!( - failure_class, - "Failed to remove consolidated product analytics recovery outbox" - ); + } else { + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(false, Ordering::Release); } } Err(failure_class) => { + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( failure_class, @@ -862,7 +1003,6 @@ pub fn init_product_session(app: &AppHandle) { ); } } - PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(false, Ordering::Release); let has_pending = !loaded.pending.is_empty(); *outbox_guard() = loaded; if has_pending { @@ -1142,6 +1282,42 @@ mod tests { assert!(decode_outbox_encryption_key("invalid").is_err()); } + #[test] + fn legacy_outbox_keys_remain_readable_during_migration() { + let first = [7_u8; 32]; + let second = [8_u8; 32]; + + assert_eq!( + decode_legacy_outbox_encryption_keys(&Value::String(BASE64.encode(first))).unwrap(), + vec![first] + ); + assert_eq!( + decode_legacy_outbox_encryption_keys(&serde_json::json!([ + BASE64.encode(first), + BASE64.encode(second) + ])) + .unwrap(), + vec![first, second] + ); + } + + #[test] + fn fallback_key_file_is_created_once_and_rejects_replacement() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("outbox-key"); + let first = [7_u8; 32]; + + persist_file_outbox_key(&path, first).unwrap(); + + assert_eq!(read_file_outbox_key(&path).unwrap(), Some(first)); + assert!(persist_file_outbox_key(&path, [8_u8; 32]).is_err()); + #[cfg(unix)] + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + #[test] fn recovery_outbox_merge_preserves_unique_events() { let first = From 20ff13524012d405061ed5bf87b6ebd5ad5e0ede Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:05:30 +0100 Subject: [PATCH 058/110] ci: scope analytics staging operators --- .github/workflows/analytics.yml | 16 ++++++++-------- ...first-party-analytics-production-readiness.md | 8 ++++++-- scripts/analytics/README.md | 4 +++- scripts/analytics/staging-ci-lib.js | 1 + scripts/analytics/tests/staging-ci.test.js | 12 ++++++++++++ .../snapshot_product_activation_daily_exact.pipe | 2 ++ ...snapshot_product_creator_retention_exact.pipe | 2 ++ .../snapshot_product_events_canonical_v1.pipe | 2 ++ .../snapshot_product_events_daily_exact.pipe | 2 ++ .../snapshot_product_events_health_hourly.pipe | 2 ++ .../snapshot_product_traffic_daily_exact.pipe | 2 ++ ...apshot_product_traffic_pages_daily_exact.pipe | 2 ++ 12 files changed, 44 insertions(+), 11 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index adbfb963a15..191dc2efdc9 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -179,8 +179,8 @@ jobs: - name: Rebuild staged decision and health copies id: staging-copies env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | @@ -230,8 +230,8 @@ jobs: - name: Rebuild promoted decision and health copies id: promoted-copies env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | @@ -260,8 +260,8 @@ jobs: id: erasure-copies if: steps.erase-identity.outcome == 'success' env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | @@ -292,8 +292,8 @@ jobs: id: cleanup-copies if: always() && steps.cleanup.outcome == 'success' env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index 7393e9ab50f..32c4e92fdde 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -32,7 +32,9 @@ Known bots, crawlers, previews, internal IP hashes, and synthetic runs are visib Vercel request-IP buckets use only the platform-owned forwarding header. A self-hosted deployment does not trust forwarding headers by default, so legacy public view tracking returns `503` until `CAP_ANALYTICS_TRUST_PROXY=1` is configured. That setting uses the first `x-forwarded-for` address and is safe only behind a reverse proxy that overwrites, rather than appends or passes through, that header. Do not set it on Vercel. -Account and organization deletion writes its durable pending marker or tombstone before erasure begins. Delivery and reconciliation reject marked identities, then deletion waits longer than the bounded in-flight delivery attempt before removing matching raw rows with a dedicated erasure token and rebuilding every replace-mode canonical, traffic, activation, retention, and health snapshot. This closes the race where a workflow that passed its identity check could otherwise append after a completed erasure. Deletion fails closed if the erasure credential or any rebuild is unavailable. The staging suite deletes a synthetic user and organization, proves their raw-health and decision state is gone, and proves an out-of-scope row sharing the anonymous ID remains until final test cleanup. +The desktop critical-event outbox uses AES-256-GCM. Its stable random key is stored in the OS keyring and in a separate app-local recovery-key file so queued events remain recoverable during temporary keyring outages; Unix recovery-key permissions are `0600`, while Windows relies on the per-user app-data ACL. This protects queue contents from accidental store disclosure but does not claim protection from a process or local account with access to both app-data files. An upgrade may read the earlier application-store fallback-key shape only to decrypt and re-encrypt pending queues, then removes that legacy value after both primary and recovery queues are safely consolidated. + +Account and organization deletion writes its durable pending marker or tombstone before erasure begins. Delivery and reconciliation reject marked identities, then deletion waits longer than the bounded in-flight delivery attempt before removing matching raw rows with a dedicated erasure token and rebuilding every replace-mode canonical, traffic, activation, retention, and health snapshot. This closes the race where a workflow that passed its identity check could otherwise append after a completed erasure. Deletion fails closed if the erasure credential or any rebuild is unavailable. The staging suite deletes a synthetic user and organization, proves their raw-health and decision state is gone, and proves an out-of-scope row sharing the anonymous ID remains until final test cleanup. Tinybird's row-deletion API currently requires the general `DATASOURCES:CREATE` scope and rejects resource-scoped creation of that operator; the token therefore has no read or deployment scope, lives only in the protected staging environment, and is used by code that accepts bounded validated deletion predicates. Its broader same-workspace mutation capability is an explicit provider limitation rather than a least-privilege claim. ## Data quality and performance gates @@ -64,7 +66,8 @@ Production remains prohibited until the final staging artifact, relevant CI, sec 2. Create least-privilege Tinybird tokens: - append-only token for `product_events_v1`; - aggregate endpoint read token with no raw or canonical datasource access; - - erasure token limited to raw deletion, job status, and the seven reviewed copy rebuild pipes; + - resource-scoped copy token for the seven reviewed copy pipes, with no raw identity datasource access; + - dedicated erasure token with Tinybird's required `DATASOURCES:CREATE` scope, no read/deploy scopes, protected as a high-impact operational secret until Tinybird offers resource-scoped row deletion; - deployment token used only by the controlled production release path. 3. Set these Vercel production variables without copying values into logs or artifacts: - `PRODUCT_ANALYTICS_TINYBIRD_HOST` @@ -88,4 +91,5 @@ Rollback is fail closed: disable collection by removing the app append token or - Anonymous traffic is abuse-resistant rather than proof of a human. - Database reconciliation proves stored signup, share, and collaboration facts; Stripe remains the source of truth for money and subscription state. - Aggregated actor counts summed across days are actor-days, not period-unique people. The admin UI labels this explicitly. +- Tinybird requires workspace-wide `DATASOURCES:CREATE` for row deletion; Cap narrows accepted predicates and isolates the credential, but the provider cannot technically restrict it to `product_events_v1` today. - No production rollout is authorized by this document or by a green staging workflow. diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index 69f1a23f5df..7700d9c3cd5 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -37,7 +37,7 @@ Cloud deployment requires: The deployed datafiles create two runtime tokens: - `product_events_ingest`: append-only access to `product_events_v1`. -- `product_events_agent_read`: read-only access to product data and saved endpoints. +- `product_events_agent_read`: read-only access to privacy-safe product endpoints and the seven reviewed copy pipes. It has no raw identity datasource scope. Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. @@ -62,3 +62,5 @@ The Analytics GitHub workflow runs static tests, Docker Compose validation, a co - The infrastructure contains no autocapture or session-replay path. Routine commands refuse destructive deployment, workspace-clear, datasource-delete, and datasource-truncate arguments. Destructive recovery is intentionally outside the normal workflow and requires separate review. + +Tinybird row deletion currently requires the general `DATASOURCES:CREATE` scope rather than a resource-scoped delete permission. Keep the dedicated erasure operator in a protected environment, give it no read or deployment scope, and expose it only to the reviewed deletion workflow. This provider limitation means the credential can technically mutate other datasources in its workspace even though Cap's workflow accepts only validated identity and synthetic-run conditions. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 0143737be14..d18dff7c670 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -507,6 +507,7 @@ export const assertWorkflowSafety = (workflow) => { "TINYBIRD_STAGING_INGEST_TOKEN", "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_CLEANUP_TOKEN", + `TINYBIRD_TOKEN: \${{ secrets.TINYBIRD_STAGING_READ_TOKEN }}`, "probe-preview", "verify-promoted", ...COPY_PIPES, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index e13bd76c329..c0700b65bf8 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -374,4 +374,16 @@ test("the analytics workflow is statically restricted to staging", () => { )?.length, 2, ); + assert.equal( + workflow.match( + /TINYBIRD_TOKEN: \$\{\{ secrets\.TINYBIRD_STAGING_READ_TOKEN \}\}/g, + )?.length, + 4, + ); + assert.equal( + workflow.match( + /TB_TOKEN: \$\{\{ secrets\.TINYBIRD_STAGING_READ_TOKEN \}\}/g, + )?.length, + 4, + ); }); diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe index b9000ce8e9b..ef115a48db7 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe @@ -1,6 +1,8 @@ DESCRIPTION > Rebuild signup activation cohorts from exact canonical server facts. +TOKEN product_events_agent_read READ + NODE snapshot_product_activation_daily_exact_node SQL > % diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe index 91761580850..922bc4e1f28 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe @@ -1,6 +1,8 @@ DESCRIPTION > Rebuild active-creator cohorts from exact successful creation and sharing events. +TOKEN product_events_agent_read READ + NODE snapshot_product_creator_retention_exact_node SQL > % diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe index 7d8d138f662..851e4c56f4f 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe @@ -1,6 +1,8 @@ DESCRIPTION > Replace the canonical stream with one stable payload per event ID and quarantine conflicts. +TOKEN product_events_agent_read READ + NODE snapshot_product_events_canonical_v1_node SQL > % diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe index 987af0a2789..7d88f84f783 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe @@ -1,6 +1,8 @@ DESCRIPTION > Rebuild exact decision metrics from one non-conflicting canonical payload per event ID. +TOKEN product_events_agent_read READ + NODE snapshot_product_events_daily_exact_node SQL > % diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe index ece93f8febe..4a9113b9580 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe @@ -1,6 +1,8 @@ DESCRIPTION > Rebuild hourly delivery health so privacy erasure retracts all derived state. +TOKEN product_events_agent_read READ + NODE snapshot_product_events_health_hourly_node SQL > % diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe index 7d0b5726fcc..8bfc529b370 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe @@ -1,6 +1,8 @@ DESCRIPTION > Rebuild exact 30-minute client-session metrics from canonical external page events. +TOKEN product_events_agent_read READ + NODE snapshot_product_traffic_daily_exact_node SQL > % diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe index dd1dfb82b85..ea787f52af0 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe @@ -1,6 +1,8 @@ DESCRIPTION > Rebuild exact external page, landing, exit, and foreground-engagement metrics. +TOKEN product_events_agent_read READ + NODE snapshot_product_traffic_pages_daily_exact_node SQL > % From e561dfbe33d1b5bb04eb0db1686b25ee4d3a2ba8 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:10:31 +0100 Subject: [PATCH 059/110] fix: adopt concurrent analytics outbox keys --- .../src-tauri/src/product_analytics.rs | 29 ++++++++----------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index ae3167a2903..cc0587c86d3 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -494,13 +494,9 @@ fn read_file_outbox_key(path: &Path) -> Result, String> { } } -fn persist_file_outbox_key(path: &Path, key: [u8; 32]) -> Result<(), String> { +fn initialize_file_outbox_key(path: &Path, key: [u8; 32]) -> Result<[u8; 32], String> { if let Some(existing) = read_file_outbox_key(path)? { - return if existing == key { - Ok(()) - } else { - Err("outbox_key_file_conflict".to_string()) - }; + return Ok(existing); } let parent = path .parent() @@ -525,14 +521,10 @@ fn persist_file_outbox_key(path: &Path, key: [u8; 32]) -> Result<(), String> { fs::File::open(parent) .and_then(|directory| directory.sync_all()) .map_err(|_| "outbox_key_file_write_failed".to_string())?; - Ok(()) + Ok(key) } Err(error) if error.error.kind() == std::io::ErrorKind::AlreadyExists => { - if read_file_outbox_key(path)? == Some(key) { - Ok(()) - } else { - Err("outbox_key_file_conflict".to_string()) - } + read_file_outbox_key(path)?.ok_or_else(|| "outbox_key_file_unavailable".to_string()) } Err(_) => Err("outbox_key_file_write_failed".to_string()), } @@ -620,7 +612,7 @@ fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { } let key_path = file_outbox_key_path(app)?; - let key = if let Some(key) = read_file_outbox_key(&key_path)? { + let candidate = if let Some(key) = read_file_outbox_key(&key_path)? { key } else if let Some(key) = keyring_outbox_encryption_keys().unwrap_or_default().first() { *key @@ -631,7 +623,7 @@ fn outbox_encryption_key(app: &AppHandle) -> Result<&'static [u8; 32], String> { .map_err(|_| "key_generation_failed".to_string())?; generated }; - persist_file_outbox_key(&key_path, key)?; + let key = initialize_file_outbox_key(&key_path, candidate)?; let _ = persist_keyring_outbox_key(PRODUCT_EVENT_OUTBOX_KEYRING_USER, key); let _ = PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY.set(key); PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY @@ -1302,15 +1294,18 @@ mod tests { } #[test] - fn fallback_key_file_is_created_once_and_rejects_replacement() { + fn fallback_key_file_is_created_once_and_concurrent_initializers_adopt_it() { let directory = tempfile::tempdir().unwrap(); let path = directory.path().join("outbox-key"); let first = [7_u8; 32]; - persist_file_outbox_key(&path, first).unwrap(); + assert_eq!(initialize_file_outbox_key(&path, first).unwrap(), first); assert_eq!(read_file_outbox_key(&path).unwrap(), Some(first)); - assert!(persist_file_outbox_key(&path, [8_u8; 32]).is_err()); + assert_eq!( + initialize_file_outbox_key(&path, [8_u8; 32]).unwrap(), + first + ); #[cfg(unix)] assert_eq!( fs::metadata(path).unwrap().permissions().mode() & 0o777, From 47a75d6960ec308dd93d90227221ed5e600fcf82 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:27:43 +0100 Subject: [PATCH 060/110] fix: bound analytics privacy scans --- packages/analytics/src/index.test.ts | 19 +++++++++ packages/analytics/src/privacy.ts | 63 +++++++++++++++++++++++++--- 2 files changed, 77 insertions(+), 5 deletions(-) diff --git a/packages/analytics/src/index.test.ts b/packages/analytics/src/index.test.ts index f8690e9e093..c23f9b086cb 100644 --- a/packages/analytics/src/index.test.ts +++ b/packages/analytics/src/index.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { CORE_EVENT_NAMES, + containsSensitiveAnalyticsContent, createProductEventPayloadHash, createProductEventRows, isCoreEventName, @@ -120,6 +121,24 @@ describe("normalizeProductEventProperties", () => { ).toBeNull(); }); + it("handles repetitive URL and filename probes in linear time", () => { + const startedAt = performance.now(); + expect(containsSensitiveAnalyticsContent(`${"a".repeat(1_000)}:`)).toBe( + false, + ); + expect( + containsSensitiveAnalyticsContent(`${"a.".repeat(500)}unknown`), + ).toBe(false); + expect( + containsSensitiveAnalyticsContent(`${"a".repeat(1_000)}://host`), + ).toBe(true); + expect(containsSensitiveAnalyticsContent(`${"a.".repeat(500)}mp4`)).toBe( + true, + ); + expect(containsSensitiveAnalyticsContent("a".repeat(100_000))).toBe(true); + expect(performance.now() - startedAt).toBeLessThan(250); + }); + it("preserves bounded campaign labels and advertising click identifiers", () => { expect( normalizeProductEventProperties("page_view", { diff --git a/packages/analytics/src/privacy.ts b/packages/analytics/src/privacy.ts index f549f122fe6..aa8a322542f 100644 --- a/packages/analytics/src/privacy.ts +++ b/packages/analytics/src/privacy.ts @@ -7,22 +7,74 @@ export type AnalyticsStringFormat = export const PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT = "[PENDING] Account deletion request"; +const MAX_SENSITIVE_ANALYTICS_SCAN_LENGTH = 1024; const SAFE_IDENTIFIER_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]*$/; const SAFE_CATEGORY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:+/-]*$/; const EMAIL_PATTERN = /\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i; const PHONE_PATTERN = /(?:^|\D)\+?\d[\d ().-]{5,}\d(?:\D|$)/; -const URL_PATTERN = /(?:[a-z][a-z0-9+.-]*:\/\/|www\.)\S+/i; const LOCAL_PATH_PATTERN = /(?:\/(?:Users|home|private|tmp|var)\/|[A-Za-z]:[\\/]|\\\\)/; const SECRET_PATTERN = /(?:Bearer\s+|(?:sk|rk|pk)_(?:live|test)_|[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,}\.[A-Za-z0-9_-]{20,})|(?:api[_-]?key|authorization|password|secret|token)\s*[:=]/i; -const CUSTOMER_FILE_PATTERN = - /[^/\\]+\.(?:avi|cap|csv|docx?|json|log|m4a|mkv|mov|mp3|mp4|pdf|txt|wav|webm)$/i; +const CUSTOMER_FILE_EXTENSIONS = new Set([ + "avi", + "cap", + "csv", + "doc", + "docx", + "json", + "log", + "m4a", + "mkv", + "mov", + "mp3", + "mp4", + "pdf", + "txt", + "wav", + "webm", +]); const IP_ADDRESS_PATTERN = /(?:^|[^\d])(?:\d{1,3}\.){3}\d{1,3}(?:[^\d]|$)|(?:^|[^0-9a-f])(?:(?:[0-9a-f]{1,4}:){7}[0-9a-f]{1,4}|(?=[0-9a-f:]*::[0-9a-f:]*)(?=[0-9a-f:]*[0-9a-f])[0-9a-f:]*::[0-9a-f:]*)(?:[^0-9a-f]|$)/i; const LONG_HEX_PATTERN = /^[0-9a-f]{16,}$/i; const MIXED_RANDOM_TOKEN_PATTERN = /^(?=.*[A-Za-z])(?=.*\d)[A-Za-z0-9_]{20,}$/; +const isAsciiLetter = (codePoint: number) => + (codePoint >= 65 && codePoint <= 90) || (codePoint >= 97 && codePoint <= 122); + +const isUrlSchemeCharacter = (codePoint: number) => + isAsciiLetter(codePoint) || + (codePoint >= 48 && codePoint <= 57) || + codePoint === 43 || + codePoint === 45 || + codePoint === 46; + +function containsUrl(value: string) { + if (value.toLowerCase().includes("www.")) return true; + let separator = value.indexOf("://"); + while (separator >= 0) { + let cursor = separator - 1; + let hasLeadingLetter = false; + while (cursor >= 0 && isUrlSchemeCharacter(value.charCodeAt(cursor))) { + hasLeadingLetter ||= isAsciiLetter(value.charCodeAt(cursor)); + cursor -= 1; + } + if (hasLeadingLetter) return true; + separator = value.indexOf("://", separator + 3); + } + return false; +} + +function containsCustomerFilename(value: string) { + const filenameStart = + Math.max(value.lastIndexOf("/"), value.lastIndexOf("\\")) + 1; + const extensionStart = value.lastIndexOf("."); + if (extensionStart <= filenameStart) return false; + return CUSTOMER_FILE_EXTENSIONS.has( + value.slice(extensionStart + 1).toLowerCase(), + ); +} + export function normalizeAnalyticsIdentifier(value: unknown, maxLength = 128) { if (typeof value !== "string") return undefined; const normalized = value.trim(); @@ -65,6 +117,7 @@ export function normalizeAnalyticsPropertyString( } export function containsSensitiveAnalyticsContent(value: string) { + if (value.length > MAX_SENSITIVE_ANALYTICS_SCAN_LENGTH) return true; return ( Array.from(value).some((character) => { const codePoint = character.codePointAt(0) ?? 0; @@ -72,10 +125,10 @@ export function containsSensitiveAnalyticsContent(value: string) { }) || EMAIL_PATTERN.test(value) || PHONE_PATTERN.test(value) || - URL_PATTERN.test(value) || + containsUrl(value) || LOCAL_PATH_PATTERN.test(value) || SECRET_PATTERN.test(value) || - CUSTOMER_FILE_PATTERN.test(value) || + containsCustomerFilename(value) || IP_ADDRESS_PATTERN.test(value) ); } From b02f04bc15df4df42246a818ed4f3a18d679441f Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:27:45 +0100 Subject: [PATCH 061/110] ci: discard failed staging before cleanup --- .github/workflows/analytics.yml | 20 ++++++++++---------- scripts/analytics/tests/staging-ci.test.js | 5 +++++ 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 191dc2efdc9..f7a0aa032e5 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -208,6 +208,16 @@ jobs: git diff --exit-code test -z "$(git status --porcelain --untracked-files=no)" docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment promote + - name: Discard an unpromoted staging deployment before cleanup + if: always() && steps.create-deployment.outcome != 'skipped' && steps.promote.outcome != 'success' + env: + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} + run: >- + docker compose --file packages/local-docker/docker-compose.yml --profile analytics + run --rm tinybird-cloud-cli --cloud deployment discard --wait - name: Prove least-privilege staging token scopes id: token-scopes env: @@ -310,16 +320,6 @@ jobs: node scripts/analytics/staging-ci.js verify-cleanup --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Discard an unpromoted staging deployment - if: always() && steps.create-deployment.outcome != 'skipped' && steps.promote.outcome != 'success' - env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} - run: >- - docker compose --file packages/local-docker/docker-compose.yml --profile analytics - run --rm tinybird-cloud-cli --cloud deployment discard --wait - name: Upload redacted staging evidence if: always() && steps.seed.outcome == 'success' uses: actions/upload-artifact@v4 diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index c0700b65bf8..a36c86873ee 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -386,4 +386,9 @@ test("the analytics workflow is statically restricted to staging", () => { )?.length, 4, ); + assert.ok( + workflow.indexOf( + "Discard an unpromoted staging deployment before cleanup", + ) < workflow.indexOf("Delete strictly scoped synthetic raw rows"), + ); }); From fc3180438babf581e65179d8325965f95ecb3bf4 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:36:22 +0100 Subject: [PATCH 062/110] ci: accept exact no-op Tinybird deployments --- .github/workflows/analytics.yml | 20 +++++--- scripts/analytics/staging-ci-lib.js | 59 ++++++++++++++++------ scripts/analytics/staging-ci.js | 10 +++- scripts/analytics/tests/staging-ci.test.js | 36 +++++++++++-- 4 files changed, 96 insertions(+), 29 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index f7a0aa032e5..b21c0f8c32e 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -139,9 +139,10 @@ jobs: TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} - run: >- - docker compose --file packages/local-docker/docker-compose.yml --profile analytics - run --rm tinybird-cloud-cli --cloud deployment create --allow-destructive-operations --wait + run: | + set -o pipefail + docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment create --allow-destructive-operations --wait | tee "$RUNNER_TEMP/tinybird-create-output.txt" + node -e 'const fs = require("node:fs"); fs.writeFileSync(process.argv[2], JSON.stringify({ output: fs.readFileSync(process.argv[1], "utf8") }));' "$RUNNER_TEMP/tinybird-create-output.txt" "$RUNNER_TEMP/tinybird-create-output.json" - name: Resolve only the deployment created by this run id: tinybird env: @@ -151,7 +152,7 @@ jobs: TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud --output json deployment ls > "$RUNNER_TEMP/tinybird-deployments.json" - node scripts/analytics/staging-ci.js select-deployment --input "$RUNNER_TEMP/tinybird-deployments.json" --minimum-created-at "$TINYBIRD_DEPLOYMENT_STARTED_AT" + node scripts/analytics/staging-ci.js select-deployment --input "$RUNNER_TEMP/tinybird-deployments.json" --create-output "$RUNNER_TEMP/tinybird-create-output.json" --minimum-created-at "$TINYBIRD_DEPLOYMENT_STARTED_AT" - name: Refuse source mutations after exact-SHA checkout run: | git diff --exit-code @@ -184,8 +185,12 @@ jobs: TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | + deployment_flag=() + if [[ "${{ steps.tinybird.outputs.needs_promotion }}" == "true" ]]; then + deployment_flag=(--staging) + fi for pipe in snapshot_product_events_canonical_v1 snapshot_product_events_daily_exact snapshot_product_traffic_daily_exact snapshot_product_traffic_pages_daily_exact snapshot_product_activation_daily_exact snapshot_product_creator_retention_exact snapshot_product_events_health_hourly; do - docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud --staging copy run "$pipe" --wait --mode replace + docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud "${deployment_flag[@]}" copy run "$pipe" --wait --mode replace done - name: Prove staged deduplication, conflict visibility, SLO, and endpoint budget id: verify @@ -198,6 +203,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Promote the verified staging deployment id: promote + if: steps.tinybird.outputs.needs_promotion == 'true' env: TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} @@ -207,9 +213,9 @@ jobs: test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" git diff --exit-code test -z "$(git status --porcelain --untracked-files=no)" - docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment promote + docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment promote --wait - name: Discard an unpromoted staging deployment before cleanup - if: always() && steps.create-deployment.outcome != 'skipped' && steps.promote.outcome != 'success' + if: always() && steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' env: TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index d18dff7c670..75dda894eff 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -105,7 +105,23 @@ export const validateTinybirdCredentials = ({ url, tokens }) => { return parsedUrl.origin; }; -export const selectStagingDeployment = (value, minimumCreatedAt) => { +const deploymentId = (deployment) => + deployment.id ?? deployment.ID ?? deployment.deployment_id; + +const deploymentState = (deployment) => + String( + deployment.status ?? + deployment.Status ?? + deployment.state ?? + deployment.environment ?? + "", + ).toLowerCase(); + +export const selectStagingDeployment = ( + value, + minimumCreatedAt, + createdDeploymentId, +) => { const candidates = Array.isArray(value) ? value : (value.deployments ?? value.data ?? value.results ?? []); @@ -113,15 +129,8 @@ export const selectStagingDeployment = (value, minimumCreatedAt) => { throw new Error("Tinybird returned an unsupported deployment list"); } const minimumTime = Date.parse(minimumCreatedAt); - const deployments = candidates + const stagingDeployments = candidates .filter((candidate) => { - const state = String( - candidate.status ?? - candidate.Status ?? - candidate.state ?? - candidate.environment ?? - "", - ).toLowerCase(); const createdAt = Date.parse( candidate.created_at ?? candidate.createdAt ?? @@ -130,7 +139,7 @@ export const selectStagingDeployment = (value, minimumCreatedAt) => { "", ); return ( - state.includes("staging") && + deploymentState(candidate).includes("staging") && Number.isFinite(createdAt) && createdAt >= minimumTime ); @@ -142,17 +151,35 @@ export const selectStagingDeployment = (value, minimumCreatedAt) => { ) - Date.parse(a.created_at ?? a.createdAt ?? a["Created at"] ?? a.created), ); - if (deployments.length !== 1) { + if (createdDeploymentId) { + const matching = stagingDeployments.filter( + (deployment) => String(deploymentId(deployment)) === createdDeploymentId, + ); + if (matching.length !== 1) { + throw new Error( + "The created Tinybird deployment is missing, stale, or ambiguous", + ); + } + return { id: createdDeploymentId, needsPromotion: true }; + } + if (stagingDeployments.length > 0) { + throw new Error( + "Tinybird reported a staging deployment that was not created by this run", + ); + } + const liveDeployments = candidates.filter((candidate) => + deploymentState(candidate).includes("live"), + ); + if (liveDeployments.length !== 1) { throw new Error( - "Expected exactly one staging deployment created by this run", + "Expected exactly one live Tinybird deployment for a no-op", ); } - const id = - deployments[0].id ?? deployments[0].ID ?? deployments[0].deployment_id; + const id = deploymentId(liveDeployments[0]); if (typeof id !== "string" && typeof id !== "number") { - throw new Error("The staging deployment does not have an ID"); + throw new Error("The live Tinybird deployment does not have an ID"); } - return String(id); + return { id: String(id), needsPromotion: false }; }; export const validateSyntheticRunId = (runId) => { diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index dca320055c3..f0887c88b73 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -1178,11 +1178,17 @@ const handlers = { "verify-credentials": async () => tinybirdEnvironment(), "verify-token-scopes": verifyTokenScopes, "select-deployment": async () => { - const id = selectStagingDeployment( + const createOutput = readJson(option("create-output")); + const createdDeploymentId = String(createOutput.output ?? "").match( + /Deployment URL:\s+\S+\/deployments\/(\d+)/, + )?.[1]; + const selection = selectStagingDeployment( readJson(option("input")), option("minimum-created-at"), + createdDeploymentId, ); - writeOutput("id", id); + writeOutput("id", selection.id); + writeOutput("needs_promotion", String(selection.needsPromotion)); }, "wait-vercel": waitForVercel, seed, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index a36c86873ee..60517b4b590 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -117,7 +117,7 @@ test("Tinybird credentials must all decode to the hard-coded staging workspace", test("deployment selection rejects stale or ambiguous staging deployments", () => { const minimum = "2026-07-31T10:00:00.000Z"; - assert.equal( + assert.deepEqual( selectStagingDeployment( { deployments: [ @@ -134,10 +134,11 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = ], }, minimum, + "current", ), - "current", + { id: "current", needsPromotion: true }, ); - assert.equal( + assert.deepEqual( selectStagingDeployment( [ { @@ -147,8 +148,9 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = }, ], minimum, + "42", ), - "42", + { id: "42", needsPromotion: true }, ); assert.throws(() => selectStagingDeployment( @@ -167,6 +169,32 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = minimum, ), ); + assert.deepEqual( + selectStagingDeployment( + [ + { + id: "live", + status: "Live", + created_at: "2026-07-31T09:00:00.000Z", + }, + ], + minimum, + ), + { id: "live", needsPromotion: false }, + ); + assert.throws(() => + selectStagingDeployment( + [ + { + id: "foreign", + status: "Staging", + created_at: "2026-07-31T10:00:01.000Z", + }, + ], + minimum, + "expected", + ), + ); }); test("synthetic fixtures are deterministic, isolated, and model duplicates and conflicts", () => { From da61fb75efb59775d93a5042e3ae6bc1b39424b6 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:38:38 +0100 Subject: [PATCH 063/110] ci: require explicit Tinybird no-op proof --- scripts/analytics/staging-ci-lib.js | 4 ++++ scripts/analytics/staging-ci.js | 7 ++++++- scripts/analytics/tests/staging-ci.test.js | 14 ++++++++++++++ 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 75dda894eff..80ab02ebdab 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -121,6 +121,7 @@ export const selectStagingDeployment = ( value, minimumCreatedAt, createdDeploymentId, + noOpConfirmed = false, ) => { const candidates = Array.isArray(value) ? value @@ -167,6 +168,9 @@ export const selectStagingDeployment = ( "Tinybird reported a staging deployment that was not created by this run", ); } + if (!noOpConfirmed) { + throw new Error("Tinybird did not prove that this was a no-op deployment"); + } const liveDeployments = candidates.filter((candidate) => deploymentState(candidate).includes("live"), ); diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index f0887c88b73..0804cbcc329 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -1179,13 +1179,18 @@ const handlers = { "verify-token-scopes": verifyTokenScopes, "select-deployment": async () => { const createOutput = readJson(option("create-output")); - const createdDeploymentId = String(createOutput.output ?? "").match( + const output = String(createOutput.output ?? ""); + const createdDeploymentId = output.match( /Deployment URL:\s+\S+\/deployments\/(\d+)/, )?.[1]; + const noOpConfirmed = + output.includes("No changes to be deployed") && + output.includes("Not deploying. No changes."); const selection = selectStagingDeployment( readJson(option("input")), option("minimum-created-at"), createdDeploymentId, + noOpConfirmed, ); writeOutput("id", selection.id); writeOutput("needs_promotion", String(selection.needsPromotion)); diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 60517b4b590..9ebb3cd571e 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -179,9 +179,23 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = }, ], minimum, + undefined, + true, ), { id: "live", needsPromotion: false }, ); + assert.throws(() => + selectStagingDeployment( + [ + { + id: "live", + status: "Live", + created_at: "2026-07-31T09:00:00.000Z", + }, + ], + minimum, + ), + ); assert.throws(() => selectStagingDeployment( [ From 3437e9f6e6c02dec71485e8801f3d37a92fbdf32 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:56:40 +0100 Subject: [PATCH 064/110] ci: run Tinybird copies with scoped API --- .github/workflows/analytics.yml | 68 +++++++++---------- scripts/analytics/README.md | 2 + scripts/analytics/staging-ci-lib.js | 76 +++++++++++++++++++++- scripts/analytics/staging-ci.js | 45 +++++++++++++ scripts/analytics/tests/staging-ci.test.js | 56 ++++++++++++++-- 5 files changed, 204 insertions(+), 43 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index b21c0f8c32e..63e49ac2e27 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -180,18 +180,14 @@ jobs: - name: Rebuild staged decision and health copies id: staging-copies env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} - run: | - deployment_flag=() - if [[ "${{ steps.tinybird.outputs.needs_promotion }}" == "true" ]]; then - deployment_flag=(--staging) - fi - for pipe in snapshot_product_events_canonical_v1 snapshot_product_events_daily_exact snapshot_product_traffic_daily_exact snapshot_product_traffic_pages_daily_exact snapshot_product_activation_daily_exact snapshot_product_creator_retention_exact snapshot_product_events_health_hourly; do - docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud "${deployment_flag[@]}" copy run "$pipe" --wait --mode replace - done + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js run-copies + --deployment-id "${{ steps.tinybird.outputs.id }}" + --phase staged + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove staged deduplication, conflict visibility, SLO, and endpoint budget id: verify env: @@ -246,14 +242,14 @@ jobs: - name: Rebuild promoted decision and health copies id: promoted-copies env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} - run: | - for pipe in snapshot_product_events_canonical_v1 snapshot_product_events_daily_exact snapshot_product_traffic_daily_exact snapshot_product_traffic_pages_daily_exact snapshot_product_activation_daily_exact snapshot_product_creator_retention_exact snapshot_product_events_health_hourly; do - docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud copy run "$pipe" --wait --mode replace - done + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js run-copies + --deployment-id "${{ steps.tinybird.outputs.id }}" + --phase promoted + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove promoted preview delivery and decision deduplication id: verify-promoted env: @@ -276,14 +272,14 @@ jobs: id: erasure-copies if: steps.erase-identity.outcome == 'success' env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} - run: | - for pipe in snapshot_product_events_canonical_v1 snapshot_product_events_daily_exact snapshot_product_traffic_daily_exact snapshot_product_traffic_pages_daily_exact snapshot_product_activation_daily_exact snapshot_product_creator_retention_exact snapshot_product_events_health_hourly; do - docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud copy run "$pipe" --wait --mode replace - done + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js run-copies + --deployment-id "${{ steps.tinybird.outputs.id }}" + --phase erasure + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove identity erasure and out-of-scope control preservation id: verify-erasure if: steps.erasure-copies.outcome == 'success' @@ -308,14 +304,14 @@ jobs: id: cleanup-copies if: always() && steps.cleanup.outcome == 'success' env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} - run: | - for pipe in snapshot_product_events_canonical_v1 snapshot_product_events_daily_exact snapshot_product_traffic_daily_exact snapshot_product_traffic_pages_daily_exact snapshot_product_activation_daily_exact snapshot_product_creator_retention_exact snapshot_product_events_health_hourly; do - docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud copy run "$pipe" --wait --mode replace - done + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js run-copies + --deployment-id "${{ steps.tinybird.outputs.id }}" + --phase cleanup + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove synthetic cleanup no longer affects queries id: verify-cleanup if: always() && steps.cleanup-copies.outcome == 'success' diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index 7700d9c3cd5..05218d06bbc 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -41,6 +41,8 @@ The deployed datafiles create two runtime tokens: Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. +The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. + Set `TINYBIRD_AGENT_TOKEN` and `TINYBIRD_URL` for the query command. Set a separate `TINYBIRD_READ_TOKEN` with workspace metadata access when running `pnpm analytics:check`; the append and deployment tokens are intentionally rejected for that task. ## Agent access diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 80ab02ebdab..9beca736e18 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -15,6 +15,8 @@ export const COPY_PIPES = [ const SHA_PATTERN = /^[0-9a-f]{40}$/; const SYNTHETIC_RUN_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; +const COPY_JOB_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; +const DEPLOYMENT_ID_PATTERN = /^[0-9]+$/; export const assertExecutionScope = ({ eventName, @@ -186,6 +188,77 @@ export const selectStagingDeployment = ( return { id: String(id), needsPromotion: false }; }; +export const runTinybirdCopyJobs = async ({ + origin, + token, + deploymentId, + request, + wait, + now = () => Date.now(), + timeoutMs = 120_000, + pipes = COPY_PIPES, +}) => { + if (!DEPLOYMENT_ID_PATTERN.test(deploymentId)) { + throw new Error("Tinybird copy jobs require a numeric deployment ID"); + } + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw new Error("Tinybird copy job timeout must be positive"); + } + const results = []; + for (const pipe of pipes) { + if (!COPY_PIPES.includes(pipe)) { + throw new Error("Tinybird copy job requested an unapproved pipe"); + } + const startedAt = now(); + const copyUrl = new URL( + `/v0/pipes/${encodeURIComponent(pipe)}/copy`, + origin, + ); + copyUrl.searchParams.set("_mode", "replace"); + copyUrl.searchParams.set("__tb__deployment", deploymentId); + const created = await request(copyUrl, { + token, + method: "POST", + attempts: 3, + }); + const jobId = String(created.data.id ?? created.data.job_id ?? ""); + if (!COPY_JOB_ID_PATTERN.test(jobId)) { + throw new Error(`Tinybird did not return a valid copy job for ${pipe}`); + } + let polls = 0; + while (true) { + polls += 1; + const job = await request( + new URL(`/v0/jobs/${encodeURIComponent(jobId)}`, origin), + { token, attempts: 3 }, + ); + const status = String(job.data.status ?? "").toLowerCase(); + if (status === "done") { + results.push({ + pipe, + jobId, + polls, + durationMs: Math.max(0, now() - startedAt), + }); + break; + } + if (["error", "cancelled"].includes(status)) { + throw new Error(`Tinybird copy job for ${pipe} ended in ${status}`); + } + if (!["waiting", "working", "cancelling"].includes(status)) { + throw new Error( + `Tinybird copy job for ${pipe} returned an invalid status`, + ); + } + if (now() - startedAt >= timeoutMs) { + throw new Error(`Tinybird copy job for ${pipe} exceeded its timeout`); + } + await wait(1_000); + } + } + return results; +}; + export const validateSyntheticRunId = (runId) => { if (!SYNTHETIC_RUN_PATTERN.test(runId)) { throw new Error("Synthetic run ID has an unsafe format"); @@ -538,10 +611,9 @@ export const assertWorkflowSafety = (workflow) => { "TINYBIRD_STAGING_INGEST_TOKEN", "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_CLEANUP_TOKEN", - `TINYBIRD_TOKEN: \${{ secrets.TINYBIRD_STAGING_READ_TOKEN }}`, + "staging-ci.js run-copies", "probe-preview", "verify-promoted", - ...COPY_PIPES, ]) { if (!workflow.includes(required)) { throw new Error(`Workflow is missing staging safeguard: ${required}`); diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 0804cbcc329..45f5a7c7571 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -17,6 +17,7 @@ import { latencySummary, normalizeCiAssertions, normalizeHealth, + runTinybirdCopyJobs, STAGING_WORKSPACE_ID, selectStagingDeployment, validateSyntheticRunId, @@ -633,6 +634,49 @@ const seed = async () => { }); }; +const runCopies = async () => { + const state = readJson(option("state")); + const artifactPath = option("artifact"); + const artifact = readJson(artifactPath); + const phase = option("phase"); + if (!["staged", "promoted", "erasure", "cleanup"].includes(phase)) { + throw new Error("Tinybird copy phase is invalid"); + } + if (String(state.deploymentId) !== option("deployment-id")) { + throw new Error("Tinybird copy deployment does not match the seeded run"); + } + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_READ_TOKEN", + ]); + try { + const jobs = await runTinybirdCopyJobs({ + origin, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + deploymentId: state.deploymentId, + request, + wait: delay, + }); + artifact.copyJobs = { + ...artifact.copyJobs, + [phase]: { status: "passed", jobs }, + }; + writeJson(artifactPath, artifact); + } catch (error) { + artifact.copyJobs = { + ...artifact.copyJobs, + [phase]: { + status: "failed", + error: + error instanceof Error + ? error.message + : "Unknown Tinybird copy error", + }, + }; + writeJson(artifactPath, artifact); + throw error; + } +}; + const verify = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); @@ -1197,6 +1241,7 @@ const handlers = { }, "wait-vercel": waitForVercel, seed, + "run-copies": runCopies, verify, "probe-preview": probePreview, "verify-promoted": verifyPromoted, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 9ebb3cd571e..7d2f2562c77 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -20,6 +20,7 @@ import { latencySummary, normalizeCiAssertions, normalizeHealth, + runTinybirdCopyJobs, STAGING_WORKSPACE_ID, selectStagingDeployment, tokenWorkspaceId, @@ -211,6 +212,51 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = ); }); +test("copy jobs use the resource-scoped API and wait for exact completion", async () => { + const requests = []; + const responses = [ + { data: { id: "copy_job_123" } }, + { data: { status: "working" } }, + { data: { status: "done" } }, + ]; + let now = 1_000; + const results = await runTinybirdCopyJobs({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + deploymentId: "6", + pipes: ["snapshot_product_events_canonical_v1"], + request: async (url, options) => { + requests.push({ url: String(url), options }); + return responses.shift(); + }, + wait: async (milliseconds) => { + now += milliseconds; + }, + now: () => now, + }); + assert.deepEqual(results, [ + { + pipe: "snapshot_product_events_canonical_v1", + jobId: "copy_job_123", + polls: 2, + durationMs: 1_000, + }, + ]); + assert.match(requests[0].url, /_mode=replace/); + assert.match(requests[0].url, /__tb__deployment=6/); + assert.equal(requests[0].options.method, "POST"); + assert.match(requests[1].url, /\/v0\/jobs\/copy_job_123$/); + await assert.rejects(() => + runTinybirdCopyJobs({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + deploymentId: "live", + request: async () => ({ data: {} }), + wait: async () => {}, + }), + ); +}); + test("synthetic fixtures are deterministic, isolated, and model duplicates and conflicts", () => { const runId = "run_12345678"; const fixture = createSyntheticEvents({ @@ -417,17 +463,17 @@ test("the analytics workflow is statically restricted to staging", () => { 2, ); assert.equal( - workflow.match( - /TINYBIRD_TOKEN: \$\{\{ secrets\.TINYBIRD_STAGING_READ_TOKEN \}\}/g, - )?.length, + workflow.match(/node scripts\/analytics\/staging-ci\.js run-copies/g) + ?.length, 4, ); assert.equal( workflow.match( - /TB_TOKEN: \$\{\{ secrets\.TINYBIRD_STAGING_READ_TOKEN \}\}/g, + /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 4, + 5, ); + assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); assert.ok( workflow.indexOf( "Discard an unpromoted staging deployment before cleanup", From b3a595622e9aff3738ebff8d0570d14f12b66126 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:06:34 +0100 Subject: [PATCH 065/110] ci: target live Tinybird copies directly --- .github/workflows/analytics.yml | 4 +++ scripts/analytics/README.md | 2 +- scripts/analytics/staging-ci-lib.js | 37 ++++++++++++++++------ scripts/analytics/staging-ci.js | 8 +++++ scripts/analytics/tests/staging-ci.test.js | 16 ++++++++++ 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 63e49ac2e27..cd64c4afcc6 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -186,6 +186,7 @@ jobs: node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase staged + --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live' }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove staged deduplication, conflict visibility, SLO, and endpoint budget @@ -248,6 +249,7 @@ jobs: node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase promoted + --target live --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove promoted preview delivery and decision deduplication @@ -278,6 +280,7 @@ jobs: node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase erasure + --target live --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove identity erasure and out-of-scope control preservation @@ -310,6 +313,7 @@ jobs: node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase cleanup + --target live --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove synthetic cleanup no longer affects queries diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index 05218d06bbc..600b1f1399e 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -41,7 +41,7 @@ The deployed datafiles create two runtime tokens: Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. -The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. +The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Candidate validation pins the staging deployment parameter, while no-op and post-promotion phases use the already verified live deployment because Tinybird rejects a deployment selector on the Copy service after promotion. Set `TINYBIRD_AGENT_TOKEN` and `TINYBIRD_URL` for the query command. Set a separate `TINYBIRD_READ_TOKEN` with workspace metadata access when running `pnpm analytics:check`; the append and deployment tokens are intentionally rejected for that task. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 9beca736e18..ab1bdd1bf37 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -197,6 +197,7 @@ export const runTinybirdCopyJobs = async ({ now = () => Date.now(), timeoutMs = 120_000, pipes = COPY_PIPES, + useDeploymentParameter = false, }) => { if (!DEPLOYMENT_ID_PATTERN.test(deploymentId)) { throw new Error("Tinybird copy jobs require a numeric deployment ID"); @@ -215,12 +216,21 @@ export const runTinybirdCopyJobs = async ({ origin, ); copyUrl.searchParams.set("_mode", "replace"); - copyUrl.searchParams.set("__tb__deployment", deploymentId); - const created = await request(copyUrl, { - token, - method: "POST", - attempts: 3, - }); + if (useDeploymentParameter) { + copyUrl.searchParams.set("__tb__deployment", deploymentId); + } + let created; + try { + created = await request(copyUrl, { + token, + method: "POST", + attempts: 3, + }); + } catch (error) { + throw new Error(`Tinybird copy submission failed for ${pipe}`, { + cause: error, + }); + } const jobId = String(created.data.id ?? created.data.job_id ?? ""); if (!COPY_JOB_ID_PATTERN.test(jobId)) { throw new Error(`Tinybird did not return a valid copy job for ${pipe}`); @@ -228,10 +238,17 @@ export const runTinybirdCopyJobs = async ({ let polls = 0; while (true) { polls += 1; - const job = await request( - new URL(`/v0/jobs/${encodeURIComponent(jobId)}`, origin), - { token, attempts: 3 }, - ); + let job; + try { + job = await request( + new URL(`/v0/jobs/${encodeURIComponent(jobId)}`, origin), + { token, attempts: 3 }, + ); + } catch (error) { + throw new Error(`Tinybird copy status read failed for ${pipe}`, { + cause: error, + }); + } const status = String(job.data.status ?? "").toLowerCase(); if (status === "done") { results.push({ diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 45f5a7c7571..3377f95a5b0 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -639,9 +639,16 @@ const runCopies = async () => { const artifactPath = option("artifact"); const artifact = readJson(artifactPath); const phase = option("phase"); + const target = option("target"); if (!["staged", "promoted", "erasure", "cleanup"].includes(phase)) { throw new Error("Tinybird copy phase is invalid"); } + if (!["live", "staging"].includes(target)) { + throw new Error("Tinybird copy target is invalid"); + } + if (target === "staging" && phase !== "staged") { + throw new Error("Only the staged copy phase can target staging"); + } if (String(state.deploymentId) !== option("deployment-id")) { throw new Error("Tinybird copy deployment does not match the seeded run"); } @@ -655,6 +662,7 @@ const runCopies = async () => { deploymentId: state.deploymentId, request, wait: delay, + useDeploymentParameter: target === "staging", }); artifact.copyJobs = { ...artifact.copyJobs, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 7d2f2562c77..8a4a8248287 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -233,6 +233,7 @@ test("copy jobs use the resource-scoped API and wait for exact completion", asyn now += milliseconds; }, now: () => now, + useDeploymentParameter: true, }); assert.deepEqual(results, [ { @@ -246,6 +247,21 @@ test("copy jobs use the resource-scoped API and wait for exact completion", asyn assert.match(requests[0].url, /__tb__deployment=6/); assert.equal(requests[0].options.method, "POST"); assert.match(requests[1].url, /\/v0\/jobs\/copy_job_123$/); + const liveRequests = []; + await runTinybirdCopyJobs({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + deploymentId: "6", + pipes: ["snapshot_product_events_canonical_v1"], + request: async (url, options) => { + liveRequests.push({ url: String(url), options }); + return liveRequests.length === 1 + ? { data: { id: "copy_job_live" } } + : { data: { status: "done" } }; + }, + wait: async () => {}, + }); + assert.doesNotMatch(liveRequests[0].url, /__tb__deployment/); await assert.rejects(() => runTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", From be8cdd3cb13370809d52502d5f25308ced1cfd78 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:11:13 +0100 Subject: [PATCH 066/110] fix: bind analytics limits to trusted networks --- ...st-party-analytics-production-readiness.md | 3 +- .../unit/product-analytics-request.test.ts | 40 ++++++++++--------- apps/web/app/api/analytics/track/route.ts | 4 +- apps/web/app/api/events/route.ts | 17 +++++--- apps/web/lib/analytics/request.ts | 23 +++++------ 5 files changed, 47 insertions(+), 40 deletions(-) diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index 32c4e92fdde..c378974b6ee 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -30,7 +30,7 @@ The collector rejects unregistered events, unknown properties, raw error fields, Known bots, crawlers, previews, internal IP hashes, and synthetic runs are visible to health monitoring but excluded from decision snapshots. Anonymous write tokens are rate-limited by both source IP and anonymous ID. Replay and automation can be made expensive and observable, but public traffic analytics cannot cryptographically guarantee a human browser. Signup, organization, billing, and other business outcomes therefore remain server-authoritative. -Vercel request-IP buckets use only the platform-owned forwarding header. A self-hosted deployment does not trust forwarding headers by default, so legacy public view tracking returns `503` until `CAP_ANALYTICS_TRUST_PROXY=1` is configured. That setting uses the first `x-forwarded-for` address and is safe only behind a reverse proxy that overwrites, rather than appends or passes through, that header. Do not set it on Vercel. +Vercel request-IP buckets use only the platform-owned forwarding header and retain only its SHA-256 hash in process memory. A self-hosted deployment does not trust forwarding headers by default, so product-event collection and legacy public view tracking return `503` until `CAP_ANALYTICS_TRUST_PROXY=1` is configured. That setting uses the first `x-forwarded-for` address and is safe only behind a reverse proxy that overwrites, rather than appends or passes through, that header. Do not set it on Vercel. Client-controlled anonymous IDs and authorization strings never become network rate-limit keys. The desktop critical-event outbox uses AES-256-GCM. Its stable random key is stored in the OS keyring and in a separate app-local recovery-key file so queued events remain recoverable during temporary keyring outages; Unix recovery-key permissions are `0600`, while Windows relies on the per-user app-data ACL. This protects queue contents from accidental store disclosure but does not claim protection from a process or local account with access to both app-data files. An upgrade may read the earlier application-store fallback-key shape only to decrypt and re-encrypt pending queues, then removes that legacy value after both primary and recovery queues are safely consolidated. @@ -77,6 +77,7 @@ Production remains prohibited until the final staging artifact, relevant CI, sec - `PRODUCT_ANALYTICS_INTERNAL_IP_HASHES` - `CRON_SECRET` - `NEXTAUTH_SECRET` (existing application secret used to sign the short-lived anonymous browser token; do not create an analytics-specific duplicate) + For a self-hosted release only, set `CAP_ANALYTICS_TRUST_PROXY=1` after verifying that the fronting reverse proxy overwrites `x-forwarded-for`. Omit it on Vercel. 4. Configure the Vercel firewall rule referenced by the collector so `/api/events` has the verified per-IP limit. The collector also applies the same rule by anonymous-ID key. Verify normal navigation is usable, the documented burst test receives `429`, forged forwarding headers do not bypass classification, and preview hostnames are excluded from decision metrics. 5. Deploy the web application through the normal production release process. Confirm the Vercel deployment Git SHA exactly matches the reviewed commit and confirm the configured Tinybird deployment ID before sending any test event. 6. Run a uniquely tagged production smoke test only if separately approved. Do not use customer identifiers. Confirm ingestion, health, decision deduplication, admin freshness, and strictly scoped cleanup. diff --git a/apps/web/__tests__/unit/product-analytics-request.test.ts b/apps/web/__tests__/unit/product-analytics-request.test.ts index 3d1b96b0d8e..21bb1e9fcfb 100644 --- a/apps/web/__tests__/unit/product-analytics-request.test.ts +++ b/apps/web/__tests__/unit/product-analytics-request.test.ts @@ -265,36 +265,38 @@ describe("ProductAnalyticsRateLimiter", () => { expect(limiter.isRateLimited("a", 1_000)).toBe(false); }); - it("uses a platform-owned proxy identity or hashed self-hosted identity", () => { - expect( + it("uses only a hashed platform-owned network identity", () => { + const first = getProductAnalyticsRateLimitKey({ + trustedNetworkProxy: true, + forwardedFor: "203.0.113.10, 10.0.0.1", + }); + expect(first).toMatch(/^network:[0-9a-f]{64}$/); + expect(first).toBe( getProductAnalyticsRateLimitKey({ - trustedVercelProxy: true, - xVercelForwardedFor: "203.0.113.10, 10.0.0.1", + trustedNetworkProxy: true, + forwardedFor: "203.0.113.10", }), - ).toBe("203.0.113.10"); - expect( + ); + expect(first).not.toBe( getProductAnalyticsRateLimitKey({ - trustedVercelProxy: false, - xVercelForwardedFor: "attacker-controlled", - fallbackIdentity: "browser-1", + trustedNetworkProxy: true, + forwardedFor: "203.0.113.11", }), - ).toMatch(/^self-hosted:[0-9a-f]{64}$/); + ); expect( getProductAnalyticsRateLimitKey({ - trustedVercelProxy: false, - fallbackIdentity: "browser-1", + trustedNetworkProxy: false, + forwardedFor: "203.0.113.10", }), - ).not.toBe( + ).toBeNull(); + expect( getProductAnalyticsRateLimitKey({ - trustedVercelProxy: false, - fallbackIdentity: "browser-2", + trustedNetworkProxy: true, + forwardedFor: "attacker-controlled", }), - ); - expect( - getProductAnalyticsRateLimitKey({ trustedVercelProxy: true }), ).toBeNull(); expect( - getProductAnalyticsRateLimitKey({ trustedVercelProxy: false }), + getProductAnalyticsRateLimitKey({ trustedNetworkProxy: true }), ).toBeNull(); }); }); diff --git a/apps/web/app/api/analytics/track/route.ts b/apps/web/app/api/analytics/track/route.ts index 9fcd778dc6c..f216bd6baac 100644 --- a/apps/web/app/api/analytics/track/route.ts +++ b/apps/web/app/api/analytics/track/route.ts @@ -87,8 +87,8 @@ export async function POST(request: NextRequest) { ); } const rateLimitKey = getProductAnalyticsRateLimitKey({ - trustedVercelProxy: trustedNetworkProxy, - xVercelForwardedFor: + trustedNetworkProxy, + forwardedFor: request.headers.get( isVercel ? "x-vercel-forwarded-for" : "x-forwarded-for", ) ?? undefined, diff --git a/apps/web/app/api/events/route.ts b/apps/web/app/api/events/route.ts index 8a302734124..e24b709a7a2 100644 --- a/apps/web/app/api/events/route.ts +++ b/apps/web/app/api/events/route.ts @@ -92,6 +92,7 @@ const RequestHeaders = Schema.Struct({ "x-vercel-ip-country-region": Schema.optional(Schema.String), "x-vercel-ip-city": Schema.optional(Schema.String), "x-vercel-forwarded-for": Schema.optional(Schema.String), + "x-forwarded-for": Schema.optional(Schema.String), "user-agent": Schema.optional(Schema.String), "x-cap-analytics-test-run": Schema.optional(Schema.String), }); @@ -132,12 +133,18 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( return yield* Effect.fail(new HttpApiError.BadRequest()); } + const isVercel = process.env.VERCEL === "1"; + const trustedNetworkProxy = + isVercel || process.env.CAP_ANALYTICS_TRUST_PROXY === "1"; + if (!trustedNetworkProxy) { + return yield* Effect.fail(new HttpApiError.ServiceUnavailable()); + } const rateLimitKey = getProductAnalyticsRateLimitKey({ - trustedVercelProxy: process.env.VERCEL === "1", - xVercelForwardedFor: headers["x-vercel-forwarded-for"], - fallbackIdentity: - (browserClaims ? browserClaims.anonymousId : undefined) ?? - headers.authorization, + trustedNetworkProxy, + forwardedFor: + headers[ + isVercel ? "x-vercel-forwarded-for" : "x-forwarded-for" + ], }); if (!rateLimitKey) { return yield* Effect.fail(new HttpApiError.BadRequest()); diff --git a/apps/web/lib/analytics/request.ts b/apps/web/lib/analytics/request.ts index 9c07d6fd04d..b541249c7e1 100644 --- a/apps/web/lib/analytics/request.ts +++ b/apps/web/lib/analytics/request.ts @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { isIP } from "node:net"; import { isServerOnlyEventName, normalizeProductEventInput, @@ -166,19 +167,13 @@ export function normalizeProductEventBatch( } export function getProductAnalyticsRateLimitKey(headers: { - trustedVercelProxy: boolean; - xVercelForwardedFor?: string; - fallbackIdentity?: string; + trustedNetworkProxy: boolean; + forwardedFor?: string; }) { - if (!headers.trustedVercelProxy) { - const identity = headers.fallbackIdentity?.trim(); - if (!identity) return null; - return `self-hosted:${createHash("sha256").update(identity).digest("hex")}`; - } - const identity = headers.xVercelForwardedFor?.split(",")[0]?.trim(); - return identity - ? identity.slice(0, PRODUCT_ANALYTICS_LIMITS.identifierLength) - : null; + if (!headers.trustedNetworkProxy) return null; + const identity = headers.forwardedFor?.split(",")[0]?.trim(); + if (!identity || isIP(identity) === 0) return null; + return `network:${createHash("sha256").update(identity).digest("hex")}`; } export function normalizeGeoHeader(value?: string, decode = false) { @@ -244,7 +239,9 @@ export function classifyAnalyticsTraffic({ .map((value) => value.trim().toLowerCase()) .filter((value) => /^[0-9a-f]{64}$/.test(value)) ?? [], ); - const requestHash = createHash("sha256").update(rateLimitKey).digest("hex"); + const requestHash = rateLimitKey.startsWith("network:") + ? rateLimitKey.slice("network:".length) + : createHash("sha256").update(rateLimitKey).digest("hex"); if (hashes.has(requestHash)) return "internal" as const; return "external" as const; } From 59397b94611aaa0ae879e554b2365773874d3301 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:21:43 +0100 Subject: [PATCH 067/110] ci: poll Tinybird copies with staging deploy token --- .github/workflows/analytics.yml | 4 ++++ scripts/analytics/README.md | 2 +- scripts/analytics/staging-ci-lib.js | 7 ++++--- scripts/analytics/staging-ci.js | 4 +++- scripts/analytics/tests/staging-ci.test.js | 15 +++++++++++---- 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index cd64c4afcc6..b6483bd8713 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -182,6 +182,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -245,6 +246,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -276,6 +278,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -309,6 +312,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index 600b1f1399e..feba383955c 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -41,7 +41,7 @@ The deployed datafiles create two runtime tokens: Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. -The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Candidate validation pins the staging deployment parameter, while no-op and post-promotion phases use the already verified live deployment because Tinybird rejects a deployment selector on the Copy service after promotion. +The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Copy submission uses that resource-scoped token, while terminal status polling uses the existing staging deployment token because Tinybird's Jobs API does not grant the submitting per-pipe token access to the resulting job. Candidate validation pins the staging deployment parameter, while no-op and post-promotion phases use the already verified live deployment because Tinybird rejects a deployment selector on the Copy service after promotion. Set `TINYBIRD_AGENT_TOKEN` and `TINYBIRD_URL` for the query command. Set a separate `TINYBIRD_READ_TOKEN` with workspace metadata access when running `pnpm analytics:check`; the append and deployment tokens are intentionally rejected for that task. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index ab1bdd1bf37..500ee688471 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -190,7 +190,8 @@ export const selectStagingDeployment = ( export const runTinybirdCopyJobs = async ({ origin, - token, + submissionToken, + statusToken, deploymentId, request, wait, @@ -222,7 +223,7 @@ export const runTinybirdCopyJobs = async ({ let created; try { created = await request(copyUrl, { - token, + token: submissionToken, method: "POST", attempts: 3, }); @@ -242,7 +243,7 @@ export const runTinybirdCopyJobs = async ({ try { job = await request( new URL(`/v0/jobs/${encodeURIComponent(jobId)}`, origin), - { token, attempts: 3 }, + { token: statusToken, attempts: 3 }, ); } catch (error) { throw new Error(`Tinybird copy status read failed for ${pipe}`, { diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 3377f95a5b0..92ce213c71d 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -654,11 +654,13 @@ const runCopies = async () => { } const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_READ_TOKEN", + "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); try { const jobs = await runTinybirdCopyJobs({ origin, - token: tokens.TINYBIRD_STAGING_READ_TOKEN, + submissionToken: tokens.TINYBIRD_STAGING_READ_TOKEN, + statusToken: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, deploymentId: state.deploymentId, request, wait: delay, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 8a4a8248287..c7b20a506ef 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -212,8 +212,10 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = ); }); -test("copy jobs use the resource-scoped API and wait for exact completion", async () => { +test("copy jobs submit with a resource token and poll with the deployment token", async () => { const requests = []; + const submissionToken = token(); + const statusToken = `${token()}.deployment`; const responses = [ { data: { id: "copy_job_123" } }, { data: { status: "working" } }, @@ -222,7 +224,8 @@ test("copy jobs use the resource-scoped API and wait for exact completion", asyn let now = 1_000; const results = await runTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", - token: token(), + submissionToken, + statusToken, deploymentId: "6", pipes: ["snapshot_product_events_canonical_v1"], request: async (url, options) => { @@ -246,11 +249,14 @@ test("copy jobs use the resource-scoped API and wait for exact completion", asyn assert.match(requests[0].url, /_mode=replace/); assert.match(requests[0].url, /__tb__deployment=6/); assert.equal(requests[0].options.method, "POST"); + assert.equal(requests[0].options.token, submissionToken); assert.match(requests[1].url, /\/v0\/jobs\/copy_job_123$/); + assert.equal(requests[1].options.token, statusToken); const liveRequests = []; await runTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", - token: token(), + submissionToken, + statusToken, deploymentId: "6", pipes: ["snapshot_product_events_canonical_v1"], request: async (url, options) => { @@ -265,7 +271,8 @@ test("copy jobs use the resource-scoped API and wait for exact completion", asyn await assert.rejects(() => runTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", - token: token(), + submissionToken, + statusToken, deploymentId: "live", request: async () => ({ data: {} }), wait: async () => {}, From 32c9b385ff764dcdcf494483e4e02d111179291f Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:40:13 +0100 Subject: [PATCH 068/110] ci: prove Tinybird copies from aggregate state --- .github/workflows/analytics.yml | 4 - scripts/analytics/README.md | 2 +- scripts/analytics/staging-ci-lib.js | 105 ++--- scripts/analytics/staging-ci.js | 403 +++++++++++++++++- scripts/analytics/tests/datafiles.test.js | 42 ++ scripts/analytics/tests/staging-ci.test.js | 102 +++-- scripts/analytics/tests/tooling.test.js | 4 +- .../product_activation_daily_exact.datasource | 3 +- ...product_creator_retention_exact.datasource | 3 +- .../product_traffic_daily_exact.datasource | 3 +- ...oduct_traffic_pages_daily_exact.datasource | 3 +- .../tinybird/pipes/product_activation.pipe | 3 +- .../product_analytics_copy_assertions.pipe | 20 + .../pipes/product_creator_retention.pipe | 3 +- .../pipes/product_traffic_countries.pipe | 3 +- .../pipes/product_traffic_overview.pipe | 3 +- .../tinybird/pipes/product_traffic_pages.pipe | 3 +- .../pipes/product_traffic_sources.pipe | 3 +- .../pipes/product_traffic_technology.pipe | 3 +- ...apshot_product_activation_daily_exact.pipe | 12 + ...pshot_product_creator_retention_exact.pipe | 13 + .../snapshot_product_traffic_daily_exact.pipe | 24 ++ ...hot_product_traffic_pages_daily_exact.pipe | 23 + scripts/analytics/tooling.js | 6 + scripts/analytics/verify-local.js | 12 + 25 files changed, 654 insertions(+), 151 deletions(-) create mode 100644 scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index b6483bd8713..cd64c4afcc6 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -182,7 +182,6 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -246,7 +245,6 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -278,7 +276,6 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -312,7 +309,6 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index feba383955c..ad2e216eb9e 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -41,7 +41,7 @@ The deployed datafiles create two runtime tokens: Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. -The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Copy submission uses that resource-scoped token, while terminal status polling uses the existing staging deployment token because Tinybird's Jobs API does not grant the submitting per-pipe token access to the resulting job. Candidate validation pins the staging deployment parameter, while no-op and post-promotion phases use the already verified live deployment because Tinybird rejects a deployment selector on the Copy service after promotion. +The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Tinybird does not grant either the resource token or the deployment token access to the resulting Jobs API record, so CI proves terminal completion from canonical, daily, and health state transitions plus a unique zero-valued marker written by every other aggregate Copy. Decision endpoints always exclude those markers. Candidate validation pins the staging deployment parameter, while no-op and post-promotion phases use the already verified live deployment because Tinybird rejects a deployment selector on the Copy service after promotion. Set `TINYBIRD_AGENT_TOKEN` and `TINYBIRD_URL` for the query command. Set a separate `TINYBIRD_READ_TOKEN` with workspace metadata access when running `pnpm analytics:check`; the append and deployment tokens are intentionally rejected for that task. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 500ee688471..3cc914ca3d9 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -12,6 +12,12 @@ export const COPY_PIPES = [ "snapshot_product_creator_retention_exact", "snapshot_product_events_health_hourly", ]; +const COPY_MARKER_PIPES = new Set([ + "snapshot_product_traffic_daily_exact", + "snapshot_product_traffic_pages_daily_exact", + "snapshot_product_activation_daily_exact", + "snapshot_product_creator_retention_exact", +]); const SHA_PATTERN = /^[0-9a-f]{40}$/; const SYNTHETIC_RUN_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; @@ -188,24 +194,20 @@ export const selectStagingDeployment = ( return { id: String(id), needsPromotion: false }; }; -export const runTinybirdCopyJobs = async ({ +export const submitTinybirdCopyJobs = async ({ origin, - submissionToken, - statusToken, + token, deploymentId, request, - wait, now = () => Date.now(), - timeoutMs = 120_000, pipes = COPY_PIPES, useDeploymentParameter = false, + copyRunId = "", }) => { if (!DEPLOYMENT_ID_PATTERN.test(deploymentId)) { throw new Error("Tinybird copy jobs require a numeric deployment ID"); } - if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { - throw new Error("Tinybird copy job timeout must be positive"); - } + if (copyRunId) validateSyntheticRunId(copyRunId); const results = []; for (const pipe of pipes) { if (!COPY_PIPES.includes(pipe)) { @@ -220,10 +222,16 @@ export const runTinybirdCopyJobs = async ({ if (useDeploymentParameter) { copyUrl.searchParams.set("__tb__deployment", deploymentId); } + if (COPY_MARKER_PIPES.has(pipe)) { + if (!copyRunId) { + throw new Error(`Tinybird copy marker is required for ${pipe}`); + } + copyUrl.searchParams.set("copy_run_id", copyRunId); + } let created; try { created = await request(copyUrl, { - token: submissionToken, + token, method: "POST", attempts: 3, }); @@ -236,43 +244,11 @@ export const runTinybirdCopyJobs = async ({ if (!COPY_JOB_ID_PATTERN.test(jobId)) { throw new Error(`Tinybird did not return a valid copy job for ${pipe}`); } - let polls = 0; - while (true) { - polls += 1; - let job; - try { - job = await request( - new URL(`/v0/jobs/${encodeURIComponent(jobId)}`, origin), - { token: statusToken, attempts: 3 }, - ); - } catch (error) { - throw new Error(`Tinybird copy status read failed for ${pipe}`, { - cause: error, - }); - } - const status = String(job.data.status ?? "").toLowerCase(); - if (status === "done") { - results.push({ - pipe, - jobId, - polls, - durationMs: Math.max(0, now() - startedAt), - }); - break; - } - if (["error", "cancelled"].includes(status)) { - throw new Error(`Tinybird copy job for ${pipe} ended in ${status}`); - } - if (!["waiting", "working", "cancelling"].includes(status)) { - throw new Error( - `Tinybird copy job for ${pipe} returned an invalid status`, - ); - } - if (now() - startedAt >= timeoutMs) { - throw new Error(`Tinybird copy job for ${pipe} exceeded its timeout`); - } - await wait(1_000); - } + results.push({ + pipe, + jobId, + submissionLatencyMs: Math.max(0, now() - startedAt), + }); } return results; }; @@ -471,6 +447,17 @@ export const normalizeCiAssertions = (payload) => { }; }; +export const normalizeCopyAssertions = (payload) => { + const row = payload?.data?.[0] ?? {}; + const number = (value) => Number(value ?? 0); + return { + trafficMarkers: number(row.traffic_markers), + trafficPageMarkers: number(row.traffic_page_markers), + activationMarkers: number(row.activation_markers), + retentionMarkers: number(row.retention_markers), + }; +}; + export const assertSyntheticDecisions = (assertions) => { assertSyntheticHealth(assertions); if (assertions.decisionEvents !== 1) { @@ -485,32 +472,6 @@ export const assertSyntheticDecisions = (assertions) => { } }; -export const assertPromotedSyntheticDecisions = (assertions) => { - const expected = { - uniqueEvents: 3, - uniquePayloads: 4, - payloadConflicts: 1, - canonicalEvents: 2, - decisionEvents: 2, - }; - for (const [name, value] of Object.entries(expected)) { - if (assertions[name] !== value) { - throw new Error( - `Promoted synthetic ${name} was ${assertions[name]}, expected ${value}`, - ); - } - } - if ( - assertions.receivedRows < 6 || - assertions.duplicateRows !== - assertions.receivedRows - assertions.uniquePayloads - ) { - throw new Error( - "Promoted synthetic assertions did not preserve retry deliveries", - ); - } -}; - export const assertSyntheticHealth = (health) => { const expected = { uniqueEvents: 2, diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 92ce213c71d..a872e4a3f8e 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -3,7 +3,6 @@ import process from "node:process"; import { assertExecutionScope, - assertPromotedSyntheticDecisions, assertSyntheticDecisions, assertSyntheticHealth, createSyntheticErasureControl, @@ -16,10 +15,11 @@ import { hashIdentifier, latencySummary, normalizeCiAssertions, + normalizeCopyAssertions, normalizeHealth, - runTinybirdCopyJobs, STAGING_WORKSPACE_ID, selectStagingDeployment, + submitTinybirdCopyJobs, validateSyntheticRunId, validateTinybirdCredentials, } from "./staging-ci-lib.js"; @@ -167,13 +167,30 @@ const healthQuery = async ({ state, deploymentId = "", appVersion }) => { ); }; -const ciAssertionsQuery = async ({ state, deploymentId = "" }) => { +const ciAssertionsQuery = async ({ + state, + deploymentId = "", + syntheticRunId = state.runId, +}) => { const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_READ_TOKEN", ]); return request( tinybirdUrl(origin, "/v0/pipes/product_analytics_ci_assertions.json", { - synthetic_run_id: state.runId, + synthetic_run_id: syntheticRunId, + __tb__deployment: deploymentId, + }), + { token: tokens.TINYBIRD_STAGING_READ_TOKEN, attempts: 3 }, + ); +}; + +const copyAssertionsQuery = async ({ copyRunId, deploymentId = "" }) => { + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_READ_TOKEN", + ]); + return request( + tinybirdUrl(origin, "/v0/pipes/product_analytics_copy_assertions.json", { + copy_run_id: copyRunId, __tb__deployment: deploymentId, }), { token: tokens.TINYBIRD_STAGING_READ_TOKEN, attempts: 3 }, @@ -378,6 +395,7 @@ const probePreview = async () => { } const occurredAt = new Date().toISOString(); const runHash = hashIdentifier(state.runId); + const previewRunId = validateSyntheticRunId(`${state.runId}_preview`); const event = { eventId: `synthetic_preview_${runHash.slice(0, 24)}`, eventName: "page_view", @@ -417,7 +435,7 @@ const probePreview = async () => { Origin: previewOrigin, "Sec-Fetch-Site": "same-origin", "User-Agent": "Cap-Analytics-Staging-E2E/1.0", - "x-cap-analytics-test-run": state.runId, + "x-cap-analytics-test-run": previewRunId, }, body, }, @@ -478,6 +496,7 @@ const probePreview = async () => { } state.previewAppVersion = event.appVersion; state.previewAcceptedRows = duplicateResponses.length + replayAccepted; + state.previewRunId = previewRunId; writeJson(statePath, state, 0o600); artifact.previewApi = { bootstrapPassed: true, @@ -634,6 +653,160 @@ const seed = async () => { }); }; +const waitForCopyVisibility = async ({ label, read, assert }) => { + const startedAt = Date.now(); + const deadline = startedAt + Number(process.env.INGESTION_SLO_MS ?? 180_000); + let lastError; + let polls = 0; + while (Date.now() < deadline) { + polls += 1; + try { + const value = await read(); + assert(value); + return { value, polls, visibilityMs: Date.now() - startedAt }; + } catch (error) { + lastError = error; + await delay(2_000); + } + } + throw new Error( + `${label} did not become visible: ${lastError instanceof Error ? lastError.message : "unknown assertion failure"}`, + ); +}; + +const phaseRunExpectations = ({ state, phase }) => { + const expectations = [ + { + runId: state.runId, + canonicalEvents: ["staged", "promoted"].includes(phase) ? 1 : 0, + decisionEvents: ["staged", "promoted"].includes(phase) ? 1 : 0, + }, + { + runId: state.loadRunId, + canonicalEvents: ["staged", "promoted"].includes(phase) + ? state.loadEventCount + : 0, + decisionEvents: ["staged", "promoted"].includes(phase) + ? state.loadEventCount + : 0, + }, + { + runId: state.erasureControlRunId, + canonicalEvents: phase === "cleanup" ? 0 : 1, + decisionEvents: phase === "cleanup" ? 0 : 1, + }, + ]; + if (phase !== "staged" && phase !== "cleanup") { + if (!state.previewRunId) { + throw new Error("The promoted copy phase is missing its preview run ID"); + } + } + if (state.previewRunId && phase !== "staged") { + expectations.push({ + runId: state.previewRunId, + canonicalEvents: phase === "cleanup" ? 0 : 1, + decisionEvents: phase === "cleanup" ? 0 : 1, + }); + } + return expectations; +}; + +const readPhaseCiAssertions = async ({ state, deploymentId, expectations }) => + Promise.all( + expectations.map(async (expectation) => ({ + ...expectation, + assertions: normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId, + syntheticRunId: expectation.runId, + }) + ).data, + ), + })), + ); + +const assertPhaseCiAssertions = (results, fieldNames) => { + for (const result of results) { + for (const fieldName of fieldNames) { + if (result.assertions[fieldName] !== result[fieldName]) { + throw new Error( + `Synthetic ${fieldName} was ${result.assertions[fieldName]}, expected ${result[fieldName]}`, + ); + } + } + } +}; + +const assertZeroHealth = (health) => { + if (Object.values(health).some((value) => value !== 0)) { + throw new Error("Synthetic health was not fully retracted"); + } +}; + +const assertSingleHealth = (health) => { + if ( + health.receivedRows < 1 || + health.uniqueEvents !== 1 || + health.uniquePayloads !== 1 || + health.payloadConflicts !== 0 + ) { + throw new Error("Synthetic single-event health is incomplete"); + } +}; + +const readAndAssertPhaseHealth = async ({ state, phase, deploymentId }) => { + const [main, load, control, preview] = await Promise.all([ + healthQuery({ state, deploymentId }), + healthQuery({ + state, + deploymentId, + appVersion: state.loadAppVersion, + }), + healthQuery({ + state, + deploymentId, + appVersion: state.erasureControlAppVersion, + }), + phase === "staged" || !state.previewAppVersion + ? Promise.resolve(null) + : healthQuery({ + state, + deploymentId, + appVersion: state.previewAppVersion, + }), + ]); + const health = { + main: normalizeHealth(main.data), + load: normalizeHealth(load.data), + control: normalizeHealth(control.data), + preview: preview ? normalizeHealth(preview.data) : null, + }; + if (["staged", "promoted"].includes(phase)) { + assertSyntheticHealth(health.main); + if ( + health.load.receivedRows < state.loadEventCount || + health.load.uniqueEvents !== state.loadEventCount || + health.load.uniquePayloads !== state.loadEventCount || + health.load.payloadConflicts !== 0 + ) { + throw new Error("Synthetic load health is incomplete"); + } + } else { + assertZeroHealth(health.main); + assertZeroHealth(health.load); + } + if (phase === "cleanup") { + assertZeroHealth(health.control); + if (health.preview) assertZeroHealth(health.preview); + } else { + assertSingleHealth(health.control); + if (health.preview) assertSingleHealth(health.preview); + } + return health; +}; + const runCopies = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); @@ -654,21 +827,134 @@ const runCopies = async () => { } const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_READ_TOKEN", - "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); + const endpointDeploymentId = target === "staging" ? state.deploymentId : ""; + const copyRunId = validateSyntheticRunId(`${state.runId}_${phase}`); + const expectations = phaseRunExpectations({ state, phase }); try { - const jobs = await runTinybirdCopyJobs({ + const canonicalJobs = await submitTinybirdCopyJobs({ origin, - submissionToken: tokens.TINYBIRD_STAGING_READ_TOKEN, - statusToken: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, deploymentId: state.deploymentId, request, - wait: delay, + pipes: ["snapshot_product_events_canonical_v1"], useDeploymentParameter: target === "staging", }); + const canonicalVisibility = await waitForCopyVisibility({ + label: "Tinybird canonical copy", + read: () => + readPhaseCiAssertions({ + state, + deploymentId: endpointDeploymentId, + expectations, + }), + assert: (results) => + assertPhaseCiAssertions(results, ["canonicalEvents"]), + }); + const downstreamJobs = []; + const downstreamVisibility = {}; + const copySteps = [ + { + pipe: "snapshot_product_events_daily_exact", + read: () => + readPhaseCiAssertions({ + state, + deploymentId: endpointDeploymentId, + expectations, + }), + assert: (results) => + assertPhaseCiAssertions(results, ["decisionEvents"]), + }, + { + pipe: "snapshot_product_traffic_daily_exact", + marker: "trafficMarkers", + }, + { + pipe: "snapshot_product_traffic_pages_daily_exact", + marker: "trafficPageMarkers", + }, + { + pipe: "snapshot_product_activation_daily_exact", + marker: "activationMarkers", + }, + { + pipe: "snapshot_product_creator_retention_exact", + marker: "retentionMarkers", + }, + { + pipe: "snapshot_product_events_health_hourly", + read: () => + readAndAssertPhaseHealth({ + state, + phase, + deploymentId: endpointDeploymentId, + }), + assert: () => undefined, + }, + ]; + for (const copyStep of copySteps) { + downstreamJobs.push( + ...(await submitTinybirdCopyJobs({ + origin, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + deploymentId: state.deploymentId, + request, + pipes: [copyStep.pipe], + useDeploymentParameter: target === "staging", + copyRunId, + })), + ); + const visibility = await waitForCopyVisibility({ + label: `Tinybird ${copyStep.pipe} copy`, + read: + copyStep.read ?? + (async () => + normalizeCopyAssertions( + ( + await copyAssertionsQuery({ + copyRunId, + deploymentId: endpointDeploymentId, + }) + ).data, + )), + assert: + copyStep.assert ?? + ((markers) => { + if (markers[copyStep.marker] !== 1) { + throw new Error( + `Tinybird ${copyStep.marker} was ${markers[copyStep.marker]}, expected 1`, + ); + } + }), + }); + downstreamVisibility[copyStep.pipe] = { + polls: visibility.polls, + visibilityMs: visibility.visibilityMs, + }; + } + const markers = normalizeCopyAssertions( + ( + await copyAssertionsQuery({ + copyRunId, + deploymentId: endpointDeploymentId, + }) + ).data, + ); + if (Object.values(markers).some((value) => value !== 1)) { + throw new Error("Not every Tinybird aggregate copy marker is visible"); + } artifact.copyJobs = { ...artifact.copyJobs, - [phase]: { status: "passed", jobs }, + [phase]: { + status: "passed", + copyRunHash: hashIdentifier(copyRunId), + jobs: [...canonicalJobs, ...downstreamJobs], + canonicalVisibility: { + polls: canonicalVisibility.polls, + visibilityMs: canonicalVisibility.visibilityMs, + }, + downstreamVisibility: { copies: downstreamVisibility, markers }, + }, }; writeJson(artifactPath, artifact); } catch (error) { @@ -966,6 +1252,22 @@ const verifySyntheticIdentityErasure = async () => { const erasedDecisions = normalizeCiAssertions( (await ciAssertionsQuery({ state })).data, ); + const previewHealth = normalizeHealth( + ( + await healthQuery({ + state, + appVersion: state.previewAppVersion, + }) + ).data, + ); + const previewDecisions = normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + syntheticRunId: state.previewRunId, + }) + ).data, + ); if ( Object.values(erasedHealth).some((value) => value !== 0) || Object.values(erasedLoadHealth).some((value) => value !== 0) || @@ -975,6 +1277,15 @@ const verifySyntheticIdentityErasure = async () => { "Synthetic identity erasure left raw-health or decision-facing state", ); } + assertSingleHealth(previewHealth); + if ( + previewDecisions.canonicalEvents !== 1 || + previewDecisions.decisionEvents !== 1 + ) { + throw new Error( + "Synthetic identity erasure corrupted the unrelated preview control", + ); + } const controlHealth = normalizeHealth( ( await healthQuery({ @@ -998,6 +1309,8 @@ const verifySyntheticIdentityErasure = async () => { erasedHealth, erasedLoadHealth, erasedDecisions, + previewHealth, + previewDecisions, controlHealth, passed: true, }; @@ -1018,10 +1331,14 @@ const cleanup = async () => { ]); validateSyntheticRunId(state.loadRunId); validateSyntheticRunId(state.erasureControlRunId); + const runIds = [state.runId, state.loadRunId, state.erasureControlRunId]; + if (state.previewRunId) { + runIds.push(validateSyntheticRunId(state.previewRunId)); + } const rowsAffected = await deleteProductEventRows({ origin, token: tokens.TINYBIRD_STAGING_CLEANUP_TOKEN, - condition: `synthetic_run_id IN ('${state.runId}', '${state.loadRunId}', '${state.erasureControlRunId}')`, + condition: `synthetic_run_id IN (${runIds.map((runId) => `'${runId}'`).join(", ")})`, }); const artifact = readJson(artifactPath); artifact.cleanup = { @@ -1036,7 +1353,11 @@ const verifyPromoted = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); const artifact = readJson(artifactPath); - if (!state.previewAppVersion || !state.previewAcceptedRows) { + if ( + !state.previewAppVersion || + !state.previewAcceptedRows || + !state.previewRunId + ) { throw new Error("The exact-SHA preview probe did not complete"); } const previewHealthResult = await healthQuery({ @@ -1055,11 +1376,35 @@ const verifyPromoted = async () => { "The promoted health snapshot did not preserve preview retry deliveries", ); } - const decisionResult = await ciAssertionsQuery({ state }); - const decisionAssertions = normalizeCiAssertions(decisionResult.data); - assertPromotedSyntheticDecisions(decisionAssertions); + const seedDecisionResult = await ciAssertionsQuery({ state }); + const seedDecisionAssertions = normalizeCiAssertions(seedDecisionResult.data); + assertSyntheticDecisions(seedDecisionAssertions); + const previewDecisionResult = await ciAssertionsQuery({ + state, + syntheticRunId: state.previewRunId, + }); + const previewDecisionAssertions = normalizeCiAssertions( + previewDecisionResult.data, + ); + if ( + previewDecisionAssertions.receivedRows < state.previewAcceptedRows || + previewDecisionAssertions.uniqueEvents !== 1 || + previewDecisionAssertions.uniquePayloads !== 1 || + previewDecisionAssertions.duplicateRows !== + previewDecisionAssertions.receivedRows - 1 || + previewDecisionAssertions.payloadConflicts !== 0 || + previewDecisionAssertions.canonicalEvents !== 1 || + previewDecisionAssertions.decisionEvents !== 1 + ) { + throw new Error( + "The promoted preview run did not preserve exact retry-deduplicated decisions", + ); + } artifact.previewApi.health = previewHealth; - artifact.previewApi.decisionAssertions = decisionAssertions; + artifact.previewApi.decisionAssertions = { + seed: seedDecisionAssertions, + preview: previewDecisionAssertions, + }; artifact.previewApi.endpointLatencyMs = previewHealthResult.latencyMs; artifact.assertions.promotedPreviewDataPassed = true; writeJson(artifactPath, artifact); @@ -1103,6 +1448,30 @@ const verifyCleanup = async () => { "Synthetic erasure control rows still affect Tinybird health after cleanup", ); } + if (state.previewAppVersion && state.previewRunId) { + const previewResult = await healthQuery({ + state, + appVersion: state.previewAppVersion, + }); + const previewHealth = normalizeHealth(previewResult.data); + if (Object.values(previewHealth).some((value) => value !== 0)) { + throw new Error( + "Synthetic preview rows still affect Tinybird health after cleanup", + ); + } + const previewDecisionResult = await ciAssertionsQuery({ + state, + syntheticRunId: state.previewRunId, + }); + const previewDecisionAssertions = normalizeCiAssertions( + previewDecisionResult.data, + ); + if (Object.values(previewDecisionAssertions).some((value) => value !== 0)) { + throw new Error( + "Synthetic preview rows still affect decisions after cleanup", + ); + } + } artifact.cleanup = { ...artifact.cleanup, passed: true, diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index 800751fcd1f..e8ff30d4e2a 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -212,6 +212,48 @@ test("copy schedules serialize canonical and derived rebuilds", () => { assert.equal(new Set(schedules.values()).size, schedules.size); }); +test("staging copy markers are excluded from every decision endpoint", () => { + for (const name of [ + "product_traffic_overview", + "product_traffic_pages", + "product_traffic_sources", + "product_traffic_countries", + "product_traffic_technology", + "product_activation", + "product_creator_retention", + ]) { + const contents = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), + "utf8", + ); + assert.match(contents, /copy_run_id = ''/); + } + for (const name of [ + "snapshot_product_traffic_daily_exact", + "snapshot_product_traffic_pages_daily_exact", + "snapshot_product_activation_daily_exact", + "snapshot_product_creator_retention_exact", + ]) { + const contents = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), + "utf8", + ); + assert.match(contents, /defined\(copy_run_id\)/); + assert.match(contents, /AS copy_run_id/); + } + const assertions = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "product_analytics_copy_assertions.pipe", + ), + "utf8", + ); + assert.match(assertions, /copy_run_id is required/); + assert.match(assertions, /traffic_markers/); + assert.match(assertions, /retention_markers/); +}); + test("health queries use stable hourly aggregates and a bounded window", () => { const contents = fs.readFileSync( path.join(TINYBIRD_PROJECT_DIR, "pipes", "product_events_health.pipe"), diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index c7b20a506ef..4c11b71c75a 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -4,7 +4,6 @@ import test from "node:test"; import { assertExecutionScope, - assertPromotedSyntheticDecisions, assertSyntheticDecisions, assertSyntheticHealth, assertWorkflowSafety, @@ -19,10 +18,11 @@ import { FEATURE_PULL_REQUEST, latencySummary, normalizeCiAssertions, + normalizeCopyAssertions, normalizeHealth, - runTinybirdCopyJobs, STAGING_WORKSPACE_ID, selectStagingDeployment, + submitTinybirdCopyJobs, tokenWorkspaceId, validateSyntheticRunId, validateTinybirdCredentials, @@ -212,70 +212,77 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = ); }); -test("copy jobs submit with a resource token and poll with the deployment token", async () => { +test("copy jobs use only approved resource-scoped submissions and bounded markers", async () => { const requests = []; - const submissionToken = token(); - const statusToken = `${token()}.deployment`; + const resourceToken = token(); const responses = [ - { data: { id: "copy_job_123" } }, - { data: { status: "working" } }, - { data: { status: "done" } }, + { data: { id: "copy_job_canonical" } }, + { data: { id: "copy_job_traffic" } }, ]; let now = 1_000; - const results = await runTinybirdCopyJobs({ + const results = await submitTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", - submissionToken, - statusToken, + token: resourceToken, deploymentId: "6", - pipes: ["snapshot_product_events_canonical_v1"], + pipes: [ + "snapshot_product_events_canonical_v1", + "snapshot_product_traffic_daily_exact", + ], request: async (url, options) => { requests.push({ url: String(url), options }); + now += 10; return responses.shift(); }, - wait: async (milliseconds) => { - now += milliseconds; - }, now: () => now, useDeploymentParameter: true, + copyRunId: "run_12345678_staged", }); assert.deepEqual(results, [ { pipe: "snapshot_product_events_canonical_v1", - jobId: "copy_job_123", - polls: 2, - durationMs: 1_000, + jobId: "copy_job_canonical", + submissionLatencyMs: 10, + }, + { + pipe: "snapshot_product_traffic_daily_exact", + jobId: "copy_job_traffic", + submissionLatencyMs: 10, }, ]); assert.match(requests[0].url, /_mode=replace/); assert.match(requests[0].url, /__tb__deployment=6/); + assert.doesNotMatch(requests[0].url, /copy_run_id/); assert.equal(requests[0].options.method, "POST"); - assert.equal(requests[0].options.token, submissionToken); - assert.match(requests[1].url, /\/v0\/jobs\/copy_job_123$/); - assert.equal(requests[1].options.token, statusToken); + assert.equal(requests[0].options.token, resourceToken); + assert.match(requests[1].url, /copy_run_id=run_12345678_staged/); + assert.ok(requests.every(({ url }) => !url.includes("/v0/jobs/"))); const liveRequests = []; - await runTinybirdCopyJobs({ + await submitTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", - submissionToken, - statusToken, + token: resourceToken, deploymentId: "6", pipes: ["snapshot_product_events_canonical_v1"], request: async (url, options) => { liveRequests.push({ url: String(url), options }); - return liveRequests.length === 1 - ? { data: { id: "copy_job_live" } } - : { data: { status: "done" } }; + return { data: { id: "copy_job_live" } }; }, - wait: async () => {}, }); assert.doesNotMatch(liveRequests[0].url, /__tb__deployment/); await assert.rejects(() => - runTinybirdCopyJobs({ + submitTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", - submissionToken, - statusToken, + token: resourceToken, deploymentId: "live", request: async () => ({ data: {} }), - wait: async () => {}, + }), + ); + await assert.rejects(() => + submitTinybirdCopyJobs({ + origin: "https://api.us-east.aws.tinybird.co", + token: resourceToken, + deploymentId: "6", + pipes: ["snapshot_product_traffic_daily_exact"], + request: async () => ({ data: { id: "copy_job_missing_marker" } }), }), ); }); @@ -457,19 +464,24 @@ test("CI assertion normalization proves decision deduplication and conflict quar ); }); -test("promoted assertions combine direct and exact-SHA preview duplicate paths", () => { - const assertions = { - receivedRows: 26, - uniqueEvents: 3, - uniquePayloads: 4, - duplicateRows: 22, - payloadConflicts: 1, - canonicalEvents: 2, - decisionEvents: 2, - }; - assert.doesNotThrow(() => assertPromotedSyntheticDecisions(assertions)); - assert.throws(() => - assertPromotedSyntheticDecisions({ ...assertions, decisionEvents: 3 }), +test("copy assertion normalization exposes every marker independently", () => { + assert.deepEqual( + normalizeCopyAssertions({ + data: [ + { + traffic_markers: "1", + traffic_page_markers: "1", + activation_markers: "1", + retention_markers: "1", + }, + ], + }), + { + trafficMarkers: 1, + trafficPageMarkers: 1, + activationMarkers: 1, + retentionMarkers: 1, + }, ); }); diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index 829a8c18286..229bc617b58 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -86,7 +86,9 @@ test("local setup builds, verifies copied endpoints and writes its deterministic assert.equal(copyCommands.length, 7); assert.ok( copyCommands.every((command) => - command.includes("--param copy_max_threads=1 --wait"), + command.includes( + "--param copy_max_threads=1 --param copy_run_id=run_local_copy_assertions --wait", + ), ), ); assert.ok(steps.some((step) => step.type === "verify-local")); diff --git a/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource index fb236fa5ca8..7b22eb959e6 100644 --- a/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource @@ -3,6 +3,7 @@ DESCRIPTION > SCHEMA > calculated_at DateTime64(3), + copy_run_id String, cohort_date Date, signups AggregateFunction(uniqExact, String), activated_creators AggregateFunction(uniqExact, String), @@ -10,6 +11,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(cohort_date) -ENGINE_SORTING_KEY cohort_date +ENGINE_SORTING_KEY (copy_run_id, cohort_date) ENGINE_TTL cohort_date + INTERVAL 400 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource b/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource index 38d7184a200..b1e23e52158 100644 --- a/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource @@ -3,6 +3,7 @@ DESCRIPTION > SCHEMA > calculated_at DateTime64(3), + copy_run_id String, cohort_date Date, activity_date Date, platform LowCardinality(String), @@ -11,6 +12,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(cohort_date) -ENGINE_SORTING_KEY (cohort_date, activity_date, platform) +ENGINE_SORTING_KEY (copy_run_id, cohort_date, activity_date, platform) ENGINE_TTL cohort_date + INTERVAL 400 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource index b6f22b8ae7f..d1269aee15f 100644 --- a/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource @@ -3,6 +3,7 @@ DESCRIPTION > SCHEMA > calculated_at DateTime64(3), + copy_run_id String, date Date, hostname LowCardinality(String), country LowCardinality(String), @@ -22,6 +23,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) +ENGINE_SORTING_KEY (copy_run_id, date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) ENGINE_TTL date + INTERVAL 400 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource index 97e5b1d6ae4..2afdcf05dc8 100644 --- a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource @@ -3,6 +3,7 @@ DESCRIPTION > SCHEMA > calculated_at DateTime64(3), + copy_run_id String, date Date, hostname LowCardinality(String), pathname String, @@ -21,6 +22,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (date, hostname, pathname, country, device, browser, os, channel) +ENGINE_SORTING_KEY (copy_run_id, date, hostname, pathname, country, device, browser, os, channel) ENGINE_TTL date + INTERVAL 400 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/pipes/product_activation.pipe b/scripts/analytics/tinybird/pipes/product_activation.pipe index bf96cb8aa11..e0e2ac8f316 100644 --- a/scripts/analytics/tinybird/pipes/product_activation.pipe +++ b/scripts/analytics/tinybird/pipes/product_activation.pipe @@ -13,7 +13,8 @@ SQL > round(if(signups = 0, 0, 100 * activated_creators / signups), 2) AS activation_rate, toUInt64(if(activated_creators = 0, 0, sum(time_to_activation_ms) / activated_creators)) AS average_time_to_activation_ms FROM product_activation_daily_exact - WHERE cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 90 DAY {% end %} + WHERE copy_run_id = '' + AND cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 90 DAY {% end %} AND cohort_date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} GROUP BY cohort_date ORDER BY cohort_date ASC diff --git a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe new file mode 100644 index 00000000000..92f84d5e2dc --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe @@ -0,0 +1,20 @@ +DESCRIPTION > + Prove each privacy-safe aggregate copy completed for one bounded staging phase. + +TOKEN product_events_agent_read READ + +NODE product_analytics_copy_assertions_node +SQL > + % + {% if not defined(copy_run_id) %} + {{ error('copy_run_id is required') }} + {% end %} + WITH {{String(copy_run_id)}} AS requested_copy_run_id + SELECT + (SELECT count() FROM product_traffic_daily_exact WHERE copy_run_id = requested_copy_run_id) AS traffic_markers, + (SELECT count() FROM product_traffic_pages_daily_exact WHERE copy_run_id = requested_copy_run_id) AS traffic_page_markers, + (SELECT count() FROM product_activation_daily_exact WHERE copy_run_id = requested_copy_run_id) AS activation_markers, + (SELECT count() FROM product_creator_retention_exact WHERE copy_run_id = requested_copy_run_id) AS retention_markers + WHERE throwIf(length(requested_copy_run_id) < 8 OR length(requested_copy_run_id) > 128, 'copy_run_id has an invalid length') = 0 + +TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_creator_retention.pipe b/scripts/analytics/tinybird/pipes/product_creator_retention.pipe index f1fc8b69ca7..870a8a9fb66 100644 --- a/scripts/analytics/tinybird/pipes/product_creator_retention.pipe +++ b/scripts/analytics/tinybird/pipes/product_creator_retention.pipe @@ -14,7 +14,8 @@ SQL > uniqExactMerge(creator_users) AS creators, uniqExactMerge(creator_organizations) AS organizations FROM product_creator_retention_exact - WHERE cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 180 DAY {% end %} + WHERE copy_run_id = '' + AND cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 180 DAY {% end %} AND cohort_date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(platform) %} AND (activity_date = cohort_date OR platform = {{String(platform)}}) {% end %} GROUP BY cohort_date, activity_date diff --git a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe index fdc015aedf5..d3d23d4d460 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe @@ -12,7 +12,8 @@ SQL > sum(visits) AS visits, sum(pageviews) AS pageviews FROM product_traffic_daily_exact - WHERE date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + WHERE copy_run_id = '' + AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} GROUP BY country diff --git a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe index ea8e4608148..2c118b91249 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe @@ -16,7 +16,8 @@ SQL > sum(visit_duration_ms) AS duration_total_ms, sum(engaged_ms) AS engagement_total_ms FROM product_traffic_daily_exact - WHERE date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + WHERE copy_run_id = '' + AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe index 51bda79028b..2f9c733c520 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe @@ -17,7 +17,8 @@ SQL > sum(engaged_ms) AS engagement_total_ms, sum(scroll_depth_sum) AS scroll_depth_total FROM product_traffic_pages_daily_exact - WHERE date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + WHERE copy_run_id = '' + AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe index 4a4daf9baca..cf618b5a03e 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe @@ -17,7 +17,8 @@ SQL > sum(pageviews) AS pageview_count, sum(bounces) AS bounce_count FROM product_traffic_daily_exact - WHERE date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + WHERE copy_run_id = '' + AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} GROUP BY channel, attribution_source, attribution_medium, campaign diff --git a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe index 815edc19a77..fda05f7508e 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe @@ -14,7 +14,8 @@ SQL > sum(visits) AS visits, sum(pageviews) AS pageviews FROM product_traffic_daily_exact - WHERE date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + WHERE copy_run_id = '' + AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe index ef115a48db7..1d1dedeb745 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe @@ -28,6 +28,7 @@ SQL > ) SELECT now64(3) AS calculated_at, + '' AS copy_run_id, toDate(signed_up_at, 'UTC') AS cohort_date, uniqExactState(signups.user_id) AS signups, uniqExactStateIf(signups.user_id, activated_at >= signed_up_at AND activated_at < signed_up_at + INTERVAL 7 DAY) AS activated_creators, @@ -35,6 +36,17 @@ SQL > FROM signups LEFT JOIN activations USING user_id GROUP BY cohort_date + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3) AS calculated_at, + {{String(copy_run_id)}} AS copy_run_id, + today() AS cohort_date, + uniqExactStateIf('', 0) AS signups, + uniqExactStateIf('', 0) AS activated_creators, + toUInt64(0) AS time_to_activation_ms + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} TYPE COPY TARGET_DATASOURCE product_activation_daily_exact diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe index 922bc4e1f28..5a7f92f56a3 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe @@ -53,6 +53,7 @@ SQL > ) SELECT now64(3) AS calculated_at, + '' AS copy_run_id, cohort_date, activity_date, platform, @@ -60,6 +61,18 @@ SQL > uniqExactStateIf(organization_id, organization_id != '') AS creator_organizations FROM cohort_activity GROUP BY cohort_date, activity_date, platform + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3) AS calculated_at, + {{String(copy_run_id)}} AS copy_run_id, + today() AS cohort_date, + today() AS activity_date, + '' AS platform, + uniqExactStateIf('', 0) AS creator_users, + uniqExactStateIf('', 0) AS creator_organizations + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} TYPE COPY TARGET_DATASOURCE product_creator_retention_exact diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe index 8bfc529b370..817a6a25361 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe @@ -43,6 +43,7 @@ SQL > ) SELECT now64(3) AS calculated_at, + '' AS copy_run_id, toDate(started_at, 'UTC') AS date, hostname, country, @@ -62,6 +63,29 @@ SQL > FROM sessions LEFT JOIN session_engagement USING session_id GROUP BY date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3) AS calculated_at, + {{String(copy_run_id)}} AS copy_run_id, + today() AS date, + '' AS hostname, + '' AS country, + '' AS device, + '' AS browser, + '' AS os, + '' AS channel, + '' AS attribution_source, + '' AS attribution_medium, + '' AS campaign, + uniqExactStateIf('', 0) AS visitors, + toUInt64(0) AS visits, + toUInt64(0) AS pageviews, + toUInt64(0) AS bounces, + toUInt64(0) AS visit_duration_ms, + toUInt64(0) AS engaged_ms + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} TYPE COPY TARGET_DATASOURCE product_traffic_daily_exact diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe index ea787f52af0..c222502f5c3 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe @@ -49,6 +49,7 @@ SQL > ) SELECT now64(3) AS calculated_at, + '' AS copy_run_id, toDate(occurred_at, 'UTC') AS date, hostname, pathname, @@ -68,6 +69,28 @@ SQL > INNER JOIN sessions ON page_views.session_id = sessions.session_id LEFT JOIN engagements ON page_views.event_id = engagements.page_view_id GROUP BY date, hostname, pathname, country, device, browser, os, channel + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3) AS calculated_at, + {{String(copy_run_id)}} AS copy_run_id, + today() AS date, + '' AS hostname, + '' AS pathname, + '' AS country, + '' AS device, + '' AS browser, + '' AS os, + '' AS channel, + uniqExactStateIf('', 0) AS visitors, + uniqExactStateIf('', 0) AS visits, + toUInt64(0) AS pageviews, + toUInt64(0) AS landings, + toUInt64(0) AS exits, + toUInt64(0) AS engaged_ms, + toFloat64(0) AS scroll_depth_sum + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} TYPE COPY TARGET_DATASOURCE product_traffic_pages_daily_exact diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index fea17c8e1a8..d8ed9455979 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -35,6 +35,7 @@ const TEST_FILES = fs .map((fileName) => path.join(MODULE_DIR, "tests", fileName)); const CLOUD_URL_DEFAULT = "https://api.tinybird.co"; const STAGING_WORKSPACE_ID = "37b8fef9-817f-4c3c-b21f-218c36a6077d"; +const LOCAL_COPY_RUN_ID = "run_local_copy_assertions"; const PRODUCT_COPY_PIPES = [ "snapshot_product_events_canonical_v1", "snapshot_product_events_daily_exact", @@ -178,6 +179,8 @@ const operationPlan = (operation) => { name, "--param", "copy_max_threads=1", + "--param", + `copy_run_id=${LOCAL_COPY_RUN_ID}`, "--wait", ), attempts: 8, @@ -221,6 +224,8 @@ const operationPlan = (operation) => { name, "--param", "copy_max_threads=1", + "--param", + `copy_run_id=${LOCAL_COPY_RUN_ID}`, "--wait", ), attempts: 8, @@ -462,6 +467,7 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { "product_creator_activity", "product_feature_adoption", "product_analytics_freshness", + "product_analytics_copy_assertions", ]) { const pipe = project.pipes.find((candidate) => candidate.name === name); if (!pipe) { diff --git a/scripts/analytics/verify-local.js b/scripts/analytics/verify-local.js index fa01f5faede..5c05f284c1a 100644 --- a/scripts/analytics/verify-local.js +++ b/scripts/analytics/verify-local.js @@ -128,6 +128,18 @@ assert.equal( ); assert.equal(Number(health[0].payload_conflicts), 0); +const copyAssertions = await query("product_analytics_copy_assertions", { + copy_run_id: "run_local_copy_assertions", +}); +assert.deepEqual(copyAssertions, [ + { + traffic_markers: 1, + traffic_page_markers: 1, + activation_markers: 1, + retention_markers: 1, + }, +]); + console.log( "Local Tinybird copies and typed endpoints match fixture semantics.", ); From b58f419e590eff918cbaeb2d019b219dbf04101c Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:43:30 +0100 Subject: [PATCH 069/110] ci: preserve analytics candidate through cleanup --- .github/workflows/analytics.yml | 24 +++++------ scripts/analytics/staging-ci.js | 46 ++++++++++++---------- scripts/analytics/tests/staging-ci.test.js | 15 +++++-- 3 files changed, 50 insertions(+), 35 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index cd64c4afcc6..60cc2e5a124 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -211,16 +211,6 @@ jobs: git diff --exit-code test -z "$(git status --porcelain --untracked-files=no)" docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment promote --wait - - name: Discard an unpromoted staging deployment before cleanup - if: always() && steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' - env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} - run: >- - docker compose --file packages/local-docker/docker-compose.yml --profile analytics - run --rm tinybird-cloud-cli --cloud deployment discard --wait - name: Prove least-privilege staging token scopes id: token-scopes env: @@ -313,7 +303,7 @@ jobs: node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase cleanup - --target live + --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' && 'staging' || 'live' }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove synthetic cleanup no longer affects queries @@ -324,8 +314,20 @@ jobs: TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} run: >- node scripts/analytics/staging-ci.js verify-cleanup + --deployment-id "${{ steps.tinybird.outputs.id }}" + --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' && 'staging' || 'live' }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Discard an unpromoted staging deployment after cleanup + if: always() && steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' + env: + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} + run: >- + docker compose --file packages/local-docker/docker-compose.yml --profile analytics + run --rm tinybird-cloud-cli --cloud deployment discard --wait - name: Upload redacted staging evidence if: always() && steps.seed.outcome == 'success' uses: actions/upload-artifact@v4 diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index a872e4a3f8e..6db3115b001 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -395,7 +395,7 @@ const probePreview = async () => { } const occurredAt = new Date().toISOString(); const runHash = hashIdentifier(state.runId); - const previewRunId = validateSyntheticRunId(`${state.runId}_preview`); + const previewRunId = validateSyntheticRunId(state.previewRunId); const event = { eventId: `synthetic_preview_${runHash.slice(0, 24)}`, eventName: "page_view", @@ -403,7 +403,7 @@ const probePreview = async () => { anonymousId, sessionId: `synthetic_preview_${runHash.slice(24, 48)}`, platform: "web", - appVersion: `staging-preview-${runHash.slice(0, 12)}`, + appVersion: state.previewAppVersion, pathname: "/analytics-synthetic", properties: { hostname: new URL(previewOrigin).hostname, @@ -494,9 +494,7 @@ const probePreview = async () => { `The exact-SHA collector p95 was ${collectorLatency.p95Ms}ms, over ${collectorP95BudgetMs}ms`, ); } - state.previewAppVersion = event.appVersion; state.previewAcceptedRows = duplicateResponses.length + replayAccepted; - state.previewRunId = previewRunId; writeJson(statePath, state, 0o600); artifact.previewApi = { bootstrapPassed: true, @@ -546,8 +544,12 @@ const seed = async () => { count: Number(process.env.PERFORMANCE_EVENT_COUNT ?? 1_000), now: startedAt, }); + const previewRunId = validateSyntheticRunId(`${runId}_preview`); + const previewAppVersion = `staging-preview-${hashIdentifier(runId).slice(0, 12)}`; const state = { runId, + previewRunId, + previewAppVersion, deploymentId, appVersion: fixture.appVersion, loadAppVersion: loadFixture.appVersion, @@ -819,8 +821,8 @@ const runCopies = async () => { if (!["live", "staging"].includes(target)) { throw new Error("Tinybird copy target is invalid"); } - if (target === "staging" && phase !== "staged") { - throw new Error("Only the staged copy phase can target staging"); + if (target === "staging" && !["staged", "cleanup"].includes(phase)) { + throw new Error("Only staged and cleanup copy phases can target staging"); } if (String(state.deploymentId) !== option("deployment-id")) { throw new Error("Tinybird copy deployment does not match the seeded run"); @@ -930,19 +932,9 @@ const runCopies = async () => { downstreamVisibility[copyStep.pipe] = { polls: visibility.polls, visibilityMs: visibility.visibilityMs, + ...(copyStep.marker ? { marker: copyStep.marker } : {}), }; } - const markers = normalizeCopyAssertions( - ( - await copyAssertionsQuery({ - copyRunId, - deploymentId: endpointDeploymentId, - }) - ).data, - ); - if (Object.values(markers).some((value) => value !== 1)) { - throw new Error("Not every Tinybird aggregate copy marker is visible"); - } artifact.copyJobs = { ...artifact.copyJobs, [phase]: { @@ -953,7 +945,7 @@ const runCopies = async () => { polls: canonicalVisibility.polls, visibilityMs: canonicalVisibility.visibilityMs, }, - downstreamVisibility: { copies: downstreamVisibility, markers }, + downstreamVisibility: { copies: downstreamVisibility }, }, }; writeJson(artifactPath, artifact); @@ -1414,14 +1406,24 @@ const verifyCleanup = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); const artifact = readJson(artifactPath); - const result = await healthQuery({ state }); + const target = option("target"); + if (!["live", "staging"].includes(target)) { + throw new Error("Tinybird cleanup verification target is invalid"); + } + if (String(state.deploymentId) !== option("deployment-id")) { + throw new Error( + "Tinybird cleanup deployment does not match the seeded run", + ); + } + const deploymentId = target === "staging" ? state.deploymentId : ""; + const result = await healthQuery({ state, deploymentId }); const health = normalizeHealth(result.data); if (Object.values(health).some((value) => value !== 0)) { throw new Error( "Synthetic rows still affect Tinybird health after cleanup", ); } - const decisionResult = await ciAssertionsQuery({ state }); + const decisionResult = await ciAssertionsQuery({ state, deploymentId }); const decisionAssertions = normalizeCiAssertions(decisionResult.data); if (Object.values(decisionAssertions).some((value) => value !== 0)) { throw new Error( @@ -1430,6 +1432,7 @@ const verifyCleanup = async () => { } const loadResult = await healthQuery({ state, + deploymentId, appVersion: state.loadAppVersion, }); const loadHealth = normalizeHealth(loadResult.data); @@ -1440,6 +1443,7 @@ const verifyCleanup = async () => { } const controlResult = await healthQuery({ state, + deploymentId, appVersion: state.erasureControlAppVersion, }); const controlHealth = normalizeHealth(controlResult.data); @@ -1451,6 +1455,7 @@ const verifyCleanup = async () => { if (state.previewAppVersion && state.previewRunId) { const previewResult = await healthQuery({ state, + deploymentId, appVersion: state.previewAppVersion, }); const previewHealth = normalizeHealth(previewResult.data); @@ -1461,6 +1466,7 @@ const verifyCleanup = async () => { } const previewDecisionResult = await ciAssertionsQuery({ state, + deploymentId, syntheticRunId: state.previewRunId, }); const previewDecisionAssertions = normalizeCiAssertions( diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 4c11b71c75a..f5211fc55d4 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -506,12 +506,19 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.match( /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 5, + 6, ); assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); assert.ok( - workflow.indexOf( - "Discard an unpromoted staging deployment before cleanup", - ) < workflow.indexOf("Delete strictly scoped synthetic raw rows"), + workflow.indexOf("Prove synthetic cleanup no longer affects queries") < + workflow.indexOf( + "Discard an unpromoted staging deployment after cleanup", + ), + ); + assert.equal( + workflow.match( + /steps\.promote\.outcome != 'success' && 'staging' \|\| 'live'/g, + )?.length, + 2, ); }); From ac90129c49615856e89f37c27bf603ff1fa55fe3 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 03:54:13 +0100 Subject: [PATCH 070/110] ci: clean partial analytics seeds --- .github/workflows/analytics.yml | 23 ++- scripts/analytics/staging-ci-lib.js | 17 +++ scripts/analytics/staging-ci.js | 170 +++++++++++++++------ scripts/analytics/tests/staging-ci.test.js | 90 ++++++++++- 4 files changed, 243 insertions(+), 57 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 60cc2e5a124..da0c384dc30 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -196,6 +196,8 @@ jobs: TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} run: >- node scripts/analytics/staging-ci.js verify + --deployment-id "${{ steps.tinybird.outputs.id }}" + --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live' }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Promote the verified staging deployment @@ -285,17 +287,24 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Delete strictly scoped synthetic raw rows id: cleanup - if: always() && steps.seed.outcome == 'success' + if: always() && steps.seed.outcome != 'skipped' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} - run: >- - node scripts/analytics/staging-ci.js cleanup - --state "$RUNNER_TEMP/analytics-staging-state.json" - --artifact "$RUNNER_TEMP/analytics-staging-report.json" + run: | + if [[ ! -f "$RUNNER_TEMP/analytics-staging-state.json" ]]; then + echo "required=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "required=true" >> "$GITHUB_OUTPUT" + node scripts/analytics/staging-ci.js cleanup \ + --deployment-id "${{ steps.tinybird.outputs.id }}" \ + --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' && 'staging' || 'live' }}" \ + --state "$RUNNER_TEMP/analytics-staging-state.json" \ + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Retract synthetic rows from every derived copy id: cleanup-copies - if: always() && steps.cleanup.outcome == 'success' + if: always() && steps.cleanup.outputs.required == 'true' && steps.cleanup.outcome == 'success' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} @@ -329,7 +338,7 @@ jobs: docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment discard --wait - name: Upload redacted staging evidence - if: always() && steps.seed.outcome == 'success' + if: always() && steps.cleanup.outputs.required == 'true' uses: actions/upload-artifact@v4 with: name: analytics-staging-${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }} diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 3cc914ca3d9..4f4f2cf46ed 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -194,6 +194,23 @@ export const selectStagingDeployment = ( return { id: String(id), needsPromotion: false }; }; +export const dataMutationDeploymentParameters = ({ + target, + deploymentId, + expectedDeploymentId, +}) => { + if (!["live", "staging"].includes(target)) { + throw new Error("Tinybird data mutation target is invalid"); + } + if ( + !DEPLOYMENT_ID_PATTERN.test(deploymentId) || + deploymentId !== expectedDeploymentId + ) { + throw new Error("Tinybird data mutation deployment is invalid"); + } + return target === "staging" ? { __tb__deployment: deploymentId } : {}; +}; + export const submitTinybirdCopyJobs = async ({ origin, token, diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 6db3115b001..95d981694e3 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -8,6 +8,7 @@ import { createSyntheticErasureControl, createSyntheticEvents, createSyntheticLoadEvents, + dataMutationDeploymentParameters, decisionEndpointQueries, evaluateBundleBudget, evaluateLatencyBudget, @@ -565,7 +566,35 @@ const seed = async () => { endTime: new Date(startedAt.getTime() + 300_000).toISOString(), }; writeJson(statePath, state, 0o600); - const deliver = async (row) => { + const artifact = { + schemaVersion: 1, + sha, + githubRun: { + id: environment("GITHUB_RUN_ID"), + attempt: environment("GITHUB_RUN_ATTEMPT"), + }, + vercel: { + deploymentId: environment("VERCEL_DEPLOYMENT_ID"), + url: environment("VERCEL_PREVIEW_URL"), + }, + tinybird: { deploymentId }, + syntheticRunHash: hashIdentifier(runId), + startedAt: state.startedAt, + delivery: { + rowsAttempted: fixture.rows.length, + rowsAccepted: 0, + }, + load: { rows: loadFixture.rows.length }, + erasure: { + controlRunHash: hashIdentifier(erasureControl.runId), + identityHash: hashIdentifier( + `${fixture.userId}:${fixture.organizationId}:${fixture.anonymousId}`, + ), + }, + assertions: { seedAccepted: false }, + }; + writeJson(artifactPath, artifact); + const deliver = async (row, fixtureRow = false) => { const result = await request( tinybirdUrl(origin, "/v0/events", { name: "product_events_v1", @@ -580,14 +609,20 @@ const seed = async () => { attempts: 4, }, ); + if (fixtureRow) { + artifact.delivery.rowsAccepted += 1; + artifact.delivery.retryAttempts = + (artifact.delivery.retryAttempts ?? 0) + result.attempt - 1; + writeJson(artifactPath, artifact); + } return { attempts: result.attempt, latencyMs: result.latencyMs }; }; const concurrentDeliveries = await Promise.all( - fixture.rows.slice(0, 2).map(deliver), + fixture.rows.slice(0, 2).map((row) => deliver(row, true)), ); const separateBatchDeliveries = []; for (const row of fixture.rows.slice(2)) { - separateBatchDeliveries.push(await deliver(row)); + separateBatchDeliveries.push(await deliver(row, true)); } const deliveries = [...concurrentDeliveries, ...separateBatchDeliveries]; const loadStartedAt = performance.now(); @@ -609,50 +644,35 @@ const seed = async () => { 1, Math.round(performance.now() - loadStartedAt), ); + artifact.load = { + rows: loadFixture.rows.length, + rowsAccepted: loadFixture.rows.length, + requestLatencyMs: loadDelivery.latencyMs, + retryAttempts: loadDelivery.attempt - 1, + rowsPerSecond: Math.round( + (loadFixture.rows.length * 1_000) / loadElapsedMs, + ), + }; + writeJson(artifactPath, artifact); const erasureControlDelivery = await deliver(erasureControl.row); - writeJson(artifactPath, { - schemaVersion: 1, - sha, - githubRun: { - id: environment("GITHUB_RUN_ID"), - attempt: environment("GITHUB_RUN_ATTEMPT"), - }, - vercel: { - deploymentId: environment("VERCEL_DEPLOYMENT_ID"), - url: environment("VERCEL_PREVIEW_URL"), - }, - tinybird: { deploymentId }, - syntheticRunHash: hashIdentifier(runId), - startedAt: state.startedAt, - delivery: { - rowsAttempted: fixture.rows.length, - rowsAccepted: deliveries.length, - requestLatency: latencySummary( - deliveries.map((delivery) => delivery.latencyMs), - ), - retryAttempts: deliveries.reduce( - (total, delivery) => total + delivery.attempts - 1, - 0, - ), - }, - load: { - rows: loadFixture.rows.length, - requestLatencyMs: loadDelivery.latencyMs, - retryAttempts: loadDelivery.attempt - 1, - rowsPerSecond: Math.round( - (loadFixture.rows.length * 1_000) / loadElapsedMs, - ), - }, - erasure: { - controlRunHash: hashIdentifier(erasureControl.runId), - identityHash: hashIdentifier( - `${fixture.userId}:${fixture.organizationId}:${fixture.anonymousId}`, - ), - controlDeliveryLatencyMs: erasureControlDelivery.latencyMs, - controlRetryAttempts: erasureControlDelivery.attempts - 1, - }, - assertions: { seedAccepted: true }, - }); + artifact.delivery = { + rowsAttempted: fixture.rows.length, + rowsAccepted: deliveries.length, + requestLatency: latencySummary( + deliveries.map((delivery) => delivery.latencyMs), + ), + retryAttempts: deliveries.reduce( + (total, delivery) => total + delivery.attempts - 1, + 0, + ), + }; + artifact.erasure = { + ...artifact.erasure, + controlDeliveryLatencyMs: erasureControlDelivery.latencyMs, + controlRetryAttempts: erasureControlDelivery.attempts - 1, + }; + artifact.assertions.seedAccepted = true; + writeJson(artifactPath, artifact); }; const waitForCopyVisibility = async ({ label, read, assert }) => { @@ -969,6 +989,12 @@ const verify = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); const artifact = readJson(artifactPath); + const target = option("target"); + dataMutationDeploymentParameters({ + target, + deploymentId: option("deployment-id"), + expectedDeploymentId: String(state.deploymentId), + }); const deadline = Date.now() + Number(process.env.INGESTION_SLO_MS ?? 180_000); let result; let health; @@ -1005,6 +1031,37 @@ const verify = async () => { "Synthetic load health did not match the accepted event set", ); } + if (target === "staging") { + const [liveHealthResult, liveLoadResult, liveControlResult, liveDecisions] = + await Promise.all([ + healthQuery({ state }), + healthQuery({ state, appVersion: state.loadAppVersion }), + healthQuery({ state, appVersion: state.erasureControlAppVersion }), + ciAssertionsQuery({ state }), + ]); + for (const liveHealth of [ + normalizeHealth(liveHealthResult.data), + normalizeHealth(liveLoadResult.data), + normalizeHealth(liveControlResult.data), + ]) { + assertZeroHealth(liveHealth); + } + if ( + Object.values(normalizeCiAssertions(liveDecisions.data)).some( + (value) => value !== 0, + ) + ) { + throw new Error( + "Candidate-only synthetic events affected live decisions", + ); + } + artifact.candidateIsolation = { + candidateDeploymentId: state.deploymentId, + candidateVisible: true, + liveVisible: false, + }; + artifact.assertions.candidateIsolationPassed = true; + } const samples = [result.latencyMs]; for (let index = 1; index < 20; index += 1) { const sample = await healthQuery({ @@ -1159,10 +1216,17 @@ const safeSyntheticIdentifier = (value, name) => { return value; }; -const deleteProductEventRows = async ({ origin, token, condition }) => { +const deleteProductEventRows = async ({ + origin, + token, + condition, + deploymentParameters = {}, +}) => { const body = new URLSearchParams({ delete_condition: condition }); const deletion = await request( - tinybirdUrl(origin, "/v0/datasources/product_events_v1/delete"), + tinybirdUrl(origin, "/v0/datasources/product_events_v1/delete", { + ...deploymentParameters, + }), { token, method: "POST", @@ -1179,7 +1243,9 @@ const deleteProductEventRows = async ({ origin, token, condition }) => { const deadline = Date.now() + 180_000; while (Date.now() < deadline) { const job = await request( - tinybirdUrl(origin, `/v0/jobs/${encodeURIComponent(jobId)}`), + tinybirdUrl(origin, `/v0/jobs/${encodeURIComponent(jobId)}`, { + ...deploymentParameters, + }), { token, attempts: 3 }, ); const status = String( @@ -1317,6 +1383,11 @@ const verifySyntheticIdentityErasure = async () => { const cleanup = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); + const deploymentParameters = dataMutationDeploymentParameters({ + target: option("target"), + deploymentId: option("deployment-id"), + expectedDeploymentId: String(state.deploymentId), + }); validateSyntheticRunId(state.runId); const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_CLEANUP_TOKEN", @@ -1331,6 +1402,7 @@ const cleanup = async () => { origin, token: tokens.TINYBIRD_STAGING_CLEANUP_TOKEN, condition: `synthetic_run_id IN (${runIds.map((runId) => `'${runId}'`).join(", ")})`, + deploymentParameters, }); const artifact = readJson(artifactPath); artifact.cleanup = { diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index f5211fc55d4..18897ddf27d 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -10,6 +10,7 @@ import { createSyntheticErasureControl, createSyntheticEvents, createSyntheticLoadEvents, + dataMutationDeploymentParameters, decisionEndpointQueries, evaluateBundleBudget, evaluateLatencyBudget, @@ -212,6 +213,36 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = ); }); +test("data mutations target only the exact staging deployment", () => { + assert.deepEqual( + dataMutationDeploymentParameters({ + target: "staging", + deploymentId: "7", + expectedDeploymentId: "7", + }), + { __tb__deployment: "7" }, + ); + assert.deepEqual( + dataMutationDeploymentParameters({ + target: "live", + deploymentId: "7", + expectedDeploymentId: "7", + }), + {}, + ); + for (const input of [ + { target: "production", deploymentId: "7", expectedDeploymentId: "7" }, + { target: "staging", deploymentId: "8", expectedDeploymentId: "7" }, + { + target: "staging", + deploymentId: "latest", + expectedDeploymentId: "latest", + }, + ]) { + assert.throws(() => dataMutationDeploymentParameters(input)); + } +}); + test("copy jobs use only approved resource-scoped submissions and bounded markers", async () => { const requests = []; const resourceToken = token(); @@ -506,7 +537,7 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.match( /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 6, + 8, ); assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); assert.ok( @@ -519,6 +550,63 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.match( /steps\.promote\.outcome != 'success' && 'staging' \|\| 'live'/g, )?.length, + 3, + ); + assert.equal( + workflow.match( + /steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'/g, + )?.length, 2, ); + assert.match(workflow, /steps\.seed\.outcome != 'skipped'/); + assert.doesNotMatch(workflow, /steps\.seed\.outcome == 'success'/); + assert.match(workflow, /echo "required=false" >> "\$GITHUB_OUTPUT"/); + assert.match(workflow, /echo "required=true" >> "\$GITHUB_OUTPUT"/); + assert.match( + workflow, + /Upload redacted staging evidence\n {8}if: always\(\) && steps\.cleanup\.outputs\.required == 'true'/, + ); +}); + +test("the seed persists cleanup state and partial evidence before ingestion", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const seedSource = source.slice( + source.indexOf("const seed = async () => {"), + source.indexOf("const waitForCopyVisibility"), + ); + const stateWrite = seedSource.indexOf("writeJson(statePath, state, 0o600)"); + const artifactWrite = seedSource.indexOf("writeJson(artifactPath, artifact)"); + const firstDelivery = seedSource.indexOf("const concurrentDeliveries"); + assert.ok(stateWrite >= 0); + assert.ok(artifactWrite > stateWrite); + assert.ok(firstDelivery > artifactWrite); + assert.match(seedSource, /assertions: \{ seedAccepted: false \}/); + assert.match( + seedSource, + /artifact\.delivery\.rowsAccepted \+= 1;[\s\S]*writeJson\(artifactPath, artifact\);/, + ); +}); + +test("synthetic deletion targets the deployment used for ingestion", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const workflow = fs.readFileSync( + new URL("../../../.github/workflows/analytics.yml", import.meta.url), + "utf8", + ); + const deleteSource = source.slice( + source.indexOf("const deleteProductEventRows"), + source.indexOf("const eraseSyntheticIdentity"), + ); + assert.doesNotMatch(deleteSource, /__tb__min_deployment/); + assert.equal(deleteSource.match(/\.\.\.deploymentParameters/g)?.length, 2); + assert.match( + workflow, + /cleanup \\\n {12}--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}" \\\n {12}--target "\$\{\{ steps\.tinybird\.outputs\.needs_promotion == 'true' && steps\.promote\.outcome != 'success' && 'staging' \|\| 'live' \}\}"/, + ); }); From 457048330f635aabf79c64cc889165f024d7cf58 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:03:02 +0100 Subject: [PATCH 071/110] ci: reconcile interrupted analytics runs --- .github/workflows/analytics.yml | 44 ++++++++++- scripts/analytics/staging-ci-lib.js | 65 +++++++++++++++ scripts/analytics/staging-ci.js | 43 +++++++++- scripts/analytics/tests/staging-ci.test.js | 92 +++++++++++++++++++++- 4 files changed, 234 insertions(+), 10 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index da0c384dc30..759165f35be 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -203,6 +203,7 @@ jobs: - name: Promote the verified staging deployment id: promote if: steps.tinybird.outputs.needs_promotion == 'true' + continue-on-error: true env: TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} @@ -213,6 +214,41 @@ jobs: git diff --exit-code test -z "$(git status --porcelain --untracked-files=no)" docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment promote --wait + - name: Resolve authoritative Tinybird state after the promotion attempt + id: deployment-state + if: always() && steps.tinybird.outcome == 'success' + env: + TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} + run: | + for resolution_attempt in $(seq 1 30); do + if ! docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud --output json deployment ls > "$RUNNER_TEMP/tinybird-final-deployments.json"; then + if [[ "$resolution_attempt" -eq 30 ]]; then + exit 1 + fi + sleep 5 + continue + fi + recover_pending=false + if [[ "$resolution_attempt" -eq 30 ]]; then + recover_pending=true + fi + if node scripts/analytics/staging-ci.js resolve-deployment-state --input "$RUNNER_TEMP/tinybird-final-deployments.json" --deployment-id "${{ steps.tinybird.outputs.id }}" --recover-pending "$recover_pending"; then + exit 0 + else + resolution_exit=$? + fi + if [[ "$resolution_exit" -ne 75 ]]; then + exit "$resolution_exit" + fi + sleep 5 + done + exit 1 + - name: Refuse to proceed without an authoritative live deployment + if: steps.deployment-state.outputs.promoted != 'true' + run: exit 1 - name: Prove least-privilege staging token scopes id: token-scopes env: @@ -299,7 +335,7 @@ jobs: echo "required=true" >> "$GITHUB_OUTPUT" node scripts/analytics/staging-ci.js cleanup \ --deployment-id "${{ steps.tinybird.outputs.id }}" \ - --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' && 'staging' || 'live' }}" \ + --target "${{ steps.deployment-state.outputs.target || (steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live') }}" \ --state "$RUNNER_TEMP/analytics-staging-state.json" \ --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Retract synthetic rows from every derived copy @@ -312,7 +348,7 @@ jobs: node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase cleanup - --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' && 'staging' || 'live' }}" + --target "${{ steps.deployment-state.outputs.target || (steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live') }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove synthetic cleanup no longer affects queries @@ -324,11 +360,11 @@ jobs: run: >- node scripts/analytics/staging-ci.js verify-cleanup --deployment-id "${{ steps.tinybird.outputs.id }}" - --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' && 'staging' || 'live' }}" + --target "${{ steps.deployment-state.outputs.target || (steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live') }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Discard an unpromoted staging deployment after cleanup - if: always() && steps.tinybird.outputs.needs_promotion == 'true' && steps.promote.outcome != 'success' + if: always() && steps.deployment-state.outputs.discard == 'true' env: TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 4f4f2cf46ed..dc835732b2c 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -211,6 +211,71 @@ export const dataMutationDeploymentParameters = ({ return target === "staging" ? { __tb__deployment: deploymentId } : {}; }; +export const resolveDeploymentState = (value, expectedDeploymentId) => { + if (!DEPLOYMENT_ID_PATTERN.test(expectedDeploymentId)) { + throw new Error("Tinybird deployment state requires a numeric ID"); + } + const deployments = Array.isArray(value) + ? value + : (value.deployments ?? value.data ?? value.results ?? []); + if (!Array.isArray(deployments)) { + throw new Error("Tinybird returned an unsupported deployment list"); + } + const matches = deployments.filter( + (deployment) => String(deploymentId(deployment)) === expectedDeploymentId, + ); + if (matches.length !== 1) { + throw new Error("The exact Tinybird deployment state is ambiguous"); + } + const state = deploymentState(matches[0]); + if (state.includes("live")) { + return { + target: "live", + discard: false, + promoted: true, + pending: false, + state: "live", + }; + } + if (state.includes("staging")) { + return { + target: "staging", + discard: true, + promoted: false, + pending: false, + state: "staging", + }; + } + if (state.includes("in progress")) { + return { + target: "staging", + discard: false, + promoted: false, + pending: true, + state: "in_progress", + }; + } + if (state.includes("failed")) { + return { + target: "staging", + discard: true, + promoted: false, + pending: false, + state: "failed", + }; + } + if (state.includes("deleted")) { + return { + target: "staging", + discard: false, + promoted: false, + pending: false, + state: "deleted", + }; + } + throw new Error(`The exact Tinybird deployment is ${state || "unknown"}`); +}; + export const submitTinybirdCopyJobs = async ({ origin, token, diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 95d981694e3..8e6004b968e 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -18,6 +18,7 @@ import { normalizeCiAssertions, normalizeCopyAssertions, normalizeHealth, + resolveDeploymentState, STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, @@ -581,20 +582,31 @@ const seed = async () => { syntheticRunHash: hashIdentifier(runId), startedAt: state.startedAt, delivery: { - rowsAttempted: fixture.rows.length, + rowsPlanned: fixture.rows.length, + rowsAttempted: 0, + rowsAccepted: 0, + }, + load: { + rowsPlanned: loadFixture.rows.length, + rowsAttempted: 0, rowsAccepted: 0, }, - load: { rows: loadFixture.rows.length }, erasure: { controlRunHash: hashIdentifier(erasureControl.runId), identityHash: hashIdentifier( `${fixture.userId}:${fixture.organizationId}:${fixture.anonymousId}`, ), + controlAttempted: false, + controlAccepted: false, }, assertions: { seedAccepted: false }, }; writeJson(artifactPath, artifact); const deliver = async (row, fixtureRow = false) => { + if (fixtureRow) { + artifact.delivery.rowsAttempted += 1; + writeJson(artifactPath, artifact); + } const result = await request( tinybirdUrl(origin, "/v0/events", { name: "product_events_v1", @@ -625,6 +637,8 @@ const seed = async () => { separateBatchDeliveries.push(await deliver(row, true)); } const deliveries = [...concurrentDeliveries, ...separateBatchDeliveries]; + artifact.load.rowsAttempted = loadFixture.rows.length; + writeJson(artifactPath, artifact); const loadStartedAt = performance.now(); const loadDelivery = await request( tinybirdUrl(origin, "/v0/events", { @@ -646,6 +660,8 @@ const seed = async () => { ); artifact.load = { rows: loadFixture.rows.length, + rowsPlanned: loadFixture.rows.length, + rowsAttempted: loadFixture.rows.length, rowsAccepted: loadFixture.rows.length, requestLatencyMs: loadDelivery.latencyMs, retryAttempts: loadDelivery.attempt - 1, @@ -654,8 +670,11 @@ const seed = async () => { ), }; writeJson(artifactPath, artifact); + artifact.erasure.controlAttempted = true; + writeJson(artifactPath, artifact); const erasureControlDelivery = await deliver(erasureControl.row); artifact.delivery = { + rowsPlanned: fixture.rows.length, rowsAttempted: fixture.rows.length, rowsAccepted: deliveries.length, requestLatency: latencySummary( @@ -668,6 +687,7 @@ const seed = async () => { }; artifact.erasure = { ...artifact.erasure, + controlAccepted: true, controlDeliveryLatencyMs: erasureControlDelivery.latencyMs, controlRetryAttempts: erasureControlDelivery.attempts - 1, }; @@ -1696,6 +1716,25 @@ const handlers = { writeOutput("id", selection.id); writeOutput("needs_promotion", String(selection.needsPromotion)); }, + "resolve-deployment-state": async () => { + const recoverPending = option("recover-pending"); + if (!["true", "false"].includes(recoverPending)) { + throw new Error("--recover-pending must be true or false"); + } + const resolution = resolveDeploymentState( + readJson(option("input")), + option("deployment-id"), + ); + if (resolution.pending && recoverPending === "false") { + process.exitCode = 75; + return; + } + writeOutput("target", resolution.target); + writeOutput("discard", resolution.pending || resolution.discard); + writeOutput("promoted", resolution.promoted); + writeOutput("state", resolution.state); + writeOutput("pending_recovery", resolution.pending); + }, "wait-vercel": waitForVercel, seed, "run-copies": runCopies, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 18897ddf27d..c6cb8ac5f41 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -21,6 +21,7 @@ import { normalizeCiAssertions, normalizeCopyAssertions, normalizeHealth, + resolveDeploymentState, STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, @@ -243,6 +244,63 @@ test("data mutations target only the exact staging deployment", () => { } }); +test("deployment state resolution binds cleanup to the exact deployment", () => { + assert.deepEqual(resolveDeploymentState([{ id: 7, status: "Live" }], "7"), { + target: "live", + discard: false, + promoted: true, + pending: false, + state: "live", + }); + assert.deepEqual( + resolveDeploymentState([{ id: 7, status: "Staging" }], "7"), + { + target: "staging", + discard: true, + promoted: false, + pending: false, + state: "staging", + }, + ); + assert.deepEqual( + resolveDeploymentState([{ id: 7, status: "In progress" }], "7"), + { + target: "staging", + discard: false, + promoted: false, + pending: true, + state: "in_progress", + }, + ); + assert.deepEqual(resolveDeploymentState([{ id: 7, status: "Failed" }], "7"), { + target: "staging", + discard: true, + promoted: false, + pending: false, + state: "failed", + }); + assert.throws(() => resolveDeploymentState([{ id: 8, status: "Live" }], "7")); + assert.throws(() => + resolveDeploymentState( + [ + { id: 7, status: "Live" }, + { id: 7, status: "Staging" }, + ], + "7", + ), + ); + assert.deepEqual( + resolveDeploymentState([{ id: 7, status: "Deleted" }], "7"), + { + target: "staging", + discard: false, + promoted: false, + pending: false, + state: "deleted", + }, + ); +}); + test("copy jobs use only approved resource-scoped submissions and bounded markers", async () => { const requests = []; const resourceToken = token(); @@ -537,7 +595,7 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.match( /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 8, + 9, ); assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); assert.ok( @@ -546,15 +604,30 @@ test("the analytics workflow is statically restricted to staging", () => { "Discard an unpromoted staging deployment after cleanup", ), ); + assert.doesNotMatch(workflow, /steps\.promote\.outcome/); + assert.match(workflow, /continue-on-error: true/); + assert.match(workflow, /staging-ci\.js resolve-deployment-state/); + assert.match(workflow, /resolution_exit=\$\?/); + assert.match(workflow, /"\$resolution_exit" -ne 75/); + assert.match(workflow, /--recover-pending "\$recover_pending"/); + assert.match(workflow, /recover_pending=true/); + assert.match( + workflow, + /Refuse to proceed without an authoritative live deployment/, + ); + assert.match( + workflow, + /Discard an unpromoted staging deployment after cleanup\n {8}if: always\(\) && steps\.deployment-state\.outputs\.discard == 'true'/, + ); assert.equal( workflow.match( - /steps\.promote\.outcome != 'success' && 'staging' \|\| 'live'/g, + /steps\.deployment-state\.outputs\.target \|\| \(steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'\)/g, )?.length, 3, ); assert.equal( workflow.match( - /steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'/g, + /--target "\$\{\{ steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live' \}\}"/g, )?.length, 2, ); @@ -584,6 +657,12 @@ test("the seed persists cleanup state and partial evidence before ingestion", () assert.ok(artifactWrite > stateWrite); assert.ok(firstDelivery > artifactWrite); assert.match(seedSource, /assertions: \{ seedAccepted: false \}/); + assert.match(seedSource, /rowsPlanned: fixture\.rows\.length/); + assert.match(seedSource, /rowsAttempted: 0/); + assert.ok( + seedSource.indexOf("artifact.delivery.rowsAttempted += 1") < + seedSource.indexOf("const result = await request"), + ); assert.match( seedSource, /artifact\.delivery\.rowsAccepted \+= 1;[\s\S]*writeJson\(artifactPath, artifact\);/, @@ -605,8 +684,13 @@ test("synthetic deletion targets the deployment used for ingestion", () => { ); assert.doesNotMatch(deleteSource, /__tb__min_deployment/); assert.equal(deleteSource.match(/\.\.\.deploymentParameters/g)?.length, 2); + assert.match( + source, + /writeOutput\("discard", resolution\.pending \|\| resolution\.discard\)/, + ); + assert.match(workflow, /node scripts\/analytics\/staging-ci\.js cleanup/); assert.match( workflow, - /cleanup \\\n {12}--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}" \\\n {12}--target "\$\{\{ steps\.tinybird\.outputs\.needs_promotion == 'true' && steps\.promote\.outcome != 'success' && 'staging' \|\| 'live' \}\}"/, + /steps\.deployment-state\.outputs\.target \|\| \(steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'\)/, ); }); From c32fd9b157c9eecca87dd8c3acc1f208448be54a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:20:34 +0100 Subject: [PATCH 072/110] fix: inline analytics copy assertion parameter --- scripts/analytics/tests/datafiles.test.js | 1 + .../pipes/product_analytics_copy_assertions.pipe | 11 +++++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index e8ff30d4e2a..b9739da6201 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -251,6 +251,7 @@ test("staging copy markers are excluded from every decision endpoint", () => { ); assert.match(assertions, /copy_run_id is required/); assert.match(assertions, /traffic_markers/); + assert.doesNotMatch(assertions, /requested_copy_run_id/); assert.match(assertions, /retention_markers/); }); diff --git a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe index 92f84d5e2dc..98e1ce2353c 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe @@ -9,12 +9,11 @@ SQL > {% if not defined(copy_run_id) %} {{ error('copy_run_id is required') }} {% end %} - WITH {{String(copy_run_id)}} AS requested_copy_run_id SELECT - (SELECT count() FROM product_traffic_daily_exact WHERE copy_run_id = requested_copy_run_id) AS traffic_markers, - (SELECT count() FROM product_traffic_pages_daily_exact WHERE copy_run_id = requested_copy_run_id) AS traffic_page_markers, - (SELECT count() FROM product_activation_daily_exact WHERE copy_run_id = requested_copy_run_id) AS activation_markers, - (SELECT count() FROM product_creator_retention_exact WHERE copy_run_id = requested_copy_run_id) AS retention_markers - WHERE throwIf(length(requested_copy_run_id) < 8 OR length(requested_copy_run_id) > 128, 'copy_run_id has an invalid length') = 0 + (SELECT count() FROM product_traffic_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS traffic_markers, + (SELECT count() FROM product_traffic_pages_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS traffic_page_markers, + (SELECT count() FROM product_activation_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS activation_markers, + (SELECT count() FROM product_creator_retention_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS retention_markers + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 TYPE ENDPOINT From 62e6dbf671dbc48859b60116d014c57d7a802e71 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 04:55:45 +0100 Subject: [PATCH 073/110] ci: bind analytics staging lifecycle to deployment --- .github/workflows/analytics.yml | 36 +- scripts/analytics/staging-ci-lib.js | 187 ++++++++- scripts/analytics/staging-ci.js | 452 ++++++++++++++++++--- scripts/analytics/tests/staging-ci.test.js | 184 ++++++++- 4 files changed, 767 insertions(+), 92 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 759165f35be..a3188d6086b 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -46,8 +46,9 @@ permissions: statuses: read concurrency: - group: analytics-staging-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: >- + ${{ ((github.event_name == 'pull_request' && github.event.pull_request.number == 2003 && github.event.pull_request.head.ref == 'codex/first-party-analytics') || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/codex/first-party-analytics')) && 'analytics-staging-37b8fef9-817f-4c3c-b21f-218c36a6077d' || format('analytics-staging-out-of-scope-{0}', github.run_id) }} + cancel-in-progress: false env: TINYBIRD_WORKSPACE_ID: 37b8fef9-817f-4c3c-b21f-218c36a6077d @@ -182,6 +183,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -205,15 +207,13 @@ jobs: if: steps.tinybird.outputs.needs_promotion == 'true' continue-on-error: true env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: | test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" git diff --exit-code test -z "$(git status --porcelain --untracked-files=no)" - docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment promote --wait + node scripts/analytics/staging-ci.js promote-deployment --deployment-id "${{ steps.tinybird.outputs.id }}" - name: Resolve authoritative Tinybird state after the promotion attempt id: deployment-state if: always() && steps.tinybird.outcome == 'success' @@ -247,7 +247,7 @@ jobs: done exit 1 - name: Refuse to proceed without an authoritative live deployment - if: steps.deployment-state.outputs.promoted != 'true' + if: steps.deployment-state.outputs.promoted != 'true' || steps.promote.outcome == 'failure' run: exit 1 - name: Prove least-privilege staging token scopes id: token-scopes @@ -273,6 +273,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -294,6 +295,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js erase-synthetic-identity --state "$RUNNER_TEMP/analytics-staging-state.json" @@ -304,6 +306,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -327,6 +330,7 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: | if [[ ! -f "$RUNNER_TEMP/analytics-staging-state.json" ]]; then echo "required=false" >> "$GITHUB_OUTPUT" @@ -344,11 +348,12 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase cleanup - --target "${{ steps.deployment-state.outputs.target || (steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live') }}" + --target "${{ steps.cleanup.outputs.target }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove synthetic cleanup no longer affects queries @@ -360,19 +365,18 @@ jobs: run: >- node scripts/analytics/staging-ci.js verify-cleanup --deployment-id "${{ steps.tinybird.outputs.id }}" - --target "${{ steps.deployment-state.outputs.target || (steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live') }}" + --target "${{ steps.cleanup-copies.outputs.target }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Discard an unpromoted staging deployment after cleanup + id: discard if: always() && steps.deployment-state.outputs.discard == 'true' env: - TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TB_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- - docker compose --file packages/local-docker/docker-compose.yml --profile analytics - run --rm tinybird-cloud-cli --cloud deployment discard --wait + node scripts/analytics/staging-ci.js discard-deployment + --deployment-id "${{ steps.tinybird.outputs.id }}" - name: Upload redacted staging evidence if: always() && steps.cleanup.outputs.required == 'true' uses: actions/upload-artifact@v4 diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index dc835732b2c..092759097bf 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -125,18 +125,58 @@ const deploymentState = (deployment) => "", ).toLowerCase(); +const deploymentsFromResponse = (value) => { + const deployments = Array.isArray(value) + ? value + : (value.deployments ?? value.data ?? value.results ?? []); + if (!Array.isArray(deployments)) { + throw new Error("Tinybird returned an unsupported deployment list"); + } + return deployments; +}; + +const isLiveDeployment = (deployment) => + deployment.live === true || deploymentState(deployment).includes("live"); + +const isStagingDeployment = (deployment) => { + const state = deploymentState(deployment); + return ( + !isLiveDeployment(deployment) && + (state === "data_ready" || state.includes("staging")) + ); +}; + +const isPendingDeployment = (deployment) => { + const state = deploymentState(deployment); + return ( + state.includes("in progress") || + state.includes("pending") || + state.includes("promot") || + ["calculating", "creating_schema", "schema_ready", "deleting"].includes( + state, + ) + ); +}; + +const exactDeployment = (value, expectedDeploymentId) => { + const deployment = value.deployment ?? value; + if ( + !deployment || + typeof deployment !== "object" || + String(deploymentId(deployment)) !== expectedDeploymentId + ) { + throw new Error("Tinybird returned the wrong exact deployment"); + } + return deployment; +}; + export const selectStagingDeployment = ( value, minimumCreatedAt, createdDeploymentId, noOpConfirmed = false, ) => { - const candidates = Array.isArray(value) - ? value - : (value.deployments ?? value.data ?? value.results ?? []); - if (!Array.isArray(candidates)) { - throw new Error("Tinybird returned an unsupported deployment list"); - } + const candidates = deploymentsFromResponse(value); const minimumTime = Date.parse(minimumCreatedAt); const stagingDeployments = candidates .filter((candidate) => { @@ -194,6 +234,118 @@ export const selectStagingDeployment = ( return { id: String(id), needsPromotion: false }; }; +export const resolveOwnedMutationTarget = (value, expectedDeploymentId) => { + if (!DEPLOYMENT_ID_PATTERN.test(expectedDeploymentId)) { + throw new Error("Tinybird mutation ownership requires a numeric ID"); + } + const deployments = deploymentsFromResponse(value); + const matches = deployments.filter( + (deployment) => String(deploymentId(deployment)) === expectedDeploymentId, + ); + if (matches.length !== 1) { + throw new Error("The owned Tinybird deployment is missing or ambiguous"); + } + if (isLiveDeployment(matches[0])) return "live"; + if (isPendingDeployment(matches[0])) return "pending"; + if (!isStagingDeployment(matches[0])) { + throw new Error("The owned Tinybird deployment is not ready for mutation"); + } + const stagingDeployments = deployments.filter(isStagingDeployment); + if ( + stagingDeployments.length !== 1 || + String(deploymentId(stagingDeployments[0])) !== expectedDeploymentId + ) { + throw new Error("The Tinybird staging alias is not owned by this run"); + } + return "staging"; +}; + +export const resolveExactPromotionPlan = (value, expectedDeploymentId) => { + if (resolveOwnedMutationTarget(value, expectedDeploymentId) !== "staging") { + throw new Error("The owned Tinybird deployment is not staging"); + } + if (resolveOwnedDiscardTarget(value, expectedDeploymentId) !== "ready") { + throw new Error("The owned Tinybird deployment is not ready for promotion"); + } + const liveDeployments = + deploymentsFromResponse(value).filter(isLiveDeployment); + if (liveDeployments.length !== 1) { + throw new Error( + "Tinybird promotion requires exactly one current live deployment", + ); + } + const previousLiveDeploymentId = String(deploymentId(liveDeployments[0])); + if ( + !DEPLOYMENT_ID_PATTERN.test(previousLiveDeploymentId) || + previousLiveDeploymentId === expectedDeploymentId + ) { + throw new Error("Tinybird returned an invalid previous live deployment"); + } + return { previousLiveDeploymentId }; +}; + +export const resolveOwnedDiscardTarget = (value, expectedDeploymentId) => { + if (!DEPLOYMENT_ID_PATTERN.test(expectedDeploymentId)) { + throw new Error("Tinybird discard ownership requires a numeric ID"); + } + const deployments = deploymentsFromResponse(value); + const matches = deployments.filter( + (deployment) => String(deploymentId(deployment)) === expectedDeploymentId, + ); + if (matches.length !== 1 || isLiveDeployment(matches[0])) { + throw new Error("The owned Tinybird deployment cannot be discarded"); + } + const state = deploymentState(matches[0]); + if ( + !isStagingDeployment(matches[0]) && + !isPendingDeployment(matches[0]) && + !state.includes("failed") + ) { + throw new Error("The owned Tinybird deployment is not discardable"); + } + const mutableDeployments = deployments.filter((deployment) => { + const candidateState = deploymentState(deployment); + return ( + !isLiveDeployment(deployment) && + !candidateState.includes("deleted") && + (isStagingDeployment(deployment) || + isPendingDeployment(deployment) || + candidateState.includes("failed")) + ); + }); + if ( + mutableDeployments.length !== 1 || + String(deploymentId(mutableDeployments[0])) !== expectedDeploymentId + ) { + throw new Error("The Tinybird discard candidate is not owned by this run"); + } + return isPendingDeployment(matches[0]) ? "pending" : "ready"; +}; + +export const resolveExactDeploymentLifecycle = ( + value, + expectedDeploymentId, +) => { + if (!DEPLOYMENT_ID_PATTERN.test(expectedDeploymentId)) { + throw new Error("Tinybird exact deployment lookup requires a numeric ID"); + } + const deployment = exactDeployment(value, expectedDeploymentId); + const state = deploymentState(deployment); + if (isLiveDeployment(deployment)) return "live"; + if (state.includes("deleted")) return "deleted"; + if (state.includes("deleting")) return "deleting"; + if (state.includes("failed")) return "failed"; + if (isPendingDeployment(deployment)) return "pending"; + if (isStagingDeployment(deployment)) return "ready"; + throw new Error(`The exact Tinybird deployment is ${state || "unknown"}`); +}; + +export const reconcileCleanupTarget = (currentTarget, resolvedTarget) => { + if (currentTarget === resolvedTarget) return currentTarget; + if (currentTarget === "staging" && resolvedTarget === "live") return "live"; + throw new Error("Tinybird cleanup target changed non-monotonically"); +}; + export const dataMutationDeploymentParameters = ({ target, deploymentId, @@ -208,19 +360,14 @@ export const dataMutationDeploymentParameters = ({ ) { throw new Error("Tinybird data mutation deployment is invalid"); } - return target === "staging" ? { __tb__deployment: deploymentId } : {}; + return target === "staging" ? { __tb__deployment: "staging" } : {}; }; export const resolveDeploymentState = (value, expectedDeploymentId) => { if (!DEPLOYMENT_ID_PATTERN.test(expectedDeploymentId)) { throw new Error("Tinybird deployment state requires a numeric ID"); } - const deployments = Array.isArray(value) - ? value - : (value.deployments ?? value.data ?? value.results ?? []); - if (!Array.isArray(deployments)) { - throw new Error("Tinybird returned an unsupported deployment list"); - } + const deployments = deploymentsFromResponse(value); const matches = deployments.filter( (deployment) => String(deploymentId(deployment)) === expectedDeploymentId, ); @@ -285,11 +432,15 @@ export const submitTinybirdCopyJobs = async ({ pipes = COPY_PIPES, useDeploymentParameter = false, copyRunId = "", + assertMutationOwnership, }) => { if (!DEPLOYMENT_ID_PATTERN.test(deploymentId)) { throw new Error("Tinybird copy jobs require a numeric deployment ID"); } if (copyRunId) validateSyntheticRunId(copyRunId); + if (typeof assertMutationOwnership !== "function") { + throw new Error("Tinybird copies require an ownership check"); + } const results = []; for (const pipe of pipes) { if (!COPY_PIPES.includes(pipe)) { @@ -302,7 +453,7 @@ export const submitTinybirdCopyJobs = async ({ ); copyUrl.searchParams.set("_mode", "replace"); if (useDeploymentParameter) { - copyUrl.searchParams.set("__tb__deployment", deploymentId); + copyUrl.searchParams.set("__tb__deployment", "staging"); } if (COPY_MARKER_PIPES.has(pipe)) { if (!copyRunId) { @@ -316,6 +467,7 @@ export const submitTinybirdCopyJobs = async ({ token, method: "POST", attempts: 3, + beforeAttempt: assertMutationOwnership, }); } catch (error) { throw new Error(`Tinybird copy submission failed for ${pipe}`, { @@ -662,11 +814,12 @@ export const assertWorkflowSafety = (workflow) => { STAGING_WORKSPACE_ID, FEATURE_BRANCH, String(FEATURE_PULL_REQUEST), - "cancel-in-progress: true", + "cancel-in-progress: false", + "analytics-staging-out-of-scope-", "pull_request.head.sha", "deployment create --allow-destructive-operations --check", - "deployment promote", - "deployment discard", + "staging-ci.js promote-deployment", + "staging-ci.js discard-deployment", "environment: staging", "TINYBIRD_STAGING_DEPLOY_TOKEN", "TINYBIRD_STAGING_INGEST_TOKEN", diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 8e6004b968e..135c7dede30 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -18,7 +18,12 @@ import { normalizeCiAssertions, normalizeCopyAssertions, normalizeHealth, + reconcileCleanupTarget, resolveDeploymentState, + resolveExactDeploymentLifecycle, + resolveExactPromotionPlan, + resolveOwnedDiscardTarget, + resolveOwnedMutationTarget, STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, @@ -96,12 +101,20 @@ const tinybirdEnvironment = (requiredTokenNames = TINYBIRD_TOKEN_NAMES) => { const request = async ( url, - { token, method = "GET", body, headers = {}, attempts = 1 } = {}, + { + token, + method = "GET", + body, + headers = {}, + attempts = 1, + beforeAttempt, + } = {}, ) => { let lastError; for (let attempt = 1; attempt <= attempts; attempt += 1) { const startedAt = performance.now(); try { + if (beforeAttempt) await beforeAttempt(); const response = await fetch(url, { method, body, @@ -199,6 +212,215 @@ const copyAssertionsQuery = async ({ copyRunId, deploymentId = "" }) => { ); }; +const ownedMutationTarget = async ({ state, origin, token }) => { + const deployments = await request(tinybirdUrl(origin, "/v1/deployments"), { + token, + attempts: 3, + }); + return resolveOwnedMutationTarget( + deployments.data, + String(state.deploymentId), + ); +}; + +const waitForOwnedMutationTarget = async ({ state, origin, token }) => { + const deadline = + Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); + let lastError; + while (Date.now() < deadline) { + const target = await ownedMutationTarget({ state, origin, token }); + if (target !== "pending") return target; + lastError = new Error("The owned Tinybird deployment is still pending"); + await delay(2_000); + } + throw new Error("Timed out waiting for the owned Tinybird deployment", { + cause: lastError, + }); +}; + +const deploymentList = async ({ origin, token }) => + request(tinybirdUrl(origin, "/v1/deployments"), { + token, + attempts: 3, + }); + +const exactDeployment = async ({ origin, token, deploymentId }) => + request( + tinybirdUrl(origin, `/v1/deployments/${encodeURIComponent(deploymentId)}`), + { token, attempts: 3 }, + ); + +const promoteOwnedDeployment = async () => { + const deploymentId = option("deployment-id"); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + ]); + const token = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; + const initial = await deploymentList({ origin, token }); + const plan = resolveExactPromotionPlan(initial.data, deploymentId); + const promotionDeadline = + Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); + let lastPromotionError; + let promotionAttempts = 0; + while (Date.now() < promotionDeadline) { + const current = await deploymentList({ origin, token }); + const target = resolveOwnedMutationTarget(current.data, deploymentId); + if (target === "live") break; + if (target === "pending") { + await delay(2_000); + continue; + } + if (promotionAttempts >= 3) { + throw new Error("The exact Tinybird deployment remained staging", { + cause: lastPromotionError, + }); + } + const currentPlan = resolveExactPromotionPlan(current.data, deploymentId); + if ( + currentPlan.previousLiveDeploymentId !== plan.previousLiveDeploymentId + ) { + throw new Error("The Tinybird live deployment changed before promotion"); + } + try { + promotionAttempts += 1; + await request( + tinybirdUrl( + origin, + `/v1/deployments/${encodeURIComponent(deploymentId)}/set-live`, + ), + { token, method: "POST" }, + ); + } catch (error) { + lastPromotionError = error; + } + await delay(2_000); + } + const promoted = await deploymentList({ origin, token }); + if (resolveOwnedMutationTarget(promoted.data, deploymentId) !== "live") { + throw new Error("The exact Tinybird deployment was not promoted", { + cause: lastPromotionError, + }); + } + const deadline = + Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); + let lastDeletionError; + while (Date.now() < deadline) { + const previous = await exactDeployment({ + origin, + token, + deploymentId: plan.previousLiveDeploymentId, + }); + const lifecycle = resolveExactDeploymentLifecycle( + previous.data, + plan.previousLiveDeploymentId, + ); + if (lifecycle === "deleted") { + writeOutput("promoted", "true"); + return; + } + if (lifecycle === "live") { + throw new Error("The previous Tinybird deployment became live again"); + } + if (lifecycle !== "deleting") { + try { + await request( + tinybirdUrl( + origin, + `/v1/deployments/${encodeURIComponent(plan.previousLiveDeploymentId)}`, + ), + { + token, + method: "DELETE", + beforeAttempt: async () => { + const ownership = await deploymentList({ origin, token }); + if ( + resolveOwnedMutationTarget(ownership.data, deploymentId) !== + "live" + ) { + throw new Error( + "The promoted Tinybird deployment is no longer live", + ); + } + const exactPrevious = await exactDeployment({ + origin, + token, + deploymentId: plan.previousLiveDeploymentId, + }); + if ( + resolveExactDeploymentLifecycle( + exactPrevious.data, + plan.previousLiveDeploymentId, + ) === "live" + ) { + throw new Error( + "Refusing to delete a live Tinybird deployment", + ); + } + }, + }, + ); + } catch (error) { + lastDeletionError = error; + } + } + await delay(2_000); + } + throw new Error("Timed out deleting the previous Tinybird deployment", { + cause: lastDeletionError, + }); +}; + +const discardOwnedDeployment = async () => { + const deploymentId = option("deployment-id"); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + ]); + const token = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; + const initial = await deploymentList({ origin, token }); + resolveOwnedDiscardTarget(initial.data, deploymentId); + const deadline = + Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); + let lastDeletionError; + while (Date.now() < deadline) { + const current = await exactDeployment({ origin, token, deploymentId }); + const lifecycle = resolveExactDeploymentLifecycle( + current.data, + deploymentId, + ); + if (lifecycle === "deleted") { + writeOutput("discarded", "true"); + return; + } + if (lifecycle === "live") { + throw new Error("Refusing to discard a live Tinybird deployment"); + } + if (lifecycle !== "deleting") { + try { + await request( + tinybirdUrl( + origin, + `/v1/deployments/${encodeURIComponent(deploymentId)}`, + ), + { + token, + method: "DELETE", + beforeAttempt: async () => { + const ownership = await deploymentList({ origin, token }); + resolveOwnedDiscardTarget(ownership.data, deploymentId); + }, + }, + ); + } catch (error) { + lastDeletionError = error; + } + } + await delay(2_000); + } + throw new Error("Timed out discarding the exact Tinybird deployment", { + cause: lastDeletionError, + }); +}; + const decisionEndpointQuery = async ({ origin, token, name, parameters }) => request(tinybirdUrl(origin, `/v0/pipes/${name}.json`, parameters), { token, @@ -854,14 +1076,14 @@ const runCopies = async () => { const artifactPath = option("artifact"); const artifact = readJson(artifactPath); const phase = option("phase"); - const target = option("target"); + const requestedTarget = option("target"); if (!["staged", "promoted", "erasure", "cleanup"].includes(phase)) { throw new Error("Tinybird copy phase is invalid"); } - if (!["live", "staging"].includes(target)) { + if (!["live", "staging"].includes(requestedTarget)) { throw new Error("Tinybird copy target is invalid"); } - if (target === "staging" && !["staged", "cleanup"].includes(phase)) { + if (requestedTarget === "staging" && !["staged", "cleanup"].includes(phase)) { throw new Error("Only staged and cleanup copy phases can target staging"); } if (String(state.deploymentId) !== option("deployment-id")) { @@ -869,11 +1091,22 @@ const runCopies = async () => { } const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_READ_TOKEN", + "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); - const endpointDeploymentId = target === "staging" ? state.deploymentId : ""; const copyRunId = validateSyntheticRunId(`${state.runId}_${phase}`); const expectations = phaseRunExpectations({ state, phase }); - try { + const executeCopies = async (target) => { + const assertMutationOwnership = async () => { + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== target + ) { + throw new Error("The owned Tinybird deployment target changed"); + } + }; const canonicalJobs = await submitTinybirdCopyJobs({ origin, token: tokens.TINYBIRD_STAGING_READ_TOKEN, @@ -881,13 +1114,14 @@ const runCopies = async () => { request, pipes: ["snapshot_product_events_canonical_v1"], useDeploymentParameter: target === "staging", + assertMutationOwnership, }); const canonicalVisibility = await waitForCopyVisibility({ label: "Tinybird canonical copy", read: () => readPhaseCiAssertions({ state, - deploymentId: endpointDeploymentId, + deploymentId: state.deploymentId, expectations, }), assert: (results) => @@ -901,7 +1135,7 @@ const runCopies = async () => { read: () => readPhaseCiAssertions({ state, - deploymentId: endpointDeploymentId, + deploymentId: state.deploymentId, expectations, }), assert: (results) => @@ -929,7 +1163,7 @@ const runCopies = async () => { readAndAssertPhaseHealth({ state, phase, - deploymentId: endpointDeploymentId, + deploymentId: state.deploymentId, }), assert: () => undefined, }, @@ -944,6 +1178,7 @@ const runCopies = async () => { pipes: [copyStep.pipe], useDeploymentParameter: target === "staging", copyRunId, + assertMutationOwnership, })), ); const visibility = await waitForCopyVisibility({ @@ -955,7 +1190,7 @@ const runCopies = async () => { ( await copyAssertionsQuery({ copyRunId, - deploymentId: endpointDeploymentId, + deploymentId: state.deploymentId, }) ).data, )), @@ -975,34 +1210,61 @@ const runCopies = async () => { ...(copyStep.marker ? { marker: copyStep.marker } : {}), }; } - artifact.copyJobs = { - ...artifact.copyJobs, - [phase]: { - status: "passed", - copyRunHash: hashIdentifier(copyRunId), - jobs: [...canonicalJobs, ...downstreamJobs], - canonicalVisibility: { - polls: canonicalVisibility.polls, - visibilityMs: canonicalVisibility.visibilityMs, - }, - downstreamVisibility: { copies: downstreamVisibility }, + await assertMutationOwnership(); + return { + status: "passed", + target, + copyRunHash: hashIdentifier(copyRunId), + jobs: [...canonicalJobs, ...downstreamJobs], + canonicalVisibility: { + polls: canonicalVisibility.polls, + visibilityMs: canonicalVisibility.visibilityMs, }, + downstreamVisibility: { copies: downstreamVisibility }, }; - writeJson(artifactPath, artifact); - } catch (error) { - artifact.copyJobs = { - ...artifact.copyJobs, - [phase]: { - status: "failed", - error: - error instanceof Error - ? error.message - : "Unknown Tinybird copy error", - }, - }; - writeJson(artifactPath, artifact); - throw error; + }; + let target = requestedTarget; + for ( + let transitionAttempt = 0; + transitionAttempt < 2; + transitionAttempt += 1 + ) { + try { + artifact.copyJobs = { + ...artifact.copyJobs, + [phase]: await executeCopies(target), + }; + if (phase === "cleanup") writeOutput("target", target); + writeJson(artifactPath, artifact); + return; + } catch (error) { + if (phase === "cleanup" && target === "staging") { + const resolvedTarget = await waitForOwnedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + }); + const nextTarget = reconcileCleanupTarget(target, resolvedTarget); + if (nextTarget !== target) { + target = nextTarget; + continue; + } + } + artifact.copyJobs = { + ...artifact.copyJobs, + [phase]: { + status: "failed", + error: + error instanceof Error + ? error.message + : "Unknown Tinybird copy error", + }, + }; + writeJson(artifactPath, artifact); + throw error; + } } + throw new Error("Tinybird cleanup copies changed target more than once"); }; const verify = async () => { @@ -1241,6 +1503,7 @@ const deleteProductEventRows = async ({ token, condition, deploymentParameters = {}, + beforeAttempt, }) => { const body = new URLSearchParams({ delete_condition: condition }); const deletion = await request( @@ -1253,6 +1516,7 @@ const deleteProductEventRows = async ({ body, headers: { "Content-Type": "application/x-www-form-urlencoded" }, attempts: 3, + beforeAttempt, }, ); const jobId = @@ -1266,7 +1530,7 @@ const deleteProductEventRows = async ({ tinybirdUrl(origin, `/v0/jobs/${encodeURIComponent(jobId)}`, { ...deploymentParameters, }), - { token, attempts: 3 }, + { token, attempts: 3, beforeAttempt }, ); const status = String( job.data.status ?? job.data.state ?? job.data.job?.status ?? "", @@ -1300,11 +1564,24 @@ const eraseSyntheticIdentity = async () => { ); const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_CLEANUP_TOKEN", + "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); + const assertLiveOwnership = async () => { + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== "live" + ) { + throw new Error("The owned Tinybird deployment is no longer live"); + } + }; const rowsAffected = await deleteProductEventRows({ origin, token: tokens.TINYBIRD_STAGING_CLEANUP_TOKEN, condition: `organization_id = '${organizationId}' OR user_id = '${userId}' OR (anonymous_id = '${anonymousId}' AND (user_id = '' OR user_id = '${userId}'))`, + beforeAttempt: assertLiveOwnership, }); artifact.erasure = { ...artifact.erasure, @@ -1318,22 +1595,31 @@ const verifySyntheticIdentityErasure = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); const artifact = readJson(artifactPath); - const erasedHealth = normalizeHealth((await healthQuery({ state })).data); + const erasedHealth = normalizeHealth( + (await healthQuery({ state, deploymentId: state.deploymentId })).data, + ); const erasedLoadHealth = normalizeHealth( ( await healthQuery({ state, + deploymentId: state.deploymentId, appVersion: state.loadAppVersion, }) ).data, ); const erasedDecisions = normalizeCiAssertions( - (await ciAssertionsQuery({ state })).data, + ( + await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + }) + ).data, ); const previewHealth = normalizeHealth( ( await healthQuery({ state, + deploymentId: state.deploymentId, appVersion: state.previewAppVersion, }) ).data, @@ -1342,6 +1628,7 @@ const verifySyntheticIdentityErasure = async () => { ( await ciAssertionsQuery({ state, + deploymentId: state.deploymentId, syntheticRunId: state.previewRunId, }) ).data, @@ -1368,6 +1655,7 @@ const verifySyntheticIdentityErasure = async () => { ( await healthQuery({ state, + deploymentId: state.deploymentId, appVersion: state.erasureControlAppVersion, }) ).data, @@ -1403,27 +1691,80 @@ const verifySyntheticIdentityErasure = async () => { const cleanup = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); - const deploymentParameters = dataMutationDeploymentParameters({ - target: option("target"), - deploymentId: option("deployment-id"), - expectedDeploymentId: String(state.deploymentId), - }); - validateSyntheticRunId(state.runId); + const requestedTarget = option("target"); + if (!["live", "staging"].includes(requestedTarget)) { + throw new Error("Tinybird cleanup target is invalid"); + } const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_CLEANUP_TOKEN", + "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); + let target = await waitForOwnedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + }); + if (requestedTarget === "live" && target !== "live") { + throw new Error("Tinybird cleanup target regressed from live to staging"); + } + validateSyntheticRunId(state.runId); validateSyntheticRunId(state.loadRunId); validateSyntheticRunId(state.erasureControlRunId); const runIds = [state.runId, state.loadRunId, state.erasureControlRunId]; if (state.previewRunId) { runIds.push(validateSyntheticRunId(state.previewRunId)); } - const rowsAffected = await deleteProductEventRows({ - origin, - token: tokens.TINYBIRD_STAGING_CLEANUP_TOKEN, - condition: `synthetic_run_id IN (${runIds.map((runId) => `'${runId}'`).join(", ")})`, - deploymentParameters, - }); + let rowsAffected; + for ( + let transitionAttempt = 0; + transitionAttempt < 2; + transitionAttempt += 1 + ) { + const deploymentParameters = dataMutationDeploymentParameters({ + target, + deploymentId: option("deployment-id"), + expectedDeploymentId: String(state.deploymentId), + }); + const assertMutationOwnership = async () => { + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== target + ) { + throw new Error("The owned Tinybird cleanup target changed"); + } + }; + try { + rowsAffected = await deleteProductEventRows({ + origin, + token: tokens.TINYBIRD_STAGING_CLEANUP_TOKEN, + condition: `synthetic_run_id IN (${runIds.map((runId) => `'${runId}'`).join(", ")})`, + deploymentParameters, + beforeAttempt: assertMutationOwnership, + }); + break; + } catch (error) { + if (target === "staging") { + const resolvedTarget = await waitForOwnedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + }); + const nextTarget = reconcileCleanupTarget(target, resolvedTarget); + if (nextTarget !== target) { + target = nextTarget; + continue; + } + } + throw error; + } + } + if (rowsAffected === undefined) { + throw new Error("Tinybird cleanup changed target more than once"); + } + writeOutput("target", target); const artifact = readJson(artifactPath); artifact.cleanup = { ...artifact.cleanup, @@ -1446,6 +1787,7 @@ const verifyPromoted = async () => { } const previewHealthResult = await healthQuery({ state, + deploymentId: state.deploymentId, appVersion: state.previewAppVersion, }); const previewHealth = normalizeHealth(previewHealthResult.data); @@ -1460,11 +1802,15 @@ const verifyPromoted = async () => { "The promoted health snapshot did not preserve preview retry deliveries", ); } - const seedDecisionResult = await ciAssertionsQuery({ state }); + const seedDecisionResult = await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + }); const seedDecisionAssertions = normalizeCiAssertions(seedDecisionResult.data); assertSyntheticDecisions(seedDecisionAssertions); const previewDecisionResult = await ciAssertionsQuery({ state, + deploymentId: state.deploymentId, syntheticRunId: state.previewRunId, }); const previewDecisionAssertions = normalizeCiAssertions( @@ -1507,7 +1853,7 @@ const verifyCleanup = async () => { "Tinybird cleanup deployment does not match the seeded run", ); } - const deploymentId = target === "staging" ? state.deploymentId : ""; + const deploymentId = state.deploymentId; const result = await healthQuery({ state, deploymentId }); const health = normalizeHealth(result.data); if (Object.values(health).some((value) => value !== 0)) { @@ -1698,6 +2044,8 @@ const handlers = { }), "verify-credentials": async () => tinybirdEnvironment(), "verify-token-scopes": verifyTokenScopes, + "promote-deployment": promoteOwnedDeployment, + "discard-deployment": discardOwnedDeployment, "select-deployment": async () => { const createOutput = readJson(option("create-output")); const output = String(createOutput.output ?? ""); diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index c6cb8ac5f41..ddd5f0fbaf5 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -21,7 +21,12 @@ import { normalizeCiAssertions, normalizeCopyAssertions, normalizeHealth, + reconcileCleanupTarget, resolveDeploymentState, + resolveExactDeploymentLifecycle, + resolveExactPromotionPlan, + resolveOwnedDiscardTarget, + resolveOwnedMutationTarget, STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, @@ -214,14 +219,14 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = ); }); -test("data mutations target only the exact staging deployment", () => { +test("data mutations validate the exact deployment before using the staging selector", () => { assert.deepEqual( dataMutationDeploymentParameters({ target: "staging", deploymentId: "7", expectedDeploymentId: "7", }), - { __tb__deployment: "7" }, + { __tb__deployment: "staging" }, ); assert.deepEqual( dataMutationDeploymentParameters({ @@ -244,6 +249,131 @@ test("data mutations target only the exact staging deployment", () => { } }); +test("mutation ownership follows only the exact staging candidate or promoted live ID", () => { + assert.equal( + resolveOwnedMutationTarget( + { + deployments: [ + { id: "6", status: "data_ready", live: true }, + { id: "7", status: "data_ready", live: false }, + ], + }, + "7", + ), + "staging", + ); + assert.equal( + resolveOwnedMutationTarget( + { + deployments: [ + { id: "7", status: "data_ready", live: true }, + { id: "8", status: "data_ready", live: false }, + ], + }, + "7", + ), + "live", + ); + assert.equal( + resolveOwnedMutationTarget( + { + deployments: [ + { id: "6", status: "data_ready", live: true }, + { id: "7", status: "creating_schema", live: false }, + ], + }, + "7", + ), + "pending", + ); + assert.equal( + resolveOwnedMutationTarget( + { + deployments: [ + { id: "6", status: "data_ready", live: true }, + { id: "7", status: "promoting", live: false }, + ], + }, + "7", + ), + "pending", + ); + for (const deployments of [ + [{ id: "8", status: "data_ready", live: false }], + [ + { id: "7", status: "data_ready", live: false }, + { id: "8", status: "data_ready", live: false }, + ], + ]) { + assert.throws(() => resolveOwnedMutationTarget({ deployments }, "7")); + } +}); + +test("cleanup target transitions only once from staging to the owned live deployment", () => { + assert.equal(reconcileCleanupTarget("staging", "staging"), "staging"); + assert.equal(reconcileCleanupTarget("staging", "live"), "live"); + assert.equal(reconcileCleanupTarget("live", "live"), "live"); + assert.throws(() => reconcileCleanupTarget("live", "staging")); + assert.throws(() => reconcileCleanupTarget("staging", "pending")); +}); + +test("promotion and discard plans stay bound to exact numeric deployments", () => { + const deployments = { + deployments: [ + { id: "6", status: "data_ready", live: true }, + { id: "7", status: "data_ready", live: false }, + ], + }; + assert.deepEqual(resolveExactPromotionPlan(deployments, "7"), { + previousLiveDeploymentId: "6", + }); + assert.equal(resolveOwnedDiscardTarget(deployments, "7"), "ready"); + assert.throws(() => resolveExactPromotionPlan(deployments, "6")); + assert.throws(() => resolveOwnedDiscardTarget(deployments, "6")); + assert.throws(() => + resolveOwnedDiscardTarget( + { + deployments: [ + ...deployments.deployments, + { id: "8", status: "creating_schema", live: false }, + ], + }, + "7", + ), + ); +}); + +test("exact deployment lifecycle refuses wrong IDs and distinguishes deletion", () => { + assert.equal( + resolveExactDeploymentLifecycle( + { deployment: { id: "7", status: "data_ready", live: true } }, + "7", + ), + "live", + ); + for (const [status, expected] of [ + ["data_ready", "ready"], + ["creating_schema", "pending"], + ["deleting", "deleting"], + ["deleted", "deleted"], + ["failed", "failed"], + ]) { + assert.equal( + resolveExactDeploymentLifecycle( + { deployment: { id: "7", status, live: false } }, + "7", + ), + expected, + ); + } + assert.throws(() => + resolveExactDeploymentLifecycle( + { deployment: { id: "8", status: "deleted" } }, + "7", + ), + ); +}); + test("deployment state resolution binds cleanup to the exact deployment", () => { assert.deepEqual(resolveDeploymentState([{ id: 7, status: "Live" }], "7"), { target: "live", @@ -309,6 +439,7 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker { data: { id: "copy_job_traffic" } }, ]; let now = 1_000; + let ownershipChecks = 0; const results = await submitTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", token: resourceToken, @@ -318,6 +449,7 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker "snapshot_product_traffic_daily_exact", ], request: async (url, options) => { + await options.beforeAttempt(); requests.push({ url: String(url), options }); now += 10; return responses.shift(); @@ -325,7 +457,11 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker now: () => now, useDeploymentParameter: true, copyRunId: "run_12345678_staged", + assertMutationOwnership: async () => { + ownershipChecks += 1; + }, }); + assert.equal(ownershipChecks, 2); assert.deepEqual(results, [ { pipe: "snapshot_product_events_canonical_v1", @@ -339,7 +475,7 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker }, ]); assert.match(requests[0].url, /_mode=replace/); - assert.match(requests[0].url, /__tb__deployment=6/); + assert.match(requests[0].url, /__tb__deployment=staging/); assert.doesNotMatch(requests[0].url, /copy_run_id/); assert.equal(requests[0].options.method, "POST"); assert.equal(requests[0].options.token, resourceToken); @@ -352,9 +488,11 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker deploymentId: "6", pipes: ["snapshot_product_events_canonical_v1"], request: async (url, options) => { + await options.beforeAttempt(); liveRequests.push({ url: String(url), options }); return { data: { id: "copy_job_live" } }; }, + assertMutationOwnership: async () => undefined, }); assert.doesNotMatch(liveRequests[0].url, /__tb__deployment/); await assert.rejects(() => @@ -372,6 +510,7 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker deploymentId: "6", pipes: ["snapshot_product_traffic_daily_exact"], request: async () => ({ data: { id: "copy_job_missing_marker" } }), + assertMutationOwnership: async () => undefined, }), ); }); @@ -595,7 +734,7 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.match( /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 9, + 11, ); assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); assert.ok( @@ -604,7 +743,7 @@ test("the analytics workflow is statically restricted to staging", () => { "Discard an unpromoted staging deployment after cleanup", ), ); - assert.doesNotMatch(workflow, /steps\.promote\.outcome/); + assert.match(workflow, /steps\.promote\.outcome == 'failure'/); assert.match(workflow, /continue-on-error: true/); assert.match(workflow, /staging-ci\.js resolve-deployment-state/); assert.match(workflow, /resolution_exit=\$\?/); @@ -617,13 +756,26 @@ test("the analytics workflow is statically restricted to staging", () => { ); assert.match( workflow, - /Discard an unpromoted staging deployment after cleanup\n {8}if: always\(\) && steps\.deployment-state\.outputs\.discard == 'true'/, + /Discard an unpromoted staging deployment after cleanup\n {8}id: discard\n {8}if: always\(\) && steps\.deployment-state\.outputs\.discard == 'true'/, + ); + assert.match( + workflow, + /staging-ci\.js promote-deployment --deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/, + ); + assert.match(workflow, /staging-ci\.js discard-deployment/); + assert.doesNotMatch( + workflow, + /tinybird-cloud-cli --cloud deployment promote/, + ); + assert.doesNotMatch( + workflow, + /tinybird-cloud-cli --cloud deployment discard/, ); assert.equal( workflow.match( /steps\.deployment-state\.outputs\.target \|\| \(steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'\)/g, )?.length, - 3, + 1, ); assert.equal( workflow.match( @@ -689,8 +841,26 @@ test("synthetic deletion targets the deployment used for ingestion", () => { /writeOutput\("discard", resolution\.pending \|\| resolution\.discard\)/, ); assert.match(workflow, /node scripts\/analytics\/staging-ci\.js cleanup/); + assert.match(source, /let target = await waitForOwnedMutationTarget/); + assert.match(source, /writeOutput\("target", target\)/); assert.match( workflow, /steps\.deployment-state\.outputs\.target \|\| \(steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'\)/, ); + assert.equal( + workflow.match(/--target "\$\{\{ steps\.cleanup\.outputs\.target \}\}"/g) + ?.length, + 1, + ); + assert.equal( + workflow.match( + /--target "\$\{\{ steps\.cleanup-copies\.outputs\.target \}\}"/g, + )?.length, + 1, + ); + assert.match(workflow, /analytics-staging-out-of-scope-/); + assert.match( + workflow, + /analytics-staging-37b8fef9-817f-4c3c-b21f-218c36a6077d/, + ); }); From 794a23f927bb74fbdb33dae208457d3871f24a90 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 05:11:15 +0100 Subject: [PATCH 074/110] ci: authorize candidate analytics copies --- .github/workflows/analytics.yml | 13 +++--- scripts/analytics/staging-ci.js | 50 +++++++++++++++++++++- scripts/analytics/tests/staging-ci.test.js | 15 ++++++- 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index a3188d6086b..83d3495a9ec 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -344,7 +344,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Retract synthetic rows from every derived copy id: cleanup-copies - if: always() && steps.cleanup.outputs.required == 'true' && steps.cleanup.outcome == 'success' + if: always() && steps.cleanup.outputs.required == 'true' && steps.cleanup.outcome == 'success' && steps.cleanup.outputs.requires_copies == 'true' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} @@ -370,13 +370,16 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Discard an unpromoted staging deployment after cleanup id: discard - if: always() && steps.deployment-state.outputs.discard == 'true' + if: always() && (steps.deployment-state.outputs.discard == 'true' || steps.cleanup.outputs.requires_discard == 'true') env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - run: >- - node scripts/analytics/staging-ci.js discard-deployment - --deployment-id "${{ steps.tinybird.outputs.id }}" + run: | + discard_args=(--deployment-id "${{ steps.tinybird.outputs.id }}") + if [[ -f "$RUNNER_TEMP/analytics-staging-report.json" ]]; then + discard_args+=(--artifact "$RUNNER_TEMP/analytics-staging-report.json") + fi + node scripts/analytics/staging-ci.js discard-deployment "${discard_args[@]}" - name: Upload redacted staging evidence if: always() && steps.cleanup.outputs.required == 'true' uses: actions/upload-artifact@v4 diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 135c7dede30..561dc4c8eb3 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -372,6 +372,7 @@ const promoteOwnedDeployment = async () => { const discardOwnedDeployment = async () => { const deploymentId = option("deployment-id"); + const artifactPath = options.get("artifact"); const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); @@ -389,6 +390,21 @@ const discardOwnedDeployment = async () => { ); if (lifecycle === "deleted") { writeOutput("discarded", "true"); + if (artifactPath && fs.existsSync(artifactPath)) { + const artifact = readJson(artifactPath); + artifact.cleanup = { + ...artifact.cleanup, + strategy: "deployment_discard", + candidateDiscarded: true, + passed: true, + verifiedAt: new Date().toISOString(), + }; + artifact.assertions = { + ...artifact.assertions, + cleanupPassed: true, + }; + writeJson(artifactPath, artifact); + } return; } if (lifecycle === "live") { @@ -1096,6 +1112,10 @@ const runCopies = async () => { const copyRunId = validateSyntheticRunId(`${state.runId}_${phase}`); const expectations = phaseRunExpectations({ state, phase }); const executeCopies = async (target) => { + const copyToken = + target === "staging" + ? tokens.TINYBIRD_STAGING_DEPLOY_TOKEN + : tokens.TINYBIRD_STAGING_READ_TOKEN; const assertMutationOwnership = async () => { if ( (await ownedMutationTarget({ @@ -1109,13 +1129,23 @@ const runCopies = async () => { }; const canonicalJobs = await submitTinybirdCopyJobs({ origin, - token: tokens.TINYBIRD_STAGING_READ_TOKEN, + token: copyToken, deploymentId: state.deploymentId, request, pipes: ["snapshot_product_events_canonical_v1"], useDeploymentParameter: target === "staging", assertMutationOwnership, }); + artifact.copyJobs = { + ...artifact.copyJobs, + [phase]: { + status: "in_progress", + target, + copyRunHash: hashIdentifier(copyRunId), + jobs: canonicalJobs, + }, + }; + writeJson(artifactPath, artifact); const canonicalVisibility = await waitForCopyVisibility({ label: "Tinybird canonical copy", read: () => @@ -1172,7 +1202,7 @@ const runCopies = async () => { downstreamJobs.push( ...(await submitTinybirdCopyJobs({ origin, - token: tokens.TINYBIRD_STAGING_READ_TOKEN, + token: copyToken, deploymentId: state.deploymentId, request, pipes: [copyStep.pipe], @@ -1253,6 +1283,7 @@ const runCopies = async () => { artifact.copyJobs = { ...artifact.copyJobs, [phase]: { + ...artifact.copyJobs?.[phase], status: "failed", error: error instanceof Error @@ -1714,6 +1745,19 @@ const cleanup = async () => { if (state.previewRunId) { runIds.push(validateSyntheticRunId(state.previewRunId)); } + if (target === "staging") { + writeOutput("target", target); + writeOutput("requires_copies", "false"); + writeOutput("requires_discard", "true"); + const artifact = readJson(artifactPath); + artifact.cleanup = { + ...artifact.cleanup, + strategy: "deployment_discard", + candidateDiscarded: false, + }; + writeJson(artifactPath, artifact); + return; + } let rowsAffected; for ( let transitionAttempt = 0; @@ -1765,6 +1809,8 @@ const cleanup = async () => { throw new Error("Tinybird cleanup changed target more than once"); } writeOutput("target", target); + writeOutput("requires_copies", "true"); + writeOutput("requires_discard", "false"); const artifact = readJson(artifactPath); artifact.cleanup = { ...artifact.cleanup, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index ddd5f0fbaf5..9d2867f0b02 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -756,7 +756,7 @@ test("the analytics workflow is statically restricted to staging", () => { ); assert.match( workflow, - /Discard an unpromoted staging deployment after cleanup\n {8}id: discard\n {8}if: always\(\) && steps\.deployment-state\.outputs\.discard == 'true'/, + /Discard an unpromoted staging deployment after cleanup\n {8}id: discard\n {8}if: always\(\) && \(steps\.deployment-state\.outputs\.discard == 'true' \|\| steps\.cleanup\.outputs\.requires_discard == 'true'\)/, ); assert.match( workflow, @@ -784,6 +784,7 @@ test("the analytics workflow is statically restricted to staging", () => { 2, ); assert.match(workflow, /steps\.seed\.outcome != 'skipped'/); + assert.match(workflow, /steps\.cleanup\.outputs\.requires_copies == 'true'/); assert.doesNotMatch(workflow, /steps\.seed\.outcome == 'success'/); assert.match(workflow, /echo "required=false" >> "\$GITHUB_OUTPUT"/); assert.match(workflow, /echo "required=true" >> "\$GITHUB_OUTPUT"/); @@ -843,6 +844,18 @@ test("synthetic deletion targets the deployment used for ingestion", () => { assert.match(workflow, /node scripts\/analytics\/staging-ci\.js cleanup/); assert.match(source, /let target = await waitForOwnedMutationTarget/); assert.match(source, /writeOutput\("target", target\)/); + assert.match( + source, + /target === "staging"[\s\S]*strategy: "deployment_discard"/, + ); + assert.match(source, /writeOutput\("requires_copies", "false"\)/); + assert.match(source, /writeOutput\("requires_copies", "true"\)/); + assert.match(source, /writeOutput\("requires_discard", "true"\)/); + assert.match(source, /writeOutput\("requires_discard", "false"\)/); + assert.match( + source, + /target === "staging"[\s\S]*tokens\.TINYBIRD_STAGING_DEPLOY_TOKEN[\s\S]*tokens\.TINYBIRD_STAGING_READ_TOKEN/, + ); assert.match( workflow, /steps\.deployment-state\.outputs\.target \|\| \(steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'\)/, From 600bbcdbee55fe0fe819acd342f3c8f6bb7f5b6d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:15:03 +0100 Subject: [PATCH 075/110] feat: harden analytics contracts and business events --- .../desktop/src-tauri/src/general_settings.rs | 4 +- .../src-tauri/src/product_analytics.rs | 665 ++- .../unit/product-analytics-delivery.test.ts | 1 + .../unit/product-analytics-first-view.test.ts | 52 + .../unit/product-analytics-queue.test.ts | 72 +- .../unit/product-analytics-scheduler.test.ts | 2 + .../unit/product-analytics-server.test.ts | 65 +- .../stripe-analytics-reconciliation.test.ts | 74 + .../subscription-analytics-webhook.test.ts | 149 +- .../unit/update-seat-quantity.test.ts | 31 +- apps/web/actions/organization/send-invites.ts | 4 + .../organization/update-seat-quantity.ts | 5 + apps/web/app/api/analytics/track/route.ts | 37 +- apps/web/app/api/invite/accept/route.ts | 6 + apps/web/app/api/webhooks/stripe/route.ts | 257 +- .../[videoId]/_components/EmbedVideo.tsx | 5 +- apps/web/app/embed/[videoId]/page.tsx | 5 +- apps/web/app/s/[videoId]/page.tsx | 2 +- apps/web/app/s/[videoId]/types.ts | 5 +- apps/web/app/utils/product-analytics.ts | 172 +- apps/web/lib/analytics/business-events.ts | 15 + apps/web/lib/analytics/first-view.ts | 23 + apps/web/lib/analytics/server-event.ts | 27 +- .../lib/analytics/stripe-business-events.ts | 355 ++ apps/web/workflows/import-loom-video.ts | 2 + .../workflows/reconcile-product-analytics.ts | 309 +- packages/analytics/src/event-registry.ts | 38 +- packages/analytics/src/index.test.ts | 19 + packages/analytics/src/index.ts | 12 +- .../migrations/0038_messy_stephen_strange.sql | 2 + .../migrations/0039_bumpy_phil_sheldon.sql | 16 + .../migrations/meta/0038_snapshot.json | 4327 ++++++++++++++++ .../migrations/meta/0039_snapshot.json | 4441 +++++++++++++++++ .../database/migrations/meta/_journal.json | 556 ++- packages/database/schema.ts | 26 + scripts/analytics/check-event-contract.js | 255 +- .../analytics/tests/event-contract.test.js | 70 + 37 files changed, 11485 insertions(+), 621 deletions(-) create mode 100644 apps/web/__tests__/unit/product-analytics-first-view.test.ts create mode 100644 apps/web/lib/analytics/first-view.ts create mode 100644 packages/database/migrations/0038_messy_stephen_strange.sql create mode 100644 packages/database/migrations/0039_bumpy_phil_sheldon.sql create mode 100644 packages/database/migrations/meta/0038_snapshot.json create mode 100644 packages/database/migrations/meta/0039_snapshot.json diff --git a/apps/desktop/src-tauri/src/general_settings.rs b/apps/desktop/src-tauri/src/general_settings.rs index 7bead04264d..e7dc285e092 100644 --- a/apps/desktop/src-tauri/src/general_settings.rs +++ b/apps/desktop/src-tauri/src/general_settings.rs @@ -430,7 +430,7 @@ impl GeneralSettingsStore { store.set("general_settings", json!(settings)); store.save().map_err(|e| e.to_string())?; - crate::product_analytics::set_telemetry_enabled(settings.enable_telemetry); + crate::product_analytics::set_telemetry_enabled(app, settings.enable_telemetry); #[cfg(target_os = "macos")] crate::permissions::sync_macos_dock_visibility(app); @@ -492,7 +492,7 @@ pub fn init(app: &AppHandle) { raw_store.set(REMOVE_TARGET_SELECT_MIGRATION_KEY, json!(true)); } - crate::product_analytics::set_telemetry_enabled(store.enable_telemetry); + crate::product_analytics::set_telemetry_enabled(app, store.enable_telemetry); register_bundled_muxer_binary(app); #[cfg(target_os = "macos")] diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index cc0587c86d3..6af232a7ec2 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -43,10 +43,21 @@ const PRODUCT_EVENT_OUTBOX_KEYRING_USER: &str = "product-analytics-outbox-v1"; const PRODUCT_EVENT_OUTBOX_LEGACY_KEYRING_USER: &str = "product-analytics-outbox-v1-backup"; const PRODUCT_EVENT_OUTBOX_CAPACITY: usize = 500; const PRODUCT_EVENT_DEAD_LETTER_CAPACITY: usize = 100; +const PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY: usize = 100; const PRODUCT_EVENT_MAX_RETRY_DELAY: Duration = Duration::from_secs(5 * 60); #[derive(Debug)] pub enum ProductAnalyticsEvent { + AnalyticsDeliveryLoss { + failure_class: String, + failed_event_name: String, + status: Option, + count: u64, + first_sequence: u64, + last_sequence: u64, + first_failed_at_ms: i64, + last_failed_at_ms: i64, + }, MultipartUploadComplete { duration: Duration, length: Duration, @@ -155,6 +166,27 @@ impl EventData { fn event_data(event: ProductAnalyticsEvent) -> EventData { match event { + ProductAnalyticsEvent::AnalyticsDeliveryLoss { + failure_class, + failed_event_name, + status, + count, + first_sequence, + last_sequence, + first_failed_at_ms, + last_failed_at_ms, + } => { + let mut data = EventData::new("analytics_delivery_loss"); + data.set("failure_class", failure_class); + data.set("failed_event_name", failed_event_name); + data.set("status", status); + data.set("count", count); + data.set("first_sequence", first_sequence); + data.set("last_sequence", last_sequence); + data.set("first_failed_at_ms", first_failed_at_ms); + data.set("last_failed_at_ms", last_failed_at_ms); + data + } ProductAnalyticsEvent::MultipartUploadComplete { duration, length, @@ -261,6 +293,7 @@ fn is_core_product_event(name: &str) -> bool { | "multipart_upload_complete" | "multipart_upload_failed" | "recording_recovery_failed" + | "analytics_delivery_loss" ) } @@ -385,10 +418,31 @@ struct ProductEventDeadLetter { failed_at: String, } +#[derive(Clone, Debug, Deserialize, Serialize)] +struct ProductEventLossSummary { + summary_id: String, + failure_class: String, + failed_event_name: String, + platform: String, + app_version: String, + anonymous_id: String, + session_id: String, + status: Option, + count: u64, + first_sequence: u64, + last_sequence: u64, + first_failed_at_ms: i64, + last_failed_at_ms: i64, +} + #[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(default)] struct ProductEventOutbox { pending: Vec, dead_letters: Vec, + loss_summaries: Vec, + loss_reports_in_flight: Vec, + next_delivery_sequence: u64, } async fn send_product_batch_once( @@ -749,6 +803,166 @@ fn load_stored_outbox(app: &AppHandle, store_key: &str) -> Result, +) -> bool { + summary.failure_class == failure_class + && summary.failed_event_name == event.event_name + && summary.platform == event.platform + && summary.app_version == event.app_version + && summary.status == status + && summary.anonymous_id == event.anonymous_id + && summary.session_id == event.session_id +} + +fn record_delivery_loss( + outbox: &mut ProductEventOutbox, + failure_class: &str, + event: &ProductEvent, + status: Option, + failed_at_ms: i64, +) { + outbox.next_delivery_sequence = outbox.next_delivery_sequence.saturating_add(1); + let sequence = outbox.next_delivery_sequence; + if let Some(summary) = outbox + .loss_summaries + .iter_mut() + .find(|summary| loss_summary_matches(summary, failure_class, event, status)) + { + summary.count = summary.count.saturating_add(1); + summary.last_sequence = sequence; + summary.last_failed_at_ms = failed_at_ms; + return; + } + + let detailed_capacity = PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY.saturating_sub(1); + if outbox.loss_summaries.len() < detailed_capacity { + outbox.loss_summaries.push(ProductEventLossSummary { + summary_id: Uuid::new_v4().to_string(), + failure_class: failure_class.to_string(), + failed_event_name: event.event_name.clone(), + platform: event.platform.clone(), + app_version: event.app_version.clone(), + anonymous_id: event.anonymous_id.clone(), + session_id: event.session_id.clone(), + status, + count: 1, + first_sequence: sequence, + last_sequence: sequence, + first_failed_at_ms: failed_at_ms, + last_failed_at_ms: failed_at_ms, + }); + return; + } + + if let Some(summary) = outbox.loss_summaries.iter_mut().find(|summary| { + summary.failure_class == "summary_capacity" + && summary.failed_event_name == "other" + && summary.platform == event.platform + && summary.app_version == event.app_version + }) { + summary.count = summary.count.saturating_add(1); + summary.last_sequence = sequence; + summary.last_failed_at_ms = failed_at_ms; + return; + } + + if outbox.loss_summaries.len() < PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY { + outbox.loss_summaries.push(ProductEventLossSummary { + summary_id: Uuid::new_v4().to_string(), + failure_class: "summary_capacity".to_string(), + failed_event_name: "other".to_string(), + platform: event.platform.clone(), + app_version: event.app_version.clone(), + anonymous_id: event.anonymous_id.clone(), + session_id: event.session_id.clone(), + status: None, + count: 1, + first_sequence: sequence, + last_sequence: sequence, + first_failed_at_ms: failed_at_ms, + last_failed_at_ms: failed_at_ms, + }); + } +} + +fn merge_loss_summary(target: &mut Vec, source: ProductEventLossSummary) { + if let Some(existing) = target + .iter_mut() + .find(|summary| summary.summary_id == source.summary_id) + { + if source.last_sequence > existing.last_sequence { + *existing = source; + } + return; + } + if target.len() < PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY { + target.push(source); + return; + } + + let fallback_index = target.iter().position(|summary| { + summary.failure_class == "summary_capacity" && summary.failed_event_name == "other" + }); + let index = fallback_index.unwrap_or_else(|| target.len().saturating_sub(1)); + let mut fallback = if fallback_index.is_some() { + target.remove(index) + } else { + let displaced = target.remove(index); + ProductEventLossSummary { + summary_id: displaced.summary_id, + failure_class: "summary_capacity".to_string(), + failed_event_name: "other".to_string(), + platform: displaced.platform, + app_version: displaced.app_version, + anonymous_id: displaced.anonymous_id, + session_id: displaced.session_id, + status: None, + count: displaced.count, + first_sequence: displaced.first_sequence, + last_sequence: displaced.last_sequence, + first_failed_at_ms: displaced.first_failed_at_ms, + last_failed_at_ms: displaced.last_failed_at_ms, + } + }; + fallback.count = fallback.count.saturating_add(source.count); + fallback.first_sequence = fallback.first_sequence.min(source.first_sequence); + fallback.last_sequence = fallback.last_sequence.max(source.last_sequence); + fallback.first_failed_at_ms = fallback.first_failed_at_ms.min(source.first_failed_at_ms); + fallback.last_failed_at_ms = fallback.last_failed_at_ms.max(source.last_failed_at_ms); + target.push(fallback); +} + +fn loss_report_from_summary(summary: ProductEventLossSummary) -> ProductEvent { + let data = event_data(ProductAnalyticsEvent::AnalyticsDeliveryLoss { + failure_class: summary.failure_class, + failed_event_name: summary.failed_event_name, + status: summary.status, + count: summary.count, + first_sequence: summary.first_sequence, + last_sequence: summary.last_sequence, + first_failed_at_ms: summary.first_failed_at_ms, + last_failed_at_ms: summary.last_failed_at_ms, + }); + let occurred_at = + chrono::DateTime::::from_timestamp_millis(summary.last_failed_at_ms) + .unwrap_or_else(chrono::Utc::now) + .to_rfc3339(); + ProductEvent { + event_id: summary.summary_id, + event_name: data.name.to_string(), + occurred_at, + anonymous_id: summary.anonymous_id, + session_id: summary.session_id, + platform: summary.platform, + app_version: summary.app_version, + properties: product_event_properties(&data), + } +} + fn merge_outbox(target: &mut ProductEventOutbox, source: ProductEventOutbox) { let mut known_event_ids = target .pending @@ -760,6 +974,12 @@ fn merge_outbox(target: &mut ProductEventOutbox, source: ProductEventOutbox) { .iter() .map(|entry| entry.event.event_id.clone()), ) + .chain( + target + .loss_reports_in_flight + .iter() + .map(|event| event.event_id.clone()), + ) .collect::>(); for event in source.pending { if known_event_ids.insert(event.event_id.clone()) { @@ -771,11 +991,39 @@ fn merge_outbox(target: &mut ProductEventOutbox, source: ProductEventOutbox) { target.dead_letters.push(entry); } } + for event in source.loss_reports_in_flight { + target + .loss_summaries + .retain(|summary| summary.summary_id != event.event_id); + if known_event_ids.insert(event.event_id.clone()) { + target.loss_reports_in_flight.push(event); + } + } + for summary in source.loss_summaries { + if !known_event_ids.contains(&summary.summary_id) { + merge_loss_summary(&mut target.loss_summaries, summary); + } + } + target.next_delivery_sequence = target + .next_delivery_sequence + .max(source.next_delivery_sequence) + .max( + target + .loss_summaries + .iter() + .map(|summary| summary.last_sequence) + .max() + .unwrap_or(0), + ); } fn persist_outbox(app: &AppHandle, outbox: &mut ProductEventOutbox) -> Result<(), String> { if !PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.load(Ordering::Acquire) { - return write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, outbox); + if write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, outbox).is_ok() { + return Ok(()); + } + PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); + return write_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY, outbox); } let Ok(mut restored) = load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY) else { @@ -799,7 +1047,7 @@ fn outbox_guard() -> std::sync::MutexGuard<'static, ProductEventOutbox> { .unwrap_or_else(std::sync::PoisonError::into_inner) } -fn persist_current_outbox(app: &AppHandle) { +fn persist_current_outbox(app: &AppHandle) -> Result<(), String> { let mut outbox = outbox_guard(); if let Err(failure_class) = persist_outbox(app, &mut outbox) { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); @@ -807,7 +1055,9 @@ fn persist_current_outbox(app: &AppHandle) { failure_class, "Failed to persist encrypted product analytics outbox" ); + return Err(failure_class); } + Ok(()) } fn remove_delivered_events(app: &AppHandle, delivered: &[ProductEvent]) { @@ -819,6 +1069,9 @@ fn remove_delivered_events(app: &AppHandle, delivered: &[ProductEvent]) { outbox .pending .retain(|event| !delivered_ids.contains(event.event_id.as_str())); + outbox + .loss_reports_in_flight + .retain(|event| !delivered_ids.contains(event.event_id.as_str())); if let Err(failure_class) = persist_outbox(app, &mut outbox) { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( @@ -828,28 +1081,78 @@ fn remove_delivered_events(app: &AppHandle, delivered: &[ProductEvent]) { } } -fn move_to_dead_letters(app: &AppHandle, events: &[ProductEvent], error: &DeliveryError) { +fn prepare_loss_report_batch(app: &AppHandle) -> Result, String> { + let mut outbox = outbox_guard(); + if outbox.loss_reports_in_flight.is_empty() && !outbox.loss_summaries.is_empty() { + let report_count = outbox.loss_summaries.len().min(PRODUCT_EVENT_BATCH_SIZE); + outbox.loss_reports_in_flight = outbox + .loss_summaries + .drain(..report_count) + .map(loss_report_from_summary) + .collect(); + persist_outbox(app, &mut outbox)?; + } + Ok(outbox.loss_reports_in_flight.clone()) +} + +fn dead_letter_events( + outbox: &mut ProductEventOutbox, + events: &[ProductEvent], + error: &DeliveryError, + failed_at_ms: i64, + failed_at: &str, +) -> (u64, u64) { let failed_ids = events .iter() .map(|event| event.event_id.as_str()) .collect::>(); - let mut outbox = outbox_guard(); outbox .pending .retain(|event| !failed_ids.contains(event.event_id.as_str())); + let mut stored = 0_u64; + let mut dropped = 0_u64; for event in events { - if outbox.dead_letters.len() >= PRODUCT_EVENT_DEAD_LETTER_CAPACITY { - outbox.dead_letters.remove(0); - PRODUCT_EVENTS_DROPPED.fetch_add(1, Ordering::Relaxed); + if outbox.dead_letters.len() < PRODUCT_EVENT_DEAD_LETTER_CAPACITY { + record_delivery_loss( + outbox, + "contract_rejected", + event, + error.status, + failed_at_ms, + ); + outbox.dead_letters.push(ProductEventDeadLetter { + event: event.clone(), + failure_class: "contract_rejected".to_string(), + status: error.status, + failed_at: failed_at.to_string(), + }); + stored = stored.saturating_add(1); + } else { + record_delivery_loss( + outbox, + "dead_letter_overflow", + event, + error.status, + failed_at_ms, + ); + dropped = dropped.saturating_add(1); } - outbox.dead_letters.push(ProductEventDeadLetter { - event: event.clone(), - failure_class: "contract_rejected".to_string(), - status: error.status, - failed_at: chrono::Utc::now().to_rfc3339(), - }); } - PRODUCT_EVENT_DEAD_LETTERS.fetch_add(events.len() as u64, Ordering::Relaxed); + (stored, dropped) +} + +fn move_to_dead_letters(app: &AppHandle, events: &[ProductEvent], error: &DeliveryError) { + let now = chrono::Utc::now(); + let mut outbox = outbox_guard(); + let (stored, dropped) = dead_letter_events( + &mut outbox, + events, + error, + now.timestamp_millis(), + &now.to_rfc3339(), + ); + PRODUCT_EVENT_DEAD_LETTERS.fetch_add(stored, Ordering::Relaxed); + PRODUCT_EVENTS_DROPPED.fetch_add(dropped, Ordering::Relaxed); if let Err(failure_class) = persist_outbox(app, &mut outbox) { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( @@ -866,14 +1169,28 @@ async fn run_product_event_worker(app: AppHandle, mut receiver: mpsc::Receiver<( if !live_telemetry_enabled(&app) { break; } - let events = { - let outbox = outbox_guard(); - outbox - .pending - .iter() - .take(PRODUCT_EVENT_BATCH_SIZE) - .cloned() - .collect::>() + let (events, delivery_loss_batch) = match prepare_loss_report_batch(&app) { + Ok(loss_reports) if !loss_reports.is_empty() => (loss_reports, true), + Ok(_) => { + let outbox = outbox_guard(); + ( + outbox + .pending + .iter() + .take(PRODUCT_EVENT_BATCH_SIZE) + .cloned() + .collect::>(), + false, + ) + } + Err(failure_class) => { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class, + "Failed to persist product analytics loss report snapshot" + ); + break; + } }; if events.is_empty() { break; @@ -900,6 +1217,14 @@ async fn run_product_event_worker(app: AppHandle, mut receiver: mpsc::Receiver<( tokio::time::sleep(delay).await; } Err(error) => { + if delivery_loss_batch { + warn!( + event_count = events.len(), + status = error.status, + "Product analytics loss report was rejected and remains encrypted for recovery" + ); + break; + } warn!( event_count = events.len(), status = error.status, @@ -913,34 +1238,69 @@ async fn run_product_event_worker(app: AppHandle, mut receiver: mpsc::Receiver<( } } -fn enqueue_product_event(app: &AppHandle, event: ProductEvent) { - let should_wake = { +fn insert_product_event( + outbox: &mut ProductEventOutbox, + event: ProductEvent, + failed_at_ms: i64, + failed_at: &str, +) -> (bool, bool, bool) { + if outbox.pending.len() < PRODUCT_EVENT_OUTBOX_CAPACITY { + outbox.pending.push(event); + return (true, false, false); + } + + if outbox.dead_letters.len() < PRODUCT_EVENT_DEAD_LETTER_CAPACITY { + record_delivery_loss( + outbox, + "queue_overflow_dead_lettered", + &event, + None, + failed_at_ms, + ); + outbox.dead_letters.push(ProductEventDeadLetter { + event, + failure_class: "queue_overflow".to_string(), + status: None, + failed_at: failed_at.to_string(), + }); + (true, true, false) + } else { + record_delivery_loss( + outbox, + "queue_overflow_unrecoverable", + &event, + None, + failed_at_ms, + ); + (true, false, true) + } +} + +fn enqueue_product_event(app: &AppHandle, event: ProductEvent) -> Result<(), String> { + let now = chrono::Utc::now(); + let (should_wake, dead_lettered, dropped) = { let mut outbox = outbox_guard(); - if outbox.pending.len() >= PRODUCT_EVENT_OUTBOX_CAPACITY { - if outbox.dead_letters.len() >= PRODUCT_EVENT_DEAD_LETTER_CAPACITY { - outbox.dead_letters.remove(0); - PRODUCT_EVENTS_DROPPED.fetch_add(1, Ordering::Relaxed); - } - outbox.dead_letters.push(ProductEventDeadLetter { - event, - failure_class: "queue_overflow".to_string(), - status: None, - failed_at: chrono::Utc::now().to_rfc3339(), - }); - PRODUCT_EVENT_DEAD_LETTERS.fetch_add(1, Ordering::Relaxed); - false - } else { - outbox.pending.push(event); - true - } + insert_product_event( + &mut outbox, + event, + now.timestamp_millis(), + &now.to_rfc3339(), + ) }; - persist_current_outbox(app); + if dead_lettered { + PRODUCT_EVENT_DEAD_LETTERS.fetch_add(1, Ordering::Relaxed); + } + if dropped { + PRODUCT_EVENTS_DROPPED.fetch_add(1, Ordering::Relaxed); + } + let persistence = persist_current_outbox(app); let sender = product_event_sender(app); if should_wake && let Err(mpsc::error::TrySendError::Closed(_)) = sender.try_send(()) { warn!("Product analytics worker is unavailable; event remains in the encrypted outbox"); } + persistence } fn product_event_sender(app: &AppHandle) -> &'static mpsc::Sender<()> { @@ -951,7 +1311,35 @@ fn product_event_sender(app: &AppHandle) -> &'static mpsc::Sender<()> { }) } +fn clear_product_event_outbox(outbox: &mut ProductEventOutbox) { + *outbox = ProductEventOutbox::default(); +} + +fn purge_stored_product_analytics(app: &AppHandle) -> Result<(), String> { + clear_product_event_outbox(&mut outbox_guard()); + let store = app + .store("store") + .map_err(|_| "store_unavailable".to_string())?; + store.delete(PRODUCT_EVENT_SESSION_STORE_KEY); + store.delete(PRODUCT_EVENT_OUTBOX_STORE_KEY); + store.delete(PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY); + store + .save() + .map_err(|_| "analytics_opt_out_purge_failed".to_string()) +} + pub fn init_product_session(app: &AppHandle) { + if !telemetry_enabled() { + if let Err(failure_class) = purge_stored_product_analytics(app) { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class, + "Failed to purge opted-out product analytics state" + ); + } + return; + } + let session_id = PRODUCT_EVENT_SESSION_ID .get_or_init(Uuid::new_v4) .to_string(); @@ -995,7 +1383,9 @@ pub fn init_product_session(app: &AppHandle) { ); } } - let has_pending = !loaded.pending.is_empty(); + let has_pending = !loaded.pending.is_empty() + || !loaded.loss_summaries.is_empty() + || !loaded.loss_reports_in_flight.is_empty(); *outbox_guard() = loaded; if has_pending { let _ = product_event_sender(app).try_send(()); @@ -1018,7 +1408,13 @@ pub fn init_product_session(app: &AppHandle) { failure_class, "Failed to restore encrypted product analytics outbox" ); - if !outbox_guard().pending.is_empty() { + let has_pending = { + let outbox = outbox_guard(); + !outbox.pending.is_empty() + || !outbox.loss_summaries.is_empty() + || !outbox.loss_reports_in_flight.is_empty() + }; + if has_pending { let _ = product_event_sender(app).try_send(()); } } @@ -1036,8 +1432,15 @@ static PRODUCT_EVENT_RETRIES: AtomicU64 = AtomicU64::new(0); static PRODUCT_EVENT_DEAD_LETTERS: AtomicU64 = AtomicU64::new(0); static PRODUCT_EVENT_PERSISTENCE_FAILURES: AtomicU64 = AtomicU64::new(0); -pub fn set_telemetry_enabled(enabled: bool) { +pub fn set_telemetry_enabled(app: &AppHandle, enabled: bool) { TELEMETRY_ENABLED.store(enabled, Ordering::Release); + if !enabled && let Err(failure_class) = purge_stored_product_analytics(app) { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class, + "Failed to purge opted-out product analytics state" + ); + } } pub fn telemetry_enabled() -> bool { @@ -1061,7 +1464,12 @@ pub fn capture_event(app: &AppHandle, event: ProductAnalyticsEvent) { let data = event_data(event); if let Some(event) = product_event(&data, anonymous_id) { - enqueue_product_event(app, event); + if let Err(failure_class) = enqueue_product_event(app, event) { + warn!( + failure_class, + "Critical product analytics event is retained only in process memory" + ); + } } } @@ -1111,8 +1519,7 @@ pub fn capture_client_product_analytics_event( app_version: env!("CARGO_PKG_VERSION").to_string(), properties: product_event_properties(&data), }, - ); - Ok(()) + ) } #[cfg(test)] @@ -1142,6 +1549,7 @@ mod tests { "multipart_upload_complete", "multipart_upload_failed", "recording_recovery_failed", + "analytics_delivery_loss", ]; let excluded = [ "recording_recovered", @@ -1231,7 +1639,7 @@ mod tests { let event_id = event.event_id.clone(); let outbox = ProductEventOutbox { pending: vec![event], - dead_letters: Vec::new(), + ..ProductEventOutbox::default() }; let key = [7_u8; 32]; @@ -1251,7 +1659,7 @@ mod tests { let event_id = event.event_id.clone(); let outbox = ProductEventOutbox { pending: vec![event], - dead_letters: Vec::new(), + ..ProductEventOutbox::default() }; let valid_key = [7_u8; 32]; let encrypted = encrypt_outbox_with_key(&outbox, &valid_key).unwrap(); @@ -1326,14 +1734,14 @@ mod tests { .unwrap(); let mut target = ProductEventOutbox { pending: vec![first.clone()], - dead_letters: Vec::new(), + ..ProductEventOutbox::default() }; merge_outbox( &mut target, ProductEventOutbox { pending: vec![duplicate, second.clone()], - dead_letters: Vec::new(), + ..ProductEventOutbox::default() }, ); @@ -1342,6 +1750,159 @@ mod tests { assert_eq!(target.pending[1].event_id, second.event_id); } + #[test] + fn dead_letter_capacity_never_evicts_recoverable_events() { + let mut outbox = ProductEventOutbox::default(); + for index in 0..PRODUCT_EVENT_DEAD_LETTER_CAPACITY { + let mut event = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + event.event_id = format!("dead-letter-{index}"); + outbox.dead_letters.push(ProductEventDeadLetter { + event, + failure_class: "contract_rejected".to_string(), + status: Some(400), + failed_at: "2026-08-01T00:00:00Z".to_string(), + }); + } + let oldest_id = outbox.dead_letters[0].event.event_id.clone(); + let event = product_event( + &event_data(recording_started()), + "overflow-install-id".to_string(), + ) + .unwrap(); + let event_id = event.event_id.clone(); + outbox.pending = (0..PRODUCT_EVENT_OUTBOX_CAPACITY) + .map(|index| { + let mut pending = event.clone(); + pending.event_id = format!("pending-{index}"); + pending + }) + .collect(); + + let (wake, dead_lettered, dropped) = insert_product_event( + &mut outbox, + event, + 1_785_542_400_000, + "2026-08-01T00:00:00Z", + ); + + assert!(wake); + assert!(!dead_lettered); + assert!(dropped); + assert_eq!( + outbox.dead_letters.len(), + PRODUCT_EVENT_DEAD_LETTER_CAPACITY + ); + assert_eq!(outbox.dead_letters[0].event.event_id, oldest_id); + assert!( + !outbox + .dead_letters + .iter() + .any(|entry| entry.event.event_id == event_id) + ); + assert_eq!(outbox.loss_summaries.len(), 1); + assert_eq!( + outbox.loss_summaries[0].failure_class, + "queue_overflow_unrecoverable" + ); + } + + #[test] + fn loss_report_payload_and_id_survive_encrypted_restart() { + let event = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + let mut outbox = ProductEventOutbox::default(); + record_delivery_loss( + &mut outbox, + "queue_overflow_unrecoverable", + &event, + None, + 1_785_542_400_000, + ); + let report = loss_report_from_summary(outbox.loss_summaries.remove(0)); + let report_id = report.event_id.clone(); + let report_json = serde_json::to_value(&report).unwrap(); + outbox.loss_reports_in_flight.push(report); + let key = [9_u8; 32]; + + let encrypted = encrypt_outbox_with_key(&outbox, &key).unwrap(); + let restored = decrypt_outbox_with_key(&encrypted, &key).unwrap(); + + assert!(!encrypted.contains("queue_overflow_unrecoverable")); + assert_eq!(restored.loss_reports_in_flight.len(), 1); + assert_eq!(restored.loss_reports_in_flight[0].event_id, report_id); + assert_eq!( + serde_json::to_value(&restored.loss_reports_in_flight[0]).unwrap(), + report_json + ); + } + + #[test] + fn recovery_merge_is_idempotent_for_loss_summaries() { + let event = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + let mut source = ProductEventOutbox::default(); + record_delivery_loss( + &mut source, + "queue_overflow_dead_lettered", + &event, + None, + 1_785_542_400_000, + ); + let mut target = ProductEventOutbox::default(); + + merge_outbox(&mut target, source.clone()); + merge_outbox(&mut target, source); + + assert_eq!(target.loss_summaries.len(), 1); + assert_eq!(target.loss_summaries[0].count, 1); + assert_eq!(target.next_delivery_sequence, 1); + } + + #[test] + fn old_encrypted_outbox_shape_defaults_new_delivery_state() { + let json = serde_json::json!({ "pending": [], "dead_letters": [] }); + let outbox = serde_json::from_value::(json).unwrap(); + + assert!(outbox.loss_summaries.is_empty()); + assert!(outbox.loss_reports_in_flight.is_empty()); + assert_eq!(outbox.next_delivery_sequence, 0); + } + + #[test] + fn opt_out_clear_removes_all_recoverable_delivery_state() { + let event = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + let mut outbox = ProductEventOutbox { + pending: vec![event.clone()], + dead_letters: vec![ProductEventDeadLetter { + event: event.clone(), + failure_class: "contract_rejected".to_string(), + status: Some(400), + failed_at: "2026-08-01T00:00:00Z".to_string(), + }], + ..ProductEventOutbox::default() + }; + record_delivery_loss( + &mut outbox, + "queue_overflow_dead_lettered", + &event, + None, + 1_785_542_400_000, + ); + outbox + .loss_reports_in_flight + .push(loss_report_from_summary(outbox.loss_summaries.remove(0))); + + clear_product_event_outbox(&mut outbox); + + assert!(outbox.pending.is_empty()); + assert!(outbox.dead_letters.is_empty()); + assert!(outbox.loss_summaries.is_empty()); + assert!(outbox.loss_reports_in_flight.is_empty()); + assert_eq!(outbox.next_delivery_sequence, 0); + } + #[test] fn product_status_retry_policy_matches_the_browser() { assert!(!should_retry_product_status(400)); diff --git a/apps/web/__tests__/unit/product-analytics-delivery.test.ts b/apps/web/__tests__/unit/product-analytics-delivery.test.ts index b65637e1aa8..048d15111a7 100644 --- a/apps/web/__tests__/unit/product-analytics-delivery.test.ts +++ b/apps/web/__tests__/unit/product-analytics-delivery.test.ts @@ -57,6 +57,7 @@ import { deliverProductAnalyticsEventStep } from "@/workflows/deliver-product-an const event = { eventId: "signup:user-1", eventName: "user_signed_up", + occurredAt: "2026-07-12T12:00:00.000Z", platform: "web", userId: "user-1", organizationId: "org-1", diff --git a/apps/web/__tests__/unit/product-analytics-first-view.test.ts b/apps/web/__tests__/unit/product-analytics-first-view.test.ts new file mode 100644 index 00000000000..42efab10d68 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-first-view.test.ts @@ -0,0 +1,52 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const where = vi.fn(); +const set = vi.fn(() => ({ where })); +const update = vi.fn(() => ({ set })); + +vi.mock("@cap/database", () => ({ db: () => ({ update }) })); +vi.mock("@cap/database/schema", () => ({ + videos: { firstExternalViewAt: "firstExternalViewAt", id: "id" }, +})); +vi.mock("drizzle-orm", () => ({ + and: vi.fn((...conditions: unknown[]) => conditions), + eq: vi.fn((field: unknown, value: unknown) => ({ field, value })), + isNull: vi.fn((field: unknown) => ({ field, isNull: true })), +})); + +describe("first external view claim", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("uses database timestamp precision for stable reconciliation payloads", async () => { + const { firstExternalViewTimestamp } = await import( + "@/lib/analytics/first-view" + ); + expect( + firstExternalViewTimestamp(1_753_000_000_987).getMilliseconds(), + ).toBe(0); + }); + + it("allows only the first conditional update to claim the milestone", async () => { + where.mockResolvedValueOnce([{ affectedRows: 1 }]); + where.mockResolvedValueOnce([{ affectedRows: 0 }]); + const { claimFirstExternalView } = await import( + "@/lib/analytics/first-view" + ); + const claimedAt = new Date("2026-07-31T12:00:00.000Z"); + + await expect( + claimFirstExternalView("video-1" as never, claimedAt), + ).resolves.toBe(true); + await expect( + claimFirstExternalView("video-1" as never, claimedAt), + ).resolves.toBe(false); + expect(set).toHaveBeenNthCalledWith(1, { + firstExternalViewAt: claimedAt, + }); + expect(set).toHaveBeenNthCalledWith(2, { + firstExternalViewAt: claimedAt, + }); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-queue.test.ts b/apps/web/__tests__/unit/product-analytics-queue.test.ts index 133dc9777f1..80e9a9d8434 100644 --- a/apps/web/__tests__/unit/product-analytics-queue.test.ts +++ b/apps/web/__tests__/unit/product-analytics-queue.test.ts @@ -46,7 +46,7 @@ describe("ProductAnalyticsQueue", () => { expect(transport.mock.calls[0]?.[0]).toHaveLength(2); expect(transport.mock.calls[0]?.[2]).toMatchObject({ attempted: 2, - accepted: 2, + accepted: 0, }); }); @@ -229,6 +229,72 @@ describe("ProductAnalyticsQueue", () => { ); expect(p95Ms).toBeLessThan(250); }); + + it("recovers an unconfirmed in-flight batch after a page restart", async () => { + vi.setSystemTime(new Date("2026-07-12T12:01:00.000Z")); + const values = new Map(); + const storage = { + getItem: (key: string) => values.get(key) ?? null, + removeItem: (key: string) => values.delete(key), + setItem: (key: string, value: string) => values.set(key, value), + }; + const abandonedTransport = vi.fn( + () => new Promise(() => {}), + ); + const abandonedQueue = new ProductAnalyticsQueue( + abandonedTransport, + setTimeout, + clearTimeout, + storage, + ); + abandonedQueue.enqueue({ + ...makeEvent(1), + properties: { hostname: "cap.so", is_session_entry: true }, + }); + void abandonedQueue.flush("unload"); + + const recoveredTransport = vi + .fn() + .mockResolvedValue("success"); + const recoveredQueue = new ProductAnalyticsQueue( + recoveredTransport, + setTimeout, + clearTimeout, + storage, + ); + expect(recoveredQueue.size).toBe(1); + await vi.advanceTimersByTimeAsync(2_000); + + expect(recoveredTransport.mock.calls[0]?.[0][0]?.eventId).toBe("event-1"); + expect(recoveredQueue.deliverySnapshot).toMatchObject({ + attempted: 2, + accepted: 1, + retried: 1, + dropped: 0, + }); + }); + + it("makes blocked queue persistence observable", () => { + const storage = { + getItem: () => { + throw new Error("blocked"); + }, + removeItem: () => { + throw new Error("blocked"); + }, + setItem: () => { + throw new Error("blocked"); + }, + }; + const queue = new ProductAnalyticsQueue( + vi.fn(), + setTimeout, + clearTimeout, + storage, + ); + queue.enqueue(makeEvent(1)); + expect(queue.deliverySnapshot.persistence_failed).toBeGreaterThan(0); + }); }); describe("browser analytics identity", () => { @@ -335,7 +401,7 @@ describe("product page views", () => { }); describe("browser product analytics transport", () => { - it("uses a beacon during unload without also fetching", async () => { + it("treats an accepted unload beacon as unconfirmed for a stable-ID retry", async () => { const fetchImpl = vi.fn(); const sendBeacon = vi.fn(() => true); await expect( @@ -343,7 +409,7 @@ describe("browser product analytics transport", () => { fetchImpl, sendBeacon, }), - ).resolves.toBe("success"); + ).resolves.toBe("retry"); expect(sendBeacon).toHaveBeenCalledOnce(); expect(fetchImpl).not.toHaveBeenCalled(); }); diff --git a/apps/web/__tests__/unit/product-analytics-scheduler.test.ts b/apps/web/__tests__/unit/product-analytics-scheduler.test.ts index fbf1ca6f623..8ffd86946d5 100644 --- a/apps/web/__tests__/unit/product-analytics-scheduler.test.ts +++ b/apps/web/__tests__/unit/product-analytics-scheduler.test.ts @@ -17,6 +17,7 @@ describe("analytics durable enqueue", () => { queueServerProductEvent({ eventId: "checkout:cs_1", eventName: "checkout_started", + occurredAt: "2026-07-12T12:00:00.000Z", anonymousId: "anonymous-1", platform: "web", properties: { price_id: "price_1", quantity: 1 }, @@ -32,6 +33,7 @@ describe("analytics durable enqueue", () => { queueServerProductEvent({ eventId: "signup:user-1", eventName: "user_signed_up", + occurredAt: "2026-07-12T12:00:00.000Z", platform: "server", userId: "user-1", }), diff --git a/apps/web/__tests__/unit/product-analytics-server.test.ts b/apps/web/__tests__/unit/product-analytics-server.test.ts index a78066402ab..590624e506d 100644 --- a/apps/web/__tests__/unit/product-analytics-server.test.ts +++ b/apps/web/__tests__/unit/product-analytics-server.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { collaborationActionCreatedEvent, + firstViewReceivedEvent, identityLinkedEvent, shareLinkCreatedEvent, userSignedUpEvent, @@ -46,34 +47,65 @@ describe("server product analytics", () => { const [row] = createServerProductEventRows({ eventId: "signup:user-1", eventName: "user_signed_up", + occurredAt: "2026-07-12T12:00:00.000Z", platform: "web", userId: "user-1", }); expect(row?.anonymous_id).toBe("user:user-1"); }); + it("marks bounded staging events as synthetic without exposing the run ID", () => { + const [row] = createServerProductEventRows({ + _syntheticRunId: "run_staging_route_123", + eventId: "signup:user-1", + eventName: "user_signed_up", + occurredAt: "2026-07-12T12:00:00.000Z", + platform: "web", + userId: "user-1", + }); + + expect(row).toMatchObject({ + synthetic_run_id: "run_staging_route_123", + traffic_class: "synthetic", + }); + expect(row?.properties).not.toContain("run_staging_route_123"); + }); + + it("rejects an invalid synthetic staging run ID", () => { + expect( + createServerProductEventRows({ + _syntheticRunId: "contains spaces", + eventId: "signup:user-1", + eventName: "user_signed_up", + occurredAt: "2026-07-12T12:00:00.000Z", + platform: "web", + userId: "user-1", + }), + ).toEqual([]); + }); + it("drops an event without any stable identity", () => { expect( createServerProductEventRows({ eventId: "event-1", eventName: "user_signed_up", + occurredAt: "2026-07-12T12:00:00.000Z", platform: "web", }), ).toEqual([]); }); - it("replaces an invalid timestamp with the receive time", () => { - const [row] = createServerProductEventRows({ - eventId: "event-1", - eventName: "purchase_completed", - occurredAt: "invalid", - platform: "server", - userId: "user-1", - properties: purchaseProperties, - }); - - expect(row?.occurred_at).toBe(row?.received_at); - expect(Number.isFinite(Date.parse(row?.occurred_at ?? ""))).toBe(true); + it("rejects an invalid timestamp before delivery", () => { + expect( + createServerProductEventRows({ + eventId: "event-1", + eventName: "purchase_completed", + occurredAt: "invalid", + platform: "server", + userId: "user-1", + properties: purchaseProperties, + }), + ).toEqual([]); }); it("rejects a property payload containing undeclared customer data", () => { @@ -84,6 +116,7 @@ describe("server product analytics", () => { unsafeCreate({ eventId: "event-1", eventName: "user_signed_up", + occurredAt: "2026-07-12T12:00:00.000Z", platform: "web", userId: "user-1", properties: { email: "private@example.com" }, @@ -99,6 +132,7 @@ describe("server product analytics", () => { unsafeCreate({ eventId: "stripe:evt_123:purchase_completed", eventName: "purchase_completed", + occurredAt: "2026-07-12T12:00:00.000Z", platform: "server", userId: "user-1", properties: { @@ -111,6 +145,7 @@ describe("server product analytics", () => { unsafeCreate({ eventId: "loom_import:video-1:failed", eventName: "loom_import_failed", + occurredAt: "2026-07-12T12:00:00.000Z", platform: "server", userId: "user-1", properties: { @@ -151,6 +186,12 @@ describe("server product analytics", () => { createdAt: "2026-07-31T10:02:00.000Z", action: "comment", }), + firstViewReceivedEvent({ + videoId: "video-1", + userId: "user-1", + organizationId: "org-1", + createdAt: "2026-07-31T10:03:00.000Z", + }), ] as const; for (const fact of facts) { diff --git a/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts b/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts index 1cbbd52689d..4021eacf45a 100644 --- a/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts +++ b/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts @@ -2,6 +2,8 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ eventsList: vi.fn(), + invoicesList: vi.fn(), + retrieveInvoice: vi.fn(), retrieveSubscription: vi.fn(), users: [] as Array<{ id: string; @@ -30,6 +32,10 @@ vi.mock("@cap/database/schema", () => ({ vi.mock("@cap/utils", () => ({ stripe: () => ({ events: { list: mocks.eventsList }, + invoices: { + list: mocks.invoicesList, + retrieve: mocks.retrieveInvoice, + }, subscriptions: { retrieve: mocks.retrieveSubscription }, }), })); @@ -72,7 +78,11 @@ const checkoutSession = { const subscription = { id: "sub_1", + customer: "cus_1", status: "active", + metadata: checkoutSession.metadata, + cancel_at_period_end: false, + ended_at: null, items: { data: [ { @@ -114,6 +124,7 @@ describe("Stripe analytics reconciliation", () => { has_more: false, })); mocks.retrieveSubscription.mockResolvedValue(subscription); + mocks.invoicesList.mockResolvedValue({ data: [], has_more: false }); }); it("rebuilds Stripe checkout facts with the original event identity", async () => { @@ -143,6 +154,17 @@ describe("Stripe analytics reconciliation", () => { }), ], }); + expect(mocks.eventsList.mock.calls.map(([input]) => input.type)).toEqual([ + "checkout.session.created", + "checkout.session.completed", + "checkout.session.async_payment_succeeded", + "charge.refunded", + "invoice.paid", + "invoice.payment_failed", + "customer.subscription.created", + "customer.subscription.updated", + "customer.subscription.deleted", + ]); }); it("rebuilds versioned checkout metadata and counts legacy sessions", async () => { @@ -259,4 +281,56 @@ describe("Stripe analytics reconciliation", () => { }), ]); }); + + it("rebuilds the first settled post-trial invoice as a purchase", async () => { + const invoice = { + id: "in_first_paid", + created: 1_752_537_600, + customer: "cus_1", + subscription: "sub_1", + billing_reason: "subscription_cycle", + amount_paid: 2_700, + amount_due: 2_700, + attempt_count: 1, + subtotal: 3_000, + currency: "usd", + total_discount_amounts: [{ amount: 300 }], + lines: { data: [{ price: { id: "price_team" } }] }, + }; + mocks.eventsList.mockImplementation(async ({ type }: { type: string }) => ({ + data: + type === "invoice.paid" + ? [ + { + id: "evt_first_paid", + created: 1_752_537_600, + type, + data: { object: invoice }, + }, + ] + : [], + has_more: false, + })); + const { loadStripeAnalyticsReconciliationEventsStep } = await import( + "@/workflows/reconcile-product-analytics" + ); + + const result = await loadStripeAnalyticsReconciliationEventsStep({ + scheduledAt: "2025-07-16T00:00:00.000Z", + lookbackHours: 48, + }); + + expect(result.events).toEqual([ + expect.objectContaining({ + eventId: "stripe:evt_first_paid:purchase_completed", + eventName: "purchase_completed", + platform: "web", + organizationId: "org-1", + properties: expect.objectContaining({ + amount_total_minor: 2_700, + is_first_purchase: true, + }), + }), + ]); + }); }); diff --git a/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts b/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts index 69231e821e7..64e579505fc 100644 --- a/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts +++ b/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts @@ -4,7 +4,10 @@ const mocks = vi.hoisted(() => ({ product: vi.fn(), constructEvent: vi.fn(), retrieveCustomer: vi.fn(), + retrieveInvoice: vi.fn(), retrieveSubscription: vi.fn(), + listInvoices: vi.fn(), + listSubscriptions: vi.fn(), })); const dbChain = { @@ -41,7 +44,11 @@ vi.mock("@cap/utils", () => ({ }, subscriptions: { retrieve: mocks.retrieveSubscription, - list: vi.fn(), + list: mocks.listSubscriptions, + }, + invoices: { + retrieve: mocks.retrieveInvoice, + list: mocks.listInvoices, }, }), })); @@ -71,13 +78,26 @@ const customer = { const subscription = { id: "sub_1", + customer: "cus_1", status: "active", + metadata: { + platform: "web", + analyticsAnonymousId: "anonymous-1", + analyticsSchemaVersion: "1", + analyticsPriceId: "price_team", + analyticsQuantity: "3", + analyticsOrganizationId: "org-immutable", + analyticsIsFirstPurchase: "true", + }, + cancel_at_period_end: false, + ended_at: null, items: { data: [ { quantity: 3, price: { id: "price_team", + currency: "usd", unit_amount: 900, recurring: { interval: "month", interval_count: 1 }, }, @@ -86,6 +106,28 @@ const subscription = { }, }; +const invoice = { + id: "in_1", + created: 1_752_537_600, + customer: "cus_1", + subscription: "sub_1", + billing_reason: "subscription_cycle", + amount_paid: 2_700, + amount_due: 2_700, + attempt_count: 1, + subtotal: 3_000, + currency: "usd", + total_discount_amounts: [{ amount: 300 }], + lines: { + data: [ + { + subscription: "sub_1", + price: { id: "price_team" }, + }, + ], + }, +}; + function session(overrides: Record = {}) { return { id: "cs_1", @@ -139,6 +181,9 @@ describe("Stripe subscription analytics", () => { dbChain.set.mockReturnValue(dbChain); mocks.retrieveCustomer.mockResolvedValue(customer); mocks.retrieveSubscription.mockResolvedValue(subscription); + mocks.retrieveInvoice.mockResolvedValue(invoice); + mocks.listInvoices.mockResolvedValue({ data: [], has_more: false }); + mocks.listSubscriptions.mockResolvedValue({ data: [subscription] }); POST = (await import("@/app/api/webhooks/stripe/route")).POST; }); @@ -362,9 +407,111 @@ describe("Stripe subscription analytics", () => { properties: { amount_refunded_minor: 300, currency: "usd", + price_id: "price_team", fully_refunded: false, }, }), ); }); + + it("counts the first positive post-trial invoice as the purchase", async () => { + mocks.constructEvent.mockReturnValue(event("invoice.paid", invoice)); + + expect((await POST(request())).status).toBe(200); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: "stripe:evt_invoice.paid:purchase_completed", + eventName: "purchase_completed", + platform: "web", + organizationId: "org-immutable", + properties: expect.objectContaining({ + amount_total_minor: 2_700, + is_first_purchase: true, + }), + }), + ); + expect(mocks.product).not.toHaveBeenCalledWith( + expect.objectContaining({ eventName: "subscription_renewed" }), + ); + }); + + it("counts a later positive subscription invoice as a renewal", async () => { + mocks.listInvoices.mockResolvedValue({ + data: [{ id: "in_prior", amount_paid: 2_700 }], + has_more: false, + }); + mocks.constructEvent.mockReturnValue(event("invoice.paid", invoice)); + + expect((await POST(request())).status).toBe(200); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: "stripe:evt_invoice.paid:subscription_renewed", + eventName: "subscription_renewed", + organizationId: "org-immutable", + }), + ); + expect(mocks.product).not.toHaveBeenCalledWith( + expect.objectContaining({ eventName: "purchase_completed" }), + ); + }); + + it("records provider-authoritative failed collection attempts", async () => { + mocks.constructEvent.mockReturnValue( + event("invoice.payment_failed", { + ...invoice, + amount_paid: 0, + amount_due: 2_700, + attempt_count: 2, + }), + ); + + expect((await POST(request())).status).toBe(200); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: + "stripe:evt_invoice.payment_failed:subscription_payment_failed", + eventName: "subscription_payment_failed", + organizationId: "org-immutable", + properties: expect.objectContaining({ + amount_due_minor: 2_700, + attempt_count: 2, + }), + }), + ); + }); + + it("emits both plan and seat changes from one Stripe update", async () => { + mocks.constructEvent.mockReturnValue({ + ...event("customer.subscription.updated", subscription), + data: { + object: subscription, + previous_attributes: { + items: { + data: [ + { + quantity: 1, + price: { id: "price_pro" }, + }, + ], + }, + }, + }, + }); + + expect((await POST(request())).status).toBe(200); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: + "stripe:evt_customer.subscription.updated:subscription_changed:plan", + properties: expect.objectContaining({ change_kind: "plan" }), + }), + ); + expect(mocks.product).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: + "stripe:evt_customer.subscription.updated:subscription_changed:seats", + properties: expect.objectContaining({ change_kind: "seats" }), + }), + ); + }); }); diff --git a/apps/web/__tests__/unit/update-seat-quantity.test.ts b/apps/web/__tests__/unit/update-seat-quantity.test.ts index 838531a96cf..3227a854128 100644 --- a/apps/web/__tests__/unit/update-seat-quantity.test.ts +++ b/apps/web/__tests__/unit/update-seat-quantity.test.ts @@ -17,6 +17,11 @@ const mockStripe = { }, }; +const queueServerProductEvent = vi.fn(async (event: { eventId: string }) => ({ + eventId: event.eventId, + runId: "run-1", +})); + vi.mock("@cap/database", () => ({ db: () => mockDb, })); @@ -52,6 +57,8 @@ vi.mock("@cap/utils", () => ({ stripe: () => mockStripe, })); +vi.mock("@/lib/analytics/server", () => ({ queueServerProductEvent })); + vi.mock("drizzle-orm", () => ({ eq: vi.fn((field: unknown, value: unknown) => ({ field, value })), })); @@ -110,7 +117,18 @@ function mockSeatLookup({ mockStripe.subscriptions.retrieve.mockResolvedValue({ id: "sub_1", items: { - data: [{ id: "si_1", quantity: currentQuantity }], + data: [ + { + id: "si_1", + quantity: currentQuantity, + price: { + id: "price_1", + unit_amount: 1200, + currency: "usd", + recurring: { interval: "month" }, + }, + }, + ], }, }); } @@ -140,6 +158,17 @@ describe("updateSeatQuantity", () => { proration_behavior: "always_invoice", }); expect(mockDb.set).toHaveBeenCalledWith({ inviteQuota: 2 }); + expect(queueServerProductEvent).toHaveBeenCalledWith( + expect.objectContaining({ + eventName: "seat_quantity_changed", + occurredAt: expect.any(String), + properties: expect.objectContaining({ + previous_quantity: 1, + new_quantity: 2, + price_id: "price_1", + }), + }), + ); }); it("does not store added seats when Stripe leaves the subscription update pending", async () => { diff --git a/apps/web/actions/organization/send-invites.ts b/apps/web/actions/organization/send-invites.ts index 33676d655ce..1cfe49027a2 100644 --- a/apps/web/actions/organization/send-invites.ts +++ b/apps/web/actions/organization/send-invites.ts @@ -149,10 +149,12 @@ export async function sendOrganizationInvites( !existingMemberEmails.has(invite.email), ); + const createdAt = new Date(); const records = invitesToSend.map((invite) => ({ id: nanoId(), email: invite.email, role: invite.role, + createdAt, })); if (records.length > 0) { @@ -163,6 +165,7 @@ export async function sendOrganizationInvites( invitedEmail: r.email, invitedByUserId: user.id, role: r.role, + createdAt: r.createdAt, })), ); } @@ -221,6 +224,7 @@ export async function sendOrganizationInvites( await queueServerProductEvent({ eventId: `organization_invites:${firstSuccessfulInvite.id}`, eventName: "organization_invite_sent", + occurredAt: firstSuccessfulInvite.createdAt.toISOString(), platform: "web", userId: user.id, organizationId, diff --git a/apps/web/actions/organization/update-seat-quantity.ts b/apps/web/actions/organization/update-seat-quantity.ts index 5a6307c4ea7..f999399d9d3 100644 --- a/apps/web/actions/organization/update-seat-quantity.ts +++ b/apps/web/actions/organization/update-seat-quantity.ts @@ -155,6 +155,9 @@ export async function updateSeatQuantity( const { subscription, subscriptionItem, proSeatsUsed, user } = await getOwnerSubscription(organizationId); const currentQuantity = subscriptionItem.quantity ?? 1; + if (newQuantity === currentQuantity) { + return { success: true, newQuantity }; + } if (newQuantity < proSeatsUsed) { throw new Error( @@ -180,6 +183,7 @@ export async function updateSeatQuantity( : {}), }, ); + const changedAt = new Date().toISOString(); if (isSeatIncrease && updatedSubscription.pending_update) { throw new Error( @@ -213,6 +217,7 @@ export async function updateSeatQuantity( await queueServerProductEvent({ eventId: `seat_quantity:${subscription.id}:${latestInvoice ?? updatedSubscription.current_period_start}:${newQuantity}`, eventName: "seat_quantity_changed", + occurredAt: changedAt, platform: "web", userId: user.id, organizationId, diff --git a/apps/web/app/api/analytics/track/route.ts b/apps/web/app/api/analytics/track/route.ts index f216bd6baac..fde38f5713c 100644 --- a/apps/web/app/api/analytics/track/route.ts +++ b/apps/web/app/api/analytics/track/route.ts @@ -6,6 +6,11 @@ import { eq } from "drizzle-orm"; import { Effect, Option } from "effect"; import type { NextRequest } from "next/server"; import UAParser from "ua-parser-js"; +import { firstViewReceivedEvent } from "@/lib/analytics/business-events"; +import { + claimFirstExternalView, + firstExternalViewTimestamp, +} from "@/lib/analytics/first-view"; import { getProductAnalyticsRateLimitKey, ProductAnalyticsRateLimiter, @@ -191,6 +196,25 @@ export async function POST(request: NextRequest) { } const tenantId = videoRecord.organizationId; + const isNewVideo = videoRecord.createdAt >= ANON_NOTIF_CUTOFF; + if (isNewVideo) { + const claimedAt = firstExternalViewTimestamp(); + const claimed = yield* Effect.tryPromise(() => + claimFirstExternalView(Video.VideoId.make(body.videoId), claimedAt), + ); + if (claimed) { + yield* Effect.tryPromise(() => + queueServerProductEvent( + firstViewReceivedEvent({ + videoId: body.videoId, + userId: videoRecord.ownerId, + organizationId: videoRecord.organizationId, + createdAt: claimedAt, + }), + ), + ); + } + } const tinybird = yield* Tinybird; yield* tinybird.appendEvents([ @@ -212,21 +236,8 @@ export async function POST(request: NextRequest) { }, ]); - const isNewVideo = - videoRecord && videoRecord.createdAt >= ANON_NOTIF_CUTOFF; const shouldSendFirstViewEmail = isNewVideo && !videoRecord.firstViewEmailSentAt; - if (shouldSendFirstViewEmail) { - yield* Effect.tryPromise(() => - queueServerProductEvent({ - eventId: `first_view:${body.videoId}`, - eventName: "first_view_received", - platform: "server", - userId: videoRecord.ownerId, - organizationId: videoRecord.organizationId, - }), - ); - } if (userId) { if (shouldSendFirstViewEmail) { diff --git a/apps/web/app/api/invite/accept/route.ts b/apps/web/app/api/invite/accept/route.ts index e396a9afb1d..71644b29d15 100644 --- a/apps/web/app/api/invite/accept/route.ts +++ b/apps/web/app/api/invite/accept/route.ts @@ -43,6 +43,7 @@ export async function POST(request: NextRequest) { let joinedMemberId: string | undefined; let joinedOrganizationId: string | undefined; let joinedRole: string | undefined; + let joinedAt: Date | undefined; let assignedProSeat = false; await db().transaction(async (tx) => { const [invite] = await tx @@ -74,6 +75,7 @@ export async function POST(request: NextRequest) { if (!existingMembership) { const newId = nanoId(); + const createdAt = new Date(); const role = normalizeAssignableOrganizationRole(invite.role) ?? "member"; await tx.insert(organizationMembers).values({ @@ -81,11 +83,13 @@ export async function POST(request: NextRequest) { organizationId: invite.organizationId, userId: user.id, role, + createdAt, }); memberId = newId; joinedMemberId = newId; joinedOrganizationId = invite.organizationId; joinedRole = role; + joinedAt = createdAt; } const [org] = await tx @@ -168,11 +172,13 @@ export async function POST(request: NextRequest) { if ( joinedMemberId && joinedOrganizationId && + joinedAt && (joinedRole === "admin" || joinedRole === "member") ) { await queueServerProductEvent({ eventId: `organization_member:${joinedMemberId}:joined`, eventName: "organization_member_joined", + occurredAt: joinedAt.toISOString(), anonymousId: readAnalyticsAnonymousId(request), platform: "web", userId: user.id, diff --git a/apps/web/app/api/webhooks/stripe/route.ts b/apps/web/app/api/webhooks/stripe/route.ts index 9ad79a07d35..b6769704116 100644 --- a/apps/web/app/api/webhooks/stripe/route.ts +++ b/apps/web/app/api/webhooks/stripe/route.ts @@ -9,9 +9,16 @@ import { NextResponse } from "next/server"; import type Stripe from "stripe"; import { queueServerProductEvent } from "@/lib/analytics/server"; import { + isFirstPositiveSubscriptionPayment, isSettledSubscriptionPurchase, queueSubscriptionCheckoutProductEvent, queueSubscriptionTrialStartedProductEvent, + subscriptionCancelledProductEvent, + subscriptionChangedProductEvents, + subscriptionInvoicePaidProductEvent, + subscriptionPaymentFailedProductEvent, + subscriptionRefundedProductEvent, + subscriptionTrialConvertedProductEvent, } from "@/lib/analytics/stripe-business-events"; import { addCreditsToAccount } from "@/lib/developer-credits"; @@ -209,6 +216,24 @@ async function findAnalyticsUserForCustomer( return findUserWithRetry(customer.email ?? "", userId, 1); } +async function chargeInvoice(charge: Stripe.Charge): Promise { + if (!charge.invoice) { + throw new Error("Subscription refund is missing its Stripe invoice"); + } + return typeof charge.invoice === "string" + ? stripe().invoices.retrieve(charge.invoice) + : charge.invoice; +} + +async function invoiceSubscription(invoice: Stripe.Invoice) { + if (!invoice.subscription) { + throw new Error("Subscription invoice is missing its subscription"); + } + return typeof invoice.subscription === "string" + ? stripe().subscriptions.retrieve(invoice.subscription) + : invoice.subscription; +} + export const POST = async (req: Request) => { console.log("Webhook received"); const buf = await req.text(); @@ -235,44 +260,42 @@ export const POST = async (req: Request) => { try { if (event.type === "invoice.paid") { const invoice = event.data.object as Stripe.Invoice; - if ( - invoice.subscription && - invoice.billing_reason === "subscription_cycle" - ) { - const dbUser = await findAnalyticsUserForCustomer(invoice.customer); - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_renewed`, - eventName: "subscription_renewed", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser?.id, - organizationId: dbUser?.activeOrganizationId, - properties: { - amount_paid_minor: invoice.amount_paid, - currency: invoice.currency, - billing_reason: "subscription_cycle", - }, - }); - } + if (!invoice.subscription) return NextResponse.json({ received: true }); + const subscription = await invoiceSubscription(invoice); + const dbUser = await findAnalyticsUserForCustomer(invoice.customer); + if (!dbUser) return retryableUserResolutionFailure(); + const invoicePaidProductEvent = subscriptionInvoicePaidProductEvent({ + eventId: event.id, + occurredAt: new Date(event.created * 1000).toISOString(), + invoice, + subscription, + user: dbUser, + firstPositivePayment: await isFirstPositiveSubscriptionPayment({ + invoice, + subscriptionId: subscription.id, + listPaidInvoices: (input) => stripe().invoices.list(input), + }), + }); + if (invoicePaidProductEvent) + await queueServerProductEvent(invoicePaidProductEvent); } if (event.type === "invoice.payment_failed") { const invoice = event.data.object as Stripe.Invoice; if (invoice.subscription) { + const subscription = await invoiceSubscription(invoice); const dbUser = await findAnalyticsUserForCustomer(invoice.customer); - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_payment_failed`, - eventName: "subscription_payment_failed", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser?.id, - organizationId: dbUser?.activeOrganizationId, - properties: { - amount_due_minor: invoice.amount_due, - currency: invoice.currency, - attempt_count: invoice.attempt_count, - }, - }); + if (!dbUser) return retryableUserResolutionFailure(); + const paymentFailedProductEvent = + subscriptionPaymentFailedProductEvent({ + eventId: event.id, + occurredAt: new Date(event.created * 1000).toISOString(), + invoice, + subscription, + user: dbUser, + }); + if (paymentFailedProductEvent) + await queueServerProductEvent(paymentFailedProductEvent); } } @@ -284,20 +307,21 @@ export const POST = async (req: Request) => { const previousAmountRefunded = previousCharge?.amount_refunded ?? 0; const refundedAmount = charge.amount_refunded - previousAmountRefunded; if (charge.invoice && refundedAmount > 0) { + const invoice = await chargeInvoice(charge); + const subscription = await invoiceSubscription(invoice); const dbUser = await findAnalyticsUserForCustomer(charge.customer); - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_refunded`, - eventName: "subscription_refunded", + if (!dbUser) return retryableUserResolutionFailure(); + const refundProductEvent = subscriptionRefundedProductEvent({ + eventId: event.id, occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser?.id, - organizationId: dbUser?.activeOrganizationId, - properties: { - amount_refunded_minor: refundedAmount, - currency: charge.currency, - fully_refunded: charge.refunded, - }, + charge, + invoice, + subscription, + user: dbUser, + refundedAmount, }); + if (refundProductEvent) + await queueServerProductEvent(refundProductEvent); } } @@ -566,115 +590,23 @@ export const POST = async (req: Request) => { }) .where(eq(users.id, dbUser.id)); - if ( - previous?.status === "trialing" && - subscription.status === "active" - ) { - await queueServerProductEvent({ - eventId: `stripe:${event.id}:trial_converted`, - eventName: "trial_converted", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser.id, - organizationId: dbUser.activeOrganizationId, - properties: { - previous_status: "trialing", - new_status: "active", - }, - }); - } - - if ( - previous?.cancel_at_period_end !== undefined && - previous.cancel_at_period_end !== subscription.cancel_at_period_end - ) { - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_changed:cancellation`, - eventName: "subscription_changed", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser.id, - organizationId: dbUser.activeOrganizationId, - properties: { - change_kind: subscription.cancel_at_period_end - ? "cancellation_scheduled" - : "cancellation_reversed", - previous_status: previous.status ?? null, - new_status: subscription.status, - previous_price_id: null, - new_price_id: null, - previous_quantity: null, - new_quantity: null, - }, - }); - } - - if (previous?.status && previous.status !== subscription.status) { - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_changed:status`, - eventName: "subscription_changed", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser.id, - organizationId: dbUser.activeOrganizationId, - properties: { - change_kind: "status", - previous_status: previous.status, - new_status: subscription.status, - previous_price_id: null, - new_price_id: null, - previous_quantity: null, - new_quantity: null, - }, - }); - } - - const previousItem = previous?.items?.data[0]; - const currentItem = subscription.items.data[0]; - if ( - previousItem && - currentItem && - previousItem.price.id !== currentItem.price.id - ) { - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_changed:plan`, - eventName: "subscription_changed", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser.id, - organizationId: dbUser.activeOrganizationId, - properties: { - change_kind: "plan", - previous_status: null, - new_status: null, - previous_price_id: previousItem.price.id, - new_price_id: currentItem.price.id, - previous_quantity: previousItem.quantity, - new_quantity: currentItem.quantity, - }, - }); - } else if ( - previousItem && - currentItem && - previousItem.quantity !== currentItem.quantity - ) { - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_changed:seats`, - eventName: "subscription_changed", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: dbUser.id, - organizationId: dbUser.activeOrganizationId, - properties: { - change_kind: "seats", - previous_status: null, - new_status: null, - previous_price_id: previousItem.price.id, - new_price_id: currentItem.price.id, - previous_quantity: previousItem.quantity, - new_quantity: currentItem.quantity, - }, - }); + const occurredAt = new Date(event.created * 1000).toISOString(); + const trialConverted = subscriptionTrialConvertedProductEvent({ + eventId: event.id, + occurredAt, + subscription, + previousStatus: previous?.status, + user: dbUser, + }); + if (trialConverted) await queueServerProductEvent(trialConverted); + for (const productEvent of subscriptionChangedProductEvents({ + eventId: event.id, + occurredAt, + subscription, + previous, + user: dbUser, + })) { + await queueServerProductEvent(productEvent); } console.log( @@ -742,19 +674,14 @@ export const POST = async (req: Request) => { }) .where(eq(users.id, foundUserId)); - await queueServerProductEvent({ - eventId: `stripe:${event.id}:subscription_cancelled`, - eventName: "subscription_cancelled", - occurredAt: new Date(event.created * 1000).toISOString(), - platform: "server", - userId: foundUserId, - organizationId: userResult[0]?.activeOrganizationId, - properties: { - status: subscription.status, - ended_at: subscription.ended_at, - cancel_at_period_end: subscription.cancel_at_period_end, - }, - }); + await queueServerProductEvent( + subscriptionCancelledProductEvent({ + eventId: event.id, + occurredAt: new Date(event.created * 1000).toISOString(), + subscription, + user: { id: foundUserId }, + }), + ); console.log("User updated successfully", { foundUserId, diff --git a/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx b/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx index e2a9a2dfebb..ec16f3902be 100644 --- a/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx +++ b/apps/web/app/embed/[videoId]/_components/EmbedVideo.tsx @@ -50,7 +50,10 @@ type CommentWithAuthor = typeof commentsSchema.$inferSelect & { export const EmbedVideo = forwardRef< HTMLVideoElement, { - data: Omit & { + data: Omit< + typeof videos.$inferSelect, + "firstExternalViewAt" | "password" + > & { hasActiveUpload: boolean | undefined; }; user: typeof userSelectProps | null; diff --git a/apps/web/app/embed/[videoId]/page.tsx b/apps/web/app/embed/[videoId]/page.tsx index e6bb947e850..3a29e553b5d 100644 --- a/apps/web/app/embed/[videoId]/page.tsx +++ b/apps/web/app/embed/[videoId]/page.tsx @@ -185,7 +185,10 @@ async function EmbedContent({ autoplay, minimal, }: { - video: Omit & { + video: Omit< + typeof videos.$inferSelect, + "firstExternalViewAt" | "password" + > & { sharedOrganization: { organizationId: Organisation.OrganisationId } | null; hasActiveUpload: boolean | undefined; orgSettings?: (typeof organizations.$inferSelect)["settings"] | null; diff --git a/apps/web/app/s/[videoId]/page.tsx b/apps/web/app/s/[videoId]/page.tsx index 668cf99b48d..cd22cf48370 100644 --- a/apps/web/app/s/[videoId]/page.tsx +++ b/apps/web/app/s/[videoId]/page.tsx @@ -405,7 +405,7 @@ async function AuthorizedContent({ }: { video: Omit< InferSelectModel, - "folderId" | "password" | "settings" | "ownerId" + "firstExternalViewAt" | "folderId" | "password" | "settings" | "ownerId" > & { owner: InferSelectModel; sharedOrganization: { organizationId: Organisation.OrganisationId } | null; diff --git a/apps/web/app/s/[videoId]/types.ts b/apps/web/app/s/[videoId]/types.ts index e41583fb3dd..6a38e29f223 100644 --- a/apps/web/app/s/[videoId]/types.ts +++ b/apps/web/app/s/[videoId]/types.ts @@ -3,7 +3,10 @@ import type { SpaceRuleSource, ViewerSettingKey } from "@cap/web-backend"; import type { ImageUpload, Organisation, User } from "@cap/web-domain"; import type { OrganizationSettings } from "@/app/(org)/dashboard/dashboard-data"; -export type VideoData = Omit & { +export type VideoData = Omit< + typeof videos.$inferSelect, + "firstExternalViewAt" | "ownerId" +> & { owner: VideoOwner; organizationMembers?: User.UserId[]; organizationId?: Organisation.OrganisationId; diff --git a/apps/web/app/utils/product-analytics.ts b/apps/web/app/utils/product-analytics.ts index b42bcc1a1aa..08d59838755 100644 --- a/apps/web/app/utils/product-analytics.ts +++ b/apps/web/app/utils/product-analytics.ts @@ -4,6 +4,7 @@ import { isCoreEventName, isServerOnlyEventName, normalizeAnalyticsIdentifier, + normalizeProductEventInput, normalizeProductEventProperties, PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE, PRODUCT_ANALYTICS_LIMITS, @@ -18,10 +19,19 @@ const FLUSH_INTERVAL_MS = 5_000; const RETRY_INTERVAL_MS = 2_000; const REQUEST_TIMEOUT_MS = 3_000; const ANONYMOUS_ID_KEY = "cap_analytics_anonymous_id_v1"; +const QUEUE_STORAGE_KEY = "cap_analytics_queue_v1"; type TransportResult = "success" | "retry" | "drop"; type TransportMode = "normal" | "unload"; type QueuedEvent = { event: ProductEventInput; attempts: number }; +type QueueStorage = Pick; + +interface PersistedQueueState { + version: 1; + queue: QueuedEvent[]; + inFlight: QueuedEvent[]; + delivery: ProductAnalyticsDeliverySnapshot; +} interface BrowserTransportDependencies { fetchImpl?: typeof fetch; @@ -42,8 +52,62 @@ export interface ProductAnalyticsDeliverySnapshot { queue_overflow: number; oversize: number; contract_rejected: number; + persistence_failed: number; } +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const normalizeDeliveryCount = (value: unknown, fallback?: number) => { + if (value === undefined && fallback !== undefined) return fallback; + if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 0) { + throw new Error("Persisted analytics delivery count is invalid"); + } + return value; +}; + +const normalizeQueuedEvent = (value: unknown): QueuedEvent => { + if (!isRecord(value) || (value.attempts !== 0 && value.attempts !== 1)) { + throw new Error("Persisted analytics queue entry is invalid"); + } + const event = normalizeProductEventInput(value.event); + if (!event) throw new Error("Persisted analytics event is invalid"); + return { event, attempts: value.attempts }; +}; + +const normalizePersistedQueueState = (value: unknown): PersistedQueueState => { + if ( + !isRecord(value) || + value.version !== 1 || + !Array.isArray(value.queue) || + !Array.isArray(value.inFlight) || + !isRecord(value.delivery) || + value.queue.length > PRODUCT_ANALYTICS_LIMITS.queueSize || + value.inFlight.length > PRODUCT_ANALYTICS_LIMITS.batchSize + ) { + throw new Error("Persisted analytics queue state is invalid"); + } + const delivery = value.delivery; + return { + version: 1, + queue: value.queue.map(normalizeQueuedEvent), + inFlight: value.inFlight.map(normalizeQueuedEvent), + delivery: { + attempted: normalizeDeliveryCount(delivery.attempted), + accepted: normalizeDeliveryCount(delivery.accepted), + retried: normalizeDeliveryCount(delivery.retried), + dropped: normalizeDeliveryCount(delivery.dropped), + queue_overflow: normalizeDeliveryCount(delivery.queue_overflow), + oversize: normalizeDeliveryCount(delivery.oversize), + contract_rejected: normalizeDeliveryCount(delivery.contract_rejected, 0), + persistence_failed: normalizeDeliveryCount( + delivery.persistence_failed, + 0, + ), + }, + }; +}; + export class ProductAnalyticsQueue { private queue: QueuedEvent[] = []; private timer: ReturnType | undefined; @@ -56,13 +120,20 @@ export class ProductAnalyticsQueue { queue_overflow: 0, oversize: 0, contract_rejected: 0, + persistence_failed: 0, }; + private persistedInFlight: QueuedEvent[] = []; constructor( private readonly transport: ProductAnalyticsTransport, private readonly schedule: typeof setTimeout = setTimeout, private readonly cancel: typeof clearTimeout = clearTimeout, - ) {} + private readonly storage?: QueueStorage | null, + ) { + if (this.storage === null) this.delivery.persistence_failed += 1; + this.restore(); + if (this.queue.length > 0) this.scheduleFlush(RETRY_INTERVAL_MS); + } enqueue(event: ProductEventInput) { if (this.queue.length >= PRODUCT_ANALYTICS_LIMITS.queueSize) { @@ -71,6 +142,7 @@ export class ProductAnalyticsQueue { this.delivery.queue_overflow += 1; } this.queue.push({ event, attempts: 0 }); + this.persist(); if (this.queue.length >= PRODUCT_ANALYTICS_LIMITS.batchSize) { void this.flush(); @@ -85,7 +157,10 @@ export class ProductAnalyticsQueue { this.clearTimer(); const batch = this.takeBatch(); - if (batch.length === 0) return Promise.resolve(); + if (batch.length === 0) { + this.persist(); + return Promise.resolve(); + } let retryScheduled = false; this.inFlight = this.send(batch, mode) .then((scheduled) => { @@ -115,19 +190,19 @@ export class ProductAnalyticsQueue { recordContractRejection() { this.delivery.dropped += 1; this.delivery.contract_rejected += 1; + this.persist(); } private async send(batch: QueuedEvent[], mode: TransportMode) { let result: TransportResult; this.delivery.attempted += batch.length; + this.persistedInFlight = batch; + this.persist(); try { result = await this.transport( batch.map(({ event }) => event), mode, - { - ...this.deliverySnapshot, - accepted: this.delivery.accepted + batch.length, - }, + this.deliverySnapshot, ); } catch { result = "retry"; @@ -135,10 +210,14 @@ export class ProductAnalyticsQueue { if (result === "success") { this.delivery.accepted += batch.length; + this.persistedInFlight = []; + this.persist(); return false; } if (result === "drop") { this.delivery.dropped += batch.length; + this.persistedInFlight = []; + this.persist(); return false; } this.delivery.retried += batch.length; @@ -148,13 +227,21 @@ export class ProductAnalyticsQueue { .map(({ event }) => ({ event, attempts: 1 })); if (retryable.length === 0) { this.delivery.dropped += batch.length; + this.persistedInFlight = []; + this.persist(); return false; } - this.queue = [...retryable, ...this.queue].slice( + const nextQueue = [...retryable, ...this.queue]; + const overflow = Math.max( 0, - PRODUCT_ANALYTICS_LIMITS.queueSize, + nextQueue.length - PRODUCT_ANALYTICS_LIMITS.queueSize, ); + this.queue = nextQueue.slice(0, PRODUCT_ANALYTICS_LIMITS.queueSize); + this.delivery.dropped += overflow; + this.delivery.queue_overflow += overflow; + this.persistedInFlight = []; + this.persist(); this.scheduleFlush(RETRY_INTERVAL_MS); return true; } @@ -201,6 +288,64 @@ export class ProductAnalyticsQueue { return batch; } + + private restore() { + if (!this.storage) return; + let state: PersistedQueueState; + try { + const serialized = this.storage.getItem(QUEUE_STORAGE_KEY); + if (!serialized) return; + const parsed = JSON.parse(serialized) as unknown; + state = normalizePersistedQueueState(parsed); + } catch { + this.delivery.persistence_failed += 1; + try { + this.storage.removeItem(QUEUE_STORAGE_KEY); + } catch {} + return; + } + this.delivery = state.delivery; + const recovered: QueuedEvent[] = []; + for (const item of state.inFlight) { + if (item.attempts === 0) { + recovered.push({ event: item.event, attempts: 1 }); + this.delivery.retried += 1; + } else { + this.delivery.dropped += 1; + } + } + this.queue = [...recovered, ...state.queue].slice( + 0, + PRODUCT_ANALYTICS_LIMITS.queueSize, + ); + this.delivery.dropped += Math.max( + 0, + recovered.length + state.queue.length - this.queue.length, + ); + this.delivery.queue_overflow += Math.max( + 0, + recovered.length + state.queue.length - this.queue.length, + ); + this.persistedInFlight = []; + this.persist(); + } + + private persist() { + if (!this.storage) return; + try { + this.storage.setItem( + QUEUE_STORAGE_KEY, + JSON.stringify({ + version: 1, + queue: this.queue, + inFlight: this.persistedInFlight, + delivery: this.delivery, + } satisfies PersistedQueueState), + ); + } catch { + this.delivery.persistence_failed += 1; + } + } } let browserQueue: ProductAnalyticsQueue | undefined; @@ -372,7 +517,14 @@ export function createProductEventId( } function getBrowserQueue() { - if (!browserQueue) browserQueue = new ProductAnalyticsQueue(browserTransport); + if (!browserQueue) { + browserQueue = new ProductAnalyticsQueue( + browserTransport, + setTimeout, + clearTimeout, + getBrowserStorage("localStorage") ?? null, + ); + } registerLifecycleListeners(); return browserQueue; } @@ -458,7 +610,7 @@ export const sendBrowserProductAnalytics = async ( new Blob([body], { type: "application/json" }), ) ) { - return "success"; + return "retry"; } } catch {} } diff --git a/apps/web/lib/analytics/business-events.ts b/apps/web/lib/analytics/business-events.ts index 5d75811e0ba..63cbc172f04 100644 --- a/apps/web/lib/analytics/business-events.ts +++ b/apps/web/lib/analytics/business-events.ts @@ -117,3 +117,18 @@ export const collaborationActionCreatedEvent = (input: { organizationId: input.organizationId ?? undefined, properties: { action: input.action }, }) satisfies ServerProductEvent; + +export const firstViewReceivedEvent = (input: { + videoId: string; + userId: string; + organizationId: string; + createdAt: Date | string; +}) => + ({ + eventId: `first_view:${input.videoId}`, + eventName: "first_view_received", + occurredAt: occurredAt(input.createdAt), + platform: "server", + userId: input.userId, + organizationId: input.organizationId, + }) satisfies ServerProductEvent; diff --git a/apps/web/lib/analytics/first-view.ts b/apps/web/lib/analytics/first-view.ts new file mode 100644 index 00000000000..ff1f372b7f8 --- /dev/null +++ b/apps/web/lib/analytics/first-view.ts @@ -0,0 +1,23 @@ +import { db } from "@cap/database"; +import { videos } from "@cap/database/schema"; +import type { Video } from "@cap/web-domain"; +import { and, eq, isNull } from "drizzle-orm"; + +const getAffectedRows = (result: unknown) => + Array.isArray(result) + ? ((result[0] as { affectedRows?: number } | undefined)?.affectedRows ?? 0) + : ((result as { affectedRows?: number } | undefined)?.affectedRows ?? 0); + +export const firstExternalViewTimestamp = (now = Date.now()) => + new Date(Math.floor(now / 1_000) * 1_000); + +export async function claimFirstExternalView( + videoId: Video.VideoId, + claimedAt: Date, +) { + const result = await db() + .update(videos) + .set({ firstExternalViewAt: claimedAt }) + .where(and(eq(videos.id, videoId), isNull(videos.firstExternalViewAt))); + return getAffectedRows(result) === 1; +} diff --git a/apps/web/lib/analytics/server-event.ts b/apps/web/lib/analytics/server-event.ts index cdaa95f9ba3..dbb1fa0be1d 100644 --- a/apps/web/lib/analytics/server-event.ts +++ b/apps/web/lib/analytics/server-event.ts @@ -9,9 +9,10 @@ import { } from "@cap/analytics"; type ServerProductEventBase = { + _syntheticRunId?: string; eventId: string; eventName: Name; - occurredAt?: string; + occurredAt: string; anonymousId?: string; platform: ProductEventPlatform; userId?: string; @@ -38,7 +39,16 @@ export function createServerProductEventRows(event: ServerProductEvent) { event.anonymousId ?? (userId ? `user:${userId}` : undefined), ); const eventId = normalizeServerIdentifier(event.eventId); - if (!anonymousId || !eventId) return []; + const syntheticRunId = normalizeServerIdentifier(event._syntheticRunId); + const occurredAt = normalizeServerOccurredAt(event.occurredAt); + if ( + !anonymousId || + !eventId || + !occurredAt || + (event._syntheticRunId !== undefined && !syntheticRunId) + ) { + return []; + } const receivedAt = new Date().toISOString(); const properties = normalizeProductEventProperties( @@ -51,7 +61,7 @@ export function createServerProductEventRows(event: ServerProductEvent) { { eventId, eventName: event.eventName, - occurredAt: normalizeServerOccurredAt(event.occurredAt, receivedAt), + occurredAt, anonymousId, platform: event.platform, ...(event.pathname ? { pathname: event.pathname } : {}), @@ -63,19 +73,18 @@ export function createServerProductEventRows(event: ServerProductEvent) { source: "server", userId, organizationId, + ...(syntheticRunId + ? { syntheticRunId, trafficClass: "synthetic" as const } + : {}), }, ); } -function normalizeServerOccurredAt( - value: string | undefined, - fallback: string, -) { - if (!value) return fallback; +function normalizeServerOccurredAt(value: string) { const timestamp = Date.parse(value); return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() - : fallback; + : undefined; } export function normalizeServerIdentifier(value?: string) { diff --git a/apps/web/lib/analytics/stripe-business-events.ts b/apps/web/lib/analytics/stripe-business-events.ts index 9e78cbcb964..f19fa93e621 100644 --- a/apps/web/lib/analytics/stripe-business-events.ts +++ b/apps/web/lib/analytics/stripe-business-events.ts @@ -7,6 +7,69 @@ type AnalyticsUser = { id: string; }; +const subscriptionAnalyticsPlatform = ( + subscription: Stripe.Subscription, +): ProductEventPlatform => { + const platform = subscription.metadata.platform; + return platform === "web" || + platform === "desktop" || + platform === "mobile" || + platform === "cli" + ? platform + : "server"; +}; + +const subscriptionOrganizationId = (subscription: Stripe.Subscription) => + subscription.metadata.analyticsOrganizationId || undefined; + +const subscriptionPrice = (subscription: Stripe.Subscription) => { + const item = subscription.items.data[0]; + if (!item?.price.id) { + throw new Error("Subscription is missing its Stripe price"); + } + return { item, price: item.price }; +}; + +export function subscriptionInvoicePriceId(invoice: Stripe.Invoice): string { + const priceId = invoice.lines.data.find((line) => line.price)?.price?.id; + if (!priceId) { + throw new Error("Subscription invoice is missing its Stripe price"); + } + return priceId; +} + +export async function isFirstPositiveSubscriptionPayment({ + invoice, + subscriptionId, + listPaidInvoices, +}: { + invoice: Stripe.Invoice; + subscriptionId: string; + listPaidInvoices: (input: { + subscription: string; + status: "paid"; + created: { lt: number }; + limit: 100; + starting_after?: string; + }) => Promise<{ data: Stripe.Invoice[]; has_more: boolean }>; +}) { + let startingAfter: string | undefined; + for (let pageNumber = 0; pageNumber < 10; pageNumber += 1) { + const page = await listPaidInvoices({ + subscription: subscriptionId, + status: "paid", + created: { lt: invoice.created }, + limit: 100, + ...(startingAfter ? { starting_after: startingAfter } : {}), + }); + if (page.data.some((candidate) => candidate.amount_paid > 0)) return false; + if (!page.has_more || page.data.length === 0) return true; + startingAfter = page.data.at(-1)?.id; + if (!startingAfter) break; + } + throw new Error("Subscription invoice history exceeded the analytics bound"); +} + export function isSettledSubscriptionPurchase( session: Stripe.Checkout.Session, ) { @@ -166,6 +229,298 @@ export function subscriptionTrialStartedProductEvent({ }; } +export function subscriptionInvoicePaidProductEvent({ + eventId, + occurredAt, + invoice, + subscription, + user, + firstPositivePayment, +}: { + eventId: string; + occurredAt: string; + invoice: Stripe.Invoice; + subscription: Stripe.Subscription; + user: AnalyticsUser; + firstPositivePayment: boolean; +}): ServerProductEvent | null { + if ( + invoice.billing_reason !== "subscription_cycle" || + invoice.amount_paid <= 0 + ) { + return null; + } + const { item, price } = subscriptionPrice(subscription); + if (!firstPositivePayment) { + return { + eventId: `stripe:${eventId}:subscription_renewed`, + eventName: "subscription_renewed", + occurredAt, + platform: "server", + userId: user.id, + organizationId: subscriptionOrganizationId(subscription), + properties: { + amount_paid_minor: invoice.amount_paid, + currency: invoice.currency, + price_id: price.id, + billing_reason: "subscription_cycle", + }, + }; + } + const discountAmount = invoice.total_discount_amounts?.reduce( + (sum, discount) => sum + discount.amount, + 0, + ); + return { + eventId: `stripe:${eventId}:purchase_completed`, + eventName: "purchase_completed", + occurredAt, + anonymousId: subscription.metadata.analyticsAnonymousId, + platform: subscriptionAnalyticsPlatform(subscription), + userId: user.id, + organizationId: subscriptionOrganizationId(subscription), + properties: { + payment_status: "paid", + subscription_status: subscription.status, + amount_total_minor: invoice.amount_paid, + amount_subtotal_minor: invoice.subtotal, + discount_amount_minor: discountAmount ?? null, + currency: invoice.currency, + unit_amount_minor: price.unit_amount, + billing_interval: price.recurring?.interval ?? null, + billing_interval_count: price.recurring?.interval_count ?? null, + invite_quota: item.quantity ?? null, + price_id: price.id, + quantity: item.quantity ?? null, + is_first_purchase: true, + is_guest_checkout: subscription.metadata.guestCheckout === "true", + is_onboarding: subscription.metadata.isOnBoarding === "true", + }, + }; +} + +export function subscriptionPaymentFailedProductEvent({ + eventId, + occurredAt, + invoice, + subscription, + user, +}: { + eventId: string; + occurredAt: string; + invoice: Stripe.Invoice; + subscription: Stripe.Subscription; + user: AnalyticsUser; +}): ServerProductEvent | null { + if (invoice.amount_due <= 0) return null; + return { + eventId: `stripe:${eventId}:subscription_payment_failed`, + eventName: "subscription_payment_failed", + occurredAt, + platform: "server", + userId: user.id, + organizationId: subscriptionOrganizationId(subscription), + properties: { + amount_due_minor: invoice.amount_due, + currency: invoice.currency, + attempt_count: invoice.attempt_count, + price_id: subscriptionInvoicePriceId(invoice), + }, + }; +} + +export function subscriptionRefundedProductEvent({ + eventId, + occurredAt, + charge, + invoice, + subscription, + user, + refundedAmount, +}: { + eventId: string; + occurredAt: string; + charge: Stripe.Charge; + invoice: Stripe.Invoice; + subscription: Stripe.Subscription; + user: AnalyticsUser; + refundedAmount: number; +}): ServerProductEvent | null { + if (refundedAmount <= 0) return null; + return { + eventId: `stripe:${eventId}:subscription_refunded`, + eventName: "subscription_refunded", + occurredAt, + platform: "server", + userId: user.id, + organizationId: subscriptionOrganizationId(subscription), + properties: { + amount_refunded_minor: refundedAmount, + currency: charge.currency, + price_id: subscriptionInvoicePriceId(invoice), + fully_refunded: charge.refunded, + }, + }; +} + +export function subscriptionTrialConvertedProductEvent({ + eventId, + occurredAt, + subscription, + previousStatus, + user, +}: { + eventId: string; + occurredAt: string; + subscription: Stripe.Subscription; + previousStatus?: Stripe.Subscription.Status; + user: AnalyticsUser; +}): ServerProductEvent | null { + if (previousStatus !== "trialing" || subscription.status !== "active") { + return null; + } + const { price } = subscriptionPrice(subscription); + return { + eventId: `stripe:${eventId}:trial_converted`, + eventName: "trial_converted", + occurredAt, + platform: "server", + userId: user.id, + organizationId: subscriptionOrganizationId(subscription), + properties: { + previous_status: "trialing", + new_status: "active", + price_id: price.id, + }, + }; +} + +export function subscriptionChangedProductEvents({ + eventId, + occurredAt, + subscription, + previous, + user, +}: { + eventId: string; + occurredAt: string; + subscription: Stripe.Subscription; + previous?: Partial; + user: AnalyticsUser; +}): ServerProductEvent[] { + const events: ServerProductEvent[] = []; + const { item: currentItem, price } = subscriptionPrice(subscription); + const organizationId = subscriptionOrganizationId(subscription); + const base = { + occurredAt, + platform: "server" as const, + userId: user.id, + organizationId, + }; + if ( + previous?.cancel_at_period_end !== undefined && + previous.cancel_at_period_end !== subscription.cancel_at_period_end + ) { + events.push({ + ...base, + eventId: `stripe:${eventId}:subscription_changed:cancellation`, + eventName: "subscription_changed", + properties: { + change_kind: subscription.cancel_at_period_end + ? "cancellation_scheduled" + : "cancellation_reversed", + previous_status: previous.status ?? null, + new_status: subscription.status, + previous_price_id: null, + new_price_id: price.id, + previous_quantity: null, + new_quantity: null, + }, + }); + } + if (previous?.status && previous.status !== subscription.status) { + events.push({ + ...base, + eventId: `stripe:${eventId}:subscription_changed:status`, + eventName: "subscription_changed", + properties: { + change_kind: "status", + previous_status: previous.status, + new_status: subscription.status, + previous_price_id: null, + new_price_id: price.id, + previous_quantity: null, + new_quantity: null, + }, + }); + } + const previousItem = previous?.items?.data[0]; + if ( + previousItem?.price.id && + previousItem.price.id !== currentItem.price.id + ) { + events.push({ + ...base, + eventId: `stripe:${eventId}:subscription_changed:plan`, + eventName: "subscription_changed", + properties: { + change_kind: "plan", + previous_status: null, + new_status: null, + previous_price_id: previousItem.price.id, + new_price_id: currentItem.price.id, + previous_quantity: previousItem.quantity ?? null, + new_quantity: currentItem.quantity ?? null, + }, + }); + } + if (previousItem && previousItem.quantity !== currentItem.quantity) { + events.push({ + ...base, + eventId: `stripe:${eventId}:subscription_changed:seats`, + eventName: "subscription_changed", + properties: { + change_kind: "seats", + previous_status: null, + new_status: null, + previous_price_id: previousItem.price.id, + new_price_id: currentItem.price.id, + previous_quantity: previousItem.quantity ?? null, + new_quantity: currentItem.quantity ?? null, + }, + }); + } + return events; +} + +export function subscriptionCancelledProductEvent({ + eventId, + occurredAt, + subscription, + user, +}: { + eventId: string; + occurredAt: string; + subscription: Stripe.Subscription; + user: AnalyticsUser; +}): ServerProductEvent { + const { price } = subscriptionPrice(subscription); + return { + eventId: `stripe:${eventId}:subscription_cancelled`, + eventName: "subscription_cancelled", + occurredAt, + platform: "server", + userId: user.id, + organizationId: subscriptionOrganizationId(subscription), + properties: { + status: subscription.status, + price_id: price.id, + ended_at: subscription.ended_at, + cancel_at_period_end: subscription.cancel_at_period_end, + }, + }; +} + export async function queueSubscriptionCheckoutProductEvent( input: Parameters[0], ) { diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts index 290bb8daf83..38f024fcc85 100644 --- a/apps/web/workflows/import-loom-video.ts +++ b/apps/web/workflows/import-loom-video.ts @@ -240,6 +240,7 @@ export async function importLoomVideoWorkflow( await enqueueProductAnalyticsEventStep({ eventId: `loom_import:${payload.videoId}:completed`, eventName: "loom_import_completed", + occurredAt: new Date().toISOString(), platform: "server", userId: analyticsContext?.ownerId ?? payload.userId, organizationId: analyticsContext?.organizationId, @@ -261,6 +262,7 @@ export async function importLoomVideoWorkflow( await enqueueProductAnalyticsEventStep({ eventId: `loom_import:${payload.videoId}:failed`, eventName: "loom_import_failed", + occurredAt: new Date().toISOString(), platform: "server", userId: analyticsContext?.ownerId ?? payload.userId, organizationId: analyticsContext?.organizationId, diff --git a/apps/web/workflows/reconcile-product-analytics.ts b/apps/web/workflows/reconcile-product-analytics.ts index aca4a6f97cc..355b41c3198 100644 --- a/apps/web/workflows/reconcile-product-analytics.ts +++ b/apps/web/workflows/reconcile-product-analytics.ts @@ -12,13 +12,21 @@ import type Stripe from "stripe"; import { checkoutStartedEvent, collaborationActionCreatedEvent, + firstViewReceivedEvent, guestCheckoutStartedEvent, shareLinkCreatedEvent, userSignedUpEvent, } from "@/lib/analytics/business-events"; import { + isFirstPositiveSubscriptionPayment, isSettledSubscriptionPurchase, + subscriptionCancelledProductEvent, + subscriptionChangedProductEvents, subscriptionCheckoutProductEvent, + subscriptionInvoicePaidProductEvent, + subscriptionPaymentFailedProductEvent, + subscriptionRefundedProductEvent, + subscriptionTrialConvertedProductEvent, subscriptionTrialStartedProductEvent, } from "@/lib/analytics/stripe-business-events"; import { enqueueReconciledProductAnalyticsEventStep } from "./deliver-product-analytics-event"; @@ -28,7 +36,12 @@ const STRIPE_ANALYTICS_EVENT_TYPES = [ "checkout.session.created", "checkout.session.completed", "checkout.session.async_payment_succeeded", + "charge.refunded", + "invoice.paid", + "invoice.payment_failed", "customer.subscription.created", + "customer.subscription.updated", + "customer.subscription.deleted", ] as const; const checkoutQuantity = (session: Stripe.Checkout.Session) => { @@ -98,65 +111,84 @@ export async function loadProductAnalyticsReconciliationEventsStep({ isNotNull(messengerSupportEmails.userId), ), ); - const [recentUsers, recentVideos, recentComments] = await Promise.all([ - db() - .select({ - id: users.id, - organizationId: users.activeOrganizationId, - createdAt: users.created_at, - }) - .from(users) - .where( - and( - gte(users.created_at, start), - lte(users.created_at, end), - notInArray(users.id, pendingDeletionUserIds), - ), - ) - .limit(RECONCILIATION_ROW_LIMIT + 1), - db() - .select({ - id: videos.id, - userId: videos.ownerId, - organizationId: videos.orgId, - createdAt: videos.createdAt, - isScreenshot: videos.isScreenshot, - source: videos.source, - metadata: videos.metadata, - }) - .from(videos) - .where( - and( - gte(videos.createdAt, start), - lte(videos.createdAt, end), - notInArray(videos.ownerId, pendingDeletionUserIds), - ), - ) - .limit(RECONCILIATION_ROW_LIMIT + 1), - db() - .select({ - id: comments.id, - authorId: comments.authorId, - organizationId: videos.orgId, - createdAt: comments.createdAt, - type: comments.type, - parentCommentId: comments.parentCommentId, - }) - .from(comments) - .leftJoin(videos, eq(comments.videoId, videos.id)) - .where( - and( - gte(comments.createdAt, start), - lte(comments.createdAt, end), - notInArray(comments.authorId, pendingDeletionUserIds), - ), - ) - .limit(RECONCILIATION_ROW_LIMIT + 1), - ]); + const [recentUsers, recentVideos, recentComments, recentFirstViews] = + await Promise.all([ + db() + .select({ + id: users.id, + organizationId: users.activeOrganizationId, + createdAt: users.created_at, + }) + .from(users) + .where( + and( + gte(users.created_at, start), + lte(users.created_at, end), + notInArray(users.id, pendingDeletionUserIds), + ), + ) + .limit(RECONCILIATION_ROW_LIMIT + 1), + db() + .select({ + id: videos.id, + userId: videos.ownerId, + organizationId: videos.orgId, + createdAt: videos.createdAt, + isScreenshot: videos.isScreenshot, + source: videos.source, + metadata: videos.metadata, + }) + .from(videos) + .where( + and( + gte(videos.createdAt, start), + lte(videos.createdAt, end), + notInArray(videos.ownerId, pendingDeletionUserIds), + ), + ) + .limit(RECONCILIATION_ROW_LIMIT + 1), + db() + .select({ + id: comments.id, + authorId: comments.authorId, + organizationId: videos.orgId, + createdAt: comments.createdAt, + type: comments.type, + parentCommentId: comments.parentCommentId, + }) + .from(comments) + .leftJoin(videos, eq(comments.videoId, videos.id)) + .where( + and( + gte(comments.createdAt, start), + lte(comments.createdAt, end), + notInArray(comments.authorId, pendingDeletionUserIds), + ), + ) + .limit(RECONCILIATION_ROW_LIMIT + 1), + db() + .select({ + id: videos.id, + userId: videos.ownerId, + organizationId: videos.orgId, + firstExternalViewAt: videos.firstExternalViewAt, + }) + .from(videos) + .where( + and( + isNotNull(videos.firstExternalViewAt), + gte(videos.firstExternalViewAt, start), + lte(videos.firstExternalViewAt, end), + notInArray(videos.ownerId, pendingDeletionUserIds), + ), + ) + .limit(RECONCILIATION_ROW_LIMIT + 1), + ]); if ( recentUsers.length > RECONCILIATION_ROW_LIMIT || recentVideos.length > RECONCILIATION_ROW_LIMIT || - recentComments.length > RECONCILIATION_ROW_LIMIT + recentComments.length > RECONCILIATION_ROW_LIMIT || + recentFirstViews.length > RECONCILIATION_ROW_LIMIT ) { throw new Error("Product analytics reconciliation row limit exceeded"); } @@ -192,6 +224,17 @@ export async function loadProductAnalyticsReconciliationEventsStep({ : "comment", }), ), + ...recentFirstViews.map((video) => { + if (!video.firstExternalViewAt) { + throw new Error("First-view reconciliation timestamp is missing"); + } + return firstViewReceivedEvent({ + videoId: video.id, + userId: video.userId, + organizationId: video.organizationId, + createdAt: video.firstExternalViewAt, + }); + }), ]; } loadProductAnalyticsReconciliationEventsStep.maxRetries = 4; @@ -238,20 +281,38 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ session: event.data.object as Stripe.Checkout.Session, })) .filter(({ session }) => session.metadata?.type !== "developer_credits"); - const createdSubscriptions = stripeEvents - .filter(({ type }) => type === "customer.subscription.created") + const subscriptionEvents = stripeEvents + .filter(({ type }) => type.startsWith("customer.subscription.")) .map((event) => ({ event, subscription: event.data.object as Stripe.Subscription, })); + const invoiceEvents = stripeEvents + .filter( + ({ type }) => + type === "invoice.paid" || type === "invoice.payment_failed", + ) + .map((event) => ({ + event, + invoice: event.data.object as Stripe.Invoice, + })); + const refundEvents = stripeEvents + .filter(({ type }) => type === "charge.refunded") + .map((event) => ({ + event, + charge: event.data.object as Stripe.Charge, + })); const customerIds = [ ...new Set( [ ...sessions.map(({ session }) => session.customer), - ...createdSubscriptions.map( - ({ subscription }) => subscription.customer, - ), - ].flatMap((customer) => (typeof customer === "string" ? [customer] : [])), + ...subscriptionEvents.map(({ subscription }) => subscription.customer), + ...invoiceEvents.map(({ invoice }) => invoice.customer), + ...refundEvents.map(({ charge }) => charge.customer), + ].flatMap((customer) => { + if (typeof customer === "string") return [customer]; + return customer?.id ? [customer.id] : []; + }), ), ]; const analyticsUsers = @@ -271,7 +332,35 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ ); const reconciled = []; let legacyStripeEventsSkipped = 0; - for (const { event, subscription } of createdSubscriptions) { + const subscriptionsById = new Map( + subscriptionEvents.map(({ subscription }) => [ + subscription.id, + subscription, + ]), + ); + const getSubscription = async ( + value: string | Stripe.Subscription | null, + ) => { + if (!value) throw new Error("Stripe event is missing a subscription"); + if (typeof value !== "string") return value; + const existing = subscriptionsById.get(value); + if (existing) return existing; + const subscription = await stripe().subscriptions.retrieve(value); + subscriptionsById.set(value, subscription); + return subscription; + }; + const getUser = ( + customer: string | Stripe.Customer | Stripe.DeletedCustomer | null, + ) => { + const customerId = typeof customer === "string" ? customer : customer?.id; + if (!customerId) throw new Error("Stripe event is missing a customer"); + const user = usersByCustomerId.get(customerId); + if (!user) throw new Error("Stripe event has no matching analytics user"); + return user; + }; + for (const { event, subscription } of subscriptionEvents.filter( + ({ event }) => event.type === "customer.subscription.created", + )) { const analyticsSchemaVersion = subscription.metadata.analyticsSchemaVersion; if (!analyticsSchemaVersion) { legacyStripeEventsSkipped += 1; @@ -298,6 +387,96 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ }); if (productEvent) reconciled.push(productEvent); } + for (const { event, subscription } of subscriptionEvents.filter( + ({ event }) => event.type === "customer.subscription.updated", + )) { + const user = getUser(subscription.customer); + const previous = event.data.previous_attributes as + | Partial + | undefined; + const occurredAt = new Date(event.created * 1_000).toISOString(); + const trialConverted = subscriptionTrialConvertedProductEvent({ + eventId: event.id, + occurredAt, + subscription, + previousStatus: previous?.status, + user, + }); + if (trialConverted) reconciled.push(trialConverted); + reconciled.push( + ...subscriptionChangedProductEvents({ + eventId: event.id, + occurredAt, + subscription, + previous, + user, + }), + ); + } + for (const { event, subscription } of subscriptionEvents.filter( + ({ event }) => event.type === "customer.subscription.deleted", + )) { + reconciled.push( + subscriptionCancelledProductEvent({ + eventId: event.id, + occurredAt: new Date(event.created * 1_000).toISOString(), + subscription, + user: getUser(subscription.customer), + }), + ); + } + for (const { event, invoice } of invoiceEvents) { + if (!invoice.subscription) continue; + const subscription = await getSubscription(invoice.subscription); + const user = getUser(invoice.customer); + const occurredAt = new Date(event.created * 1_000).toISOString(); + if (event.type === "invoice.paid") { + const productEvent = subscriptionInvoicePaidProductEvent({ + eventId: event.id, + occurredAt, + invoice, + subscription, + user, + firstPositivePayment: await isFirstPositiveSubscriptionPayment({ + invoice, + subscriptionId: subscription.id, + listPaidInvoices: (input) => stripe().invoices.list(input), + }), + }); + if (productEvent) reconciled.push(productEvent); + continue; + } + const productEvent = subscriptionPaymentFailedProductEvent({ + eventId: event.id, + occurredAt, + invoice, + subscription, + user, + }); + if (productEvent) reconciled.push(productEvent); + } + for (const { event, charge } of refundEvents) { + if (!charge.invoice) continue; + const invoice = + typeof charge.invoice === "string" + ? await stripe().invoices.retrieve(charge.invoice) + : charge.invoice; + if (!invoice.subscription) continue; + const subscription = await getSubscription(invoice.subscription); + const previous = event.data.previous_attributes as + | Partial + | undefined; + const productEvent = subscriptionRefundedProductEvent({ + eventId: event.id, + occurredAt: new Date(event.created * 1_000).toISOString(), + charge, + invoice, + subscription, + user: getUser(charge.customer), + refundedAmount: charge.amount_refunded - (previous?.amount_refunded ?? 0), + }); + if (productEvent) reconciled.push(productEvent); + } for (const { event, session } of sessions) { if (String(event.type) === "checkout.session.created") { const analyticsSchemaVersion = session.metadata?.analyticsSchemaVersion; @@ -348,7 +527,9 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ ? "desktop" : session.metadata?.platform === "mobile" ? "mobile" - : "web", + : session.metadata?.platform === "cli" + ? "cli" + : "web", userId: user.id, organizationId: session.metadata?.analyticsOrganizationId, anonymousId, diff --git a/packages/analytics/src/event-registry.ts b/packages/analytics/src/event-registry.ts index 065dc8e9c42..1289007d7ea 100644 --- a/packages/analytics/src/event-registry.ts +++ b/packages/analytics/src/event-registry.ts @@ -301,6 +301,28 @@ export const EVENT_REGISTRY = { }, }, }, + analytics_delivery_loss: { + version: 1, + delivery: "critical", + authority: "client", + platforms: ["desktop"], + semantic: + "A durable native-client delivery or persistence failure summary became deliverable. Each summary has a stable event ID, bounded categorical dimensions, and a sequence range; it is retried until accepted and never recursively reports its own delivery failure.", + properties: { + failure_class: { type: "string", required: true, format: "category" }, + failed_event_name: { + type: "string", + required: true, + format: "category", + }, + status: { type: "number", nullable: true }, + count: { type: "number", required: true }, + first_sequence: { type: "number", required: true }, + last_sequence: { type: "number", required: true }, + first_failed_at_ms: { type: "number", required: true }, + last_failed_at_ms: { type: "number", required: true }, + }, + }, onboarding_milestone_completed: { ...bestEffortClient, platforms: ["web"], @@ -447,9 +469,10 @@ export const EVENT_REGISTRY = { }, purchase_completed: { ...criticalServer, + version: 3, platforms: ["web", "desktop", "mobile", "cli", "server"], semantic: - "Stripe reports a checkout whose immutable payment status is paid. Trials without a settled payment are not purchases.", + "Stripe reports either a paid checkout or the first positive settled subscription-cycle invoice after a trial. Immediate checkout invoices are not counted again, and trials without settled money are never purchases.", properties: { payment_status: { type: "string", required: true, values: ["paid"] }, subscription_status: { @@ -496,12 +519,14 @@ export const EVENT_REGISTRY = { }, subscription_renewed: { ...criticalServer, + version: 2, platforms: ["server"], semantic: "Stripe reports a paid subscription-cycle invoice. Revenue is recorded in minor units and never mixed across currencies.", properties: { amount_paid_minor: { type: "number", required: true }, currency: { type: "string", required: true, format: "category" }, + price_id: { type: "string", required: true, format: "identifier" }, billing_reason: { type: "string", required: true, @@ -511,6 +536,7 @@ export const EVENT_REGISTRY = { }, trial_converted: { ...criticalServer, + version: 2, platforms: ["server"], semantic: "An authoritative Stripe subscription transition changed from trialing to active. This does not itself imply a paid invoice.", @@ -525,10 +551,12 @@ export const EVENT_REGISTRY = { required: true, values: ["active"], }, + price_id: { type: "string", required: true, format: "identifier" }, }, }, subscription_changed: { ...criticalServer, + version: 2, platforms: ["server"], semantic: "Stripe changed a subscription's status, cancellation schedule, plan, or seat quantity. The typed change kind defines the decision meaning.", @@ -553,7 +581,7 @@ export const EVENT_REGISTRY = { }, new_price_id: { type: "string", - nullable: true, + required: true, format: "identifier", }, previous_quantity: { type: "number", nullable: true }, @@ -562,28 +590,33 @@ export const EVENT_REGISTRY = { }, subscription_cancelled: { ...criticalServer, + version: 2, platforms: ["server"], semantic: "Stripe reports that a subscription has terminated. This is the churn boundary, distinct from scheduling cancellation.", properties: { status: { type: "string", required: true, format: "category" }, + price_id: { type: "string", required: true, format: "identifier" }, ended_at: { type: "number", nullable: true }, cancel_at_period_end: { type: "boolean", required: true }, }, }, subscription_refunded: { ...criticalServer, + version: 2, platforms: ["server"], semantic: "Stripe reports an incremental increase in money refunded against a settled charge. Each event contributes only the delta since the previous charge state.", properties: { amount_refunded_minor: { type: "number", required: true }, currency: { type: "string", required: true, format: "category" }, + price_id: { type: "string", required: true, format: "identifier" }, fully_refunded: { type: "boolean", required: true }, }, }, subscription_payment_failed: { ...criticalServer, + version: 2, platforms: ["server"], semantic: "Stripe reports that collection failed for a subscription invoice. Attempt count is provider-authoritative.", @@ -591,6 +624,7 @@ export const EVENT_REGISTRY = { amount_due_minor: { type: "number", required: true }, currency: { type: "string", required: true, format: "category" }, attempt_count: { type: "number", required: true }, + price_id: { type: "string", required: true, format: "identifier" }, }, }, organization_invite_sent: { diff --git a/packages/analytics/src/index.test.ts b/packages/analytics/src/index.test.ts index c23f9b086cb..32db7866266 100644 --- a/packages/analytics/src/index.test.ts +++ b/packages/analytics/src/index.test.ts @@ -6,6 +6,7 @@ import { createProductEventRows, isCoreEventName, isServerOnlyEventName, + normalizeAcquisitionChannel, normalizeProductEventInput, normalizeProductEventProperties, PRODUCT_ANALYTICS_LIMITS, @@ -304,6 +305,24 @@ describe("normalizeProductEventInput", () => { }); describe("createProductEventRows", () => { + it("classifies the current session touch without leaking lifetime first touch", () => { + expect( + normalizeAcquisitionChannel({ + first_touch_gclid: "old-paid-click", + first_touch_source: "google", + }), + ).toBe("direct"); + expect( + normalizeAcquisitionChannel({ + first_touch_source: "newsletter", + session_touch_medium: "email", + }), + ).toBe("email"); + expect( + normalizeAcquisitionChannel({ session_touch_gclid: "current-click" }), + ).toBe("paid_search"); + }); + it("fingerprints canonical payloads independent of object key order", () => { expect(createProductEventPayloadHash({ a: 1, b: 2 })).toBe( createProductEventPayloadHash({ b: 2, a: 1 }), diff --git a/packages/analytics/src/index.ts b/packages/analytics/src/index.ts index 55ba8e78f02..44f9c1a14d8 100644 --- a/packages/analytics/src/index.ts +++ b/packages/analytics/src/index.ts @@ -432,15 +432,13 @@ export function normalizeAcquisitionChannel( properties: ProductEventProperties | undefined, referrer?: string, ) { - if (properties?.first_touch_gclid || properties?.session_touch_gclid) { + if (properties?.session_touch_gclid) { return "paid_search"; } - if (properties?.first_touch_fbclid || properties?.session_touch_fbclid) { + if (properties?.session_touch_fbclid) { return "paid_social"; } - const medium = String( - properties?.session_touch_medium ?? properties?.first_touch_medium ?? "", - ).toLowerCase(); + const medium = String(properties?.session_touch_medium ?? "").toLowerCase(); if (medium.includes("email")) return "email"; if (medium.includes("affiliate")) return "affiliate"; if ( @@ -450,9 +448,7 @@ export function normalizeAcquisitionChannel( ) { return "paid_other"; } - const source = String( - properties?.session_touch_source ?? properties?.first_touch_source ?? "", - ).toLowerCase(); + const source = String(properties?.session_touch_source ?? "").toLowerCase(); const hostname = referrer?.toLowerCase() ?? ""; if (!source && !hostname) return "direct"; if ( diff --git a/packages/database/migrations/0038_messy_stephen_strange.sql b/packages/database/migrations/0038_messy_stephen_strange.sql new file mode 100644 index 00000000000..113a7e51a00 --- /dev/null +++ b/packages/database/migrations/0038_messy_stephen_strange.sql @@ -0,0 +1,2 @@ +ALTER TABLE `videos` ADD `firstExternalViewAt` timestamp;--> statement-breakpoint +CREATE INDEX `first_external_view_at_idx` ON `videos` (`firstExternalViewAt`); \ No newline at end of file diff --git a/packages/database/migrations/0039_bumpy_phil_sheldon.sql b/packages/database/migrations/0039_bumpy_phil_sheldon.sql new file mode 100644 index 00000000000..fcd0851e6be --- /dev/null +++ b/packages/database/migrations/0039_bumpy_phil_sheldon.sql @@ -0,0 +1,16 @@ +CREATE TABLE `product_analytics_erasure_leases` ( + `name` varchar(64) NOT NULL, + `ownerId` varchar(64), + `requestId` varchar(64), + `fencingToken` bigint unsigned NOT NULL DEFAULT 0, + `leaseExpiresAt` timestamp, + `phase` varchar(32) NOT NULL DEFAULT 'idle', + `pausedPipes` json, + `userId` varchar(255), + `organizationId` varchar(255), + `attemptCount` int NOT NULL DEFAULT 0, + `lastErrorCode` varchar(64), + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `product_analytics_erasure_leases_name` PRIMARY KEY(`name`) +); diff --git a/packages/database/migrations/meta/0038_snapshot.json b/packages/database/migrations/meta/0038_snapshot.json new file mode 100644 index 00000000000..fadf6b52fcb --- /dev/null +++ b/packages/database/migrations/meta/0038_snapshot.json @@ -0,0 +1,4327 @@ +{ + "version": "5", + "dialect": "mysql", + "id": "794035aa-b67e-4430-8034-d3f6210b5262", + "prevId": "76d6a0f0-dc80-439a-9632-7d808d45e548", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_in": { + "name": "expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_in": { + "name": "refresh_token_expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "tempColumn": { + "name": "tempColumn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "provider_account_id_idx": { + "name": "provider_account_id_idx", + "columns": [ + "providerAccountId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accounts_id": { + "name": "accounts_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_authorization_codes": { + "name": "agent_api_authorization_codes", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeHash": { + "name": "codeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeChallenge": { + "name": "codeChallenge", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirectUri": { + "name": "redirectUri", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consumedAt": { + "name": "consumedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "code_hash_idx": { + "name": "code_hash_idx", + "columns": [ + "codeHash" + ], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expiresAt" + ], + "isUnique": false + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_authorization_codes_id": { + "name": "agent_api_authorization_codes_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_idempotency": { + "name": "agent_api_idempotency", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestHash": { + "name": "requestHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "statusCode": { + "name": "statusCode", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response": { + "name": "response", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_operation_key_idx": { + "name": "user_operation_key_idx", + "columns": [ + "userId", + "operation", + "keyHash" + ], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expiresAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_idempotency_id": { + "name": "agent_api_idempotency_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_keys": { + "name": "agent_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tokenHash": { + "name": "tokenHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Cap CLI'" + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "token_hash_idx": { + "name": "token_hash_idx", + "columns": [ + "tokenHash" + ], + "isUnique": true + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expiresAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_keys_id": { + "name": "agent_api_keys_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_operations": { + "name": "agent_api_operations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resourceId": { + "name": "resourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resultResourceId": { + "name": "resultResourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + }, + "state_updated_at_idx": { + "name": "state_updated_at_idx", + "columns": [ + "state", + "updatedAt" + ], + "isUnique": false + }, + "resource_id_idx": { + "name": "resource_id_idx", + "columns": [ + "resourceId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_operations_id": { + "name": "agent_api_operations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "auth_api_keys": { + "name": "auth_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_id_created_at_idx": { + "name": "user_id_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "auth_api_keys_id": { + "name": "auth_api_keys_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "comments": { + "name": "comments", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authorId": { + "name": "authorId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "parentCommentId": { + "name": "parentCommentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "video_type_created_idx": { + "name": "video_type_created_idx", + "columns": [ + "videoId", + "type", + "createdAt", + "id" + ], + "isUnique": false + }, + "author_id_idx": { + "name": "author_id_idx", + "columns": [ + "authorId" + ], + "isUnique": false + }, + "parent_comment_id_idx": { + "name": "parent_comment_id_idx", + "columns": [ + "parentCommentId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "comments_id": { + "name": "comments_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_api_keys": { + "name": "developer_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyType": { + "name": "keyType", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyPrefix": { + "name": "keyPrefix", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedKey": { + "name": "encryptedKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "key_hash_idx": { + "name": "key_hash_idx", + "columns": [ + "keyHash" + ], + "isUnique": true + }, + "app_key_type_idx": { + "name": "app_key_type_idx", + "columns": [ + "appId", + "keyType" + ], + "isUnique": false + } + }, + "foreignKeys": { + "developer_api_keys_appId_developer_apps_id_fk": { + "name": "developer_api_keys_appId_developer_apps_id_fk", + "tableFrom": "developer_api_keys", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_api_keys_id": { + "name": "developer_api_keys_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_app_domains": { + "name": "developer_app_domains", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_app_domains_appId_developer_apps_id_fk": { + "name": "developer_app_domains_appId_developer_apps_id_fk", + "tableFrom": "developer_app_domains", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_app_domains_id": { + "name": "developer_app_domains_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "app_domain_unique": { + "name": "app_domain_unique", + "columns": [ + "appId", + "domain" + ] + } + }, + "checkConstraint": {} + }, + "developer_apps": { + "name": "developer_apps", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logoUrl": { + "name": "logoUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_deleted_idx": { + "name": "owner_deleted_idx", + "columns": [ + "ownerId", + "deletedAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "developer_apps_id": { + "name": "developer_apps_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_accounts": { + "name": "developer_credit_accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceMicroCredits": { + "name": "balanceMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripePaymentMethodId": { + "name": "stripePaymentMethodId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoTopUpEnabled": { + "name": "autoTopUpEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "autoTopUpThresholdMicroCredits": { + "name": "autoTopUpThresholdMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoTopUpAmountCents": { + "name": "autoTopUpAmountCents", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_id_unique": { + "name": "app_id_unique", + "columns": [ + "appId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "developer_credit_accounts_appId_developer_apps_id_fk": { + "name": "developer_credit_accounts_appId_developer_apps_id_fk", + "tableFrom": "developer_credit_accounts", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_accounts_id": { + "name": "developer_credit_accounts_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_transactions": { + "name": "developer_credit_transactions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accountId": { + "name": "accountId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amountMicroCredits": { + "name": "amountMicroCredits", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceAfterMicroCredits": { + "name": "balanceAfterMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referenceType": { + "name": "referenceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "account_type_created_idx": { + "name": "account_type_created_idx", + "columns": [ + "accountId", + "type", + "createdAt" + ], + "isUnique": false + }, + "account_ref_dedup_idx": { + "name": "account_ref_dedup_idx", + "columns": [ + "accountId", + "referenceId", + "referenceType" + ], + "isUnique": false + } + }, + "foreignKeys": { + "dev_credit_txn_account_fk": { + "name": "dev_credit_txn_account_fk", + "tableFrom": "developer_credit_transactions", + "tableTo": "developer_credit_accounts", + "columnsFrom": [ + "accountId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_transactions_id": { + "name": "developer_credit_transactions_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_daily_storage_snapshots": { + "name": "developer_daily_storage_snapshots", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshotDate": { + "name": "snapshotDate", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalDurationMinutes": { + "name": "totalDurationMinutes", + "type": "float", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "videoCount": { + "name": "videoCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "microCreditsCharged": { + "name": "microCreditsCharged", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { + "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", + "tableFrom": "developer_daily_storage_snapshots", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_daily_storage_snapshots_id": { + "name": "developer_daily_storage_snapshots_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "app_date_unique": { + "name": "app_date_unique", + "columns": [ + "appId", + "snapshotDate" + ] + } + }, + "checkConstraint": {} + }, + "developer_videos": { + "name": "developer_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalUserId": { + "name": "externalUserId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3Key": { + "name": "s3Key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_created_idx": { + "name": "app_created_idx", + "columns": [ + "appId", + "createdAt" + ], + "isUnique": false + }, + "app_user_idx": { + "name": "app_user_idx", + "columns": [ + "appId", + "externalUserId" + ], + "isUnique": false + }, + "app_deleted_idx": { + "name": "app_deleted_idx", + "columns": [ + "appId", + "deletedAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "developer_videos_appId_developer_apps_id_fk": { + "name": "developer_videos_appId_developer_apps_id_fk", + "tableFrom": "developer_videos", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_videos_id": { + "name": "developer_videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "folders": { + "name": "folders", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": [ + "createdById" + ], + "isUnique": false + }, + "parent_id_idx": { + "name": "parent_id_idx", + "columns": [ + "parentId" + ], + "isUnique": false + }, + "space_id_idx": { + "name": "space_id_idx", + "columns": [ + "spaceId" + ], + "isUnique": false + }, + "public_parent_id_idx": { + "name": "public_parent_id_idx", + "columns": [ + "public", + "parentId" + ], + "isUnique": false + }, + "public_space_parent_id_idx": { + "name": "public_space_parent_id_idx", + "columns": [ + "public", + "spaceId", + "parentId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "folders_id": { + "name": "folders_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "imported_videos": { + "name": "imported_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "imported_videos_orgId_source_source_id_pk": { + "name": "imported_videos_orgId_source_source_id_pk", + "columns": [ + "orgId", + "source", + "source_id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "integration_installations": { + "name": "integration_installations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalId": { + "name": "externalId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "installedByUserId": { + "name": "installedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedCredentials": { + "name": "encryptedCredentials", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "provider_external_id_idx": { + "name": "provider_external_id_idx", + "columns": [ + "provider", + "externalId" + ], + "isUnique": true + }, + "organization_provider_display_name_idx": { + "name": "organization_provider_display_name_idx", + "columns": [ + "organizationId", + "provider", + "displayName" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "integration_installations_id": { + "name": "integration_installations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_conversations": { + "name": "messenger_conversations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverByUserId": { + "name": "takeoverByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverAt": { + "name": "takeoverAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastMessageAt": { + "name": "lastMessageAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_last_message_idx": { + "name": "user_last_message_idx", + "columns": [ + "userId", + "lastMessageAt" + ], + "isUnique": false + }, + "anonymous_last_message_idx": { + "name": "anonymous_last_message_idx", + "columns": [ + "anonymousId", + "lastMessageAt" + ], + "isUnique": false + }, + "mode_last_message_idx": { + "name": "mode_last_message_idx", + "columns": [ + "mode", + "lastMessageAt" + ], + "isUnique": false + }, + "updated_at_idx": { + "name": "updated_at_idx", + "columns": [ + "updatedAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "messenger_conversations_id": { + "name": "messenger_conversations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_messages": { + "name": "messenger_messages", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "conversation_created_at_idx": { + "name": "conversation_created_at_idx", + "columns": [ + "conversationId", + "createdAt" + ], + "isUnique": false + }, + "role_created_at_idx": { + "name": "role_created_at_idx", + "columns": [ + "role", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messenger_messages_conversationId_messenger_conversations_id_fk": { + "name": "messenger_messages_conversationId_messenger_conversations_id_fk", + "tableFrom": "messenger_messages", + "tableTo": "messenger_conversations", + "columnsFrom": [ + "conversationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_messages_id": { + "name": "messenger_messages_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_support_emails": { + "name": "messenger_support_emails", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "support_email_user_created_at_idx": { + "name": "support_email_user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + }, + "support_email_conversation_created_at_idx": { + "name": "support_email_conversation_created_at_idx", + "columns": [ + "conversationId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "support_email_conversation_fk": { + "name": "support_email_conversation_fk", + "tableFrom": "messenger_support_emails", + "tableTo": "messenger_conversations", + "columnsFrom": [ + "conversationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_support_emails_id": { + "name": "messenger_support_emails_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipientId": { + "name": "recipientId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "org_id_idx": { + "name": "org_id_idx", + "columns": [ + "orgId" + ], + "isUnique": false + }, + "type_idx": { + "name": "type_idx", + "columns": [ + "type" + ], + "isUnique": false + }, + "read_at_idx": { + "name": "read_at_idx", + "columns": [ + "readAt" + ], + "isUnique": false + }, + "created_at_idx": { + "name": "created_at_idx", + "columns": [ + "createdAt" + ], + "isUnique": false + }, + "recipient_read_idx": { + "name": "recipient_read_idx", + "columns": [ + "recipientId", + "readAt" + ], + "isUnique": false + }, + "recipient_created_idx": { + "name": "recipient_created_idx", + "columns": [ + "recipientId", + "createdAt" + ], + "isUnique": false + }, + "dedup_key_idx": { + "name": "dedup_key_idx", + "columns": [ + "dedupKey" + ], + "isUnique": true + }, + "type_recipient_created_idx": { + "name": "type_recipient_created_idx", + "columns": [ + "type", + "recipientId", + "createdAt" + ], + "isUnique": false + }, + "type_recipient_video_created_idx": { + "name": "type_recipient_video_created_idx", + "columns": [ + "type", + "recipientId", + "videoId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "notifications_id": { + "name": "notifications_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_invites": { + "name": "organization_invites", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmail": { + "name": "invitedEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedByUserId": { + "name": "invitedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "invited_email_idx": { + "name": "invited_email_idx", + "columns": [ + "invitedEmail" + ], + "isUnique": false + }, + "invited_by_user_id_idx": { + "name": "invited_by_user_id_idx", + "columns": [ + "invitedByUserId" + ], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_invites_id": { + "name": "organization_invites_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hasProSeat": { + "name": "hasProSeat", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "user_id_organization_id_idx": { + "name": "user_id_organization_id_idx", + "columns": [ + "userId", + "organizationId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_members_id": { + "name": "organization_members_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tombstoneAt": { + "name": "tombstoneAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowedEmailDomain": { + "name": "allowedEmailDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customDomain": { + "name": "customDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domainVerified": { + "name": "domainVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "shareableLinkIconUrl": { + "name": "shareableLinkIconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "workosOrganizationId": { + "name": "workosOrganizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workosConnectionId": { + "name": "workosConnectionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "owner_id_tombstone_idx": { + "name": "owner_id_tombstone_idx", + "columns": [ + "ownerId", + "tombstoneAt" + ], + "isUnique": false + }, + "custom_domain_idx": { + "name": "custom_domain_idx", + "columns": [ + "customDomain" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organizations_id": { + "name": "organizations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "s3_buckets": { + "name": "s3_buckets", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucketName": { + "name": "bucketName", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessKeyId": { + "name": "accessKeyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretAccessKey": { + "name": "secretAccessKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('aws')" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_organization_idx": { + "name": "owner_organization_idx", + "columns": [ + "ownerId", + "organizationId" + ], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": [ + "organizationId", + "active", + "updatedAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "s3_buckets_id": { + "name": "s3_buckets_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "session_token_idx": { + "name": "session_token_idx", + "columns": [ + "sessionToken" + ], + "isUnique": true + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sessions_id": { + "name": "sessions_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "shared_videos": { + "name": "shared_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedByUserId": { + "name": "sharedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedAt": { + "name": "sharedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": [ + "folderId" + ], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "shared_by_user_id_idx": { + "name": "shared_by_user_id_idx", + "columns": [ + "sharedByUserId" + ], + "isUnique": false + }, + "video_id_organization_id_idx": { + "name": "video_id_organization_id_idx", + "columns": [ + "videoId", + "organizationId" + ], + "isUnique": false + }, + "video_id_folder_id_idx": { + "name": "video_id_folder_id_idx", + "columns": [ + "videoId", + "folderId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "shared_videos_id": { + "name": "shared_videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_members_id": { + "name": "space_members_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "space_id_user_id_unique": { + "name": "space_id_user_id_unique", + "columns": [ + "spaceId", + "userId" + ] + } + }, + "checkConstraint": {} + }, + "space_videos": { + "name": "space_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedById": { + "name": "addedById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": [ + "folderId" + ], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": [ + "videoId" + ], + "isUnique": false + }, + "added_by_id_idx": { + "name": "added_by_id_idx", + "columns": [ + "addedById" + ], + "isUnique": false + }, + "space_id_video_id_idx": { + "name": "space_id_video_id_idx", + "columns": [ + "spaceId", + "videoId" + ], + "isUnique": false + }, + "space_id_folder_id_idx": { + "name": "space_id_folder_id_idx", + "columns": [ + "spaceId", + "folderId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_videos_id": { + "name": "space_videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "primary": { + "name": "primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "privacy": { + "name": "privacy", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Private'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": [ + "createdById" + ], + "isUnique": false + }, + "public_organization_id_idx": { + "name": "public_organization_id_idx", + "columns": [ + "public", + "organizationId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "spaces_id": { + "name": "spaces_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_integrations": { + "name": "storage_integrations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "encryptedConfig": { + "name": "encryptedConfig", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "googleDriveAccessToken": { + "name": "googleDriveAccessToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveAccessTokenExpiresAt": { + "name": "googleDriveAccessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseId": { + "name": "googleDriveTokenRefreshLeaseId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseExpiresAt": { + "name": "googleDriveTokenRefreshLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveStorageQuotaCache": { + "name": "googleDriveStorageQuotaCache", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_provider_idx": { + "name": "owner_provider_idx", + "columns": [ + "ownerId", + "provider" + ], + "isUnique": false + }, + "owner_active_idx": { + "name": "owner_active_idx", + "columns": [ + "ownerId", + "active" + ], + "isUnique": false + }, + "organization_provider_idx": { + "name": "organization_provider_idx", + "columns": [ + "organizationId", + "provider" + ], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": [ + "organizationId", + "active", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "storage_integrations_id": { + "name": "storage_integrations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_objects": { + "name": "storage_objects", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrationId": { + "name": "integrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "objectKey": { + "name": "objectKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "objectKeyHash": { + "name": "objectKeyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerObjectId": { + "name": "providerObjectId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploadSessionUrl": { + "name": "uploadSessionUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploadStatus": { + "name": "uploadStatus", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "contentType": { + "name": "contentType", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentLength": { + "name": "contentLength", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "integration_key_hash_idx": { + "name": "integration_key_hash_idx", + "columns": [ + "integrationId", + "objectKeyHash" + ], + "isUnique": true + }, + "integration_status_idx": { + "name": "integration_status_idx", + "columns": [ + "integrationId", + "uploadStatus" + ], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": [ + "videoId" + ], + "isUnique": false + }, + "owner_id_idx": { + "name": "owner_id_idx", + "columns": [ + "ownerId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "storage_objects_integrationId_storage_integrations_id_fk": { + "name": "storage_objects_integrationId_storage_integrations_id_fk", + "tableFrom": "storage_objects", + "tableTo": "storage_integrations", + "columnsFrom": [ + "integrationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "storage_objects_id": { + "name": "storage_objects_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastName": { + "name": "lastName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionId": { + "name": "stripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thirdPartyStripeSubscriptionId": { + "name": "thirdPartyStripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionStatus": { + "name": "stripeSubscriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionPriceId": { + "name": "stripeSubscriptionPriceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferences": { + "name": "preferences", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "('null')" + }, + "activeOrganizationId": { + "name": "activeOrganizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "onboardingSteps": { + "name": "onboardingSteps", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customBucket": { + "name": "customBucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "inviteQuota": { + "name": "inviteQuota", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "defaultOrgId": { + "name": "defaultOrgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authSessionVersion": { + "name": "authSessionVersion", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "email_idx": { + "name": "email_idx", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "identifier": { + "name": "identifier", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier": { + "name": "verification_tokens_identifier", + "columns": [ + "identifier" + ] + } + }, + "uniqueConstraints": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": [ + "token" + ] + } + }, + "checkConstraint": {} + }, + "video_edits": { + "name": "video_edits", + "columns": { + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceKey": { + "name": "sourceKey", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "editSpec": { + "name": "editSpec", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "video_edits_videoId_videos_id_fk": { + "name": "video_edits_videoId_videos_id_fk", + "tableFrom": "video_edits", + "tableTo": "videos", + "columnsFrom": [ + "videoId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "video_edits_videoId": { + "name": "video_edits_videoId", + "columns": [ + "videoId" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "video_uploads": { + "name": "video_uploads", + "columns": { + "video_id": { + "name": "video_id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded": { + "name": "uploaded", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "mode": { + "name": "mode", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'uploading'" + }, + "processing_progress": { + "name": "processing_progress", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processing_message": { + "name": "processing_message", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_file_key": { + "name": "raw_file_key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "phase_updated_at_video_id_idx": { + "name": "phase_updated_at_video_id_idx", + "columns": [ + "phase", + "updated_at", + "video_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "video_uploads_video_id": { + "name": "video_uploads_video_id", + "columns": [ + "video_id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'My Video'" + }, + "bucket": { + "name": "bucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageIntegrationId": { + "name": "storageIntegrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('{\"type\":\"MediaConvert\"}')" + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "effectiveCreatedAt": { + "name": "effectiveCreatedAt", + "type": "datetime", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "generated": { + "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", + "type": "stored" + } + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "xStreamInfo": { + "name": "xStreamInfo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstExternalViewAt": { + "name": "firstExternalViewAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstViewEmailSentAt": { + "name": "firstViewEmailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isScreenshot": { + "name": "isScreenshot", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "awsRegion": { + "name": "awsRegion", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "awsBucket": { + "name": "awsBucket", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoStartTime": { + "name": "videoStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audioStartTime": { + "name": "audioStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobId": { + "name": "jobId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobStatus": { + "name": "jobStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipProcessing": { + "name": "skipProcessing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "owner_id_idx": { + "name": "owner_id_idx", + "columns": [ + "ownerId" + ], + "isUnique": false + }, + "is_public_idx": { + "name": "is_public_idx", + "columns": [ + "public" + ], + "isUnique": false + }, + "folder_id_idx": { + "name": "folder_id_idx", + "columns": [ + "folderId" + ], + "isUnique": false + }, + "storage_integration_id_idx": { + "name": "storage_integration_id_idx", + "columns": [ + "storageIntegrationId" + ], + "isUnique": false + }, + "first_external_view_at_idx": { + "name": "first_external_view_at_idx", + "columns": [ + "firstExternalViewAt" + ], + "isUnique": false + }, + "org_owner_folder_idx": { + "name": "org_owner_folder_idx", + "columns": [ + "orgId", + "ownerId", + "folderId" + ], + "isUnique": false + }, + "org_effective_created_idx": { + "name": "org_effective_created_idx", + "columns": [ + "orgId", + "effectiveCreatedAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "videos_storageIntegrationId_storage_integrations_id_fk": { + "name": "videos_storageIntegrationId_storage_integrations_id_fk", + "tableFrom": "videos", + "tableTo": "storage_integrations", + "columnsFrom": [ + "storageIntegrationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "videos_id": { + "name": "videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/database/migrations/meta/0039_snapshot.json b/packages/database/migrations/meta/0039_snapshot.json new file mode 100644 index 00000000000..369efc9972e --- /dev/null +++ b/packages/database/migrations/meta/0039_snapshot.json @@ -0,0 +1,4441 @@ +{ + "version": "5", + "dialect": "mysql", + "id": "f7a1a6b4-42da-4813-9462-a09d30766433", + "prevId": "794035aa-b67e-4430-8034-d3f6210b5262", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_in": { + "name": "expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_in": { + "name": "refresh_token_expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "tempColumn": { + "name": "tempColumn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": [ + "userId" + ], + "isUnique": false + }, + "provider_account_id_idx": { + "name": "provider_account_id_idx", + "columns": [ + "providerAccountId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accounts_id": { + "name": "accounts_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_authorization_codes": { + "name": "agent_api_authorization_codes", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeHash": { + "name": "codeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeChallenge": { + "name": "codeChallenge", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirectUri": { + "name": "redirectUri", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consumedAt": { + "name": "consumedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "code_hash_idx": { + "name": "code_hash_idx", + "columns": [ + "codeHash" + ], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expiresAt" + ], + "isUnique": false + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_authorization_codes_id": { + "name": "agent_api_authorization_codes_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_idempotency": { + "name": "agent_api_idempotency", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestHash": { + "name": "requestHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "statusCode": { + "name": "statusCode", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response": { + "name": "response", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_operation_key_idx": { + "name": "user_operation_key_idx", + "columns": [ + "userId", + "operation", + "keyHash" + ], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expiresAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_idempotency_id": { + "name": "agent_api_idempotency_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_keys": { + "name": "agent_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tokenHash": { + "name": "tokenHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Cap CLI'" + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "token_hash_idx": { + "name": "token_hash_idx", + "columns": [ + "tokenHash" + ], + "isUnique": true + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": [ + "expiresAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_keys_id": { + "name": "agent_api_keys_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_operations": { + "name": "agent_api_operations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resourceId": { + "name": "resourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resultResourceId": { + "name": "resultResourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + }, + "state_updated_at_idx": { + "name": "state_updated_at_idx", + "columns": [ + "state", + "updatedAt" + ], + "isUnique": false + }, + "resource_id_idx": { + "name": "resource_id_idx", + "columns": [ + "resourceId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_operations_id": { + "name": "agent_api_operations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "auth_api_keys": { + "name": "auth_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_id_created_at_idx": { + "name": "user_id_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "auth_api_keys_id": { + "name": "auth_api_keys_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "comments": { + "name": "comments", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authorId": { + "name": "authorId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "parentCommentId": { + "name": "parentCommentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "video_type_created_idx": { + "name": "video_type_created_idx", + "columns": [ + "videoId", + "type", + "createdAt", + "id" + ], + "isUnique": false + }, + "author_id_idx": { + "name": "author_id_idx", + "columns": [ + "authorId" + ], + "isUnique": false + }, + "parent_comment_id_idx": { + "name": "parent_comment_id_idx", + "columns": [ + "parentCommentId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "comments_id": { + "name": "comments_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_api_keys": { + "name": "developer_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyType": { + "name": "keyType", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyPrefix": { + "name": "keyPrefix", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedKey": { + "name": "encryptedKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "key_hash_idx": { + "name": "key_hash_idx", + "columns": [ + "keyHash" + ], + "isUnique": true + }, + "app_key_type_idx": { + "name": "app_key_type_idx", + "columns": [ + "appId", + "keyType" + ], + "isUnique": false + } + }, + "foreignKeys": { + "developer_api_keys_appId_developer_apps_id_fk": { + "name": "developer_api_keys_appId_developer_apps_id_fk", + "tableFrom": "developer_api_keys", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_api_keys_id": { + "name": "developer_api_keys_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_app_domains": { + "name": "developer_app_domains", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_app_domains_appId_developer_apps_id_fk": { + "name": "developer_app_domains_appId_developer_apps_id_fk", + "tableFrom": "developer_app_domains", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_app_domains_id": { + "name": "developer_app_domains_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "app_domain_unique": { + "name": "app_domain_unique", + "columns": [ + "appId", + "domain" + ] + } + }, + "checkConstraint": {} + }, + "developer_apps": { + "name": "developer_apps", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logoUrl": { + "name": "logoUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_deleted_idx": { + "name": "owner_deleted_idx", + "columns": [ + "ownerId", + "deletedAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "developer_apps_id": { + "name": "developer_apps_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_accounts": { + "name": "developer_credit_accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceMicroCredits": { + "name": "balanceMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripePaymentMethodId": { + "name": "stripePaymentMethodId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoTopUpEnabled": { + "name": "autoTopUpEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "autoTopUpThresholdMicroCredits": { + "name": "autoTopUpThresholdMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoTopUpAmountCents": { + "name": "autoTopUpAmountCents", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_id_unique": { + "name": "app_id_unique", + "columns": [ + "appId" + ], + "isUnique": true + } + }, + "foreignKeys": { + "developer_credit_accounts_appId_developer_apps_id_fk": { + "name": "developer_credit_accounts_appId_developer_apps_id_fk", + "tableFrom": "developer_credit_accounts", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_accounts_id": { + "name": "developer_credit_accounts_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_transactions": { + "name": "developer_credit_transactions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accountId": { + "name": "accountId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amountMicroCredits": { + "name": "amountMicroCredits", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceAfterMicroCredits": { + "name": "balanceAfterMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referenceType": { + "name": "referenceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "account_type_created_idx": { + "name": "account_type_created_idx", + "columns": [ + "accountId", + "type", + "createdAt" + ], + "isUnique": false + }, + "account_ref_dedup_idx": { + "name": "account_ref_dedup_idx", + "columns": [ + "accountId", + "referenceId", + "referenceType" + ], + "isUnique": false + } + }, + "foreignKeys": { + "dev_credit_txn_account_fk": { + "name": "dev_credit_txn_account_fk", + "tableFrom": "developer_credit_transactions", + "tableTo": "developer_credit_accounts", + "columnsFrom": [ + "accountId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_transactions_id": { + "name": "developer_credit_transactions_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_daily_storage_snapshots": { + "name": "developer_daily_storage_snapshots", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshotDate": { + "name": "snapshotDate", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalDurationMinutes": { + "name": "totalDurationMinutes", + "type": "float", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "videoCount": { + "name": "videoCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "microCreditsCharged": { + "name": "microCreditsCharged", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { + "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", + "tableFrom": "developer_daily_storage_snapshots", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_daily_storage_snapshots_id": { + "name": "developer_daily_storage_snapshots_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "app_date_unique": { + "name": "app_date_unique", + "columns": [ + "appId", + "snapshotDate" + ] + } + }, + "checkConstraint": {} + }, + "developer_videos": { + "name": "developer_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalUserId": { + "name": "externalUserId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3Key": { + "name": "s3Key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_created_idx": { + "name": "app_created_idx", + "columns": [ + "appId", + "createdAt" + ], + "isUnique": false + }, + "app_user_idx": { + "name": "app_user_idx", + "columns": [ + "appId", + "externalUserId" + ], + "isUnique": false + }, + "app_deleted_idx": { + "name": "app_deleted_idx", + "columns": [ + "appId", + "deletedAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "developer_videos_appId_developer_apps_id_fk": { + "name": "developer_videos_appId_developer_apps_id_fk", + "tableFrom": "developer_videos", + "tableTo": "developer_apps", + "columnsFrom": [ + "appId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_videos_id": { + "name": "developer_videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "folders": { + "name": "folders", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": [ + "createdById" + ], + "isUnique": false + }, + "parent_id_idx": { + "name": "parent_id_idx", + "columns": [ + "parentId" + ], + "isUnique": false + }, + "space_id_idx": { + "name": "space_id_idx", + "columns": [ + "spaceId" + ], + "isUnique": false + }, + "public_parent_id_idx": { + "name": "public_parent_id_idx", + "columns": [ + "public", + "parentId" + ], + "isUnique": false + }, + "public_space_parent_id_idx": { + "name": "public_space_parent_id_idx", + "columns": [ + "public", + "spaceId", + "parentId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "folders_id": { + "name": "folders_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "imported_videos": { + "name": "imported_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "imported_videos_orgId_source_source_id_pk": { + "name": "imported_videos_orgId_source_source_id_pk", + "columns": [ + "orgId", + "source", + "source_id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "integration_installations": { + "name": "integration_installations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalId": { + "name": "externalId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "installedByUserId": { + "name": "installedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedCredentials": { + "name": "encryptedCredentials", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "provider_external_id_idx": { + "name": "provider_external_id_idx", + "columns": [ + "provider", + "externalId" + ], + "isUnique": true + }, + "organization_provider_display_name_idx": { + "name": "organization_provider_display_name_idx", + "columns": [ + "organizationId", + "provider", + "displayName" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "integration_installations_id": { + "name": "integration_installations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_conversations": { + "name": "messenger_conversations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverByUserId": { + "name": "takeoverByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverAt": { + "name": "takeoverAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastMessageAt": { + "name": "lastMessageAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_last_message_idx": { + "name": "user_last_message_idx", + "columns": [ + "userId", + "lastMessageAt" + ], + "isUnique": false + }, + "anonymous_last_message_idx": { + "name": "anonymous_last_message_idx", + "columns": [ + "anonymousId", + "lastMessageAt" + ], + "isUnique": false + }, + "mode_last_message_idx": { + "name": "mode_last_message_idx", + "columns": [ + "mode", + "lastMessageAt" + ], + "isUnique": false + }, + "updated_at_idx": { + "name": "updated_at_idx", + "columns": [ + "updatedAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "messenger_conversations_id": { + "name": "messenger_conversations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_messages": { + "name": "messenger_messages", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "conversation_created_at_idx": { + "name": "conversation_created_at_idx", + "columns": [ + "conversationId", + "createdAt" + ], + "isUnique": false + }, + "role_created_at_idx": { + "name": "role_created_at_idx", + "columns": [ + "role", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messenger_messages_conversationId_messenger_conversations_id_fk": { + "name": "messenger_messages_conversationId_messenger_conversations_id_fk", + "tableFrom": "messenger_messages", + "tableTo": "messenger_conversations", + "columnsFrom": [ + "conversationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_messages_id": { + "name": "messenger_messages_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_support_emails": { + "name": "messenger_support_emails", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "support_email_user_created_at_idx": { + "name": "support_email_user_created_at_idx", + "columns": [ + "userId", + "createdAt" + ], + "isUnique": false + }, + "support_email_conversation_created_at_idx": { + "name": "support_email_conversation_created_at_idx", + "columns": [ + "conversationId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "support_email_conversation_fk": { + "name": "support_email_conversation_fk", + "tableFrom": "messenger_support_emails", + "tableTo": "messenger_conversations", + "columnsFrom": [ + "conversationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_support_emails_id": { + "name": "messenger_support_emails_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipientId": { + "name": "recipientId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "org_id_idx": { + "name": "org_id_idx", + "columns": [ + "orgId" + ], + "isUnique": false + }, + "type_idx": { + "name": "type_idx", + "columns": [ + "type" + ], + "isUnique": false + }, + "read_at_idx": { + "name": "read_at_idx", + "columns": [ + "readAt" + ], + "isUnique": false + }, + "created_at_idx": { + "name": "created_at_idx", + "columns": [ + "createdAt" + ], + "isUnique": false + }, + "recipient_read_idx": { + "name": "recipient_read_idx", + "columns": [ + "recipientId", + "readAt" + ], + "isUnique": false + }, + "recipient_created_idx": { + "name": "recipient_created_idx", + "columns": [ + "recipientId", + "createdAt" + ], + "isUnique": false + }, + "dedup_key_idx": { + "name": "dedup_key_idx", + "columns": [ + "dedupKey" + ], + "isUnique": true + }, + "type_recipient_created_idx": { + "name": "type_recipient_created_idx", + "columns": [ + "type", + "recipientId", + "createdAt" + ], + "isUnique": false + }, + "type_recipient_video_created_idx": { + "name": "type_recipient_video_created_idx", + "columns": [ + "type", + "recipientId", + "videoId", + "createdAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "notifications_id": { + "name": "notifications_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_invites": { + "name": "organization_invites", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmail": { + "name": "invitedEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedByUserId": { + "name": "invitedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "invited_email_idx": { + "name": "invited_email_idx", + "columns": [ + "invitedEmail" + ], + "isUnique": false + }, + "invited_by_user_id_idx": { + "name": "invited_by_user_id_idx", + "columns": [ + "invitedByUserId" + ], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": [ + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_invites_id": { + "name": "organization_invites_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hasProSeat": { + "name": "hasProSeat", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "user_id_organization_id_idx": { + "name": "user_id_organization_id_idx", + "columns": [ + "userId", + "organizationId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_members_id": { + "name": "organization_members_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tombstoneAt": { + "name": "tombstoneAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowedEmailDomain": { + "name": "allowedEmailDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customDomain": { + "name": "customDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domainVerified": { + "name": "domainVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "shareableLinkIconUrl": { + "name": "shareableLinkIconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "workosOrganizationId": { + "name": "workosOrganizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workosConnectionId": { + "name": "workosConnectionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "owner_id_tombstone_idx": { + "name": "owner_id_tombstone_idx", + "columns": [ + "ownerId", + "tombstoneAt" + ], + "isUnique": false + }, + "custom_domain_idx": { + "name": "custom_domain_idx", + "columns": [ + "customDomain" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organizations_id": { + "name": "organizations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_leases": { + "name": "product_analytics_erasure_leases", + "columns": { + "name": { + "name": "name", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requestId": { + "name": "requestId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fencingToken": { + "name": "fencingToken", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "pausedPipes": { + "name": "pausedPipes", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_leases_name": { + "name": "product_analytics_erasure_leases_name", + "columns": [ + "name" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "s3_buckets": { + "name": "s3_buckets", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucketName": { + "name": "bucketName", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessKeyId": { + "name": "accessKeyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretAccessKey": { + "name": "secretAccessKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('aws')" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_organization_idx": { + "name": "owner_organization_idx", + "columns": [ + "ownerId", + "organizationId" + ], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": [ + "organizationId", + "active", + "updatedAt" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "s3_buckets_id": { + "name": "s3_buckets_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "session_token_idx": { + "name": "session_token_idx", + "columns": [ + "sessionToken" + ], + "isUnique": true + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sessions_id": { + "name": "sessions_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "shared_videos": { + "name": "shared_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedByUserId": { + "name": "sharedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedAt": { + "name": "sharedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": [ + "folderId" + ], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "shared_by_user_id_idx": { + "name": "shared_by_user_id_idx", + "columns": [ + "sharedByUserId" + ], + "isUnique": false + }, + "video_id_organization_id_idx": { + "name": "video_id_organization_id_idx", + "columns": [ + "videoId", + "organizationId" + ], + "isUnique": false + }, + "video_id_folder_id_idx": { + "name": "video_id_folder_id_idx", + "columns": [ + "videoId", + "folderId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "shared_videos_id": { + "name": "shared_videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": [ + "userId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_members_id": { + "name": "space_members_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": { + "space_id_user_id_unique": { + "name": "space_id_user_id_unique", + "columns": [ + "spaceId", + "userId" + ] + } + }, + "checkConstraint": {} + }, + "space_videos": { + "name": "space_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedById": { + "name": "addedById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": [ + "folderId" + ], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": [ + "videoId" + ], + "isUnique": false + }, + "added_by_id_idx": { + "name": "added_by_id_idx", + "columns": [ + "addedById" + ], + "isUnique": false + }, + "space_id_video_id_idx": { + "name": "space_id_video_id_idx", + "columns": [ + "spaceId", + "videoId" + ], + "isUnique": false + }, + "space_id_folder_id_idx": { + "name": "space_id_folder_id_idx", + "columns": [ + "spaceId", + "folderId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_videos_id": { + "name": "space_videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "primary": { + "name": "primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "privacy": { + "name": "privacy", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Private'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": [ + "organizationId" + ], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": [ + "createdById" + ], + "isUnique": false + }, + "public_organization_id_idx": { + "name": "public_organization_id_idx", + "columns": [ + "public", + "organizationId" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "spaces_id": { + "name": "spaces_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_integrations": { + "name": "storage_integrations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "encryptedConfig": { + "name": "encryptedConfig", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "googleDriveAccessToken": { + "name": "googleDriveAccessToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveAccessTokenExpiresAt": { + "name": "googleDriveAccessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseId": { + "name": "googleDriveTokenRefreshLeaseId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseExpiresAt": { + "name": "googleDriveTokenRefreshLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveStorageQuotaCache": { + "name": "googleDriveStorageQuotaCache", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_provider_idx": { + "name": "owner_provider_idx", + "columns": [ + "ownerId", + "provider" + ], + "isUnique": false + }, + "owner_active_idx": { + "name": "owner_active_idx", + "columns": [ + "ownerId", + "active" + ], + "isUnique": false + }, + "organization_provider_idx": { + "name": "organization_provider_idx", + "columns": [ + "organizationId", + "provider" + ], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": [ + "organizationId", + "active", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "storage_integrations_id": { + "name": "storage_integrations_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_objects": { + "name": "storage_objects", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrationId": { + "name": "integrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "objectKey": { + "name": "objectKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "objectKeyHash": { + "name": "objectKeyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerObjectId": { + "name": "providerObjectId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploadSessionUrl": { + "name": "uploadSessionUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploadStatus": { + "name": "uploadStatus", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "contentType": { + "name": "contentType", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentLength": { + "name": "contentLength", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "integration_key_hash_idx": { + "name": "integration_key_hash_idx", + "columns": [ + "integrationId", + "objectKeyHash" + ], + "isUnique": true + }, + "integration_status_idx": { + "name": "integration_status_idx", + "columns": [ + "integrationId", + "uploadStatus" + ], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": [ + "videoId" + ], + "isUnique": false + }, + "owner_id_idx": { + "name": "owner_id_idx", + "columns": [ + "ownerId" + ], + "isUnique": false + } + }, + "foreignKeys": { + "storage_objects_integrationId_storage_integrations_id_fk": { + "name": "storage_objects_integrationId_storage_integrations_id_fk", + "tableFrom": "storage_objects", + "tableTo": "storage_integrations", + "columnsFrom": [ + "integrationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "storage_objects_id": { + "name": "storage_objects_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastName": { + "name": "lastName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionId": { + "name": "stripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thirdPartyStripeSubscriptionId": { + "name": "thirdPartyStripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionStatus": { + "name": "stripeSubscriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionPriceId": { + "name": "stripeSubscriptionPriceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferences": { + "name": "preferences", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "('null')" + }, + "activeOrganizationId": { + "name": "activeOrganizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "onboardingSteps": { + "name": "onboardingSteps", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customBucket": { + "name": "customBucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "inviteQuota": { + "name": "inviteQuota", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "defaultOrgId": { + "name": "defaultOrgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authSessionVersion": { + "name": "authSessionVersion", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "email_idx": { + "name": "email_idx", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "identifier": { + "name": "identifier", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier": { + "name": "verification_tokens_identifier", + "columns": [ + "identifier" + ] + } + }, + "uniqueConstraints": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": [ + "token" + ] + } + }, + "checkConstraint": {} + }, + "video_edits": { + "name": "video_edits", + "columns": { + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceKey": { + "name": "sourceKey", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "editSpec": { + "name": "editSpec", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "video_edits_videoId_videos_id_fk": { + "name": "video_edits_videoId_videos_id_fk", + "tableFrom": "video_edits", + "tableTo": "videos", + "columnsFrom": [ + "videoId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "video_edits_videoId": { + "name": "video_edits_videoId", + "columns": [ + "videoId" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "video_uploads": { + "name": "video_uploads", + "columns": { + "video_id": { + "name": "video_id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded": { + "name": "uploaded", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "mode": { + "name": "mode", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'uploading'" + }, + "processing_progress": { + "name": "processing_progress", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processing_message": { + "name": "processing_message", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_file_key": { + "name": "raw_file_key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "phase_updated_at_video_id_idx": { + "name": "phase_updated_at_video_id_idx", + "columns": [ + "phase", + "updated_at", + "video_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "video_uploads_video_id": { + "name": "video_uploads_video_id", + "columns": [ + "video_id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'My Video'" + }, + "bucket": { + "name": "bucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageIntegrationId": { + "name": "storageIntegrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('{\"type\":\"MediaConvert\"}')" + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "effectiveCreatedAt": { + "name": "effectiveCreatedAt", + "type": "datetime", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "generated": { + "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", + "type": "stored" + } + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "xStreamInfo": { + "name": "xStreamInfo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstExternalViewAt": { + "name": "firstExternalViewAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstViewEmailSentAt": { + "name": "firstViewEmailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isScreenshot": { + "name": "isScreenshot", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "awsRegion": { + "name": "awsRegion", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "awsBucket": { + "name": "awsBucket", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoStartTime": { + "name": "videoStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audioStartTime": { + "name": "audioStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobId": { + "name": "jobId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobStatus": { + "name": "jobStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipProcessing": { + "name": "skipProcessing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "owner_id_idx": { + "name": "owner_id_idx", + "columns": [ + "ownerId" + ], + "isUnique": false + }, + "is_public_idx": { + "name": "is_public_idx", + "columns": [ + "public" + ], + "isUnique": false + }, + "folder_id_idx": { + "name": "folder_id_idx", + "columns": [ + "folderId" + ], + "isUnique": false + }, + "storage_integration_id_idx": { + "name": "storage_integration_id_idx", + "columns": [ + "storageIntegrationId" + ], + "isUnique": false + }, + "first_external_view_at_idx": { + "name": "first_external_view_at_idx", + "columns": [ + "firstExternalViewAt" + ], + "isUnique": false + }, + "org_owner_folder_idx": { + "name": "org_owner_folder_idx", + "columns": [ + "orgId", + "ownerId", + "folderId" + ], + "isUnique": false + }, + "org_effective_created_idx": { + "name": "org_effective_created_idx", + "columns": [ + "orgId", + "effectiveCreatedAt" + ], + "isUnique": false + } + }, + "foreignKeys": { + "videos_storageIntegrationId_storage_integrations_id_fk": { + "name": "videos_storageIntegrationId_storage_integrations_id_fk", + "tableFrom": "videos", + "tableTo": "storage_integrations", + "columnsFrom": [ + "storageIntegrationId" + ], + "columnsTo": [ + "id" + ], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "videos_id": { + "name": "videos_id", + "columns": [ + "id" + ] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/database/migrations/meta/_journal.json b/packages/database/migrations/meta/_journal.json index 6c7a67c1a6c..60232fd0eb6 100644 --- a/packages/database/migrations/meta/_journal.json +++ b/packages/database/migrations/meta/_journal.json @@ -1,272 +1,286 @@ { - "version": "5", - "dialect": "mysql", - "entries": [ - { - "idx": 0, - "version": "5", - "when": 1743020179593, - "tag": "0000_brown_sunfire", - "breakpoints": true - }, - { - "idx": 1, - "version": "5", - "when": 1749268354138, - "tag": "0001_white_young_avengers", - "breakpoints": true - }, - { - "idx": 2, - "version": "5", - "when": 1750935538683, - "tag": "0002_dusty_maginty", - "breakpoints": true - }, - { - "idx": 3, - "version": "5", - "when": 1761710697286, - "tag": "0003_thin_gressill", - "breakpoints": true - }, - { - "idx": 4, - "version": "5", - "when": 1761711378574, - "tag": "0004_video-org-id", - "breakpoints": true - }, - { - "idx": 5, - "version": "5", - "when": 1761711605408, - "tag": "0005_video-org-id-required", - "breakpoints": true - }, - { - "idx": 6, - "version": "5", - "when": 1762410865419, - "tag": "0006_hesitant_stone_men", - "breakpoints": true - }, - { - "idx": 7, - "version": "5", - "when": 1762428551824, - "tag": "0007_public_toxin", - "breakpoints": true - }, - { - "idx": 8, - "version": "5", - "when": 1762428905323, - "tag": "0008_fat_ender_wiggin", - "breakpoints": true - }, - { - "idx": 9, - "version": "5", - "when": 1768139432782, - "tag": "0009_easy_ulik", - "breakpoints": true - }, - { - "idx": 10, - "version": "5", - "when": 1738505600000, - "tag": "0010_video_uploads_bigint", - "breakpoints": true - }, - { - "idx": 11, - "version": "5", - "when": 1771325011010, - "tag": "0011_public_inhumans", - "breakpoints": true - }, - { - "idx": 12, - "version": "5", - "when": 1772534950328, - "tag": "0012_lethal_lilith", - "breakpoints": true - }, - { - "idx": 13, - "version": "5", - "when": 1772573402457, - "tag": "0013_faithful_marvex", - "breakpoints": true - }, - { - "idx": 14, - "version": "5", - "when": 1772576220593, - "tag": "0014_modern_triton", - "breakpoints": true - }, - { - "idx": 15, - "version": "5", - "when": 1772582468257, - "tag": "0015_closed_tigra", - "breakpoints": true - }, - { - "idx": 16, - "version": "5", - "when": 1772641549840, - "tag": "0016_bouncy_hellcat", - "breakpoints": true - }, - { - "idx": 17, - "version": "5", - "when": 1778099532336, - "tag": "0017_productive_betty_brant", - "breakpoints": true - }, - { - "idx": 18, - "version": "5", - "when": 1778153157657, - "tag": "0018_loud_mongu", - "breakpoints": true - }, - { - "idx": 19, - "version": "5", - "when": 1778154209547, - "tag": "0019_solid_paibok", - "breakpoints": true - }, - { - "idx": 20, - "version": "5", - "when": 1778157016497, - "tag": "0020_orange_talkback", - "breakpoints": true - }, - { - "idx": 21, - "version": "5", - "when": 1778249922872, - "tag": "0021_glorious_khan", - "breakpoints": true - }, - { - "idx": 22, - "version": "5", - "when": 1778252207674, - "tag": "0022_dazzling_namor", - "breakpoints": true - }, - { - "idx": 23, - "version": "5", - "when": 1778434170430, - "tag": "0023_misty_luckman", - "breakpoints": true - }, - { - "idx": 24, - "version": "5", - "when": 1778776694053, - "tag": "0024_many_speed", - "breakpoints": true - }, - { - "idx": 25, - "version": "5", - "when": 1778858762935, - "tag": "0025_demonic_mother_askani", - "breakpoints": true - }, - { - "idx": 26, - "version": "5", - "when": 1779283712737, - "tag": "0026_pretty_the_professor", - "breakpoints": true - }, - { - "idx": 27, - "version": "5", - "when": 1780932760351, - "tag": "0027_giant_shinko_yamashiro", - "breakpoints": true - }, - { - "idx": 28, - "version": "5", - "when": 1780937790068, - "tag": "0028_nebulous_madame_masque", - "breakpoints": true - }, - { - "idx": 29, - "version": "5", - "when": 1780937848351, - "tag": "0029_blushing_pretty_boy", - "breakpoints": true - }, - { - "idx": 30, - "version": "5", - "when": 1780940100362, - "tag": "0030_add_folder_settings", - "breakpoints": true - }, - { - "idx": 31, - "version": "5", - "when": 1781271269353, - "tag": "0031_add_auth_api_keys_user_created_index", - "breakpoints": true - }, - { - "idx": 32, - "version": "5", - "when": 1782850635482, - "tag": "0032_past_lester", - "breakpoints": true - }, - { - "idx": 33, - "version": "5", - "when": 1782853232527, - "tag": "0033_fluffy_gamora", - "breakpoints": true - }, - { - "idx": 34, - "version": "5", - "when": 1782854064505, - "tag": "0034_fluffy_falcon", - "breakpoints": true - }, - { - "idx": 35, - "version": "5", - "when": 1784393810560, - "tag": "0035_agent_api", - "breakpoints": true - }, - { - "idx": 36, - "version": "5", - "when": 1784402487715, - "tag": "0036_premium_master_chief", - "breakpoints": true - }, - { - "idx": 37, - "version": "5", - "when": 1785452585612, - "tag": "0037_integration_installations", - "breakpoints": true - } - ] -} + "version": "5", + "dialect": "mysql", + "entries": [ + { + "idx": 0, + "version": "5", + "when": 1743020179593, + "tag": "0000_brown_sunfire", + "breakpoints": true + }, + { + "idx": 1, + "version": "5", + "when": 1749268354138, + "tag": "0001_white_young_avengers", + "breakpoints": true + }, + { + "idx": 2, + "version": "5", + "when": 1750935538683, + "tag": "0002_dusty_maginty", + "breakpoints": true + }, + { + "idx": 3, + "version": "5", + "when": 1761710697286, + "tag": "0003_thin_gressill", + "breakpoints": true + }, + { + "idx": 4, + "version": "5", + "when": 1761711378574, + "tag": "0004_video-org-id", + "breakpoints": true + }, + { + "idx": 5, + "version": "5", + "when": 1761711605408, + "tag": "0005_video-org-id-required", + "breakpoints": true + }, + { + "idx": 6, + "version": "5", + "when": 1762410865419, + "tag": "0006_hesitant_stone_men", + "breakpoints": true + }, + { + "idx": 7, + "version": "5", + "when": 1762428551824, + "tag": "0007_public_toxin", + "breakpoints": true + }, + { + "idx": 8, + "version": "5", + "when": 1762428905323, + "tag": "0008_fat_ender_wiggin", + "breakpoints": true + }, + { + "idx": 9, + "version": "5", + "when": 1768139432782, + "tag": "0009_easy_ulik", + "breakpoints": true + }, + { + "idx": 10, + "version": "5", + "when": 1738505600000, + "tag": "0010_video_uploads_bigint", + "breakpoints": true + }, + { + "idx": 11, + "version": "5", + "when": 1771325011010, + "tag": "0011_public_inhumans", + "breakpoints": true + }, + { + "idx": 12, + "version": "5", + "when": 1772534950328, + "tag": "0012_lethal_lilith", + "breakpoints": true + }, + { + "idx": 13, + "version": "5", + "when": 1772573402457, + "tag": "0013_faithful_marvex", + "breakpoints": true + }, + { + "idx": 14, + "version": "5", + "when": 1772576220593, + "tag": "0014_modern_triton", + "breakpoints": true + }, + { + "idx": 15, + "version": "5", + "when": 1772582468257, + "tag": "0015_closed_tigra", + "breakpoints": true + }, + { + "idx": 16, + "version": "5", + "when": 1772641549840, + "tag": "0016_bouncy_hellcat", + "breakpoints": true + }, + { + "idx": 17, + "version": "5", + "when": 1778099532336, + "tag": "0017_productive_betty_brant", + "breakpoints": true + }, + { + "idx": 18, + "version": "5", + "when": 1778153157657, + "tag": "0018_loud_mongu", + "breakpoints": true + }, + { + "idx": 19, + "version": "5", + "when": 1778154209547, + "tag": "0019_solid_paibok", + "breakpoints": true + }, + { + "idx": 20, + "version": "5", + "when": 1778157016497, + "tag": "0020_orange_talkback", + "breakpoints": true + }, + { + "idx": 21, + "version": "5", + "when": 1778249922872, + "tag": "0021_glorious_khan", + "breakpoints": true + }, + { + "idx": 22, + "version": "5", + "when": 1778252207674, + "tag": "0022_dazzling_namor", + "breakpoints": true + }, + { + "idx": 23, + "version": "5", + "when": 1778434170430, + "tag": "0023_misty_luckman", + "breakpoints": true + }, + { + "idx": 24, + "version": "5", + "when": 1778776694053, + "tag": "0024_many_speed", + "breakpoints": true + }, + { + "idx": 25, + "version": "5", + "when": 1778858762935, + "tag": "0025_demonic_mother_askani", + "breakpoints": true + }, + { + "idx": 26, + "version": "5", + "when": 1779283712737, + "tag": "0026_pretty_the_professor", + "breakpoints": true + }, + { + "idx": 27, + "version": "5", + "when": 1780932760351, + "tag": "0027_giant_shinko_yamashiro", + "breakpoints": true + }, + { + "idx": 28, + "version": "5", + "when": 1780937790068, + "tag": "0028_nebulous_madame_masque", + "breakpoints": true + }, + { + "idx": 29, + "version": "5", + "when": 1780937848351, + "tag": "0029_blushing_pretty_boy", + "breakpoints": true + }, + { + "idx": 30, + "version": "5", + "when": 1780940100362, + "tag": "0030_add_folder_settings", + "breakpoints": true + }, + { + "idx": 31, + "version": "5", + "when": 1781271269353, + "tag": "0031_add_auth_api_keys_user_created_index", + "breakpoints": true + }, + { + "idx": 32, + "version": "5", + "when": 1782850635482, + "tag": "0032_past_lester", + "breakpoints": true + }, + { + "idx": 33, + "version": "5", + "when": 1782853232527, + "tag": "0033_fluffy_gamora", + "breakpoints": true + }, + { + "idx": 34, + "version": "5", + "when": 1782854064505, + "tag": "0034_fluffy_falcon", + "breakpoints": true + }, + { + "idx": 35, + "version": "5", + "when": 1784393810560, + "tag": "0035_agent_api", + "breakpoints": true + }, + { + "idx": 36, + "version": "5", + "when": 1784402487715, + "tag": "0036_premium_master_chief", + "breakpoints": true + }, + { + "idx": 37, + "version": "5", + "when": 1785452585612, + "tag": "0037_integration_installations", + "breakpoints": true + }, + { + "idx": 38, + "version": "5", + "when": 1785568617812, + "tag": "0038_messy_stephen_strange", + "breakpoints": true + }, + { + "idx": 39, + "version": "5", + "when": 1785569502530, + "tag": "0039_bumpy_phil_sheldon", + "breakpoints": true + } + ] +} \ No newline at end of file diff --git a/packages/database/schema.ts b/packages/database/schema.ts index d59bb3455ed..6996889e80f 100644 --- a/packages/database/schema.ts +++ b/packages/database/schema.ts @@ -185,6 +185,30 @@ export const verificationTokens = mysqlTable("verification_tokens", { updated_at: timestamp("updated_at").notNull().defaultNow().onUpdateNow(), }); +export const productAnalyticsErasureLeases = mysqlTable( + "product_analytics_erasure_leases", + { + name: varchar("name", { length: 64 }).notNull().primaryKey(), + ownerId: varchar("ownerId", { length: 64 }), + requestId: varchar("requestId", { length: 64 }), + fencingToken: bigint("fencingToken", { + mode: "number", + unsigned: true, + }) + .notNull() + .default(0), + leaseExpiresAt: timestamp("leaseExpiresAt"), + phase: varchar("phase", { length: 32 }).notNull().default("idle"), + pausedPipes: json("pausedPipes").$type(), + userId: varchar("userId", { length: 255 }), + organizationId: varchar("organizationId", { length: 255 }), + attemptCount: int("attemptCount").notNull().default(0), + lastErrorCode: varchar("lastErrorCode", { length: 64 }), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow().onUpdateNow(), + }, +); + export const organizations = mysqlTable( "organizations", { @@ -407,6 +431,7 @@ export const videos = mysqlTable( password: encryptedTextNullable("password"), // LEGACY xStreamInfo: text("xStreamInfo"), + firstExternalViewAt: timestamp("firstExternalViewAt"), firstViewEmailSentAt: timestamp("firstViewEmailSentAt"), isScreenshot: boolean("isScreenshot").notNull().default(false), // DEPRECATED @@ -423,6 +448,7 @@ export const videos = mysqlTable( index("is_public_idx").on(table.public), index("folder_id_idx").on(table.folderId), index("storage_integration_id_idx").on(table.storageIntegrationId), + index("first_external_view_at_idx").on(table.firstExternalViewAt), index("org_owner_folder_idx").on( table.orgId, table.ownerId, diff --git a/scripts/analytics/check-event-contract.js b/scripts/analytics/check-event-contract.js index d0bdd760539..ac248a8bf9f 100644 --- a/scripts/analytics/check-event-contract.js +++ b/scripts/analytics/check-event-contract.js @@ -126,6 +126,14 @@ const CAPTURE_MODULES = new Map([ platforms: ["server"], }, ], + [ + "firstViewReceivedEvent", + { + kind: "helper", + eventName: "first_view_received", + platforms: ["server"], + }, + ], ]), ], [ @@ -139,6 +147,54 @@ const CAPTURE_MODULES = new Map([ platforms: ["web", "desktop", "mobile", "cli", "server"], }, ], + [ + "subscriptionInvoicePaidProductEvent", + { + kind: "helper-set", + eventNames: ["purchase_completed", "subscription_renewed"], + platforms: ["server"], + }, + ], + [ + "subscriptionPaymentFailedProductEvent", + { + kind: "helper", + eventName: "subscription_payment_failed", + platforms: ["server"], + }, + ], + [ + "subscriptionRefundedProductEvent", + { + kind: "helper", + eventName: "subscription_refunded", + platforms: ["server"], + }, + ], + [ + "subscriptionTrialConvertedProductEvent", + { + kind: "helper", + eventName: "trial_converted", + platforms: ["server"], + }, + ], + [ + "subscriptionChangedProductEvents", + { + kind: "helper", + eventName: "subscription_changed", + platforms: ["server"], + }, + ], + [ + "subscriptionCancelledProductEvent", + { + kind: "helper", + eventName: "subscription_cancelled", + platforms: ["server"], + }, + ], ]), ], [ @@ -223,6 +279,14 @@ const CAPTURE_MODULES = new Map([ platforms: ["server"], }, ], + [ + "firstViewReceivedEvent", + { + kind: "helper", + eventName: "first_view_received", + platforms: ["server"], + }, + ], ]), ], [ @@ -236,6 +300,54 @@ const CAPTURE_MODULES = new Map([ platforms: ["web", "desktop", "mobile", "cli", "server"], }, ], + [ + "subscriptionInvoicePaidProductEvent", + { + kind: "helper-set", + eventNames: ["purchase_completed", "subscription_renewed"], + platforms: ["server"], + }, + ], + [ + "subscriptionPaymentFailedProductEvent", + { + kind: "helper", + eventName: "subscription_payment_failed", + platforms: ["server"], + }, + ], + [ + "subscriptionRefundedProductEvent", + { + kind: "helper", + eventName: "subscription_refunded", + platforms: ["server"], + }, + ], + [ + "subscriptionTrialConvertedProductEvent", + { + kind: "helper", + eventName: "trial_converted", + platforms: ["server"], + }, + ], + [ + "subscriptionChangedProductEvents", + { + kind: "helper", + eventName: "subscription_changed", + platforms: ["server"], + }, + ], + [ + "subscriptionCancelledProductEvent", + { + kind: "helper", + eventName: "subscription_cancelled", + platforms: ["server"], + }, + ], ]), ], [ @@ -345,6 +457,50 @@ function staticStringUnion(expression, initializers, seen = new Set()) { new Set([...seen, value.text]), ); } + if ( + ts.isCallExpression(value) && + ts.isIdentifier(value.expression) && + !seen.has(value.expression.text) + ) { + const initializer = initializers.get(value.expression.text); + if ( + !initializer || + (!ts.isArrowFunction(initializer) && + !ts.isFunctionExpression(initializer)) + ) { + return undefined; + } + const nextSeen = new Set([...seen, value.expression.text]); + if (!ts.isBlock(initializer.body)) { + return staticStringUnion(initializer.body, initializers, nextSeen); + } + const returns = []; + let invalid = false; + const visit = (node) => { + if (invalid) return; + if (ts.isFunctionLike(node) && node !== initializer) return; + if (ts.isReturnStatement(node)) { + if (!node.expression) { + invalid = true; + return; + } + const values = staticStringUnion( + node.expression, + initializers, + nextSeen, + ); + if (!values) { + invalid = true; + return; + } + returns.push(...values); + return; + } + ts.forEachChild(node, visit); + }; + ts.forEachChild(initializer.body, visit); + return invalid || returns.length === 0 ? undefined : [...new Set(returns)]; + } return undefined; } @@ -626,8 +782,24 @@ function helperPlatforms(descriptor, call, initializers) { return staticStringUnion(property.initializer, initializers) ?? []; } -function eventNameProperty(expression, bindings, initializers) { +function eventNameProperty( + expression, + bindings, + initializers, + seen = new Set(), +) { const value = unwrapExpression(expression); + if (ts.isIdentifier(value) && !seen.has(value.text)) { + const initializer = initializers.get(value.text); + if (initializer) { + return eventNameProperty( + initializer, + bindings, + initializers, + new Set([...seen, value.text]), + ); + } + } if (ts.isCallExpression(value)) { const descriptor = callDescriptor(value.expression, bindings); if (descriptor?.kind === "helper") { @@ -638,6 +810,14 @@ function eventNameProperty(expression, bindings, initializers) { platforms: helperPlatforms(descriptor, value, initializers), }; } + if (descriptor?.kind === "helper-set") { + return { + eventNames: descriptor.eventNames, + kind: "static-set", + node: value, + platforms: descriptor.platforms, + }; + } } if (!ts.isObjectLiteralExpression(value)) return { kind: "non-object", node: value }; @@ -655,6 +835,30 @@ function eventNameProperty(expression, bindings, initializers) { return { kind: "missing", node: value }; } +function forOfHelperBinding(identifier, call, bindings) { + if (!ts.isIdentifier(identifier)) return false; + let current = call.parent; + while (current) { + if (ts.isForOfStatement(current)) { + const declaration = current.initializer.declarations?.find( + (candidate) => + ts.isIdentifier(candidate.name) && + candidate.name.text === identifier.text, + ); + const expression = unwrapExpression(current.expression); + if ( + declaration && + ts.isCallExpression(expression) && + callDescriptor(expression.expression, bindings) + ) { + return true; + } + } + current = current.parent; + } + return false; +} + export function analyzeTypeScriptSource({ sourceText, file, @@ -763,7 +967,14 @@ export function analyzeTypeScriptSource({ result.platforms ?? eventPlatforms(argument, bindings, initializers), ); - } else { + } else if (result.kind === "static-set") { + for (const eventName of result.eventNames) { + registerEmission(eventName, result.node, result.platforms); + } + } else if ( + result.kind !== "non-object" || + !forOfHelperBinding(unwrapExpression(argument), node, bindings) + ) { const messages = { "non-object": "Analytics event objects must be inline object literals", @@ -1164,8 +1375,39 @@ function rustTokensWithoutTestItems(tokens) { ); } -export function rustProductionVariantUses(sourceText) { - const tokens = rustTokensWithoutTestItems(tokenizeRust(sourceText)); +function rustTokensWithoutFunctions(tokens, functionNames) { + const excluded = []; + for (let index = 0; index < tokens.length - 2; index += 1) { + if ( + tokens[index]?.value !== "fn" || + !functionNames.has(tokens[index + 1]?.value) + ) { + continue; + } + let opening = index + 2; + while (opening < tokens.length && tokens[opening]?.value !== "{") + opening += 1; + if (opening >= tokens.length) continue; + const closing = matchingTokenIndex(tokens, opening, "{", "}"); + if (closing === undefined) continue; + excluded.push([index, closing]); + index = closing; + } + return tokens.filter( + (_token, index) => + !excluded.some(([start, end]) => index >= start && index <= end), + ); +} + +export function rustProductionVariantUses( + sourceText, + { excludeFunctions = [] } = {}, +) { + const productionTokens = rustTokensWithoutTestItems(tokenizeRust(sourceText)); + const tokens = rustTokensWithoutFunctions( + productionTokens, + new Set(excludeFunctions), + ); const variants = new Set(); for (let index = 0; index < tokens.length - 2; index += 1) { if (isTokenSequence(tokens, index, ["ProductAnalyticsEvent", "::"])) { @@ -1303,6 +1545,11 @@ export function runEventContractCheck({ }); diagnostics.push(...native.diagnostics); const usedNativeVariants = new Map(); + for (const variant of rustProductionVariantUses(nativeSource, { + excludeFunctions: ["event_data"], + })) { + usedNativeVariants.set(variant, new Set(["desktop"])); + } for (const relativePath of files) { if ( !relativePath.endsWith(".rs") || diff --git a/scripts/analytics/tests/event-contract.test.js b/scripts/analytics/tests/event-contract.test.js index 1aa4d2e6d1d..81986d1b944 100644 --- a/scripts/analytics/tests/event-contract.test.js +++ b/scripts/analytics/tests/event-contract.test.js @@ -277,9 +277,22 @@ test("credits only the literal platform passed to routed business factories", () file: "apps/web/app/api/desktop/subscribe.ts", registeredEvents: new Set(["checkout_started"]), }); + const boundedHelper = analyzeTypeScriptSource({ + sourceText: ` + import { shareLinkCreatedEvent } from "@/lib/analytics/business-events"; + const resolvePlatform = (mobile: boolean) => { + if (mobile) return "mobile" as const; + return "server" as const; + }; + shareLinkCreatedEvent({ platform: resolvePlatform(true) }); + `, + file: "apps/web/workflows/reconcile.ts", + registeredEvents: new Set(["share_link_created"]), + }); assert.deepEqual(literal.emissions[0]?.platforms, ["mobile"]); assert.deepEqual(dynamic.emissions[0]?.platforms, []); assert.deepEqual(bounded.emissions[0]?.platforms, ["mobile", "desktop"]); + assert.deepEqual(boundedHelper.emissions[0]?.platforms, ["mobile", "server"]); }); test("accepts a bounded helper that emits one of a declared event set", () => { @@ -310,6 +323,47 @@ test("accepts a bounded helper that emits one of a declared event set", () => { ); }); +test("resolves queued events created by a registered optional factory", () => { + const result = analyzeTypeScriptSource({ + sourceText: ` + import { subscriptionPaymentFailedProductEvent } from "@/lib/analytics/stripe-business-events"; + import { queueServerProductEvent } from "@/lib/analytics/server"; + const paymentFailed = subscriptionPaymentFailedProductEvent({ eventId: "evt_1" }); + if (paymentFailed) queueServerProductEvent(paymentFailed); + `, + file: "apps/web/app/api/webhooks/stripe/route.ts", + registeredEvents: new Set(["subscription_payment_failed"]), + }); + assert.deepEqual(result.diagnostics, []); + assert.ok( + result.emissions.every( + (emission) => + emission.eventName === "subscription_payment_failed" && + emission.platforms.includes("server"), + ), + ); +}); + +test("accepts queued events iterated from a registered factory", () => { + const result = analyzeTypeScriptSource({ + sourceText: ` + import { subscriptionChangedProductEvents } from "@/lib/analytics/stripe-business-events"; + import { queueServerProductEvent } from "@/lib/analytics/server"; + for (const productEvent of subscriptionChangedProductEvents({ eventId: "evt_1" })) { + queueServerProductEvent(productEvent); + } + `, + file: "apps/web/app/api/webhooks/stripe/route.ts", + registeredEvents: new Set(["subscription_changed"]), + }); + assert.deepEqual(result.diagnostics, []); + assert.ok( + result.emissions.some( + (emission) => emission.eventName === "subscription_changed", + ), + ); +}); + test("Rust tokenization excludes comments", () => { const tokens = tokenizeRust(` // EventData::new("comment_event") @@ -340,6 +394,22 @@ fn production() { assert.deepEqual([...variants], ["RecordingStarted"]); }); +test("native emitter discovery can exclude registry mapping functions", () => { + const variants = rustProductionVariantUses( + ` +fn event_data() { + ProductAnalyticsEvent::RecordingStarted; +} + +fn internally_emitted() { + ProductAnalyticsEvent::AnalyticsDeliveryLoss; +} +`, + { excludeFunctions: ["event_data"] }, + ); + assert.deepEqual([...variants], ["AnalyticsDeliveryLoss"]); +}); + test("native Rust mappings match variants, registry and core catalog", () => { const result = analyzeRustNativeContract({ file: "product_analytics.rs", From 1723c7e21f546c7e30a38338de818d404ae38b5e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:15:08 +0100 Subject: [PATCH 076/110] fix: make analytics delivery and erasure durable --- .../unit/product-analytics-erasure.test.ts | 369 +++++++++++++++-- .../route.ts | 35 ++ apps/web/vercel.json | 4 + packages/env/server.ts | 4 + .../ProductAnalyticsErasureLeaseRepo.ts | 279 +++++++++++++ packages/web-backend/src/Tinybird/index.ts | 383 +++++++++++++++--- packages/web-backend/src/index.ts | 6 + 7 files changed, 984 insertions(+), 96 deletions(-) create mode 100644 apps/web/app/api/cron/recover-product-analytics-erasure/route.ts create mode 100644 packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts diff --git a/apps/web/__tests__/unit/product-analytics-erasure.test.ts b/apps/web/__tests__/unit/product-analytics-erasure.test.ts index 137f9550b89..9bcb1e92c73 100644 --- a/apps/web/__tests__/unit/product-analytics-erasure.test.ts +++ b/apps/web/__tests__/unit/product-analytics-erasure.test.ts @@ -1,14 +1,28 @@ -import { Tinybird } from "@cap/web-backend"; -import { Effect } from "effect"; +import { + ProductAnalyticsErasureLeaseRepo, + type ProductAnalyticsErasureLeaseStore, + Tinybird, +} from "@cap/web-backend"; +import { Effect, Layer } from "effect"; import { afterEach, describe, expect, it, vi } from "vitest"; const serviceEnvironment = vi.hoisted( (): { + PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN: string | undefined; + PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN: string | undefined; + PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN: string | undefined; PRODUCT_ANALYTICS_TINYBIRD_HOST: string | undefined; + PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN: string | undefined; + PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN: string | undefined; TINYBIRD_HOST: undefined; TINYBIRD_TOKEN: undefined; } => ({ + PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN: "copy-token", + PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN: "lookup-token", + PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN: "erasure-token", PRODUCT_ANALYTICS_TINYBIRD_HOST: "https://staging.tinybird.test", + PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN: "read-token", + PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN: "scheduler-token", TINYBIRD_HOST: undefined, TINYBIRD_TOKEN: undefined, }), @@ -16,44 +30,106 @@ const serviceEnvironment = vi.hoisted( vi.mock("@cap/env", () => ({ serverEnv: () => serviceEnvironment })); -const originalEnvironment = { - erasureToken: process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN, +const tinybirdTestLayer = ( + overrides: Partial = {}, +) => { + const store = { + claimNew: (scope) => + Effect.succeed({ + ownerId: "owner-1", + requestId: "request-1", + fencingToken: 1, + phase: "claimed" as const, + pausedPipes: [], + scope, + }), + claimRecovery: () => Effect.succeed(null), + heartbeat: () => Effect.succeed(true), + advance: () => Effect.succeed(true), + complete: () => Effect.succeed(true), + fail: () => Effect.succeed(true), + ...overrides, + } satisfies ProductAnalyticsErasureLeaseStore; + return Tinybird.DefaultWithoutDependencies.pipe( + Layer.provide( + Layer.succeed(ProductAnalyticsErasureLeaseRepo, { + _tag: "ProductAnalyticsErasureLeaseRepo", + ...store, + }), + ), + ); +}; + +const copyScheduleResponse = (url: URL, pausedPipes: Set) => { + const match = url.pathname.match( + /^\/v0\/pipes\/([A-Za-z0-9_]+)(?:\/copy\/(cancel|resume))?$/, + ); + if (!match?.[1]) return undefined; + const pipe = match[1]; + if (match[2] === "cancel") pausedPipes.add(pipe); + if (match[2] === "resume") pausedPipes.delete(pipe); + return Response.json({ + schedule: { status: pausedPipes.has(pipe) ? "paused" : "scheduled" }, + }); }; afterEach(() => { vi.unstubAllGlobals(); + serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN = "copy-token"; + serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN = + "lookup-token"; + serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN = "erasure-token"; serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_HOST = "https://staging.tinybird.test"; - if (originalEnvironment.erasureToken === undefined) { - delete process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN; - } else { - process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN = - originalEnvironment.erasureToken; - } + serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN = "read-token"; + serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN = + "scheduler-token"; }); describe.sequential("product analytics erasure", () => { it("deletes linked identities and rebuilds every derived snapshot", async () => { - process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN = "erasure-token"; const requests: Array<{ url: URL; init: RequestInit }> = []; + const pausedPipes = new Set(); + let deleted = false; vi.stubGlobal( "fetch", vi.fn(async (input: string | URL | Request, init: RequestInit = {}) => { const url = new URL(String(input)); requests.push({ url, init }); + const scheduleResponse = copyScheduleResponse(url, pausedPipes); + if (scheduleResponse) return scheduleResponse; if (url.pathname === "/v0/sql") { - return Response.json({ data: [{ anonymous_id: "anonymous-1" }] }); + const query = url.searchParams.get("q") ?? ""; + if (query.startsWith("SELECT anonymous_id")) { + return Response.json({ data: [{ anonymous_id: "anonymous-1" }] }); + } + return Response.json({ + data: [{ matching_rows: deleted ? 0 : 1 }], + }); } if (url.pathname.includes("/delete")) { + deleted = true; return Response.json({ mutation: { is_done: true } }); } + if (url.pathname.endsWith("product_analytics_copy_assertions.json")) { + return Response.json({ + data: [ + { + activation_markers: 1, + decision_markers: 1, + health_markers: 1, + identity_markers: 1, + retention_markers: 1, + traffic_markers: 1, + traffic_page_markers: 1, + }, + ], + }); + } if (url.pathname.includes("/copy")) { const pipe = url.pathname.split("/").at(-2); return Response.json({ job_id: `${pipe}-job` }); } - if (url.pathname.includes("/jobs/")) { - return Response.json({ status: "done" }); - } return Response.json({}); }), ); @@ -65,25 +141,34 @@ describe.sequential("product analytics erasure", () => { userId: "user-1", organizationId: "organization-1", }); - }).pipe(Effect.provide(Tinybird.Default)), + }).pipe(Effect.provide(tinybirdTestLayer())), ); - expect(requests).toHaveLength(16); - expect(requests[0]?.url.pathname).toBe("/v0/sql"); - expect(requests[0]?.url.searchParams.get("q")).toContain( + const identityLookup = requests.find(({ url }) => + (url.searchParams.get("q") ?? "").startsWith("SELECT anonymous_id"), + ); + expect(identityLookup?.url.pathname).toBe("/v0/sql"); + expect(identityLookup?.url.searchParams.get("q")).toContain( "countIf(user_id != '' AND user_id != 'user-1') = 0", ); - expect(requests[1]?.url.pathname).toBe( + const deletion = requests.find(({ url }) => + url.pathname.includes("/delete"), + ); + expect(deletion?.url.pathname).toBe( "/v1/datasources/product_events_v1/delete", ); - const deleteBody = new URLSearchParams(String(requests[1]?.init.body)).get( + const deleteBody = new URLSearchParams(String(deletion?.init.body)).get( "delete_condition", ); expect(deleteBody).toContain("organization_id"); expect(deleteBody).toContain("user_id"); expect(deleteBody).toContain("anonymous_id"); expect(deleteBody).toContain("AND (user_id = '' OR user_id = 'user-1')"); - expect(requests.slice(2).map(({ url }) => url.pathname)).toEqual( + expect( + requests + .filter(({ url }) => url.pathname.endsWith("/copy")) + .map(({ url }) => url.pathname), + ).toEqual( [ "snapshot_product_events_canonical_v1", "snapshot_product_events_daily_exact", @@ -91,44 +176,56 @@ describe.sequential("product analytics erasure", () => { "snapshot_product_traffic_pages_daily_exact", "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", + "snapshot_product_identity_funnel_exact", "snapshot_product_events_health_hourly", - ].flatMap((pipe) => [`/v0/pipes/${pipe}/copy`, `/v0/jobs/${pipe}-job`]), + ].map((pipe) => `/v0/pipes/${pipe}/copy`), ); for (const request of requests.filter(({ url }) => url.pathname.endsWith("/copy"), )) { expect(request.url.searchParams.get("_mode")).toBe("replace"); } - for (const request of requests) { - expect(new Headers(request.init.headers).get("Authorization")).toBe( - "Bearer erasure-token", - ); - } + expect(requests.some(({ url }) => url.pathname.includes("/jobs/"))).toBe( + false, + ); + expect( + requests + .filter(({ url }) => url.pathname === "/v0/sql") + .every( + ({ init }) => + new Headers(init.headers).get("Authorization") === + "Bearer lookup-token", + ), + ).toBe(true); + expect(new Headers(deletion?.init.headers).get("Authorization")).toBe( + "Bearer erasure-token", + ); }); it("fails closed when the erasure credential is missing", async () => { - delete process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN; + serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN = undefined; const error = await Effect.runPromise( Effect.gen(function* () { const tinybird = yield* Tinybird; yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); - }).pipe(Effect.provide(Tinybird.Default), Effect.flip), + }).pipe(Effect.provide(tinybirdTestLayer()), Effect.flip), ); expect(error).toBeInstanceOf(Error); - expect(error.message).toBe("Product analytics erasure is not configured"); + expect(error.message).toBe( + "Product analytics Copy schedule control is not configured", + ); }); it("fails closed when the erasure host is missing", async () => { - process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN = "erasure-token"; serviceEnvironment.PRODUCT_ANALYTICS_TINYBIRD_HOST = undefined; const error = await Effect.runPromise( Effect.gen(function* () { const tinybird = yield* Tinybird; yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); - }).pipe(Effect.provide(Tinybird.Default), Effect.flip), + }).pipe(Effect.provide(tinybirdTestLayer()), Effect.flip), ); expect(error).toBeInstanceOf(Error); @@ -138,13 +235,15 @@ describe.sequential("product analytics erasure", () => { }); it("fails closed when Tinybird does not confirm deletion", async () => { - process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN = "erasure-token"; + const pausedPipes = new Set(); vi.stubGlobal( "fetch", vi.fn(async (input: string | URL | Request) => { const url = new URL(String(input)); + const scheduleResponse = copyScheduleResponse(url, pausedPipes); + if (scheduleResponse) return scheduleResponse; if (url.pathname === "/v0/sql") { - return Response.json({ data: [] }); + return Response.json({ data: [{ matching_rows: 1 }] }); } return Response.json({}); }), @@ -154,10 +253,208 @@ describe.sequential("product analytics erasure", () => { Effect.gen(function* () { const tinybird = yield* Tinybird; yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); - }).pipe(Effect.provide(Tinybird.Default), Effect.flip), + }).pipe(Effect.provide(tinybirdTestLayer()), Effect.flip), ); expect(error).toBeInstanceOf(Error); expect(error.message).toBe("Product analytics deletion did not finish"); }); + + it("rebuilds every snapshot when a retry finds no remaining identity rows", async () => { + const requests: URL[] = []; + const pausedPipes = new Set(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + requests.push(url); + const scheduleResponse = copyScheduleResponse(url, pausedPipes); + if (scheduleResponse) return scheduleResponse; + if (url.pathname === "/v0/sql") { + return Response.json({ data: [{ matching_rows: 0 }] }); + } + if (url.pathname.endsWith("product_analytics_copy_assertions.json")) { + return Response.json({ + data: [ + { + activation_markers: 1, + decision_markers: 1, + health_markers: 1, + identity_markers: 1, + retention_markers: 1, + traffic_markers: 1, + traffic_page_markers: 1, + }, + ], + }); + } + if (url.pathname.endsWith("/copy")) { + return Response.json({ job_id: "copy-job" }); + } + return Response.json({}); + }), + ); + + await Effect.runPromise( + Effect.gen(function* () { + const tinybird = yield* Tinybird; + yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); + }).pipe(Effect.provide(tinybirdTestLayer())), + ); + + expect(requests.some((url) => url.pathname.includes("/delete"))).toBe( + false, + ); + expect( + requests + .filter((url) => url.pathname.endsWith("/copy")) + .map((url) => url.pathname), + ).toHaveLength(8); + }); + + it("resumes schedules already paused when a later pause fails", async () => { + const requests: URL[] = []; + const pausedPipes = new Set(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + requests.push(url); + if ( + url.pathname === + "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel" + ) { + return new Response("pause failed", { status: 500 }); + } + const scheduleResponse = copyScheduleResponse(url, pausedPipes); + if (scheduleResponse) return scheduleResponse; + return Response.json({}); + }), + ); + + const error = await Effect.runPromise( + Effect.gen(function* () { + const tinybird = yield* Tinybird; + yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); + }).pipe(Effect.provide(tinybirdTestLayer()), Effect.flip), + ); + + expect(error).toBeInstanceOf(Error); + expect( + requests + .map((url) => url.pathname) + .filter((pathname) => /\/copy\/(cancel|resume)$/.test(pathname)), + ).toEqual([ + "/v0/pipes/snapshot_product_events_canonical_v1/copy/cancel", + "/v0/pipes/snapshot_product_events_daily_exact/copy/cancel", + "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", + "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", + "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", + "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", + "/v0/pipes/snapshot_product_events_canonical_v1/copy/resume", + "/v0/pipes/snapshot_product_events_daily_exact/copy/resume", + ]); + }); + + it("fails visibly without touching Tinybird when another owner holds the lease", async () => { + const fetch = vi.fn(); + vi.stubGlobal("fetch", fetch); + + const error = await Effect.runPromise( + Effect.gen(function* () { + const tinybird = yield* Tinybird; + yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); + }).pipe( + Effect.provide( + tinybirdTestLayer({ + claimNew: () => Effect.succeed(null), + claimRecovery: () => Effect.succeed(null), + }), + ), + Effect.flip, + ), + ); + + expect(error).toBeInstanceOf(Error); + expect(error.message).toBe( + "Product analytics erasure is already in progress", + ); + expect(fetch).not.toHaveBeenCalled(); + }); + + it("recovers a fenced scope and resumes persisted schedules before rebuilding", async () => { + const requests: URL[] = []; + const pausedPipes = new Set(["snapshot_product_events_canonical_v1"]); + const complete = vi.fn(() => Effect.succeed(true)); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + requests.push(url); + const scheduleResponse = copyScheduleResponse(url, pausedPipes); + if (scheduleResponse) return scheduleResponse; + if (url.pathname === "/v0/sql") { + return Response.json({ data: [{ matching_rows: 0 }] }); + } + if (url.pathname.endsWith("product_analytics_copy_assertions.json")) { + return Response.json({ + data: [ + { + activation_markers: 1, + decision_markers: 1, + health_markers: 1, + identity_markers: 1, + retention_markers: 1, + traffic_markers: 1, + traffic_page_markers: 1, + }, + ], + }); + } + if (url.pathname.endsWith("/copy")) { + return Response.json({ job_id: "copy-job" }); + } + return Response.json({}); + }), + ); + + const result = await Effect.runPromise( + Effect.gen(function* () { + const tinybird = yield* Tinybird; + return yield* tinybird.recoverProductAnalyticsErasure; + }).pipe( + Effect.provide( + tinybirdTestLayer({ + claimRecovery: () => + Effect.succeed({ + ownerId: "recovery-owner", + requestId: "persisted-request", + fencingToken: 2, + phase: "failed", + pausedPipes: ["snapshot_product_events_canonical_v1"], + scope: { organizationId: "persisted-org" }, + }), + complete, + }), + ), + ), + ); + + expect(result).toEqual({ + recovered: true, + requestId: "persisted-request", + }); + expect(requests[0]?.pathname).toBe( + "/v0/pipes/snapshot_product_events_canonical_v1/copy/resume", + ); + expect( + requests.some((url) => + (url.searchParams.get("q") ?? "").includes( + "organization_id = 'persisted-org'", + ), + ), + ).toBe(true); + expect(complete).toHaveBeenCalledOnce(); + expect(pausedPipes.size).toBe(0); + }); }); diff --git a/apps/web/app/api/cron/recover-product-analytics-erasure/route.ts b/apps/web/app/api/cron/recover-product-analytics-erasure/route.ts new file mode 100644 index 00000000000..5b077a9b562 --- /dev/null +++ b/apps/web/app/api/cron/recover-product-analytics-erasure/route.ts @@ -0,0 +1,35 @@ +import { timingSafeEqual } from "node:crypto"; +import { Tinybird } from "@cap/web-backend"; +import { Effect } from "effect"; +import { NextResponse } from "next/server"; +import { runPromise } from "@/lib/server"; + +export const dynamic = "force-dynamic"; + +export async function GET(request: Request) { + const cronSecret = process.env.CRON_SECRET; + if (!cronSecret) { + return NextResponse.json( + { error: "Server misconfiguration" }, + { status: 500 }, + ); + } + + const authorization = request.headers.get("authorization"); + const expected = `Bearer ${cronSecret}`; + if ( + !authorization || + authorization.length !== expected.length || + !timingSafeEqual(Buffer.from(authorization), Buffer.from(expected)) + ) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const result = await runPromise( + Effect.gen(function* () { + const tinybird = yield* Tinybird; + return yield* tinybird.recoverProductAnalyticsErasure; + }), + ); + return NextResponse.json({ success: true, ...result }); +} diff --git a/apps/web/vercel.json b/apps/web/vercel.json index a638b30aca2..fd194c2e332 100644 --- a/apps/web/vercel.json +++ b/apps/web/vercel.json @@ -16,6 +16,10 @@ { "path": "/api/cron/reconcile-product-analytics", "schedule": "43 4 * * *" + }, + { + "path": "/api/cron/recover-product-analytics-erasure", + "schedule": "*/10 * * * *" } ], "functions": { diff --git a/packages/env/server.ts b/packages/env/server.ts index 7485a945fef..0a492bdf0de 100644 --- a/packages/env/server.ts +++ b/packages/env/server.ts @@ -117,7 +117,11 @@ function createServerEnv() { PRODUCT_ANALYTICS_TINYBIRD_TOKEN: z.string().optional(), PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN: z.string().optional(), PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN: z.string().optional(), + PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN: z.string().optional(), + PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN: z.string().optional(), + PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN: z.string().optional(), PRODUCT_ANALYTICS_INTERNAL_IP_HASHES: z.string().optional(), + CAP_ANALYTICS_STAGING_TEST_SECRET: z.string().optional(), VERCEL_ENV: z .union([ z.literal("production"), diff --git a/packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts b/packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts new file mode 100644 index 00000000000..a4618f29fc2 --- /dev/null +++ b/packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts @@ -0,0 +1,279 @@ +import { randomUUID } from "node:crypto"; +import * as Db from "@cap/database/schema"; +import * as Dz from "drizzle-orm"; +import { Effect } from "effect"; + +import { Database } from "../Database.ts"; + +const LEASE_NAME = "global"; +const LEASE_DURATION_MS = 5 * 60 * 1_000; + +export type ProductAnalyticsErasurePhase = + | "claimed" + | "pausing" + | "deleting" + | "rebuilding" + | "resuming" + | "failed"; + +export type ProductAnalyticsErasureScope = { + userId?: string; + organizationId?: string; +}; + +export type ProductAnalyticsErasureLease = { + ownerId: string; + requestId: string; + fencingToken: number; + phase: ProductAnalyticsErasurePhase; + pausedPipes: string[]; + scope: ProductAnalyticsErasureScope; +}; + +export interface ProductAnalyticsErasureLeaseStore { + claimNew: ( + scope: ProductAnalyticsErasureScope, + ) => Effect.Effect; + claimRecovery: () => Effect.Effect< + ProductAnalyticsErasureLease | null, + Error + >; + heartbeat: ( + lease: ProductAnalyticsErasureLease, + ) => Effect.Effect; + advance: ( + lease: ProductAnalyticsErasureLease, + phase: ProductAnalyticsErasurePhase, + pausedPipes: readonly string[], + ) => Effect.Effect; + complete: ( + lease: ProductAnalyticsErasureLease, + ) => Effect.Effect; + fail: ( + lease: ProductAnalyticsErasureLease, + pausedPipes: readonly string[], + lastErrorCode: "erase_failed" | "resume_failed", + ) => Effect.Effect; +} + +const affectedRows = (result: unknown) => { + if (Array.isArray(result)) { + return ( + (result[0] as { affectedRows?: number } | undefined)?.affectedRows ?? 0 + ); + } + return (result as { affectedRows?: number } | undefined)?.affectedRows ?? 0; +}; + +const leaseExpiry = () => new Date(Date.now() + LEASE_DURATION_MS); + +export class ProductAnalyticsErasureLeaseRepo extends Effect.Service()( + "ProductAnalyticsErasureLeaseRepo", + { + effect: Effect.gen(function* () { + const database = yield* Database; + + const readOwned = (ownerId: string) => + database.use(async (db) => { + const [row] = await db + .select() + .from(Db.productAnalyticsErasureLeases) + .where( + Dz.and( + Dz.eq(Db.productAnalyticsErasureLeases.name, LEASE_NAME), + Dz.eq(Db.productAnalyticsErasureLeases.ownerId, ownerId), + ), + ) + .limit(1); + if ( + !row?.ownerId || + !row.requestId || + row.phase === "idle" || + !Number.isSafeInteger(row.fencingToken) + ) { + return null; + } + return { + ownerId: row.ownerId, + requestId: row.requestId, + fencingToken: row.fencingToken, + phase: row.phase as ProductAnalyticsErasurePhase, + pausedPipes: Array.isArray(row.pausedPipes) + ? row.pausedPipes.filter( + (value): value is string => typeof value === "string", + ) + : [], + scope: { + userId: row.userId ?? undefined, + organizationId: row.organizationId ?? undefined, + }, + } satisfies ProductAnalyticsErasureLease; + }); + + const claimNew: ProductAnalyticsErasureLeaseStore["claimNew"] = (scope) => + Effect.gen(function* () { + const ownerId = randomUUID(); + const requestId = randomUUID(); + const available = Dz.sql`${Db.productAnalyticsErasureLeases.ownerId} IS NULL AND ${Db.productAnalyticsErasureLeases.phase} = 'idle'`; + yield* database.use((db) => + db + .insert(Db.productAnalyticsErasureLeases) + .values({ + name: LEASE_NAME, + ownerId, + requestId, + fencingToken: 1, + leaseExpiresAt: leaseExpiry(), + phase: "claimed", + pausedPipes: [], + userId: scope.userId ?? null, + organizationId: scope.organizationId ?? null, + attemptCount: 1, + }) + .onDuplicateKeyUpdate({ + set: { + ownerId: Dz.sql`IF(${available}, ${ownerId}, ${Db.productAnalyticsErasureLeases.ownerId})`, + requestId: Dz.sql`IF(${available}, ${requestId}, ${Db.productAnalyticsErasureLeases.requestId})`, + fencingToken: Dz.sql`IF(${available}, ${Db.productAnalyticsErasureLeases.fencingToken} + 1, ${Db.productAnalyticsErasureLeases.fencingToken})`, + leaseExpiresAt: Dz.sql`IF(${available}, ${leaseExpiry()}, ${Db.productAnalyticsErasureLeases.leaseExpiresAt})`, + phase: Dz.sql`IF(${available}, 'claimed', ${Db.productAnalyticsErasureLeases.phase})`, + pausedPipes: Dz.sql`IF(${available}, JSON_ARRAY(), ${Db.productAnalyticsErasureLeases.pausedPipes})`, + userId: Dz.sql`IF(${available}, ${scope.userId ?? null}, ${Db.productAnalyticsErasureLeases.userId})`, + organizationId: Dz.sql`IF(${available}, ${scope.organizationId ?? null}, ${Db.productAnalyticsErasureLeases.organizationId})`, + attemptCount: Dz.sql`IF(${available}, ${Db.productAnalyticsErasureLeases.attemptCount} + 1, ${Db.productAnalyticsErasureLeases.attemptCount})`, + lastErrorCode: Dz.sql`IF(${available}, NULL, ${Db.productAnalyticsErasureLeases.lastErrorCode})`, + updatedAt: Dz.sql`IF(${available}, CURRENT_TIMESTAMP, ${Db.productAnalyticsErasureLeases.updatedAt})`, + }, + }), + ); + return yield* readOwned(ownerId); + }); + + const claimRecovery: ProductAnalyticsErasureLeaseStore["claimRecovery"] = + () => + Effect.gen(function* () { + const ownerId = randomUUID(); + const result = yield* database.use((db) => + db + .update(Db.productAnalyticsErasureLeases) + .set({ + ownerId, + fencingToken: Dz.sql`${Db.productAnalyticsErasureLeases.fencingToken} + 1`, + leaseExpiresAt: leaseExpiry(), + attemptCount: Dz.sql`${Db.productAnalyticsErasureLeases.attemptCount} + 1`, + updatedAt: new Date(), + }) + .where( + Dz.and( + Dz.eq(Db.productAnalyticsErasureLeases.name, LEASE_NAME), + Dz.ne(Db.productAnalyticsErasureLeases.phase, "idle"), + Dz.or( + Dz.isNull(Db.productAnalyticsErasureLeases.ownerId), + Dz.isNull( + Db.productAnalyticsErasureLeases.leaseExpiresAt, + ), + Dz.sql`${Db.productAnalyticsErasureLeases.leaseExpiresAt} < CURRENT_TIMESTAMP`, + ), + ), + ), + ); + if (affectedRows(result) === 0) return null; + return yield* readOwned(ownerId); + }); + + const ownedFence = (lease: ProductAnalyticsErasureLease) => + Dz.and( + Dz.eq(Db.productAnalyticsErasureLeases.name, LEASE_NAME), + Dz.eq(Db.productAnalyticsErasureLeases.ownerId, lease.ownerId), + Dz.eq( + Db.productAnalyticsErasureLeases.fencingToken, + lease.fencingToken, + ), + ); + + const heartbeat: ProductAnalyticsErasureLeaseStore["heartbeat"] = ( + lease, + ) => + Effect.gen(function* () { + const result = yield* database.use((db) => + db + .update(Db.productAnalyticsErasureLeases) + .set({ leaseExpiresAt: leaseExpiry(), updatedAt: new Date() }) + .where(ownedFence(lease)), + ); + if (affectedRows(result) > 0) return true; + return Boolean(yield* readOwned(lease.ownerId)); + }); + + const advance: ProductAnalyticsErasureLeaseStore["advance"] = ( + lease, + phase, + pausedPipes, + ) => + Effect.gen(function* () { + const result = yield* database.use((db) => + db + .update(Db.productAnalyticsErasureLeases) + .set({ + phase, + pausedPipes: [...pausedPipes], + leaseExpiresAt: leaseExpiry(), + updatedAt: new Date(), + }) + .where(ownedFence(lease)), + ); + if (affectedRows(result) > 0) return true; + return Boolean(yield* readOwned(lease.ownerId)); + }); + + const complete: ProductAnalyticsErasureLeaseStore["complete"] = (lease) => + database.use(async (db) => { + const result = await db + .update(Db.productAnalyticsErasureLeases) + .set({ + ownerId: null, + requestId: null, + leaseExpiresAt: null, + phase: "idle", + pausedPipes: [], + userId: null, + organizationId: null, + lastErrorCode: null, + updatedAt: new Date(), + }) + .where(ownedFence(lease)); + return affectedRows(result) > 0; + }); + + const fail: ProductAnalyticsErasureLeaseStore["fail"] = ( + lease, + pausedPipes, + lastErrorCode, + ) => + database.use(async (db) => { + const result = await db + .update(Db.productAnalyticsErasureLeases) + .set({ + ownerId: null, + leaseExpiresAt: null, + phase: "failed", + pausedPipes: [...pausedPipes], + lastErrorCode, + updatedAt: new Date(), + }) + .where(ownedFence(lease)); + return affectedRows(result) > 0; + }); + + return { + claimNew, + claimRecovery, + heartbeat, + advance, + complete, + fail, + } satisfies ProductAnalyticsErasureLeaseStore; + }), + dependencies: [Database.Default], + }, +) {} diff --git a/packages/web-backend/src/Tinybird/index.ts b/packages/web-backend/src/Tinybird/index.ts index bc74cee867d..da606a13f67 100644 --- a/packages/web-backend/src/Tinybird/index.ts +++ b/packages/web-backend/src/Tinybird/index.ts @@ -1,6 +1,11 @@ import { serverEnv } from "@cap/env"; import { Effect } from "effect"; +import { + type ProductAnalyticsErasureLease, + ProductAnalyticsErasureLeaseRepo, +} from "./ProductAnalyticsErasureLeaseRepo.ts"; + const DEFAULT_DATASOURCE = "analytics_events"; const PRODUCT_ANALYTICS_REBUILD_PIPES = [ "snapshot_product_events_canonical_v1", @@ -9,6 +14,7 @@ const PRODUCT_ANALYTICS_REBUILD_PIPES = [ "snapshot_product_traffic_pages_daily_exact", "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", + "snapshot_product_identity_funnel_exact", "snapshot_product_events_health_hourly", ] as const; @@ -32,19 +38,24 @@ interface TinybirdJobResponse { }; } +interface TinybirdPipeResponse { + schedule?: { status?: string }; +} + const tinybirdJobId = (response: TinybirdJobResponse) => { const id = response.job_id ?? response.job?.id ?? response.id; return typeof id === "string" && id ? id : undefined; }; -const tinybirdJobStatus = (response: TinybirdJobResponse) => { - const status = - response.status ?? - response.state ?? - response.job?.status ?? - response.job?.state; - return typeof status === "string" ? status.toLowerCase() : ""; -}; +const PRODUCT_ANALYTICS_COPY_MARKERS = { + snapshot_product_events_daily_exact: "decision_markers", + snapshot_product_traffic_daily_exact: "traffic_markers", + snapshot_product_traffic_pages_daily_exact: "traffic_page_markers", + snapshot_product_activation_daily_exact: "activation_markers", + snapshot_product_creator_retention_exact: "retention_markers", + snapshot_product_identity_funnel_exact: "identity_markers", + snapshot_product_events_health_hourly: "health_markers", +} as const; export interface TinybirdEventRow { timestamp: string; @@ -65,12 +76,19 @@ export interface TinybirdEventRow { export class Tinybird extends Effect.Service()("Tinybird", { effect: Effect.gen(function* () { + const erasureLeases = yield* ProductAnalyticsErasureLeaseRepo; const env = serverEnv(); const token = env.TINYBIRD_TOKEN; const host = env.TINYBIRD_HOST; const productAnalyticsHost = env.PRODUCT_ANALYTICS_TINYBIRD_HOST; const productAnalyticsErasureToken = - process.env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN; + env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN; + const productAnalyticsErasureLookupToken = + env.PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN; + const productAnalyticsCopyToken = env.PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN; + const productAnalyticsReadToken = env.PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN; + const productAnalyticsSchedulerToken = + env.PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN; const enabled = Boolean(token && host); @@ -330,15 +348,19 @@ export class Tinybird extends Effect.Service()("Tinybird", { }).pipe(Effect.asVoid); }; - const productAnalyticsRequest = (path: string, init?: RequestInit) => { + const productAnalyticsRequest = ( + path: string, + auth: { token: string | undefined; purpose: string }, + init?: RequestInit, + ) => { if (!productAnalyticsHost) { return Effect.fail( new Error("Product analytics erasure host is not configured"), ); } - if (!productAnalyticsErasureToken) { + if (!auth.token) { return Effect.fail( - new Error("Product analytics erasure is not configured"), + new Error(`Product analytics ${auth.purpose} is not configured`), ); } return Effect.tryPromise({ @@ -346,7 +368,7 @@ export class Tinybird extends Effect.Service()("Tinybird", { const response = await fetch(`${productAnalyticsHost}${path}`, { ...init, headers: { - Authorization: `Bearer ${productAnalyticsErasureToken}`, + Authorization: `Bearer ${auth.token}`, ...(init?.headers ?? {}), }, signal: AbortSignal.timeout(65_000), @@ -370,6 +392,10 @@ export class Tinybird extends Effect.Service()("Tinybird", { ) => productAnalyticsRequest<{ mutation?: { is_done?: boolean } }>( `/v1/datasources/${encodeURIComponent(name)}/delete?wait=true&wait_max_seconds=60`, + { + token: productAnalyticsErasureToken, + purpose: "erasure deletion", + }, { method: "POST", body: new URLSearchParams({ delete_condition: deleteCondition }), @@ -387,61 +413,195 @@ export class Tinybird extends Effect.Service()("Tinybird", { ), ); - const runProductAnalyticsCopyPipe = (name: string) => + const runProductAnalyticsCopyPipe = (name: string, copyRunId?: string) => { + const search = new URLSearchParams({ _mode: "replace" }); + if (copyRunId) search.set("copy_run_id", copyRunId); + return productAnalyticsRequest( + `/v0/pipes/${encodeURIComponent(name)}/copy?${search.toString()}`, + { token: productAnalyticsCopyToken, purpose: "Copy execution" }, + { method: "POST" }, + ).pipe( + Effect.flatMap((copy) => + tinybirdJobId(copy) + ? Effect.void + : Effect.fail( + new Error("Product analytics copy did not return a job ID"), + ), + ), + ); + }; + + const queryProductAnalyticsSql = (sql: string) => + productAnalyticsRequest<{ data: T[] }>( + `/v0/sql?q=${encodeURIComponent(sql)}&format=JSON`, + { + token: productAnalyticsErasureLookupToken, + purpose: "erasure lookup", + }, + ).pipe(Effect.map((result) => result.data ?? [])); + + const setProductAnalyticsCopySchedulePaused = ( + name: (typeof PRODUCT_ANALYTICS_REBUILD_PIPES)[number], + paused: boolean, + ) => { + const auth = { + token: productAnalyticsSchedulerToken, + purpose: "Copy schedule control", + }; + const attempt = productAnalyticsRequest( + `/v0/pipes/${encodeURIComponent(name)}/copy/${paused ? "cancel" : "resume"}`, + auth, + { method: "POST" }, + ).pipe( + Effect.either, + Effect.flatMap((mutation) => + productAnalyticsRequest( + `/v0/pipes/${encodeURIComponent(name)}`, + auth, + ).pipe( + Effect.flatMap((pipe) => { + const status = pipe.schedule?.status?.toLowerCase() ?? ""; + const matches = paused + ? status === "paused" + : status === "scheduled" || status === "active"; + if (matches) return Effect.void; + if (mutation._tag === "Left") return Effect.fail(mutation.left); + return Effect.fail( + new Error( + `Product analytics schedule state did not become ${paused ? "paused" : "active"}`, + ), + ); + }), + ), + ), + ); + return attempt.pipe(Effect.retry({ times: 3 })); + }; + + const resumeProductAnalyticsCopySchedules = ( + names: ReadonlyArray<(typeof PRODUCT_ANALYTICS_REBUILD_PIPES)[number]>, + ) => + Effect.forEach( + names, + (name) => + setProductAnalyticsCopySchedulePaused(name, false).pipe( + Effect.either, + ), + { concurrency: 1 }, + ).pipe( + Effect.flatMap((outcomes) => { + const failure = outcomes.find((outcome) => outcome._tag === "Left"); + return failure?._tag === "Left" + ? Effect.fail(failure.left) + : Effect.void; + }), + ); + + const pauseProductAnalyticsCopySchedules = ( + onPaused: ( + paused: ReadonlyArray<(typeof PRODUCT_ANALYTICS_REBUILD_PIPES)[number]>, + ) => Effect.Effect, + ) => Effect.gen(function* () { - const copy = yield* productAnalyticsRequest( - `/v0/pipes/${encodeURIComponent(name)}/copy?_mode=replace`, - { method: "POST" }, - ); - const jobId = tinybirdJobId(copy); - if (!jobId) { - return yield* Effect.fail( - new Error("Product analytics copy did not return a job ID"), - ); + const paused: Array<(typeof PRODUCT_ANALYTICS_REBUILD_PIPES)[number]> = + []; + for (const name of PRODUCT_ANALYTICS_REBUILD_PIPES) { + const outcome = yield* setProductAnalyticsCopySchedulePaused( + name, + true, + ).pipe(Effect.either); + if (outcome._tag === "Left") { + const resumed = yield* resumeProductAnalyticsCopySchedules( + paused, + ).pipe(Effect.either); + if (resumed._tag === "Right") yield* onPaused([]); + return yield* Effect.fail(outcome.left); + } + paused.push(name); + yield* onPaused(paused); } + return paused; + }); + const waitForProductAnalytics = ( + label: string, + read: () => Effect.Effect, + accept: (value: T) => boolean, + ) => + Effect.gen(function* () { for (let attempt = 0; attempt < 90; attempt += 1) { - const job = yield* productAnalyticsRequest( - `/v0/jobs/${encodeURIComponent(jobId)}`, - ); - const status = tinybirdJobStatus(job); - if (["done", "success", "finished", "completed"].includes(status)) { - return; - } - if (["failed", "error", "cancelled", "canceled"].includes(status)) { - return yield* Effect.fail( - new Error(`Product analytics copy job ended in ${status}`), - ); - } + const value = yield* read(); + if (accept(value)) return value; yield* Effect.sleep(2_000); } - return yield* Effect.fail( - new Error("Product analytics copy job timed out"), + new Error(`Product analytics ${label} timed out`), ); }); - const queryProductAnalyticsSql = (sql: string) => - productAnalyticsRequest<{ data: T[] }>( - `/v0/sql?q=${encodeURIComponent(sql)}&format=JSON`, - ).pipe(Effect.map((result) => result.data ?? [])); + const queryProductAnalyticsCopyMarker = ( + copyRunId: string, + marker: string, + ) => + productAnalyticsRequest<{ data?: Array> }>( + `/v0/pipes/product_analytics_copy_assertions.json?copy_run_id=${encodeURIComponent(copyRunId)}`, + { + token: productAnalyticsReadToken, + purpose: "aggregate read", + }, + ).pipe(Effect.map((result) => Number(result.data?.[0]?.[marker] ?? 0))); - const eraseProductAnalytics = ({ - userId, - organizationId, - }: { - userId?: string; - organizationId?: string; - }) => - Effect.gen(function* () { + const runProductAnalyticsErasure = ( + lease: ProductAnalyticsErasureLease, + ) => { + let pausedPipes: Array<(typeof PRODUCT_ANALYTICS_REBUILD_PIPES)[number]> = + lease.pausedPipes.filter( + (name): name is (typeof PRODUCT_ANALYTICS_REBUILD_PIPES)[number] => + PRODUCT_ANALYTICS_REBUILD_PIPES.some((known) => known === name), + ); + + const requireLeaseUpdate = (updated: Effect.Effect) => + updated.pipe( + Effect.flatMap((owned) => + owned + ? Effect.void + : Effect.fail( + new Error("Product analytics erasure lease was fenced"), + ), + ), + ); + + const advance = ( + phase: "pausing" | "deleting" | "rebuilding" | "resuming", + paused: readonly string[] = pausedPipes, + ) => requireLeaseUpdate(erasureLeases.advance(lease, phase, paused)); + + const heartbeat = Effect.forever( + Effect.sleep(20_000).pipe( + Effect.zipRight(requireLeaseUpdate(erasureLeases.heartbeat(lease))), + ), + ); + + const operation = Effect.gen(function* () { + if (pausedPipes.length > 0) { + yield* advance("resuming"); + yield* resumeProductAnalyticsCopySchedules(pausedPipes); + pausedPipes = []; + yield* advance("pausing", []); + } + pausedPipes = yield* pauseProductAnalyticsCopySchedules((paused) => { + pausedPipes = [...paused]; + return advance("pausing", paused); + }); + yield* advance("deleting"); const conditions: string[] = []; - if (organizationId) { + if (lease.scope.organizationId) { conditions.push( - `organization_id = '${escapeTinybirdString(organizationId)}'`, + `organization_id = '${escapeTinybirdString(lease.scope.organizationId)}'`, ); } - if (userId) { - const escapedUserId = escapeTinybirdString(userId); + if (lease.scope.userId) { + const escapedUserId = escapeTinybirdString(lease.scope.userId); const anonymousRows = yield* queryProductAnalyticsSql<{ anonymous_id: string; }>( @@ -473,15 +633,116 @@ export class Tinybird extends Effect.Service()("Tinybird", { new Error("Product analytics erasure requires an identity"), ); } - yield* deleteProductAnalyticsData( - "product_events_v1", - `(${conditions.join(" OR ")})`, + const deleteCondition = `(${conditions.join(" OR ")})`; + const countRows = (datasource: string) => + queryProductAnalyticsSql<{ matching_rows: number | string }>( + `SELECT count() AS matching_rows FROM ${datasource} WHERE ${deleteCondition}`, + ).pipe(Effect.map((rows) => Number(rows[0]?.matching_rows ?? 0))); + const rawRows = yield* countRows("product_events_v1"); + if (rawRows > 0) { + yield* advance("deleting"); + yield* deleteProductAnalyticsData( + "product_events_v1", + deleteCondition, + ); + yield* waitForProductAnalytics( + "raw erasure visibility", + () => countRows("product_events_v1"), + (count) => count === 0, + ); + } + yield* advance("rebuilding"); + yield* runProductAnalyticsCopyPipe( + "snapshot_product_events_canonical_v1", + ); + yield* waitForProductAnalytics( + "canonical erasure visibility", + () => countRows("product_events_canonical_v1"), + (count) => count === 0, ); - yield* Effect.forEach( - PRODUCT_ANALYTICS_REBUILD_PIPES, - runProductAnalyticsCopyPipe, - { concurrency: 1 }, + const copyRunId = `erasure_${lease.requestId.replaceAll("-", "")}`; + for (const [pipe, marker] of Object.entries( + PRODUCT_ANALYTICS_COPY_MARKERS, + )) { + yield* advance("rebuilding"); + yield* runProductAnalyticsCopyPipe(pipe, copyRunId); + yield* waitForProductAnalytics( + `${pipe} visibility`, + () => queryProductAnalyticsCopyMarker(copyRunId, marker), + (count) => count === 1, + ); + } + yield* waitForProductAnalytics( + "final erasure verification", + () => + Effect.all([ + countRows("product_events_v1"), + countRows("product_events_canonical_v1"), + ]), + ([rawCount, canonicalCount]) => + rawCount === 0 && canonicalCount === 0, ); + yield* advance("resuming"); + yield* resumeProductAnalyticsCopySchedules(pausedPipes); + pausedPipes = []; + yield* advance("resuming", []); + yield* requireLeaseUpdate(erasureLeases.complete(lease)); + }); + + const recoverableOperation = Effect.gen(function* () { + const outcome = yield* operation.pipe(Effect.either); + if (outcome._tag === "Right") return; + const resumed = yield* resumeProductAnalyticsCopySchedules( + pausedPipes, + ).pipe(Effect.either); + if (resumed._tag === "Right") pausedPipes = []; + const failed = yield* erasureLeases.fail( + lease, + pausedPipes, + resumed._tag === "Left" ? "resume_failed" : "erase_failed", + ); + if (!failed) { + return yield* Effect.fail( + new Error("Product analytics erasure lease was fenced"), + ); + } + return yield* Effect.fail(outcome.left); + }); + + return Effect.raceFirst(recoverableOperation, heartbeat); + }; + + const recoverProductAnalyticsErasure = Effect.gen(function* () { + const lease = yield* erasureLeases.claimRecovery(); + if (!lease) return { recovered: false as const }; + yield* runProductAnalyticsErasure(lease); + return { recovered: true as const, requestId: lease.requestId }; + }); + + const eraseProductAnalytics = ({ + userId, + organizationId, + }: { + userId?: string; + organizationId?: string; + }) => + Effect.gen(function* () { + if (!userId && !organizationId) { + return yield* Effect.fail( + new Error("Product analytics erasure requires an identity"), + ); + } + let lease = yield* erasureLeases.claimNew({ userId, organizationId }); + if (!lease) { + yield* recoverProductAnalyticsErasure; + lease = yield* erasureLeases.claimNew({ userId, organizationId }); + } + if (!lease) { + return yield* Effect.fail( + new Error("Product analytics erasure is already in progress"), + ); + } + yield* runProductAnalyticsErasure(lease); }); return { @@ -494,6 +755,8 @@ export class Tinybird extends Effect.Service()("Tinybird", { deleteProductAnalyticsData, runProductAnalyticsCopyPipe, eraseProductAnalytics, + recoverProductAnalyticsErasure, } as const; }), + dependencies: [ProductAnalyticsErasureLeaseRepo.Default], }) {} diff --git a/packages/web-backend/src/index.ts b/packages/web-backend/src/index.ts index d74bf98060b..7067cb60259 100644 --- a/packages/web-backend/src/index.ts +++ b/packages/web-backend/src/index.ts @@ -35,6 +35,12 @@ export { StorageRepo, } from "./Storage/StorageRepo.ts"; export { Tinybird } from "./Tinybird/index.ts"; +export { + type ProductAnalyticsErasureLease, + ProductAnalyticsErasureLeaseRepo, + type ProductAnalyticsErasureLeaseStore, + type ProductAnalyticsErasureScope, +} from "./Tinybird/ProductAnalyticsErasureLeaseRepo.ts"; export { Users } from "./Users/index.ts"; export { collectPasswordHashes, From d351872b90514bb5b346e74aa167c338c1eb545b Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:15:19 +0100 Subject: [PATCH 077/110] feat: add exact Tinybird decision aggregates --- .../unit/admin-product-analytics.test.ts | 84 ++++++ apps/web/app/admin/analytics/page.tsx | 272 +++++++++++++++++- apps/web/app/admin/analytics/tinybird.ts | 168 ++++++++++- scripts/analytics/tests/datafiles.test.js | 124 +++++++- scripts/analytics/tests/tooling.test.js | 123 +++++++- .../product_activation_daily_exact.datasource | 9 +- ...product_creator_retention_exact.datasource | 9 +- .../product_events_canonical_v1.datasource | 5 +- .../product_events_daily_exact.datasource | 31 +- ...duct_events_health_hourly_exact.datasource | 6 +- .../datasources/product_events_v1.datasource | 3 +- .../product_identity_funnel_exact.datasource | 34 +++ .../product_traffic_daily_exact.datasource | 9 +- ...oduct_traffic_pages_daily_exact.datasource | 9 +- .../tinybird/pipes/product_activation.pipe | 8 +- .../product_analytics_ci_assertions.pipe | 90 +++++- .../product_analytics_copy_assertions.pipe | 5 +- .../pipes/product_analytics_freshness.pipe | 6 +- .../pipes/product_creator_activity.pipe | 9 +- .../pipes/product_creator_retention.pipe | 6 + .../tinybird/pipes/product_events_daily.pipe | 47 ++- .../tinybird/pipes/product_events_health.pipe | 3 +- .../pipes/product_feature_adoption.pipe | 10 +- .../pipes/product_identity_funnel.pipe | 58 ++++ .../pipes/product_traffic_countries.pipe | 6 + .../pipes/product_traffic_overview.pipe | 8 +- .../tinybird/pipes/product_traffic_pages.pipe | 6 + .../pipes/product_traffic_sources.pipe | 6 + .../pipes/product_traffic_technology.pipe | 6 + ...apshot_product_activation_daily_exact.pipe | 14 +- ...pshot_product_creator_retention_exact.pipe | 19 +- .../snapshot_product_events_canonical_v1.pipe | 2 +- .../snapshot_product_events_daily_exact.pipe | 61 +++- ...snapshot_product_events_health_hourly.pipe | 21 +- ...napshot_product_identity_funnel_exact.pipe | 234 +++++++++++++++ .../snapshot_product_traffic_daily_exact.pipe | 18 +- ...hot_product_traffic_pages_daily_exact.pipe | 23 +- scripts/analytics/tooling.js | 149 +++++++++- scripts/analytics/verify-local.js | 1 + 39 files changed, 1606 insertions(+), 96 deletions(-) create mode 100644 scripts/analytics/tinybird/datasources/product_identity_funnel_exact.datasource create mode 100644 scripts/analytics/tinybird/pipes/product_identity_funnel.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe diff --git a/apps/web/__tests__/unit/admin-product-analytics.test.ts b/apps/web/__tests__/unit/admin-product-analytics.test.ts index e9430d3c82c..985999405e0 100644 --- a/apps/web/__tests__/unit/admin-product-analytics.test.ts +++ b/apps/web/__tests__/unit/admin-product-analytics.test.ts @@ -5,9 +5,13 @@ vi.mock("server-only", () => ({})); import { AdminAnalyticsConfigurationError, AdminAnalyticsRequestError, + assertAdminAnalyticsDateRange, buildAdminAnalyticsEndpointUrl, calculateHealthWindowStart, + decodeAnalyticsFreshnessResponse, + decodeProductEventsResponse, decodeTrafficOverviewResponse, + fetchOptionalRollbackEndpoint, } from "@/app/admin/analytics/tinybird"; describe("admin analytics Tinybird client", () => { @@ -54,6 +58,86 @@ describe("admin analytics Tinybird client", () => { ); }); + it("allows two year-over-year windows but rejects dates beyond retention", () => { + expect(() => + assertAdminAnalyticsDateRange("2024-06-01", "2026-07-31"), + ).not.toThrow(); + expect(() => + assertAdminAnalyticsDateRange("2024-05-01", "2026-07-31"), + ).toThrow("no longer than 800 UTC days"); + }); + + it("degrades only a missing rollback-era optional endpoint", async () => { + const missing = async () => { + throw new AdminAnalyticsRequestError("missing", 404); + }; + await expect( + fetchOptionalRollbackEndpoint( + missing, + "product_identity_funnel", + {}, + () => ({ linkedUsers: 1 }), + ), + ).resolves.toEqual({ available: false, rows: [] }); + + const unavailable = async () => { + throw new AdminAnalyticsRequestError("unavailable", 503); + }; + await expect( + fetchOptionalRollbackEndpoint( + unavailable, + "product_identity_funnel", + {}, + () => ({ linkedUsers: 1 }), + ), + ).rejects.toMatchObject({ status: 503 }); + }); + + it("decodes the retained aggregate schema during rollback", () => { + expect( + decodeProductEventsResponse({ + data: [ + { + date: "2026-07-31", + event_name: "purchase_completed", + source: "server", + platform: "web", + app_version: "web", + hostname: "cap.so", + country: "GB", + device: "desktop", + browser: "Chrome", + os: "macOS", + channel: "direct", + plan_id: "price_pro", + payment_status: "paid", + subscription_status: "active", + currency: "GBP", + billing_interval: "month", + events: 1, + actors: 1, + users: 1, + organizations: 1, + revenue_minor: 2_500, + }, + ], + }), + ).toMatchObject([{ changeKind: "", revenueMinor: 2_500 }]); + expect( + decodeAnalyticsFreshnessResponse({ + data: [ + { + latest_received_hour: "2026-07-31 10:00:00", + health_freshness_ms: 1_000, + product_calculated_at: "2026-07-31 10:01:00", + traffic_calculated_at: "2026-07-31 10:01:00", + retention_calculated_at: "2026-07-31 10:01:00", + }, + ], + }), + ).toMatchObject([{ identityCalculatedAt: "" }]); + }); + it("decodes numeric Tinybird values without accepting malformed rows", () => { expect( decodeTrafficOverviewResponse({ diff --git a/apps/web/app/admin/analytics/page.tsx b/apps/web/app/admin/analytics/page.tsx index 20a299b8141..11c59d6eed3 100644 --- a/apps/web/app/admin/analytics/page.tsx +++ b/apps/web/app/admin/analytics/page.tsx @@ -207,6 +207,43 @@ function eventCount(data: AdminAnalyticsDashboard, eventName: string): number { ); } +function eventValue( + data: AdminAnalyticsDashboard, + eventName: string, + field: "attemptCount" | "seatDelta", +): number { + return sumBy( + data.productEvents.filter((row) => row.eventName === eventName), + (row) => row[field] * row.events, + ); +} + +function eventMaxValue( + data: AdminAnalyticsDashboard, + eventName: string, + field: "attemptCount", +): number { + return Math.max( + 0, + ...data.productEvents + .filter((row) => row.eventName === eventName) + .map((row) => row[field]), + ); +} + +function eventCountWhere( + data: AdminAnalyticsDashboard, + eventName: string, + predicate: (row: AdminAnalyticsDashboard["productEvents"][number]) => boolean, +): number { + return sumBy( + data.productEvents.filter( + (row) => row.eventName === eventName && predicate(row), + ), + (row) => row.events, + ); +} + function retentionRate( data: AdminAnalyticsDashboard, cohortDay: number, @@ -787,6 +824,16 @@ function RevenueSection({ data }: { data: AdminAnalyticsDashboard }) { label="Purchases" value={formatInteger(eventCount(data, "purchase_completed"))} /> + row.firstPurchase === "true", + ), + )} + /> row.changeKind === "plan" || row.changeKind === "seats", + ), + )} + /> + + row.changeKind === "cancellation_scheduled", + ), + )} /> + row.fullyRefunded === "true", + ), + )} + /> + {revenue.map(([currency, minorUnits]) => ( (); + for (const row of data.productEvents) { + if (row.eventName !== "experiment_exposed" || !row.experimentId) continue; + const key = [ + row.experimentId, + row.experimentVariant, + row.assignmentVersion, + ].join("\u0000"); + const current = exposures.get(key) ?? { + experimentId: row.experimentId, + variant: row.experimentVariant, + assignmentVersion: row.assignmentVersion, + exposures: 0, + actorDays: 0, + userDays: 0, + }; + current.exposures += row.events; + current.actorDays += row.actors; + current.userDays += row.users; + exposures.set(key, current); + } + const rows = [...exposures.values()].sort( + (left, right) => + left.experimentId.localeCompare(right.experimentId) || + left.assignmentVersion.localeCompare(right.assignmentVersion) || + left.variant.localeCompare(right.variant), + ); + + return ( +
+ {rows.length === 0 ? ( + No experiment exposures in this period. + ) : ( +
+ + + + + + + + + + + + + {rows.map((row) => ( + + + + + + + + + ))} + +
ExperimentVersionVariantExposuresActor-daysUser-days
+ {row.experimentId} + + {row.assignmentVersion} + {row.variant} + {formatInteger(row.exposures)} + + {formatInteger(row.actorDays)} + + {formatInteger(row.userDays)} +
+
+ )} +
+ ); +} + +function IdentityFunnelSection({ data }: { data: AdminAnalyticsDashboard }) { + if (!data.identityFunnelAvailable) { + return ( +
+ + Identity funnel metrics are unavailable on this deployment. + +
+ ); + } + if (data.identityFunnel.length === 0) { + return ( +
+ No identity funnel activity in this period. +
+ ); + } + const total = (field: keyof (typeof data.identityFunnel)[number]) => + data.identityFunnel.reduce((sum, row) => { + const value = row[field]; + return sum + (typeof value === "number" ? value : 0); + }, 0); + return ( +
+
+ + + + + + + + +
+
+ ); +} + function QualitySection({ data }: { data: AdminAnalyticsDashboard }) { const freshness = data.freshness[0]; const health = data.health[0]; const status = qualityStatus(data); const lag = health?.ingestionLagMs ?? []; + const reportedDeliveryLosses = sumBy( + data.productEvents, + (row) => row.deliveryLossCount, + ); return (
+ @@ -900,6 +1155,11 @@ function QualitySection({ data }: { data: AdminAnalyticsDashboard }) { label="Retention calculated" value={formatTimestamp(freshness?.retentionCalculatedAt ?? "")} /> +
); @@ -936,10 +1196,18 @@ function Definitions() { "Revenue", "Server-authoritative tracked revenue in original-currency minor units. No FX or decimal conversion is applied here.", ], + [ + "Identity stitching", + "A server-authoritative link or settled guest purchase connects one anonymous acquisition identity to an authenticated user. The endpoint returns cohort counts only, never the mapping.", + ], [ "Deduplication", "Decision-facing endpoints are built from stable event IDs. Duplicate deliveries remain visible in health but count once in metrics.", ], + [ + "Experiment exposure", + "A typed, stable assignment rendered to an actor. Variants are read only from exposure events and are never inferred from conversion behavior.", + ], [ "Privacy", "This page queries aggregate endpoints only and never renders actor, user, organization, network, or raw user-agent identifiers.", @@ -1026,8 +1294,10 @@ export default async function AdminAnalyticsPage({ + + diff --git a/apps/web/app/admin/analytics/tinybird.ts b/apps/web/app/admin/analytics/tinybird.ts index d0fb6b95a33..e25f9b2a9b1 100644 --- a/apps/web/app/admin/analytics/tinybird.ts +++ b/apps/web/app/admin/analytics/tinybird.ts @@ -9,6 +9,7 @@ const ADMIN_ANALYTICS_ENDPOINTS = [ "product_activation", "product_creator_activity", "product_creator_retention", + "product_identity_funnel", "product_events_daily", "product_feature_adoption", "product_events_health", @@ -112,6 +113,7 @@ export type CreatorRetentionRow = { export type ProductEventRow = { date: string; eventName: string; + schemaVersion: number; source: string; platform: string; appVersion: string; @@ -126,6 +128,27 @@ export type ProductEventRow = { subscriptionStatus: string; currency: string; billingInterval: string; + changeKind: string; + previousStatus: string; + newStatus: string; + previousPlanId: string; + quantity: number; + previousQuantity: number; + newQuantity: number; + seatDelta: number; + firstPurchase: string; + guestCheckout: string; + onboarding: string; + cancelAtPeriodEnd: string; + fullyRefunded: string; + endedAt: number; + trialEndAt: number; + amountDueMinor: number; + attemptCount: number; + experimentId: string; + experimentVariant: string; + assignmentVersion: string; + deliveryLossCount: number; events: number; actors: number; users: number; @@ -133,6 +156,24 @@ export type ProductEventRow = { revenueMinor: number; }; +export type IdentityFunnelRow = { + linkedVisitors: number; + linkedUsers: number; + signupUsers: number; + organizations: number; + guestCheckoutVisitors: number; + guestPurchasers: number; + authenticatedCheckoutUsers: number; + webCheckoutUsers: number; + desktopCheckoutUsers: number; + mobileCheckoutUsers: number; + crossDeviceCheckoutUsers: number; + trialUsers: number; + purchasers: number; + signupRate: number; + purchaseRate: number; +}; + export type FeatureAdoptionRow = { eventName: string; events: number; @@ -159,6 +200,7 @@ export type AnalyticsFreshnessRow = { productCalculatedAt: string; trafficCalculatedAt: string; retentionCalculatedAt: string; + identityCalculatedAt: string; }; export type AdminAnalyticsDashboard = { @@ -170,6 +212,8 @@ export type AdminAnalyticsDashboard = { activation: ActivationRow[]; creatorActivity: CreatorActivityRow[]; creatorRetention: CreatorRetentionRow[]; + identityFunnel: IdentityFunnelRow[]; + identityFunnelAvailable: boolean; productEvents: ProductEventRow[]; featureAdoption: FeatureAdoptionRow[]; health: ProductEventsHealthRow[]; @@ -185,7 +229,10 @@ export class AdminAnalyticsConfigurationError extends Error { } export class AdminAnalyticsRequestError extends Error { - constructor(message: string) { + constructor( + message: string, + readonly status?: number, + ) { super(message); this.name = "AdminAnalyticsRequestError"; } @@ -205,6 +252,12 @@ function readString(row: UnknownRecord, key: string): string { return value; } +function readOptionalString(row: UnknownRecord, key: string): string { + const value = row[key]; + if (value === undefined || value === null) return ""; + return readString(row, key); +} + function readNumber(row: UnknownRecord, key: string): number { const value = row[key]; const parsed = typeof value === "number" ? value : Number(value); @@ -216,6 +269,12 @@ function readNumber(row: UnknownRecord, key: string): number { return parsed; } +function readOptionalNumber(row: UnknownRecord, key: string): number { + const value = row[key]; + if (value === undefined || value === null) return 0; + return readNumber(row, key); +} + function readNumberArray(row: UnknownRecord, key: string): number[] { const value = row[key]; if (!Array.isArray(value)) { @@ -376,10 +435,31 @@ function decodeCreatorRetentionRow(row: UnknownRecord): CreatorRetentionRow { }; } +function decodeIdentityFunnelRow(row: UnknownRecord): IdentityFunnelRow { + return { + linkedVisitors: readNumber(row, "linked_visitors"), + linkedUsers: readNumber(row, "linked_users"), + signupUsers: readNumber(row, "signup_users"), + organizations: readNumber(row, "organizations"), + guestCheckoutVisitors: readNumber(row, "guest_checkout_visitors"), + guestPurchasers: readNumber(row, "guest_purchasers"), + authenticatedCheckoutUsers: readNumber(row, "authenticated_checkout_users"), + webCheckoutUsers: readNumber(row, "web_checkout_users"), + desktopCheckoutUsers: readNumber(row, "desktop_checkout_users"), + mobileCheckoutUsers: readNumber(row, "mobile_checkout_users"), + crossDeviceCheckoutUsers: readNumber(row, "cross_device_checkout_users"), + trialUsers: readNumber(row, "trial_users"), + purchasers: readNumber(row, "purchasers"), + signupRate: readNumber(row, "signup_rate"), + purchaseRate: readNumber(row, "purchase_rate"), + }; +} + function decodeProductEventRow(row: UnknownRecord): ProductEventRow { return { date: readString(row, "date"), eventName: readString(row, "event_name"), + schemaVersion: readOptionalNumber(row, "schema_version"), source: readString(row, "source"), platform: readString(row, "platform"), appVersion: readString(row, "app_version"), @@ -394,6 +474,27 @@ function decodeProductEventRow(row: UnknownRecord): ProductEventRow { subscriptionStatus: readString(row, "subscription_status"), currency: readString(row, "currency"), billingInterval: readString(row, "billing_interval"), + changeKind: readOptionalString(row, "change_kind"), + previousStatus: readOptionalString(row, "previous_status"), + newStatus: readOptionalString(row, "new_status"), + previousPlanId: readOptionalString(row, "previous_plan_id"), + quantity: readOptionalNumber(row, "quantity"), + previousQuantity: readOptionalNumber(row, "previous_quantity"), + newQuantity: readOptionalNumber(row, "new_quantity"), + seatDelta: readOptionalNumber(row, "seat_delta"), + firstPurchase: readOptionalString(row, "first_purchase"), + guestCheckout: readOptionalString(row, "guest_checkout"), + onboarding: readOptionalString(row, "onboarding"), + cancelAtPeriodEnd: readOptionalString(row, "cancel_at_period_end"), + fullyRefunded: readOptionalString(row, "fully_refunded"), + endedAt: readOptionalNumber(row, "ended_at"), + trialEndAt: readOptionalNumber(row, "trial_end_at"), + amountDueMinor: readOptionalNumber(row, "amount_due_minor"), + attemptCount: readOptionalNumber(row, "attempt_count"), + experimentId: readOptionalString(row, "experiment_id"), + experimentVariant: readOptionalString(row, "experiment_variant"), + assignmentVersion: readOptionalString(row, "assignment_version"), + deliveryLossCount: readOptionalNumber(row, "delivery_loss_count"), events: readNumber(row, "events"), actors: readNumber(row, "actors"), users: readNumber(row, "users"), @@ -437,9 +538,20 @@ function decodeAnalyticsFreshnessRow( productCalculatedAt: readString(row, "product_calculated_at"), trafficCalculatedAt: readString(row, "traffic_calculated_at"), retentionCalculatedAt: readString(row, "retention_calculated_at"), + identityCalculatedAt: readOptionalString(row, "identity_calculated_at"), }; } +export function decodeProductEventsResponse(value: unknown): ProductEventRow[] { + return decodeTinybirdRows(value, decodeProductEventRow); +} + +export function decodeAnalyticsFreshnessResponse( + value: unknown, +): AnalyticsFreshnessRow[] { + return decodeTinybirdRows(value, decodeAnalyticsFreshnessRow); +} + type FetchEndpoint = ( endpoint: AdminAnalyticsEndpoint, params: QueryParams, @@ -477,6 +589,7 @@ function createFetchEndpoint(): FetchEndpoint { if (!response.ok) { throw new AdminAnalyticsRequestError( `Tinybird endpoint ${endpoint} returned HTTP ${response.status}`, + response.status, ); } @@ -492,6 +605,25 @@ function createFetchEndpoint(): FetchEndpoint { }; } +export async function fetchOptionalRollbackEndpoint( + fetchEndpoint: FetchEndpoint, + endpoint: AdminAnalyticsEndpoint, + params: QueryParams, + decodeRow: (row: UnknownRecord) => T, +): Promise<{ available: boolean; rows: T[] }> { + try { + return { + available: true, + rows: await fetchEndpoint(endpoint, params, decodeRow), + }; + } catch (error) { + if (error instanceof AdminAnalyticsRequestError && error.status === 404) { + return { available: false, rows: [] }; + } + throw error; + } +} + export function calculateHealthWindowStart( startDate: string, endDate: string, @@ -505,21 +637,28 @@ export function calculateHealthWindowStart( .slice(0, 10); } -export async function fetchAdminAnalyticsDashboard( - filters: AdminAnalyticsFilters, -): Promise { - const rangeStart = new Date(`${filters.startDate}T00:00:00.000Z`); - const rangeEnd = new Date(`${filters.endDate}T00:00:00.000Z`); +export function assertAdminAnalyticsDateRange( + startDate: string, + endDate: string, +) { + const rangeStart = new Date(`${startDate}T00:00:00.000Z`); + const rangeEnd = new Date(`${endDate}T00:00:00.000Z`); if ( !Number.isFinite(rangeStart.getTime()) || !Number.isFinite(rangeEnd.getTime()) || rangeEnd < rangeStart || - rangeEnd.getTime() - rangeStart.getTime() > 399 * 86_400_000 + rangeEnd.getTime() - rangeStart.getTime() > 799 * 86_400_000 ) { throw new AdminAnalyticsRequestError( - "The analytics date range must be valid, ordered, and no longer than 400 UTC days.", + "The analytics date range must be valid, ordered, and no longer than 800 UTC days.", ); } +} + +export async function fetchAdminAnalyticsDashboard( + filters: AdminAnalyticsFilters, +): Promise { + assertAdminAnalyticsDateRange(filters.startDate, filters.endDate); const fetchEndpoint = createFetchEndpoint(); const dateParams = { @@ -540,6 +679,7 @@ export async function fetchAdminAnalyticsDashboard( activation, creatorActivity, creatorRetention, + identityFunnelResult, productEvents, featureAdoption, health, @@ -585,6 +725,16 @@ export async function fetchAdminAnalyticsDashboard( }, decodeCreatorRetentionRow, ), + fetchOptionalRollbackEndpoint( + fetchEndpoint, + "product_identity_funnel", + { + ...dateParams, + source: filters.source, + country: filters.country, + }, + decodeIdentityFunnelRow, + ), fetchEndpoint( "product_events_daily", { @@ -647,6 +797,8 @@ export async function fetchAdminAnalyticsDashboard( activation, creatorActivity, creatorRetention, + identityFunnel: identityFunnelResult.rows, + identityFunnelAvailable: identityFunnelResult.available, productEvents, featureAdoption, health, diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index b9739da6201..4c9ddbc3250 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -64,9 +64,10 @@ test("product datasource matches the runtime event contract", () => { assert.equal(datasource.sortingKey, "(received_at, event_id)"); assert.equal(datasource.versionColumn, null); assert.equal(datasource.partitionKey, "toYYYYMM(received_at)"); - assert.equal(datasource.ttl, "toDateTime(received_at) + INTERVAL 400 DAY"); + assert.equal(datasource.ttl, "toDateTime(received_at) + INTERVAL 800 DAY"); assert.deepEqual(datasource.tokens, [ { name: "product_events_ingest", scope: "APPEND" }, + { name: "product_events_erasure_lookup", scope: "READ" }, ]); assert.equal( datasource.tokens.some( @@ -75,6 +76,56 @@ test("product datasource matches the runtime event contract", () => { ), false, ); + const retainedDatasources = [ + "product_events_canonical_v1", + "product_events_daily_exact", + "product_traffic_daily_exact", + "product_traffic_pages_daily_exact", + "product_activation_daily_exact", + "product_creator_retention_exact", + "product_identity_funnel_exact", + ]; + for (const name of retainedDatasources) { + const retained = project.datasources.find( + (candidate) => candidate.name === name, + ); + assert.ok(retained); + assert.match(retained.ttl ?? "", /\+ INTERVAL 800 DAY$/); + } + const copyTargets = [ + "product_events_canonical_v1", + "product_events_daily_exact", + "product_traffic_daily_exact", + "product_traffic_pages_daily_exact", + "product_activation_daily_exact", + "product_creator_retention_exact", + "product_identity_funnel_exact", + "product_events_health_hourly_exact", + ]; + for (const name of copyTargets) { + const target = project.datasources.find( + (candidate) => candidate.name === name, + ); + assert.ok(target); + assert.ok( + target.tokens.some( + (token) => + token.name === "product_events_copy_runner" && + token.scope === "APPEND", + ), + ); + } + const canonical = project.datasources.find( + (candidate) => candidate.name === "product_events_canonical_v1", + ); + assert.ok(canonical); + assert.ok( + canonical.tokens.some( + (token) => + token.name === "product_events_erasure_lookup" && + token.scope === "READ", + ), + ); }); test("existing viewer resources remain in the Tinybird project", () => { @@ -144,6 +195,31 @@ test("retention merges identities across activity platforms", () => { assert.match(contents, /uniqExactMerge\(creator_users\) AS creators/); }); +test("identity cohorts stitch acquisition and guest purchases without exposing mappings", () => { + const snapshot = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "snapshot_product_identity_funnel_exact.pipe", + ), + "utf8", + ); + const endpoint = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", "product_identity_funnel.pipe"), + "utf8", + ); + assert.match(snapshot, /event_name = 'identity_linked'/); + assert.match(snapshot, /event_name = 'guest_checkout_started'/); + assert.match(snapshot, /FROM guest_checkouts\n\s+LEFT JOIN guest_paths/); + assert.match(snapshot, /JSONExtractBool\(properties, 'is_guest_checkout'\)/); + assert.match(snapshot, /organization_first_links/); + assert.match(snapshot, /sum\(linked_organization\)/); + assert.match(snapshot, /cross_device_checkout_users/); + assert.doesNotMatch(endpoint, /user_id|anonymous_id|organization_id/); + assert.match(endpoint, /FROM product_identity_funnel_exact/); + assert.doesNotMatch(endpoint, /GROUP BY|LIMIT/); +}); + test("daily snapshot quarantines payload conflicts and rebuilds exact metrics", () => { const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); const canonical = project.pipes.find( @@ -200,6 +276,7 @@ test("copy schedules serialize canonical and derived rebuilds", () => { ["snapshot_product_traffic_pages_daily_exact", "4-59/8 * * * *"], ["snapshot_product_activation_daily_exact", "5-59/8 * * * *"], ["snapshot_product_creator_retention_exact", "6-59/8 * * * *"], + ["snapshot_product_identity_funnel_exact", "7-59/8 * * * *"], ]); for (const [name, schedule] of schedules) { const contents = fs.readFileSync( @@ -207,20 +284,51 @@ test("copy schedules serialize canonical and derived rebuilds", () => { "utf8", ); assert.ok(contents.split("\n").includes(`COPY_SCHEDULE ${schedule}`)); + assert.match(contents, /^TOKEN product_events_copy_runner READ$/m); + assert.doesNotMatch(contents, /^TOKEN product_events_agent_read READ$/m); assert.match(contents, /\{\{max_threads\(Int32\(copy_max_threads\)\)\}\}/); } assert.equal(new Set(schedules.values()).size, schedules.size); }); -test("staging copy markers are excluded from every decision endpoint", () => { - for (const name of [ +test("decision endpoints cannot be executed with the Copy runner token", () => { + const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); + for (const pipe of project.pipes.filter( + (candidate) => + candidate.name.startsWith("product_") && candidate.type === "endpoint", + )) { + assert.deepEqual(pipe.tokens, [ + { name: "product_events_agent_read", scope: "READ" }, + ]); + } +}); + +test("staging copy markers and synthetic rows are excluded from decision endpoints", () => { + const syntheticEndpoints = [ "product_traffic_overview", "product_traffic_pages", "product_traffic_sources", "product_traffic_countries", "product_traffic_technology", "product_activation", + "product_creator_activity", "product_creator_retention", + "product_events_daily", + "product_feature_adoption", + "product_identity_funnel", + ]; + for (const name of syntheticEndpoints) { + const contents = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), + "utf8", + ); + assert.match(contents, /synthetic_run_id = ''/); + assert.match(contents, /defined\(synthetic_run_id\)/); + assert.match(contents, /synthetic_run_id has an invalid length/); + } + for (const name of [ + ...syntheticEndpoints.slice(0, 8), + "product_identity_funnel", ]) { const contents = fs.readFileSync( path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), @@ -233,6 +341,7 @@ test("staging copy markers are excluded from every decision endpoint", () => { "snapshot_product_traffic_pages_daily_exact", "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", + "snapshot_product_identity_funnel_exact", ]) { const contents = fs.readFileSync( path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), @@ -240,6 +349,7 @@ test("staging copy markers are excluded from every decision endpoint", () => { ); assert.match(contents, /defined\(copy_run_id\)/); assert.match(contents, /AS copy_run_id/); + assert.match(contents, /AS synthetic_run_id/); } const assertions = fs.readFileSync( path.join( @@ -253,6 +363,7 @@ test("staging copy markers are excluded from every decision endpoint", () => { assert.match(assertions, /traffic_markers/); assert.doesNotMatch(assertions, /requested_copy_run_id/); assert.match(assertions, /retention_markers/); + assert.match(assertions, /identity_markers/); }); test("health queries use stable hourly aggregates and a bounded window", () => { @@ -290,8 +401,15 @@ test("staging assertions prove canonical decisions without returning raw IDs", ( assert.match(contents, /uniqExact\(payload_hash\) AS payloads/); assert.match(contents, /FROM product_events_canonical_v1/); assert.match(contents, /FROM product_events_daily_exact/); + assert.match(contents, /FROM product_traffic_daily_exact/); + assert.match(contents, /FROM product_traffic_pages_daily_exact/); + assert.match(contents, /FROM product_activation_daily_exact/); + assert.match(contents, /FROM product_creator_retention_exact/); assert.match(contents, /canonical_events/); assert.match(contents, /decision_events/); + assert.match(contents, /traffic_visitors/); + assert.match(contents, /activated_creators/); + assert.match(contents, /retention_organizations/); assert.match(contents, /FROM product_events_v1/); assert.doesNotMatch(contents, /event_id AS/); assert.doesNotMatch(contents, /TOKEN product_events_ingest READ/); diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index 229bc617b58..e73617d7054 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -78,12 +78,12 @@ test("local setup builds, verifies copied endpoints and writes its deterministic .map((step, index) => ({ index, command: step.args?.join(" ") ?? "" })) .filter(({ command }) => command.includes("--local copy pause")) .map(({ index }) => index); - assert.equal(pauseIndexes.length, 7); + assert.equal(pauseIndexes.length, 8); assert.ok(pauseIndexes.every((index) => index < appendIndex)); const copyCommands = commands.filter((command) => command.includes("--local copy run"), ); - assert.equal(copyCommands.length, 7); + assert.equal(copyCommands.length, 8); assert.ok( copyCommands.every((command) => command.includes( @@ -431,6 +431,125 @@ test("project validation rejects agent access to raw product events", () => { } }); +test("project validation rejects the agent token on Copy Pipes", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "cap-analytics-")); + const projectDir = path.join(tempRoot, "tinybird"); + try { + fs.cpSync(TINYBIRD_PROJECT_DIR, projectDir, { recursive: true }); + const pipePath = path.join( + projectDir, + "pipes", + "snapshot_product_events_canonical_v1.pipe", + ); + const contents = fs.readFileSync(pipePath, "utf8"); + fs.writeFileSync( + pipePath, + contents.replace( + "TOKEN product_events_copy_runner READ", + "TOKEN product_events_agent_read READ", + ), + ); + const issues = validateAnalyticsProject(projectDir); + assert.ok( + issues.some((issue) => + issue.includes("missing its execution-only Copy token"), + ), + ); + assert.ok( + issues.some((issue) => + issue.includes("must not grant Copy execution to the agent token"), + ), + ); + } finally { + fs.rmSync(tempRoot, { force: true, recursive: true }); + } +}); + +test("project validation rejects the Copy runner token on decision endpoints", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "cap-analytics-")); + const projectDir = path.join(tempRoot, "tinybird"); + try { + fs.cpSync(TINYBIRD_PROJECT_DIR, projectDir, { recursive: true }); + const pipePath = path.join( + projectDir, + "pipes", + "product_events_health.pipe", + ); + const contents = fs.readFileSync(pipePath, "utf8"); + fs.writeFileSync( + pipePath, + contents.replace( + "TOKEN product_events_agent_read READ", + "TOKEN product_events_copy_runner READ", + ), + ); + const issues = validateAnalyticsProject(projectDir); + assert.ok( + issues.some((issue) => + issue.includes("missing its read-only agent token"), + ), + ); + assert.ok( + issues.some((issue) => + issue.includes("must not be queryable by the Copy runner token"), + ), + ); + } finally { + fs.rmSync(tempRoot, { force: true, recursive: true }); + } +}); + +test("project validation rejects extra Copy and erasure lookup grants", () => { + const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "cap-analytics-")); + const projectDir = path.join(tempRoot, "tinybird"); + try { + fs.cpSync(TINYBIRD_PROJECT_DIR, projectDir, { recursive: true }); + const datasourcePath = path.join( + projectDir, + "datasources", + "product_events_v1.datasource", + ); + const contents = fs.readFileSync(datasourcePath, "utf8"); + fs.writeFileSync( + datasourcePath, + contents.replace( + "TOKEN product_events_ingest APPEND", + "TOKEN product_events_ingest APPEND\nTOKEN product_events_copy_runner READ", + ), + ); + const endpointPath = path.join( + projectDir, + "pipes", + "product_events_health.pipe", + ); + const endpoint = fs.readFileSync(endpointPath, "utf8"); + fs.writeFileSync( + endpointPath, + endpoint.replace( + "TOKEN product_events_agent_read READ", + "TOKEN product_events_agent_read READ\nTOKEN product_events_erasure_lookup READ", + ), + ); + const issues = validateAnalyticsProject(projectDir); + assert.ok( + issues.some((issue) => + issue.includes( + "product_events_copy_runner has unexpected datasource:product_events_v1:READ", + ), + ), + ); + assert.ok( + issues.some((issue) => + issue.includes( + "product_events_erasure_lookup has unexpected pipe:product_events_health:READ", + ), + ), + ); + } finally { + fs.rmSync(tempRoot, { force: true, recursive: true }); + } +}); + test("project validation rejects duplicate Tinybird resource names", () => { const tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), "cap-analytics-")); const projectDir = path.join(tempRoot, "tinybird"); diff --git a/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource index 7b22eb959e6..98d5a0421df 100644 --- a/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_activation_daily_exact.datasource @@ -1,9 +1,12 @@ DESCRIPTION > - Four-hundred-day signup cohorts and the defensible activation milestone of creating a first share link within seven UTC days. + Eight-hundred-day signup cohorts and the defensible activation milestone of creating a first share link within seven UTC days. + +TOKEN product_events_copy_runner APPEND SCHEMA > calculated_at DateTime64(3), copy_run_id String, + synthetic_run_id String, cohort_date Date, signups AggregateFunction(uniqExact, String), activated_creators AggregateFunction(uniqExact, String), @@ -11,6 +14,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(cohort_date) -ENGINE_SORTING_KEY (copy_run_id, cohort_date) -ENGINE_TTL cohort_date + INTERVAL 400 DAY +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, cohort_date) +ENGINE_TTL cohort_date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource b/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource index b1e23e52158..c4ce9f96279 100644 --- a/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_creator_retention_exact.datasource @@ -1,9 +1,12 @@ DESCRIPTION > - Four-hundred-day creator and organization activity cohorts using successful creation-value events. + Eight-hundred-day creator and organization activity cohorts using successful creation-value events. + +TOKEN product_events_copy_runner APPEND SCHEMA > calculated_at DateTime64(3), copy_run_id String, + synthetic_run_id String, cohort_date Date, activity_date Date, platform LowCardinality(String), @@ -12,6 +15,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(cohort_date) -ENGINE_SORTING_KEY (copy_run_id, cohort_date, activity_date, platform) -ENGINE_TTL cohort_date + INTERVAL 400 DAY +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, cohort_date, activity_date, platform) +ENGINE_TTL cohort_date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_events_canonical_v1.datasource b/scripts/analytics/tinybird/datasources/product_events_canonical_v1.datasource index 078b5ae0a4e..8f975667910 100644 --- a/scripts/analytics/tinybird/datasources/product_events_canonical_v1.datasource +++ b/scripts/analytics/tinybird/datasources/product_events_canonical_v1.datasource @@ -1,6 +1,9 @@ DESCRIPTION > Canonical non-conflicting product events rebuilt from append-only delivery attempts. +TOKEN product_events_copy_runner APPEND +TOKEN product_events_erasure_lookup READ + SCHEMA > event_id String, payload_hash FixedString(32), @@ -32,5 +35,5 @@ SCHEMA > ENGINE MergeTree ENGINE_PARTITION_KEY toYYYYMM(received_at) ENGINE_SORTING_KEY (occurred_at, event_name, event_id) -ENGINE_TTL toDateTime(received_at) + INTERVAL 400 DAY +ENGINE_TTL toDateTime(received_at) + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 8192 diff --git a/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource index df540f49f35..9d450bf6833 100644 --- a/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource @@ -1,10 +1,14 @@ DESCRIPTION > - Four-hundred-day exact daily product decision metrics rebuilt from canonical event IDs. + Eight-hundred-day exact daily product decision metrics rebuilt from canonical event IDs. + +TOKEN product_events_copy_runner APPEND SCHEMA > calculated_at DateTime64(3), + copy_run_id String, date Date, event_name LowCardinality(String), + schema_version UInt16, source LowCardinality(String), platform LowCardinality(String), app_version LowCardinality(String), @@ -21,6 +25,27 @@ SCHEMA > subscription_status LowCardinality(String), currency LowCardinality(String), billing_interval LowCardinality(String), + change_kind LowCardinality(String), + previous_status LowCardinality(String), + new_status LowCardinality(String), + previous_plan_id LowCardinality(String), + quantity Int64, + previous_quantity Int64, + new_quantity Int64, + seat_delta Int64, + first_purchase LowCardinality(String), + guest_checkout LowCardinality(String), + onboarding LowCardinality(String), + cancel_at_period_end LowCardinality(String), + fully_refunded LowCardinality(String), + ended_at Int64, + trial_end_at Int64, + amount_due_minor Int64, + attempt_count Int64, + experiment_id LowCardinality(String), + experiment_variant LowCardinality(String), + assignment_version LowCardinality(String), + delivery_loss_count UInt64, events UInt64, actors AggregateFunction(uniqExact, String), users AggregateFunction(uniqExact, String), @@ -29,6 +54,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (date, event_name, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, payment_status, subscription_status, currency, billing_interval) -ENGINE_TTL date + INTERVAL 400 DAY +ENGINE_SORTING_KEY (copy_run_id, date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version) +ENGINE_TTL date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_events_health_hourly_exact.datasource b/scripts/analytics/tinybird/datasources/product_events_health_hourly_exact.datasource index cf4bf8ae2e9..161018ff4b7 100644 --- a/scripts/analytics/tinybird/datasources/product_events_health_hourly_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_events_health_hourly_exact.datasource @@ -1,8 +1,12 @@ DESCRIPTION > Hourly delivery aggregates for stable, bounded product analytics health checks. +TOKEN product_events_copy_runner APPEND + SCHEMA > hour DateTime, + calculated_at DateTime64(3), + copy_run_id String, platform LowCardinality(String), app_version LowCardinality(String), received_rows SimpleAggregateFunction(sum, UInt64), @@ -15,6 +19,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(hour) -ENGINE_SORTING_KEY (hour, platform, app_version) +ENGINE_SORTING_KEY (copy_run_id, hour, platform, app_version) ENGINE_TTL hour + INTERVAL 90 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_events_v1.datasource b/scripts/analytics/tinybird/datasources/product_events_v1.datasource index b5cd53ccc65..4d4e58c3b04 100644 --- a/scripts/analytics/tinybird/datasources/product_events_v1.datasource +++ b/scripts/analytics/tinybird/datasources/product_events_v1.datasource @@ -2,6 +2,7 @@ DESCRIPTION > Provider-neutral product analytics events emitted by Cap. TOKEN product_events_ingest APPEND +TOKEN product_events_erasure_lookup READ SCHEMA > event_id String `json:$.event_id`, payload_hash FixedString(32) `json:$.payload_hash`, @@ -33,5 +34,5 @@ SCHEMA > ENGINE MergeTree ENGINE_PARTITION_KEY toYYYYMM(received_at) ENGINE_SORTING_KEY (received_at, event_id) -ENGINE_TTL toDateTime(received_at) + INTERVAL 400 DAY +ENGINE_TTL toDateTime(received_at) + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 8192 diff --git a/scripts/analytics/tinybird/datasources/product_identity_funnel_exact.datasource b/scripts/analytics/tinybird/datasources/product_identity_funnel_exact.datasource new file mode 100644 index 00000000000..ece805e2398 --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_identity_funnel_exact.datasource @@ -0,0 +1,34 @@ +DESCRIPTION > + Privacy-safe anonymous acquisition to authenticated conversion cohorts rebuilt from canonical identity links. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + calculated_at DateTime64(3), + copy_run_id String, + synthetic_run_id String, + cohort_date Date, + channel LowCardinality(String), + attribution_source LowCardinality(String), + attribution_medium LowCardinality(String), + campaign LowCardinality(String), + country LowCardinality(String), + linked_visitors UInt64, + linked_users UInt64, + signup_users UInt64, + organizations UInt64, + guest_checkout_visitors UInt64, + guest_purchasers UInt64, + authenticated_checkout_users UInt64, + web_checkout_users UInt64, + desktop_checkout_users UInt64, + mobile_checkout_users UInt64, + cross_device_checkout_users UInt64, + trial_users UInt64, + purchasers UInt64 + +ENGINE SummingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(cohort_date) +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, cohort_date, channel, attribution_source, attribution_medium, campaign, country) +ENGINE_TTL cohort_date + INTERVAL 800 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource index d1269aee15f..8ca28cdb67b 100644 --- a/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource @@ -1,9 +1,12 @@ DESCRIPTION > - Four-hundred-day privacy-safe daily visit metrics with exact visitor states and no raw identifiers. + Eight-hundred-day privacy-safe daily visit metrics with exact visitor states and no raw identifiers. + +TOKEN product_events_copy_runner APPEND SCHEMA > calculated_at DateTime64(3), copy_run_id String, + synthetic_run_id String, date Date, hostname LowCardinality(String), country LowCardinality(String), @@ -23,6 +26,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (copy_run_id, date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) -ENGINE_TTL date + INTERVAL 400 DAY +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) +ENGINE_TTL date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource index 2afdcf05dc8..267fe688ee6 100644 --- a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource @@ -1,9 +1,12 @@ DESCRIPTION > - Four-hundred-day privacy-safe page, landing, exit, and foreground-engagement metrics. + Eight-hundred-day privacy-safe page, landing, exit, and foreground-engagement metrics. + +TOKEN product_events_copy_runner APPEND SCHEMA > calculated_at DateTime64(3), copy_run_id String, + synthetic_run_id String, date Date, hostname LowCardinality(String), pathname String, @@ -22,6 +25,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (copy_run_id, date, hostname, pathname, country, device, browser, os, channel) -ENGINE_TTL date + INTERVAL 400 DAY +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, date, hostname, pathname, country, device, browser, os, channel) +ENGINE_TTL date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/pipes/product_activation.pipe b/scripts/analytics/tinybird/pipes/product_activation.pipe index e0e2ac8f316..9b37b82c9fb 100644 --- a/scripts/analytics/tinybird/pipes/product_activation.pipe +++ b/scripts/analytics/tinybird/pipes/product_activation.pipe @@ -14,10 +14,16 @@ SQL > toUInt64(if(activated_creators = 0, 0, sum(time_to_activation_ms) / activated_creators)) AS average_time_to_activation_ms FROM product_activation_daily_exact WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} AND cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 90 DAY {% end %} AND cohort_date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} GROUP BY cohort_date ORDER BY cohort_date ASC - LIMIT 400 + LIMIT 800 TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe b/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe index bae6292bee3..acfcd58f294 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe @@ -31,9 +31,60 @@ SQL > FROM product_events_canonical_v1 WHERE synthetic_run_id = {{String(synthetic_run_id)}} ), decision_metrics AS ( - SELECT toUInt64(coalesce(sum(events), 0)) AS decision_events + SELECT + toUInt64(coalesce(sum(events), 0)) AS decision_events, + toInt64(coalesce(sum(revenue_minor), 0)) AS decision_revenue_minor FROM product_events_daily_exact WHERE synthetic_run_id = {{String(synthetic_run_id)}} + ), traffic_metrics AS ( + SELECT + uniqExactMerge(visitors) AS traffic_visitors, + toUInt64(sum(visits)) AS traffic_visits, + toUInt64(sum(pageviews)) AS traffic_pageviews, + toUInt64(sum(bounces)) AS traffic_bounces, + toUInt64(sum(visit_duration_ms)) AS traffic_duration_ms + FROM product_traffic_daily_exact + WHERE synthetic_run_id = {{String(synthetic_run_id)}} + ), page_metrics AS ( + SELECT + uniqExactMerge(visitors) AS page_visitors, + uniqExactMerge(visits) AS page_visits, + toUInt64(sum(pageviews)) AS pageviews, + toUInt64(sum(landings)) AS page_landings, + toUInt64(sum(exits)) AS page_exits, + toUInt64(sum(engaged_ms)) AS page_engaged_ms, + toFloat64(sum(scroll_depth_sum)) AS page_scroll_depth + FROM product_traffic_pages_daily_exact + WHERE synthetic_run_id = {{String(synthetic_run_id)}} + ), activation_metrics AS ( + SELECT + uniqExactMerge(signups) AS activation_signups, + uniqExactMerge(activated_creators) AS activated_creators + FROM product_activation_daily_exact + WHERE synthetic_run_id = {{String(synthetic_run_id)}} + ), retention_metrics AS ( + SELECT + uniqExactMerge(creator_users) AS retention_creators, + uniqExactMerge(creator_organizations) AS retention_organizations + FROM product_creator_retention_exact + WHERE synthetic_run_id = {{String(synthetic_run_id)}} + ), identity_metrics AS ( + SELECT + toUInt64(coalesce(sum(linked_visitors), 0)) AS identity_linked_visitors, + toUInt64(coalesce(sum(linked_users), 0)) AS identity_linked_users, + toUInt64(coalesce(sum(signup_users), 0)) AS identity_signup_users, + toUInt64(coalesce(sum(organizations), 0)) AS identity_organizations, + toUInt64(coalesce(sum(guest_checkout_visitors), 0)) AS identity_guest_checkout_visitors, + toUInt64(coalesce(sum(guest_purchasers), 0)) AS identity_guest_purchasers, + toUInt64(coalesce(sum(authenticated_checkout_users), 0)) AS identity_authenticated_checkout_users, + toUInt64(coalesce(sum(web_checkout_users), 0)) AS identity_web_checkout_users, + toUInt64(coalesce(sum(desktop_checkout_users), 0)) AS identity_desktop_checkout_users, + toUInt64(coalesce(sum(mobile_checkout_users), 0)) AS identity_mobile_checkout_users, + toUInt64(coalesce(sum(cross_device_checkout_users), 0)) AS identity_cross_device_checkout_users, + toUInt64(coalesce(sum(trial_users), 0)) AS identity_trial_users, + toUInt64(coalesce(sum(purchasers), 0)) AS identity_purchasers + FROM product_identity_funnel_exact + WHERE synthetic_run_id = {{String(synthetic_run_id)}} ) SELECT raw_received_rows AS received_rows, @@ -42,9 +93,44 @@ SQL > raw_duplicate_rows AS duplicate_rows, raw_payload_conflicts AS payload_conflicts, canonical_events, - decision_events + decision_events, + decision_revenue_minor, + traffic_visitors, + traffic_visits, + traffic_pageviews, + traffic_bounces, + traffic_duration_ms, + page_visitors, + page_visits, + pageviews, + page_landings, + page_exits, + page_engaged_ms, + page_scroll_depth, + activation_signups, + activated_creators, + retention_creators, + retention_organizations, + identity_linked_visitors, + identity_linked_users, + identity_signup_users, + identity_organizations, + identity_guest_checkout_visitors, + identity_guest_purchasers, + identity_authenticated_checkout_users, + identity_web_checkout_users, + identity_desktop_checkout_users, + identity_mobile_checkout_users, + identity_cross_device_checkout_users, + identity_trial_users, + identity_purchasers FROM raw_metrics CROSS JOIN canonical_metrics CROSS JOIN decision_metrics + CROSS JOIN traffic_metrics + CROSS JOIN page_metrics + CROSS JOIN activation_metrics + CROSS JOIN retention_metrics + CROSS JOIN identity_metrics TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe index 98e1ce2353c..06682f6d7b2 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe @@ -11,9 +11,12 @@ SQL > {% end %} SELECT (SELECT count() FROM product_traffic_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS traffic_markers, + (SELECT count() FROM product_events_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS decision_markers, (SELECT count() FROM product_traffic_pages_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS traffic_page_markers, (SELECT count() FROM product_activation_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS activation_markers, - (SELECT count() FROM product_creator_retention_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS retention_markers + (SELECT count() FROM product_creator_retention_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS retention_markers, + (SELECT count() FROM product_identity_funnel_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS identity_markers, + (SELECT count() FROM product_events_health_hourly_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS health_markers WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe index 4cb69f0003f..66d3295e55f 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe @@ -8,9 +8,11 @@ SQL > SELECT max(hour) AS latest_received_hour, dateDiff('millisecond', max(hour), now64(3)) AS health_freshness_ms, - (SELECT max(calculated_at) FROM product_events_daily_exact) AS product_calculated_at, + (SELECT max(calculated_at) FROM product_events_daily_exact WHERE copy_run_id = '') AS product_calculated_at, (SELECT max(calculated_at) FROM product_traffic_daily_exact) AS traffic_calculated_at, - (SELECT max(calculated_at) FROM product_creator_retention_exact) AS retention_calculated_at + (SELECT max(calculated_at) FROM product_creator_retention_exact) AS retention_calculated_at, + (SELECT max(calculated_at) FROM product_identity_funnel_exact) AS identity_calculated_at FROM product_events_health_hourly_exact + WHERE copy_run_id = '' TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_creator_activity.pipe b/scripts/analytics/tinybird/pipes/product_creator_activity.pipe index 16ce4a644a5..e6aed7458d9 100644 --- a/scripts/analytics/tinybird/pipes/product_creator_activity.pipe +++ b/scripts/analytics/tinybird/pipes/product_creator_activity.pipe @@ -17,7 +17,14 @@ SQL > round(if(wau = 0, 0, 100 * dau / wau), 2) AS dau_wau_stickiness, round(if(mau = 0, 0, 100 * dau / mau), 2) AS dau_mau_stickiness FROM product_creator_retention_exact - WHERE activity_date >= as_of_date - INTERVAL 29 DAY + WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} + AND activity_date >= as_of_date - INTERVAL 29 DAY AND activity_date <= as_of_date {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_creator_retention.pipe b/scripts/analytics/tinybird/pipes/product_creator_retention.pipe index 870a8a9fb66..6dea19f4d76 100644 --- a/scripts/analytics/tinybird/pipes/product_creator_retention.pipe +++ b/scripts/analytics/tinybird/pipes/product_creator_retention.pipe @@ -15,6 +15,12 @@ SQL > uniqExactMerge(creator_organizations) AS organizations FROM product_creator_retention_exact WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} AND cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 180 DAY {% end %} AND cohort_date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(platform) %} AND (activity_date = cohort_date OR platform = {{String(platform)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_events_daily.pipe b/scripts/analytics/tinybird/pipes/product_events_daily.pipe index ec3f11fb7cc..714bd2eb476 100644 --- a/scripts/analytics/tinybird/pipes/product_events_daily.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_daily.pipe @@ -9,6 +9,7 @@ SQL > SELECT date, event_name, + schema_version, source, platform, app_version, @@ -23,17 +24,45 @@ SQL > subscription_status, currency, billing_interval, + change_kind, + previous_status, + new_status, + previous_plan_id, + quantity, + previous_quantity, + new_quantity, + seat_delta, + first_purchase, + guest_checkout, + onboarding, + cancel_at_period_end, + fully_refunded, + ended_at, + trial_end_at, + amount_due_minor, + attempt_count, + experiment_id, + experiment_variant, + assignment_version, + delivery_loss_count, events, uniqExactMerge(actors) AS actors, uniqExactMerge(users) AS users, uniqExactMerge(organizations) AS organizations, revenue_minor FROM product_events_daily_exact - WHERE traffic_class = 'external' - AND synthetic_run_id = '' + WHERE copy_run_id = '' + AND {% if defined(synthetic_run_id) %} + synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + traffic_class = 'external' + AND synthetic_run_id = '' + {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(event_name) %} AND event_name = {{String(event_name)}} {% end %} + {% if defined(schema_version) %} AND schema_version = {{UInt16(schema_version)}} {% end %} {% if defined(source) %} AND source = {{String(source)}} {% end %} {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} @@ -47,8 +76,18 @@ SQL > {% if defined(payment_status) %} AND payment_status = {{String(payment_status)}} {% end %} {% if defined(subscription_status) %} AND subscription_status = {{String(subscription_status)}} {% end %} {% if defined(currency) %} AND currency = upper({{String(currency)}}) {% end %} - GROUP BY date, event_name, source, platform, app_version, hostname, country, device, browser, os, channel, plan_id, payment_status, subscription_status, currency, billing_interval, events, revenue_minor, calculated_at - ORDER BY date DESC, event_name ASC, source ASC, platform ASC, app_version ASC, hostname ASC, country ASC, device ASC, browser ASC, os ASC, channel ASC, plan_id ASC, payment_status ASC, subscription_status ASC, currency ASC, billing_interval ASC + {% if defined(change_kind) %} AND change_kind = {{String(change_kind)}} {% end %} + {% if defined(previous_status) %} AND previous_status = {{String(previous_status)}} {% end %} + {% if defined(new_status) %} AND new_status = {{String(new_status)}} {% end %} + {% if defined(first_purchase) %} AND first_purchase = {{String(first_purchase)}} {% end %} + {% if defined(guest_checkout) %} AND guest_checkout = {{String(guest_checkout)}} {% end %} + {% if defined(cancel_at_period_end) %} AND cancel_at_period_end = {{String(cancel_at_period_end)}} {% end %} + {% if defined(fully_refunded) %} AND fully_refunded = {{String(fully_refunded)}} {% end %} + {% if defined(experiment_id) %} AND experiment_id = {{String(experiment_id)}} {% end %} + {% if defined(experiment_variant) %} AND experiment_variant = {{String(experiment_variant)}} {% end %} + {% if defined(assignment_version) %} AND assignment_version = {{String(assignment_version)}} {% end %} + GROUP BY date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, plan_id, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version, delivery_loss_count, events, revenue_minor, calculated_at + ORDER BY date DESC, event_name ASC, schema_version DESC, source ASC, platform ASC, app_version ASC, hostname ASC, country ASC, device ASC, browser ASC, os ASC, channel ASC, plan_id ASC, payment_status ASC, subscription_status ASC, currency ASC, billing_interval ASC, change_kind ASC, previous_status ASC, new_status ASC, previous_plan_id ASC, experiment_id ASC, experiment_variant ASC, assignment_version ASC LIMIT greatest(1, least({{Int32(limit, 1000)}}, 1000)) TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_events_health.pipe b/scripts/analytics/tinybird/pipes/product_events_health.pipe index 39c4077a685..ee3edb4e628 100644 --- a/scripts/analytics/tinybird/pipes/product_events_health.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_health.pipe @@ -23,7 +23,8 @@ SQL > sum(missing_identity_events) AS missing_identity_events, quantilesTDigestMerge(0.5, 0.95, 0.99)(ingestion_lag_ms) AS ingestion_lag_ms FROM product_events_health_hourly_exact - WHERE hour >= toStartOfHour(toDateTime64({{DateTime64(start_time)}}, 3)) + WHERE copy_run_id = '' + AND hour >= toStartOfHour(toDateTime64({{DateTime64(start_time)}}, 3)) AND hour <= toDateTime64({{DateTime64(end_time)}}, 3) {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe b/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe index 5dbd250e450..81bf7610554 100644 --- a/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe +++ b/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe @@ -15,8 +15,13 @@ SQL > uniqExactMerge(users) AS users, uniqExactMerge(organizations) AS organizations FROM product_events_daily_exact - WHERE traffic_class = 'external' - AND synthetic_run_id = '' + WHERE {% if defined(synthetic_run_id) %} + synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + traffic_class = 'external' + AND synthetic_run_id = '' + {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(event_name) %} AND event_name = {{String(event_name)}} {% end %} @@ -33,6 +38,7 @@ SQL > {% if defined(payment_status) %} AND payment_status = {{String(payment_status)}} {% end %} {% if defined(subscription_status) %} AND subscription_status = {{String(subscription_status)}} {% end %} {% if defined(currency) %} AND currency = upper({{String(currency)}}) {% end %} + {% if defined(change_kind) %} AND change_kind = {{String(change_kind)}} {% end %} GROUP BY date, event_name ) SELECT diff --git a/scripts/analytics/tinybird/pipes/product_identity_funnel.pipe b/scripts/analytics/tinybird/pipes/product_identity_funnel.pipe new file mode 100644 index 00000000000..2ad1bede8e1 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_identity_funnel.pipe @@ -0,0 +1,58 @@ +DESCRIPTION > + Query complete privacy-safe anonymous acquisition to authenticated conversion totals without raw identifiers. + +TOKEN product_events_agent_read READ + +NODE product_identity_funnel_node +SQL > + % + WITH totals AS ( + SELECT + sum(linked_visitors) AS linked_visitors, + sum(linked_users) AS linked_users, + sum(signup_users) AS signup_users, + sum(organizations) AS organizations, + sum(guest_checkout_visitors) AS guest_checkout_visitors, + sum(guest_purchasers) AS guest_purchasers, + sum(authenticated_checkout_users) AS authenticated_checkout_users, + sum(web_checkout_users) AS web_checkout_users, + sum(desktop_checkout_users) AS desktop_checkout_users, + sum(mobile_checkout_users) AS mobile_checkout_users, + sum(cross_device_checkout_users) AS cross_device_checkout_users, + sum(trial_users) AS trial_users, + sum(purchasers) AS purchasers + FROM product_identity_funnel_exact + WHERE copy_run_id = '' + AND {% if defined(synthetic_run_id) %} + synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + synthetic_run_id = '' + {% end %} + AND cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + AND cohort_date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(channel) %} AND channel = {{String(channel)}} {% end %} + {% if defined(source) %} AND attribution_source = {{String(source)}} {% end %} + {% if defined(medium) %} AND attribution_medium = {{String(medium)}} {% end %} + {% if defined(campaign) %} AND campaign = {{String(campaign)}} {% end %} + {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} + ) + SELECT + toUInt64(linked_visitors) AS linked_visitors, + toUInt64(linked_users) AS linked_users, + toUInt64(signup_users) AS signup_users, + toUInt64(organizations) AS organizations, + toUInt64(guest_checkout_visitors) AS guest_checkout_visitors, + toUInt64(guest_purchasers) AS guest_purchasers, + toUInt64(authenticated_checkout_users) AS authenticated_checkout_users, + toUInt64(web_checkout_users) AS web_checkout_users, + toUInt64(desktop_checkout_users) AS desktop_checkout_users, + toUInt64(mobile_checkout_users) AS mobile_checkout_users, + toUInt64(cross_device_checkout_users) AS cross_device_checkout_users, + toUInt64(trial_users) AS trial_users, + toUInt64(purchasers) AS purchasers, + round(if(linked_users = 0, 0, signup_users * 100.0 / linked_users), 2) AS signup_rate, + round(if(linked_users = 0, 0, purchasers * 100.0 / linked_users), 2) AS purchase_rate + FROM totals + +TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe index d3d23d4d460..c51cc33e973 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe @@ -13,6 +13,12 @@ SQL > sum(pageviews) AS pageviews FROM product_traffic_daily_exact WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe index 2c118b91249..b9710e0ba12 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe @@ -17,6 +17,12 @@ SQL > sum(engaged_ms) AS engagement_total_ms FROM product_traffic_daily_exact WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} @@ -38,6 +44,6 @@ SQL > engagement_total_ms AS engaged_ms FROM daily ORDER BY date ASC - LIMIT 400 + LIMIT 800 TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe index 2f9c733c520..fc8ca3a429b 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe @@ -18,6 +18,12 @@ SQL > sum(scroll_depth_sum) AS scroll_depth_total FROM product_traffic_pages_daily_exact WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe index cf618b5a03e..ed75f6476f8 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe @@ -18,6 +18,12 @@ SQL > sum(bounces) AS bounce_count FROM product_traffic_daily_exact WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe index fda05f7508e..c2438c5f7af 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe @@ -15,6 +15,12 @@ SQL > sum(pageviews) AS pageviews FROM product_traffic_daily_exact WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe index 1d1dedeb745..a222ee9d0bb 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe @@ -1,7 +1,7 @@ DESCRIPTION > Rebuild signup activation cohorts from exact canonical server facts. -TOKEN product_events_agent_read READ +TOKEN product_events_copy_runner READ NODE snapshot_product_activation_daily_exact_node SQL > @@ -11,36 +11,40 @@ SQL > {% end %} WITH signups AS ( SELECT + synthetic_run_id, user_id, min(occurred_at) AS signed_up_at FROM product_events_canonical_v1 WHERE event_name = 'user_signed_up' AND user_id != '' - GROUP BY user_id + GROUP BY synthetic_run_id, user_id ), activations AS ( SELECT + synthetic_run_id, user_id, min(occurred_at) AS activated_at FROM product_events_canonical_v1 WHERE event_name = 'share_link_created' AND user_id != '' - GROUP BY user_id + GROUP BY synthetic_run_id, user_id ) SELECT now64(3) AS calculated_at, '' AS copy_run_id, + synthetic_run_id, toDate(signed_up_at, 'UTC') AS cohort_date, uniqExactState(signups.user_id) AS signups, uniqExactStateIf(signups.user_id, activated_at >= signed_up_at AND activated_at < signed_up_at + INTERVAL 7 DAY) AS activated_creators, toUInt64(sum(if(activated_at >= signed_up_at AND activated_at < signed_up_at + INTERVAL 7 DAY, dateDiff('millisecond', signed_up_at, activated_at), 0))) AS time_to_activation_ms FROM signups - LEFT JOIN activations USING user_id - GROUP BY cohort_date + LEFT JOIN activations USING (synthetic_run_id, user_id) + GROUP BY synthetic_run_id, cohort_date {% if defined(copy_run_id) %} UNION ALL SELECT now64(3) AS calculated_at, {{String(copy_run_id)}} AS copy_run_id, + '' AS synthetic_run_id, today() AS cohort_date, uniqExactStateIf('', 0) AS signups, uniqExactStateIf('', 0) AS activated_creators, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe index 5a7f92f56a3..93ee5eda729 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe @@ -1,7 +1,7 @@ DESCRIPTION > Rebuild active-creator cohorts from exact successful creation and sharing events. -TOKEN product_events_agent_read READ +TOKEN product_events_copy_runner READ NODE snapshot_product_creator_retention_exact_node SQL > @@ -11,6 +11,7 @@ SQL > {% end %} WITH activity AS ( SELECT DISTINCT + synthetic_run_id, user_id, organization_id, toDate(occurred_at, 'UTC') AS activity_date, @@ -21,51 +22,57 @@ SQL > AND (event_name != 'recording_completed' OR JSONExtractString(properties, 'status') IN ('success', 'degraded')) ), user_cohorts AS ( SELECT + synthetic_run_id, user_id, min(activity_date) AS cohort_date FROM activity - GROUP BY user_id + GROUP BY synthetic_run_id, user_id ), organization_cohorts AS ( SELECT + synthetic_run_id, organization_id, min(activity_date) AS cohort_date FROM activity WHERE organization_id != '' - GROUP BY organization_id + GROUP BY synthetic_run_id, organization_id ), cohort_activity AS ( SELECT + synthetic_run_id, cohort_date, activity_date, platform, user_id, '' AS organization_id FROM activity - INNER JOIN user_cohorts USING user_id + INNER JOIN user_cohorts USING (synthetic_run_id, user_id) UNION ALL SELECT + synthetic_run_id, cohort_date, activity_date, platform, '' AS user_id, organization_id FROM activity - INNER JOIN organization_cohorts USING organization_id + INNER JOIN organization_cohorts USING (synthetic_run_id, organization_id) ) SELECT now64(3) AS calculated_at, '' AS copy_run_id, + synthetic_run_id, cohort_date, activity_date, platform, uniqExactStateIf(user_id, user_id != '') AS creator_users, uniqExactStateIf(organization_id, organization_id != '') AS creator_organizations FROM cohort_activity - GROUP BY cohort_date, activity_date, platform + GROUP BY synthetic_run_id, cohort_date, activity_date, platform {% if defined(copy_run_id) %} UNION ALL SELECT now64(3) AS calculated_at, {{String(copy_run_id)}} AS copy_run_id, + '' AS synthetic_run_id, today() AS cohort_date, today() AS activity_date, '' AS platform, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe index 851e4c56f4f..ac23b9a92e9 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe @@ -1,7 +1,7 @@ DESCRIPTION > Replace the canonical stream with one stable payload per event ID and quarantine conflicts. -TOKEN product_events_agent_read READ +TOKEN product_events_copy_runner READ NODE snapshot_product_events_canonical_v1_node SQL > diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe index 7d88f84f783..44434f70f6e 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe @@ -1,7 +1,7 @@ DESCRIPTION > Rebuild exact decision metrics from one non-conflicting canonical payload per event ID. -TOKEN product_events_agent_read READ +TOKEN product_events_copy_runner READ NODE snapshot_product_events_daily_exact_node SQL > @@ -11,8 +11,10 @@ SQL > {% end %} SELECT now64(3) AS calculated_at, + '' AS copy_run_id, toDate(occurred_at, 'UTC') AS date, event_name, + schema_version, source, platform, app_version, @@ -24,11 +26,38 @@ SQL > channel, traffic_class, synthetic_run_id, - JSONExtractString(properties, 'price_id') AS plan_id, + if(event_name = 'subscription_changed', JSONExtractString(properties, 'new_price_id'), JSONExtractString(properties, 'price_id')) AS plan_id, if(event_name = 'purchase_completed', JSONExtractString(properties, 'payment_status'), '') AS payment_status, - if(event_name IN ('purchase_completed', 'trial_started'), JSONExtractString(properties, 'subscription_status'), '') AS subscription_status, - if(event_name IN ('purchase_completed', 'trial_started', 'subscription_renewed', 'subscription_refunded'), upper(JSONExtractString(properties, 'currency')), '') AS currency, + multiIf( + event_name IN ('purchase_completed', 'trial_started'), JSONExtractString(properties, 'subscription_status'), + event_name = 'trial_converted', JSONExtractString(properties, 'new_status'), + event_name = 'subscription_changed', JSONExtractString(properties, 'new_status'), + event_name = 'subscription_cancelled', JSONExtractString(properties, 'status'), + '' + ) AS subscription_status, + if(event_name IN ('purchase_completed', 'trial_started', 'subscription_renewed', 'subscription_refunded', 'subscription_payment_failed'), upper(JSONExtractString(properties, 'currency')), '') AS currency, if(event_name IN ('purchase_completed', 'trial_started'), JSONExtractString(properties, 'billing_interval'), '') AS billing_interval, + if(event_name = 'subscription_changed', JSONExtractString(properties, 'change_kind'), '') AS change_kind, + if(event_name IN ('trial_converted', 'subscription_changed'), JSONExtractString(properties, 'previous_status'), '') AS previous_status, + if(event_name IN ('trial_converted', 'subscription_changed'), JSONExtractString(properties, 'new_status'), '') AS new_status, + if(event_name = 'subscription_changed', JSONExtractString(properties, 'previous_price_id'), '') AS previous_plan_id, + if(event_name IN ('checkout_started', 'guest_checkout_started', 'purchase_completed', 'trial_started'), JSONExtractInt(properties, 'quantity'), 0) AS quantity, + if(event_name = 'subscription_changed', JSONExtractInt(properties, 'previous_quantity'), 0) AS previous_quantity, + if(event_name = 'subscription_changed', JSONExtractInt(properties, 'new_quantity'), 0) AS new_quantity, + if(event_name = 'subscription_changed' AND JSONExtractString(properties, 'change_kind') = 'seats', JSONExtractInt(properties, 'new_quantity') - JSONExtractInt(properties, 'previous_quantity'), 0) AS seat_delta, + if(event_name = 'purchase_completed', if(JSONExtractBool(properties, 'is_first_purchase'), 'true', 'false'), '') AS first_purchase, + if(event_name IN ('purchase_completed', 'trial_started'), if(JSONExtractBool(properties, 'is_guest_checkout'), 'true', 'false'), '') AS guest_checkout, + if(event_name IN ('checkout_started', 'purchase_completed', 'trial_started'), if(JSONExtractBool(properties, 'is_onboarding'), 'true', 'false'), '') AS onboarding, + if(event_name = 'subscription_cancelled', if(JSONExtractBool(properties, 'cancel_at_period_end'), 'true', 'false'), '') AS cancel_at_period_end, + if(event_name = 'subscription_refunded', if(JSONExtractBool(properties, 'fully_refunded'), 'true', 'false'), '') AS fully_refunded, + if(event_name = 'subscription_cancelled', JSONExtractInt(properties, 'ended_at'), 0) AS ended_at, + if(event_name = 'trial_started', JSONExtractInt(properties, 'trial_end_at'), 0) AS trial_end_at, + if(event_name = 'subscription_payment_failed', JSONExtractInt(properties, 'amount_due_minor'), 0) AS amount_due_minor, + if(event_name = 'subscription_payment_failed', JSONExtractInt(properties, 'attempt_count'), 0) AS attempt_count, + if(event_name = 'experiment_exposed', JSONExtractString(properties, 'experiment_id'), '') AS experiment_id, + if(event_name = 'experiment_exposed', JSONExtractString(properties, 'variant'), '') AS experiment_variant, + if(event_name = 'experiment_exposed', JSONExtractString(properties, 'assignment_version'), '') AS assignment_version, + toUInt64(sum(if(event_name = 'analytics_delivery_loss', greatest(JSONExtractInt(properties, 'count'), 0), 0))) AS delivery_loss_count, toUInt64(count()) AS events, uniqExactState(if(user_id != '', concat('user:', user_id), if(anonymous_id != '', concat('anonymous:', anonymous_id), concat('session:', session_id)))) AS actors, uniqExactStateIf(user_id, user_id != '') AS users, @@ -40,7 +69,29 @@ SQL > 0 ))) AS revenue_minor FROM product_events_canonical_v1 - GROUP BY date, event_name, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, payment_status, subscription_status, currency, billing_interval + GROUP BY date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3), + {{String(copy_run_id)}}, + today(), + '', + toUInt16(0), + '', '', '', '', '', '', '', '', '', '', + '', '', '', '', '', '', '', '', '', '', + toInt64(0), toInt64(0), toInt64(0), toInt64(0), + '', '', '', '', '', + toInt64(0), toInt64(0), toInt64(0), toInt64(0), + '', '', '', + toUInt64(0), + toUInt64(0), + uniqExactStateIf('', 0), + uniqExactStateIf('', 0), + uniqExactStateIf('', 0), + toInt64(0) + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} TYPE COPY TARGET_DATASOURCE product_events_daily_exact diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe index 4a9113b9580..5e3e9c2274e 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe @@ -1,7 +1,7 @@ DESCRIPTION > Rebuild hourly delivery health so privacy erasure retracts all derived state. -TOKEN product_events_agent_read READ +TOKEN product_events_copy_runner READ NODE snapshot_product_events_health_hourly_node SQL > @@ -11,6 +11,8 @@ SQL > {% end %} SELECT toStartOfHour(received_at) AS hour, + now64(3) AS calculated_at, + '' AS copy_run_id, platform, app_version, count() AS received_rows, @@ -22,6 +24,23 @@ SQL > quantilesTDigestState(0.5, 0.95, 0.99)(toFloat64(dateDiff('millisecond', occurred_at, received_at))) AS ingestion_lag_ms FROM product_events_v1 GROUP BY hour, platform, app_version + {% if defined(copy_run_id) %} + UNION ALL + SELECT + toStartOfHour(now()), + now64(3), + {{String(copy_run_id)}}, + '', + '', + toUInt64(0), + uniqExactStateIf('', 0), + uniqExactStateIf(tuple('', toFixedString('', 32)), 0), + toUInt64(0), + toUInt64(0), + toUInt64(0), + quantilesTDigestState(0.5, 0.95, 0.99)(toFloat64(0)) + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} TYPE COPY TARGET_DATASOURCE product_events_health_hourly_exact diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe new file mode 100644 index 00000000000..ee3d7fadb3e --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe @@ -0,0 +1,234 @@ +DESCRIPTION > + Rebuild privacy-safe cohorts that join pre-auth acquisition to authoritative user outcomes without exposing identity mappings. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_identity_funnel_exact_node +SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + WITH identity_links AS ( + SELECT synthetic_run_id, user_id, anonymous_id, organization_id, occurred_at + FROM product_events_canonical_v1 + WHERE event_name = 'identity_linked' + AND user_id != '' + AND anonymous_id != '' + ), paid_guest_links AS ( + SELECT synthetic_run_id, user_id, anonymous_id, organization_id, occurred_at + FROM product_events_canonical_v1 + WHERE event_name = 'purchase_completed' + AND JSONExtractString(properties, 'payment_status') = 'paid' + AND JSONExtractBool(properties, 'is_guest_checkout') + AND user_id != '' + AND anonymous_id != '' + ), link_candidates AS ( + SELECT * FROM identity_links + UNION ALL + SELECT * FROM paid_guest_links + ), first_links AS ( + SELECT + synthetic_run_id, + user_id, + argMin(anonymous_id, occurred_at) AS anonymous_id, + argMinIf(organization_id, occurred_at, organization_id != '') AS organization_id, + min(occurred_at) AS linked_at + FROM link_candidates + GROUP BY synthetic_run_id, user_id + ), organization_first_links AS ( + SELECT + synthetic_run_id, + organization_id, + argMin(user_id, linked_at) AS first_user_id + FROM first_links + WHERE organization_id != '' + GROUP BY synthetic_run_id, organization_id + ), acquisition AS ( + SELECT + synthetic_run_id, + anonymous_id, + min(occurred_at) AS acquired_at, + argMin(channel, occurred_at) AS channel, + argMin(JSONExtractString(properties, 'session_touch_source'), occurred_at) AS attribution_source, + argMin(JSONExtractString(properties, 'session_touch_medium'), occurred_at) AS attribution_medium, + argMin(JSONExtractString(properties, 'session_touch_campaign'), occurred_at) AS campaign, + argMin(country, occurred_at) AS country + FROM product_events_canonical_v1 + WHERE event_name = 'page_view' + AND anonymous_id != '' + AND (traffic_class = 'external' OR synthetic_run_id != '') + GROUP BY synthetic_run_id, anonymous_id + ), outcomes AS ( + SELECT + synthetic_run_id, + user_id, + toUInt64(countIf(event_name = 'user_signed_up') > 0) AS signed_up, + toUInt64(countIf(event_name = 'checkout_started') > 0) AS authenticated_checkout, + toUInt64(countIf(event_name = 'checkout_started' AND platform = 'web') > 0) AS web_checkout, + toUInt64(countIf(event_name = 'checkout_started' AND platform = 'desktop') > 0) AS desktop_checkout, + toUInt64(countIf(event_name = 'checkout_started' AND platform = 'mobile') > 0) AS mobile_checkout, + toUInt64(uniqExactIf(platform, event_name = 'checkout_started') > 1) AS cross_device_checkout, + toUInt64(countIf(event_name = 'trial_started') > 0) AS trial_started, + toUInt64(countIf(event_name = 'purchase_completed' AND JSONExtractString(properties, 'payment_status') = 'paid') > 0) AS purchased, + toUInt64(countIf(event_name = 'purchase_completed' AND JSONExtractString(properties, 'payment_status') = 'paid' AND JSONExtractBool(properties, 'is_guest_checkout')) > 0) AS guest_purchased + FROM product_events_canonical_v1 + WHERE user_id != '' + GROUP BY synthetic_run_id, user_id + ), guest_checkouts AS ( + SELECT + synthetic_run_id, + anonymous_id, + min(occurred_at) AS checkout_at, + toUInt64(count() > 0) AS guest_checkout + FROM product_events_canonical_v1 + WHERE event_name = 'guest_checkout_started' + AND anonymous_id != '' + GROUP BY synthetic_run_id, anonymous_id + ), guest_path_candidates AS ( + SELECT + link_candidates.synthetic_run_id AS synthetic_run_id, + link_candidates.user_id AS user_id, + link_candidates.anonymous_id AS anonymous_id, + link_candidates.organization_id AS organization_id, + guest_checkouts.checkout_at AS path_at, + toUInt64(0) AS paid_guest_purchase + FROM link_candidates + INNER JOIN guest_checkouts ON link_candidates.synthetic_run_id = guest_checkouts.synthetic_run_id AND link_candidates.anonymous_id = guest_checkouts.anonymous_id + WHERE guest_checkouts.checkout_at <= link_candidates.occurred_at + UNION ALL + SELECT + synthetic_run_id, + user_id, + anonymous_id, + organization_id, + occurred_at AS path_at, + toUInt64(1) AS paid_guest_purchase + FROM paid_guest_links + ), guest_paths AS ( + SELECT + synthetic_run_id, + user_id, + argMin(anonymous_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at)) AS anonymous_id, + argMin(organization_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at)) AS organization_id, + argMin(path_at, tuple(if(paid_guest_purchase > 0, 0, 1), path_at)) AS linked_at, + toUInt64(max(paid_guest_purchase) > 0) AS guest_purchased + FROM guest_path_candidates + GROUP BY synthetic_run_id, user_id + ), user_linked_outcomes AS ( + SELECT + first_links.synthetic_run_id AS synthetic_run_id, + first_links.user_id AS user_id, + first_links.anonymous_id AS anonymous_id, + first_links.organization_id AS organization_id, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > first_links.linked_at, first_links.linked_at, acquisition.acquired_at) AS acquired_at, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > first_links.linked_at, '', acquisition.channel) AS channel, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > first_links.linked_at, '', acquisition.attribution_source) AS attribution_source, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > first_links.linked_at, '', acquisition.attribution_medium) AS attribution_medium, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > first_links.linked_at, '', acquisition.campaign) AS campaign, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > first_links.linked_at, '', acquisition.country) AS country, + toUInt64(1) AS linked_user, + toUInt64(first_links.organization_id != '' AND organization_first_links.first_user_id = first_links.user_id) AS linked_organization, + outcomes.signed_up AS signed_up, + toUInt64(0) AS guest_checkout, + toUInt64(0) AS guest_purchased, + outcomes.authenticated_checkout AS authenticated_checkout, + outcomes.web_checkout AS web_checkout, + outcomes.desktop_checkout AS desktop_checkout, + outcomes.mobile_checkout AS mobile_checkout, + outcomes.cross_device_checkout AS cross_device_checkout, + outcomes.trial_started AS trial_started, + outcomes.purchased AS purchased + FROM first_links + LEFT JOIN acquisition ON first_links.synthetic_run_id = acquisition.synthetic_run_id AND first_links.anonymous_id = acquisition.anonymous_id + LEFT JOIN organization_first_links ON first_links.synthetic_run_id = organization_first_links.synthetic_run_id AND first_links.organization_id = organization_first_links.organization_id + LEFT JOIN outcomes ON first_links.synthetic_run_id = outcomes.synthetic_run_id AND first_links.user_id = outcomes.user_id + ), guest_linked_outcomes AS ( + SELECT + guest_checkouts.synthetic_run_id AS synthetic_run_id, + guest_paths.user_id AS user_id, + guest_checkouts.anonymous_id AS anonymous_id, + guest_paths.organization_id AS organization_id, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > guest_checkouts.checkout_at, guest_checkouts.checkout_at, acquisition.acquired_at) AS acquired_at, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > guest_checkouts.checkout_at, '', acquisition.channel) AS channel, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > guest_checkouts.checkout_at, '', acquisition.attribution_source) AS attribution_source, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > guest_checkouts.checkout_at, '', acquisition.attribution_medium) AS attribution_medium, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > guest_checkouts.checkout_at, '', acquisition.campaign) AS campaign, + if(acquisition.anonymous_id = '' OR acquisition.acquired_at > guest_checkouts.checkout_at, '', acquisition.country) AS country, + toUInt64(0) AS linked_user, + toUInt64(0) AS linked_organization, + toUInt64(0) AS signed_up, + guest_checkouts.guest_checkout AS guest_checkout, + guest_paths.guest_purchased AS guest_purchased, + toUInt64(0) AS authenticated_checkout, + toUInt64(0) AS web_checkout, + toUInt64(0) AS desktop_checkout, + toUInt64(0) AS mobile_checkout, + toUInt64(0) AS cross_device_checkout, + toUInt64(0) AS trial_started, + toUInt64(0) AS purchased + FROM guest_checkouts + LEFT JOIN guest_paths ON guest_checkouts.synthetic_run_id = guest_paths.synthetic_run_id AND guest_checkouts.anonymous_id = guest_paths.anonymous_id + LEFT JOIN acquisition ON guest_checkouts.synthetic_run_id = acquisition.synthetic_run_id AND guest_checkouts.anonymous_id = acquisition.anonymous_id + ), linked_outcomes AS ( + SELECT * FROM user_linked_outcomes + UNION ALL + SELECT * FROM guest_linked_outcomes + ) + SELECT + now64(3) AS calculated_at, + '' AS copy_run_id, + synthetic_run_id, + toDate(acquired_at, 'UTC') AS cohort_date, + channel, + attribution_source, + attribution_medium, + campaign, + country, + toUInt64(uniqExactIf(anonymous_id, linked_user > 0)) AS linked_visitors, + toUInt64(sum(linked_user)) AS linked_users, + toUInt64(sum(signed_up)) AS signup_users, + toUInt64(sum(linked_organization)) AS organizations, + toUInt64(uniqExactIf(anonymous_id, guest_checkout > 0)) AS guest_checkout_visitors, + toUInt64(sum(guest_purchased)) AS guest_purchasers, + toUInt64(sum(authenticated_checkout)) AS authenticated_checkout_users, + toUInt64(sum(web_checkout)) AS web_checkout_users, + toUInt64(sum(desktop_checkout)) AS desktop_checkout_users, + toUInt64(sum(mobile_checkout)) AS mobile_checkout_users, + toUInt64(sum(cross_device_checkout)) AS cross_device_checkout_users, + toUInt64(sum(trial_started)) AS trial_users, + toUInt64(sum(purchased)) AS purchasers + FROM linked_outcomes + GROUP BY synthetic_run_id, cohort_date, channel, attribution_source, attribution_medium, campaign, country + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3) AS calculated_at, + {{String(copy_run_id)}} AS copy_run_id, + '' AS synthetic_run_id, + today() AS cohort_date, + '' AS channel, + '' AS attribution_source, + '' AS attribution_medium, + '' AS campaign, + '' AS country, + toUInt64(0) AS linked_visitors, + toUInt64(0) AS linked_users, + toUInt64(0) AS signup_users, + toUInt64(0) AS organizations, + toUInt64(0) AS guest_checkout_visitors, + toUInt64(0) AS guest_purchasers, + toUInt64(0) AS authenticated_checkout_users, + toUInt64(0) AS web_checkout_users, + toUInt64(0) AS desktop_checkout_users, + toUInt64(0) AS mobile_checkout_users, + toUInt64(0) AS cross_device_checkout_users, + toUInt64(0) AS trial_users, + toUInt64(0) AS purchasers + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} + +TYPE COPY +TARGET_DATASOURCE product_identity_funnel_exact +COPY_MODE replace +COPY_SCHEDULE 7-59/8 * * * * diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe index 817a6a25361..2d5eb2bc4ff 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe @@ -1,7 +1,7 @@ DESCRIPTION > Rebuild exact 30-minute client-session metrics from canonical external page events. -TOKEN product_events_agent_read READ +TOKEN product_events_copy_runner READ NODE snapshot_product_traffic_daily_exact_node SQL > @@ -11,16 +11,18 @@ SQL > {% end %} WITH session_engagement AS ( SELECT + synthetic_run_id, session_id, toUInt64(sum(greatest(0, JSONExtractInt(properties, 'engaged_ms')))) AS session_engaged_ms, max(greatest(0, least(100, JSONExtractInt(properties, 'max_scroll_depth')))) AS session_max_scroll_depth FROM product_events_canonical_v1 WHERE event_name = 'page_engagement' - AND traffic_class = 'external' + AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' - GROUP BY session_id + GROUP BY synthetic_run_id, session_id ), sessions AS ( SELECT + synthetic_run_id, session_id, concat('anonymous:', argMin(anonymous_id, occurred_at)) AS visitor_id, min(occurred_at) AS started_at, @@ -36,14 +38,15 @@ SQL > toUInt64(count()) AS session_pageviews FROM product_events_canonical_v1 WHERE event_name = 'page_view' - AND traffic_class = 'external' + AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' AND anonymous_id != '' - GROUP BY session_id + GROUP BY synthetic_run_id, session_id ) SELECT now64(3) AS calculated_at, '' AS copy_run_id, + synthetic_run_id, toDate(started_at, 'UTC') AS date, hostname, country, @@ -61,13 +64,14 @@ SQL > toUInt64(sum(coalesce(session_engaged_ms, 0))) AS visit_duration_ms, toUInt64(sum(coalesce(session_engaged_ms, 0))) AS engaged_ms FROM sessions - LEFT JOIN session_engagement USING session_id - GROUP BY date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign + LEFT JOIN session_engagement USING (synthetic_run_id, session_id) + GROUP BY synthetic_run_id, date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign {% if defined(copy_run_id) %} UNION ALL SELECT now64(3) AS calculated_at, {{String(copy_run_id)}} AS copy_run_id, + '' AS synthetic_run_id, today() AS date, '' AS hostname, '' AS country, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe index c222502f5c3..e173d6fd16f 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe @@ -1,7 +1,7 @@ DESCRIPTION > Rebuild exact external page, landing, exit, and foreground-engagement metrics. -TOKEN product_events_agent_read READ +TOKEN product_events_copy_runner READ NODE snapshot_product_traffic_pages_daily_exact_node SQL > @@ -11,25 +11,28 @@ SQL > {% end %} WITH sessions AS ( SELECT + synthetic_run_id, session_id, argMin(event_id, occurred_at) AS landing_event_id, argMax(event_id, occurred_at) AS exit_event_id FROM product_events_canonical_v1 WHERE event_name = 'page_view' - AND traffic_class = 'external' + AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' - GROUP BY session_id + GROUP BY synthetic_run_id, session_id ), engagements AS ( SELECT + synthetic_run_id, JSONExtractString(properties, 'page_view_id') AS page_view_id, toUInt64(sum(greatest(0, JSONExtractInt(properties, 'engaged_ms')))) AS engaged_ms, toFloat64(max(greatest(0, least(100, JSONExtractInt(properties, 'max_scroll_depth'))))) AS scroll_depth FROM product_events_canonical_v1 WHERE event_name = 'page_engagement' - AND traffic_class = 'external' - GROUP BY page_view_id + AND (traffic_class = 'external' OR synthetic_run_id != '') + GROUP BY synthetic_run_id, page_view_id ), page_views AS ( SELECT + synthetic_run_id, event_id, occurred_at, session_id, @@ -43,13 +46,14 @@ SQL > channel FROM product_events_canonical_v1 WHERE event_name = 'page_view' - AND traffic_class = 'external' + AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' AND anonymous_id != '' ) SELECT now64(3) AS calculated_at, '' AS copy_run_id, + page_views.synthetic_run_id AS synthetic_run_id, toDate(occurred_at, 'UTC') AS date, hostname, pathname, @@ -66,14 +70,15 @@ SQL > toUInt64(sum(coalesce(engaged_ms, 0))) AS engaged_ms, toFloat64(sum(coalesce(scroll_depth, 0))) AS scroll_depth_sum FROM page_views - INNER JOIN sessions ON page_views.session_id = sessions.session_id - LEFT JOIN engagements ON page_views.event_id = engagements.page_view_id - GROUP BY date, hostname, pathname, country, device, browser, os, channel + INNER JOIN sessions ON page_views.synthetic_run_id = sessions.synthetic_run_id AND page_views.session_id = sessions.session_id + LEFT JOIN engagements ON page_views.synthetic_run_id = engagements.synthetic_run_id AND page_views.event_id = engagements.page_view_id + GROUP BY page_views.synthetic_run_id, date, hostname, pathname, country, device, browser, os, channel {% if defined(copy_run_id) %} UNION ALL SELECT now64(3) AS calculated_at, {{String(copy_run_id)}} AS copy_run_id, + '' AS synthetic_run_id, today() AS date, '' AS hostname, '' AS pathname, diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index d8ed9455979..40498d10647 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -43,8 +43,19 @@ const PRODUCT_COPY_PIPES = [ "snapshot_product_traffic_pages_daily_exact", "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", + "snapshot_product_identity_funnel_exact", "snapshot_product_events_health_hourly", ]; +const PRODUCT_COPY_TARGETS = [ + "product_events_canonical_v1", + "product_events_daily_exact", + "product_traffic_daily_exact", + "product_traffic_pages_daily_exact", + "product_activation_daily_exact", + "product_creator_retention_exact", + "product_identity_funnel_exact", + "product_events_health_hourly_exact", +]; const WORKSPACE_ID_SOURCE = "[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; const WORKSPACE_ID_PATTERN = new RegExp(`^${WORKSPACE_ID_SOURCE}$`, "i"); @@ -264,6 +275,31 @@ const operationPlan = (operation) => { const hasToken = (resource, name, scope) => resource.tokens.some((token) => token.name === name && token.scope === scope); +const tokenGrants = (project, tokenName) => [ + ...project.datasources.flatMap((datasource) => + datasource.tokens + .filter(({ name }) => name === tokenName) + .map(({ scope }) => `datasource:${datasource.name}:${scope}`), + ), + ...project.pipes.flatMap((pipe) => + pipe.tokens + .filter(({ name }) => name === tokenName) + .map(({ scope }) => `pipe:${pipe.name}:${scope}`), + ), +]; + +const validateExactTokenGrants = (project, tokenName, expected, issues) => { + const actual = new Set(tokenGrants(project, tokenName)); + for (const grant of expected) { + if (!actual.has(grant)) issues.push(`${tokenName} is missing ${grant}`); + } + for (const grant of actual) { + if (!expected.has(grant)) { + issues.push(`${tokenName} has unexpected ${grant}`); + } + } +}; + const validateFixtures = (projectDir, issues) => { const fixturePath = path.join( projectDir, @@ -357,6 +393,24 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { } const project = loadTinybirdProject(projectDir); + validateExactTokenGrants( + project, + "product_events_copy_runner", + new Set([ + ...PRODUCT_COPY_TARGETS.map((name) => `datasource:${name}:APPEND`), + ...PRODUCT_COPY_PIPES.map((name) => `pipe:${name}:READ`), + ]), + issues, + ); + validateExactTokenGrants( + project, + "product_events_erasure_lookup", + new Set([ + "datasource:product_events_v1:READ", + "datasource:product_events_canonical_v1:READ", + ]), + issues, + ); const datasourceNames = new Set( project.datasources.map((datasource) => datasource.name), ); @@ -399,12 +453,15 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { if (product.partitionKey !== "toYYYYMM(received_at)") { issues.push("product_events_v1 must use monthly receipt-time partitions"); } - if (product.ttl !== "toDateTime(received_at) + INTERVAL 400 DAY") { - issues.push("product_events_v1 must retain raw deliveries for 400 days"); + if (product.ttl !== "toDateTime(received_at) + INTERVAL 800 DAY") { + issues.push("product_events_v1 must retain raw deliveries for 800 days"); } if (!hasToken(product, "product_events_ingest", "APPEND")) { issues.push("product_events_v1 is missing its append-only token"); } + if (!hasToken(product, "product_events_erasure_lookup", "READ")) { + issues.push("product_events_v1 is missing its erasure lookup token"); + } if (hasToken(product, "product_events_agent_read", "READ")) { issues.push( "product_events_v1 must not expose raw identity data to agents", @@ -417,30 +474,49 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { ); if (!daily || daily.engine !== "AggregatingMergeTree") { issues.push("Missing product_events_daily_exact snapshot datasource"); - } else if (hasToken(daily, "product_events_agent_read", "READ")) { - issues.push("Product event aggregate states must be endpoint-only"); + } else { + if (hasToken(daily, "product_events_agent_read", "READ")) { + issues.push("Product event aggregate states must be endpoint-only"); + } + if (daily.ttl !== "date + INTERVAL 800 DAY") { + issues.push("Product event aggregates must retain 800 days"); + } } const canonical = project.datasources.find( (datasource) => datasource.name === "product_events_canonical_v1", ); if (!canonical || canonical.engine !== "MergeTree") { issues.push("Missing product_events_canonical_v1 datasource"); - } else if (hasToken(canonical, "product_events_agent_read", "READ")) { - issues.push("Canonical product events must not be readable by agents"); + } else { + if (hasToken(canonical, "product_events_agent_read", "READ")) { + issues.push("Canonical product events must not be readable by agents"); + } + if (canonical.ttl !== "toDateTime(received_at) + INTERVAL 800 DAY") { + issues.push("Canonical product events must retain 800 days"); + } + if (!hasToken(canonical, "product_events_erasure_lookup", "READ")) { + issues.push("Canonical product events are missing erasure lookup access"); + } } - for (const name of [ - "product_traffic_daily_exact", - "product_traffic_pages_daily_exact", - "product_activation_daily_exact", - "product_creator_retention_exact", + for (const [name, engine] of [ + ["product_traffic_daily_exact", "AggregatingMergeTree"], + ["product_traffic_pages_daily_exact", "AggregatingMergeTree"], + ["product_activation_daily_exact", "AggregatingMergeTree"], + ["product_creator_retention_exact", "AggregatingMergeTree"], + ["product_identity_funnel_exact", "SummingMergeTree"], ]) { const datasource = project.datasources.find( (candidate) => candidate.name === name, ); - if (!datasource || datasource.engine !== "AggregatingMergeTree") { + if (!datasource || datasource.engine !== engine) { issues.push(`Missing privacy-safe aggregate datasource ${name}`); - } else if (hasToken(datasource, "product_events_agent_read", "READ")) { - issues.push(`${name} aggregate states must be endpoint-only`); + } else { + if (hasToken(datasource, "product_events_agent_read", "READ")) { + issues.push(`${name} aggregate states must be endpoint-only`); + } + if (!datasource.ttl?.endsWith("+ INTERVAL 800 DAY")) { + issues.push(`${name} must retain 800 days`); + } } } const healthHourly = project.datasources.find( @@ -453,8 +529,47 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { } else if (hasToken(healthHourly, "product_events_agent_read", "READ")) { issues.push("Product event health states must be endpoint-only"); } + for (const name of PRODUCT_COPY_TARGETS) { + const datasource = project.datasources.find( + (candidate) => candidate.name === name, + ); + if ( + !datasource || + !hasToken(datasource, "product_events_copy_runner", "APPEND") + ) { + issues.push(`${name} is missing Copy runner append access`); + } + } + for (const datasource of project.datasources) { + if ( + !PRODUCT_COPY_TARGETS.includes(datasource.name) && + hasToken(datasource, "product_events_copy_runner", "APPEND") + ) { + issues.push(`${datasource.name} grants unexpected Copy runner access`); + } + if ( + !["product_events_v1", "product_events_canonical_v1"].includes( + datasource.name, + ) && + hasToken(datasource, "product_events_erasure_lookup", "READ") + ) { + issues.push(`${datasource.name} grants unexpected erasure lookup access`); + } + } + for (const name of PRODUCT_COPY_PIPES) { + const pipe = project.pipes.find((candidate) => candidate.name === name); + if (!pipe) { + issues.push(`Missing product analytics pipe ${name}`); + continue; + } + if (!hasToken(pipe, "product_events_copy_runner", "READ")) { + issues.push(`${name} is missing its execution-only Copy token`); + } + if (hasToken(pipe, "product_events_agent_read", "READ")) { + issues.push(`${name} must not grant Copy execution to the agent token`); + } + } for (const name of [ - ...PRODUCT_COPY_PIPES, "product_events_daily", "product_events_health", "product_traffic_overview", @@ -466,6 +581,7 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { "product_creator_retention", "product_creator_activity", "product_feature_adoption", + "product_identity_funnel", "product_analytics_freshness", "product_analytics_copy_assertions", ]) { @@ -481,6 +597,9 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { ) { issues.push(`${name} is missing its read-only agent token`); } + if (hasToken(pipe, "product_events_copy_runner", "READ")) { + issues.push(`${name} must not be queryable by the Copy runner token`); + } } validateFixtures(projectDir, issues); diff --git a/scripts/analytics/verify-local.js b/scripts/analytics/verify-local.js index 5c05f284c1a..23b4c8713cd 100644 --- a/scripts/analytics/verify-local.js +++ b/scripts/analytics/verify-local.js @@ -137,6 +137,7 @@ assert.deepEqual(copyAssertions, [ traffic_page_markers: 1, activation_markers: 1, retention_markers: 1, + identity_markers: 1, }, ]); From d2a45c69f12aac6031eb8030c376f2e6ccea50ac Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:15:36 +0100 Subject: [PATCH 078/110] ci: prove analytics against exact staging deployments --- .github/workflows/analytics.yml | 134 +- ...st-party-analytics-production-readiness.md | 21 +- .../e2e/analytics-staging.spec.ts | 264 +++ .../app/api/analytics/staging-test/route.ts | 216 ++ package.json | 2 +- .../analytics/src/staging-fixtures.test.js | 67 + scripts/analytics/README.md | 14 +- scripts/analytics/staging-ci-lib.js | 1735 ++++++++++++++- scripts/analytics/staging-ci.js | 1917 +++++++++++++++-- scripts/analytics/tests/staging-ci.test.js | 960 ++++++++- 10 files changed, 5088 insertions(+), 242 deletions(-) create mode 100644 apps/chrome-extension/e2e/analytics-staging.spec.ts create mode 100644 apps/web/app/api/analytics/staging-test/route.ts create mode 100644 packages/analytics/src/staging-fixtures.test.js diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 83d3495a9ec..978c7054e17 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -112,6 +112,9 @@ jobs: env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} + TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} TINYBIRD_STAGING_INGEST_TOKEN: ${{ secrets.TINYBIRD_STAGING_INGEST_TOKEN }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} @@ -178,24 +181,28 @@ jobs: --deployment-id "${{ steps.tinybird.outputs.id }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Rebuild staged decision and health copies + - name: Rebuild no-op live decision and health copies id: staging-copies + if: steps.tinybird.outputs.needs_promotion != 'true' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" --phase staged - --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live' }}" + --target live --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Prove staged deduplication, conflict visibility, SLO, and endpoint budget + - name: Prove candidate isolation or no-op live decision quality id: verify env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- node scripts/analytics/staging-ci.js verify --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -253,13 +260,27 @@ jobs: id: token-scopes env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TINYBIRD_STAGING_INGEST_TOKEN: ${{ secrets.TINYBIRD_STAGING_INGEST_TOKEN }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} + TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_INGEST_TOKEN: ${{ secrets.TINYBIRD_STAGING_INGEST_TOKEN }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} run: >- node scripts/analytics/staging-ci.js verify-token-scopes --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Prove the exact-SHA deployed browser tracker + id: browser-e2e + env: + ANALYTICS_PREVIEW_URL: ${{ steps.vercel.outputs.url }} + ANALYTICS_BROWSER_RUN_ID: ${{ env.ANALYTICS_TEST_RUN_ID }}_preview + ANALYTICS_STATE_PATH: ${{ runner.temp }}/analytics-staging-state.json + ANALYTICS_ARTIFACT_PATH: ${{ runner.temp }}/analytics-staging-report.json + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + run: | + pnpm --dir apps/chrome-extension exec playwright install --with-deps chromium + pnpm --dir apps/chrome-extension exec playwright test e2e/analytics-staging.spec.ts - name: Probe the exact-SHA Vercel browser collector and staging rate limit id: preview-api env: @@ -268,10 +289,22 @@ jobs: node scripts/analytics/staging-ci.js probe-preview --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Prove exact-SHA durable server delivery + id: server-delivery + env: + CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + run: >- + node scripts/analytics/staging-ci.js probe-server + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Rebuild promoted decision and health copies id: promoted-copies env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- @@ -281,42 +314,52 @@ jobs: --target live --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Prove promoted preview delivery and decision deduplication - id: verify-promoted + - name: Measure populated decision endpoint performance + id: populated-performance env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} run: >- - node scripts/analytics/staging-ci.js verify-promoted + node scripts/analytics/staging-ci.js verify + --deployment-id "${{ steps.tinybird.outputs.id }}" + --baseline-deployment-id "${{ steps.promote.outputs.previous_live_id || steps.tinybird.outputs.id }}" + --target live --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Delete the scoped synthetic user and organization identity - id: erase-identity + - name: Prove promoted delivery, business values, and decision deduplication + id: verify-promoted env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} - TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} run: >- - node scripts/analytics/staging-ci.js erase-synthetic-identity + node scripts/analytics/staging-ci.js verify-promoted --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Retract the erased identity from every derived copy - id: erasure-copies - if: steps.erase-identity.outcome == 'success' + - name: Prove exact staging rollback and restoration + id: rollback-drill + if: steps.verify-promoted.outcome == 'success' && steps.promote.outputs.promoted == 'true' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} run: >- - node scripts/analytics/staging-ci.js run-copies + node scripts/analytics/staging-ci.js drill-rollback --deployment-id "${{ steps.tinybird.outputs.id }}" - --phase erasure - --target live + --previous-live-id "${{ steps.promote.outputs.previous_live_id }}" + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Delete the scoped identity through the exact-SHA application path + id: erase-identity + env: + CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + run: >- + node scripts/analytics/staging-ci.js erase-synthetic-identity --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Prove identity erasure and out-of-scope control preservation id: verify-erasure - if: steps.erasure-copies.outcome == 'success' + if: steps.erase-identity.outcome == 'success' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} @@ -324,6 +367,18 @@ jobs: node scripts/analytics/staging-ci.js verify-synthetic-identity-erasure --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Quiesce scheduled and active Copy jobs before final cleanup + id: pause-copies + if: always() && steps.seed.outcome != 'skipped' + env: + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js set-copy-schedules + --action pause + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Delete strictly scoped synthetic raw rows id: cleanup if: always() && steps.seed.outcome != 'skipped' @@ -347,6 +402,7 @@ jobs: if: always() && steps.cleanup.outputs.required == 'true' && steps.cleanup.outcome == 'success' && steps.cleanup.outputs.requires_copies == 'true' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- @@ -368,9 +424,45 @@ jobs: --target "${{ steps.cleanup-copies.outputs.target }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Resume reviewed Copy schedules + id: resume-copies + if: always() && steps.pause-copies.outcome != 'skipped' + env: + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js set-copy-schedules + --action resume + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Finalize the fully verified staging promotion + id: finalize + if: steps.verify-cleanup.outcome == 'success' && steps.rollback-drill.outcome == 'success' && steps.resume-copies.outcome == 'success' + env: + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js finalize-promotion + --deployment-id "${{ steps.tinybird.outputs.id }}" + --previous-live-id "${{ steps.promote.outputs.previous_live_id }}" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Restore the previous staging deployment on failure + id: rollback + if: always() && steps.promote.outputs.previous_live_id != '' && steps.finalize.outcome != 'success' && steps.rollback-drill.outputs.rollback_target_usable != 'false' + env: + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js rollback-promotion + --deployment-id "${{ steps.tinybird.outputs.id }}" + --previous-live-id "${{ steps.promote.outputs.previous_live_id }}" + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Discard an unpromoted staging deployment after cleanup id: discard - if: always() && (steps.deployment-state.outputs.discard == 'true' || steps.cleanup.outputs.requires_discard == 'true') + if: always() && steps.rollback.outcome != 'success' && (steps.deployment-state.outputs.discard == 'true' || steps.cleanup.outputs.requires_discard == 'true') env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index c378974b6ee..08b2460d28f 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -14,7 +14,7 @@ Desktop critical events enter a bounded encrypted local outbox before network de - Visitor: a first-party pseudonymous browser ID stored in a persistent cookie/local storage. It is not a claim that the actor is a person. - Session: a browser visit shared across tabs and renewed after 30 minutes of inactivity. Reloads and SPA navigation retain the session; activity at 29 minutes extends it; a return after more than 30 minutes starts a new session. Hidden time is not engagement time. -- Actor: `user_id` when authenticated, otherwise the anonymous visitor ID, otherwise the session ID. Anonymous and authenticated IDs are carried together during signup and checkout so the journey can be stitched without replacing the authenticated source of truth. +- Actor: `user_id` when authenticated, otherwise the anonymous visitor ID, otherwise the session ID. A canonical `identity_linked` event maps pre-auth acquisition to signup, while a settled guest purchase may establish the same mapping from its preserved anonymous checkout identity. The privacy-safe funnel materialization returns cohort counts only and never exposes that mapping. - User: an authenticated Cap account. Cross-device creator metrics use `user_id`. - Organization: the authoritative organization attached to a server fact. Organization metrics never infer membership from an anonymous browser event. @@ -36,9 +36,11 @@ The desktop critical-event outbox uses AES-256-GCM. Its stable random key is sto Account and organization deletion writes its durable pending marker or tombstone before erasure begins. Delivery and reconciliation reject marked identities, then deletion waits longer than the bounded in-flight delivery attempt before removing matching raw rows with a dedicated erasure token and rebuilding every replace-mode canonical, traffic, activation, retention, and health snapshot. This closes the race where a workflow that passed its identity check could otherwise append after a completed erasure. Deletion fails closed if the erasure credential or any rebuild is unavailable. The staging suite deletes a synthetic user and organization, proves their raw-health and decision state is gone, and proves an out-of-scope row sharing the anonymous ID remains until final test cleanup. Tinybird's row-deletion API currently requires the general `DATASOURCES:CREATE` scope and rejects resource-scoped creation of that operator; the token therefore has no read or deployment scope, lives only in the protected staging environment, and is used by code that accepts bounded validated deletion predicates. Its broader same-workspace mutation capability is an explicit provider limitation rather than a least-privilege claim. +Raw, canonical, and decision data share an 800-day TTL. This supports two complete 365-day comparison windows plus a rebuild buffer for year-over-year decisions. The identity-bearing source is intentionally retained for the same horizon as its exact aggregates so a later account or organization deletion can still retract every historical contribution. Keeping only irreversible long-lived counts would use less storage but would break the derived-erasure guarantee. Health-detail aggregates remain limited to 90 days. This retention choice requires privacy-owner approval before production rollout and must be disclosed with the persistent first-party identifier model. + ## Data quality and performance gates -The staging workflow is restricted to PR 2003 and `codex/first-party-analytics`, hard-codes the staging Tinybird workspace ID, checks the exact Git SHA, waits for the exact-SHA Vercel preview, creates an isolated Tinybird staging deployment, runs fixture and synthetic tests, promotes only a verified deployment inside that staging workspace, and discards an unpromoted deployment on failure. It has no push trigger, production environment, production token, or production deployment command. +The staging workflow is restricted to PR 2003 and `codex/first-party-analytics`, hard-codes the staging Tinybird workspace ID, checks the exact Git SHA, waits for the exact-SHA Vercel preview, creates an isolated Tinybird staging deployment, runs fixture and synthetic tests, and promotes only a verified deployment inside that staging workspace. Candidate reads use the exact numeric deployment ID and prove raw delivery, isolation, endpoint execution, and absolute latency before promotion. Tinybird's Copy API does not reliably route an on-demand Copy mutation into a candidate, so Copy mutations are prohibited until the exact candidate is live in staging. The prior staging deployment remains available until promoted Copy results, public business values, retained-deployment and representative-volume performance, erasure, cleanup, and a live rollback-and-restoration drill pass. The first rollout of a new endpoint compares only shared endpoints to the retained deployment, records the new endpoint as having no historical baseline, and still applies candidate and representative absolute budgets. The rollback drill executes every shared typed endpoint while the prior deployment is live; the exact-SHA admin client treats only an absent identity-funnel endpoint as optional, then the drill restores the candidate and re-proves the full endpoint suite, health, and synthetic business responses. Ambiguous live-switch responses are reconciled against the exact deployment pair before recovery continues. Any earlier failure restores a data-plane-proven prior deployment and removes the rejected candidate. The workflow has no push trigger, production environment, production token, or production deployment command. The redacted evidence artifact records the Git SHA, GitHub run, Vercel and Tinybird deployment IDs, hashed synthetic-run identity, delivery attempts, ingestion throughput, endpoint and full-dashboard p50/p95/p99, measured-baseline regressions, visibility time, health totals, decision-dedup assertions, conflict quarantine, least-privilege token checks, scoped identity erasure, and cleanup. Synthetic rows are excluded from normal metrics and cleanup is verified after every promoted run. @@ -52,21 +54,27 @@ Required live gates are: - raw and all derived synthetic state is removed successfully; - an erased identity disappears from raw and derived results while an out-of-scope control remains; - aggregate read tokens cannot query raw identifiers or append, and append/cleanup tokens cannot read aggregate endpoints; +- a bounded five-event fixture produces exact non-zero traffic, page, activation, and retention values while remaining absent from normal decision endpoints; +- a populated-table performance pass measures every typed decision endpoint and full dashboard fanout after materialization; - wrong workspace, stale SHA, missing credentials, partial execution, failed promotion, and failed cleanup all fail closed. -Copy-backed decision tables rebuild on serialized eight-minute schedules to avoid competing for the same Tinybird worker pool. Exact-SHA staging runs invoke the seven copies in dependency order immediately after seeding, promotion, identity erasure, and final cleanup, so the staging visibility SLO measures the deployed data path rather than waiting for the periodic schedule. Dashboard freshness exposes the most recent aggregate timestamps; scheduled production freshness is therefore bounded by the documented copy cadence plus provider execution time. +Copy-backed decision tables rebuild on serialized eight-minute schedules to avoid competing for the same Tinybird worker pool. Exact-SHA staging runs invoke the eight copies in dependency order after an exact candidate is promoted, after identity erasure, and after final cleanup. A no-op deployment invokes them immediately after seeding. Candidate ingestion visibility is measured directly against the exact candidate and promoted Copy visibility is measured separately, so neither proof waits for the periodic schedule. Dashboard freshness exposes product, traffic, retention, and identity-funnel aggregate timestamps; scheduled production freshness is therefore bounded by the documented copy cadence plus provider execution time. -The first staging run establishes absolute ingestion and endpoint baselines on the exact deployed code. The final branch must then commit regression budgets derived from those measurements and rerun at representative and larger bounded volumes. A green static/unit run alone is not rollout evidence. +The staging run records absolute ingestion budgets, compares shared typed endpoints and dashboard fanout against the retained prior deployment, and separately measures bounded 1,000-row and 10,000-row mixed high-cardinality corpora with nonzero traffic, activation, retention, identity, product, and revenue assertions. It measures exact-SHA browser main-thread cost and append-batch p50, p95, p99, error rate, and throughput. A newly introduced endpoint without an honest prior baseline is labeled as such and must pass measured and representative absolute budgets. It reports p50, p95, and p99 for baseline, measured, and representative samples, and applies retained-baseline regression budgets to representative samples wherever a baseline exists. A green static/unit run alone is not rollout evidence. ## Production rollout checklist Production remains prohibited until the final staging artifact, relevant CI, security/data/performance reviews, and Greptile are green for the same branch SHA. +The preview-only `/api/analytics/staging-test` route must not receive a production secret. Its `CAP_ANALYTICS_STAGING_TEST_SECRET` exists only in the Vercel Preview environment and the protected GitHub staging environment; the route returns `404` outside `VERCEL_ENV=preview` and requires the exact Vercel Git SHA. + 1. Create production Tinybird resources from the reviewed datafiles with a deploy credential scoped only to the production workspace. Run `deployment create --check`, review destructive/schema changes, create an isolated deployment, run fixture tests, rebuild every copy, query all aggregate endpoints, then promote. Record the deployment ID. Do not reuse the staging workspace or tokens. 2. Create least-privilege Tinybird tokens: - append-only token for `product_events_v1`; - aggregate endpoint read token with no raw or canonical datasource access; - - resource-scoped copy token for the seven reviewed copy pipes, with no raw identity datasource access; + - resource-scoped copy token for the eight reviewed copy pipes, with no raw identity datasource access; + - erasure-lookup token limited to read access on `product_events_v1` and `product_events_canonical_v1`, protected from all agent and admin surfaces; + - schedule-controller token limited to cancelling, pausing, and resuming the eight reviewed Copy Pipes; - dedicated erasure token with Tinybird's required `DATASOURCES:CREATE` scope, no read/deploy scopes, protected as a high-impact operational secret until Tinybird offers resource-scoped row deletion; - deployment token used only by the controlled production release path. 3. Set these Vercel production variables without copying values into logs or artifacts: @@ -74,6 +82,9 @@ Production remains prohibited until the final staging artifact, relevant CI, sec - `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` - `PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN` - `PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN` + - `PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN` + - `PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN` + - `PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN` - `PRODUCT_ANALYTICS_INTERNAL_IP_HASHES` - `CRON_SECRET` - `NEXTAUTH_SECRET` (existing application secret used to sign the short-lived anonymous browser token; do not create an analytics-specific duplicate) diff --git a/apps/chrome-extension/e2e/analytics-staging.spec.ts b/apps/chrome-extension/e2e/analytics-staging.spec.ts new file mode 100644 index 00000000000..11eb7a6d5a4 --- /dev/null +++ b/apps/chrome-extension/e2e/analytics-staging.spec.ts @@ -0,0 +1,264 @@ +import fs from "node:fs"; +import { expect, test } from "@playwright/test"; + +type CapturedEvent = { + eventId: string; + eventName: string; + sessionId: string; + properties?: { is_session_entry?: boolean }; +}; + +const requestEvents = (request: { + postData: () => string | null; +}): CapturedEvent[] => { + try { + const payload = JSON.parse(request.postData() ?? "{}") as { + events?: CapturedEvent[]; + }; + return payload.events ?? []; + } catch { + return []; + } +}; + +const requiredEnvironment = (name: string) => { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +}; + +test("exact-SHA browser tracker preserves sessions, retries, unloads, and its main-thread budget", async ({ + browser, +}) => { + const previewUrl = requiredEnvironment("ANALYTICS_PREVIEW_URL"); + const runId = requiredEnvironment("ANALYTICS_BROWSER_RUN_ID"); + const statePath = requiredEnvironment("ANALYTICS_STATE_PATH"); + const artifactPath = requiredEnvironment("ANALYTICS_ARTIFACT_PATH"); + const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim(); + const context = await browser.newContext({ + baseURL: previewUrl, + extraHTTPHeaders: { + "x-cap-analytics-test-run": runId, + ...(bypass + ? { + "x-vercel-protection-bypass": bypass, + "x-vercel-set-bypass-cookie": "true", + } + : {}), + }, + }); + const captured: CapturedEvent[] = []; + const acceptedEventIds = new Set(); + let acceptedRequests = 0; + let failedRequests = 0; + context.on("request", (request) => { + if (!request.url().endsWith("/api/events") || request.method() !== "POST") { + return; + } + captured.push(...requestEvents(request)); + }); + context.on("response", (response) => { + if ( + response.url().endsWith("/api/events") && + response.request().method() === "POST" + ) { + if (response.ok()) { + acceptedRequests += 1; + for (const event of requestEvents(response.request())) { + acceptedEventIds.add(event.eventId); + } + } + } + }); + context.on("requestfailed", (request) => { + if (request.url().endsWith("/api/events")) failedRequests += 1; + }); + + const page = await context.newPage(); + const cdp = await context.newCDPSession(page); + await cdp.send("Performance.enable"); + const taskDuration = async () => { + const metrics = await cdp.send("Performance.getMetrics"); + return ( + metrics.metrics.find((metric) => metric.name === "TaskDuration")?.value ?? + 0 + ); + }; + const beforeTaskDuration = await taskDuration(); + await page.goto("/?utm_source=staging-browser&utm_medium=e2e", { + waitUntil: "networkidle", + }); + await expect + .poll( + () => captured.filter((event) => event.eventName === "page_view").length, + ) + .toBeGreaterThanOrEqual(1); + const firstPageView = captured.find( + (event) => event.eventName === "page_view", + ); + expect(firstPageView?.properties?.is_session_entry).toBe(true); + + await page.reload({ waitUntil: "networkidle" }); + await expect + .poll( + () => captured.filter((event) => event.eventName === "page_view").length, + ) + .toBeGreaterThanOrEqual(2); + const reloadPageView = captured + .filter((event) => event.eventName === "page_view") + .at(-1); + expect(reloadPageView?.sessionId).toBe(firstPageView?.sessionId); + expect(reloadPageView?.properties?.is_session_entry).toBe(false); + + await page.evaluate(() => { + const key = "cap_analytics_session_v2"; + const value = JSON.parse(localStorage.getItem(key) ?? "null") as { + lastActivityAt?: number; + } | null; + if (!value || typeof value.lastActivityAt !== "number") { + throw new Error("Browser analytics session was not persisted"); + } + value.lastActivityAt = Date.now() - 29 * 60 * 1_000; + localStorage.setItem(key, JSON.stringify(value)); + }); + await page.reload({ waitUntil: "networkidle" }); + await expect + .poll( + () => captured.filter((event) => event.eventName === "page_view").length, + ) + .toBeGreaterThanOrEqual(3); + const activePageView = captured + .filter((event) => event.eventName === "page_view") + .at(-1); + expect(activePageView?.sessionId).toBe(firstPageView?.sessionId); + + const secondPage = await context.newPage(); + await secondPage.goto("/pricing", { waitUntil: "networkidle" }); + await expect + .poll( + () => captured.filter((event) => event.eventName === "page_view").length, + ) + .toBeGreaterThanOrEqual(4); + const secondTabPageView = captured + .filter((event) => event.eventName === "page_view") + .at(-1); + expect(secondTabPageView?.sessionId).toBe(firstPageView?.sessionId); + await secondPage.close(); + + await page.evaluate(() => { + const key = "cap_analytics_session_v2"; + const value = JSON.parse(localStorage.getItem(key) ?? "null") as { + lastActivityAt?: number; + } | null; + if (!value || typeof value.lastActivityAt !== "number") { + throw new Error("Browser analytics session was not persisted"); + } + value.lastActivityAt = Date.now() - (30 * 60 * 1_000 + 1); + localStorage.setItem(key, JSON.stringify(value)); + }); + await page.reload({ waitUntil: "networkidle" }); + await expect + .poll( + () => captured.filter((event) => event.eventName === "page_view").length, + ) + .toBeGreaterThanOrEqual(5); + const returnedPageView = captured + .filter((event) => event.eventName === "page_view") + .at(-1); + expect(returnedPageView?.sessionId).not.toBe(firstPageView?.sessionId); + expect(returnedPageView?.properties?.is_session_entry).toBe(true); + + let abortNextCollectorRequest = true; + await context.route("**/api/events", async (route) => { + if (abortNextCollectorRequest) { + abortNextCollectorRequest = false; + await route.abort("internetdisconnected"); + return; + } + await route.continue(); + }); + await page + .locator('a[href="/pricing"]') + .first() + .evaluate((element) => { + (element as HTMLAnchorElement).click(); + }); + await expect(page).toHaveURL(/\/pricing/); + await expect + .poll(() => failedRequests, { timeout: 15_000 }) + .toBeGreaterThanOrEqual(1); + await context.unroute("**/api/events"); + await expect + .poll(() => acceptedEventIds.size, { timeout: 15_000 }) + .toBeGreaterThanOrEqual(6); + await page.evaluate(() => window.dispatchEvent(new Event("pointerdown"))); + await page.waitForTimeout(100); + await page.evaluate(() => + window.dispatchEvent(new PageTransitionEvent("pagehide")), + ); + await expect + .poll( + () => + captured + .filter((event) => event.eventName === "page_engagement") + .some((event) => acceptedEventIds.has(event.eventId)), + { timeout: 15_000 }, + ) + .toBe(true); + + const afterTaskDuration = await taskDuration(); + const interactionTaskDurationMs = Math.max( + 0, + Math.round((afterTaskDuration - beforeTaskDuration) * 1_000), + ); + const taskDurationBudgetMs = Number( + process.env.ANALYTICS_BROWSER_TASK_BUDGET_MS ?? 1_500, + ); + expect(interactionTaskDurationMs).toBeLessThanOrEqual(taskDurationBudgetMs); + const uniqueEventIds = new Set(captured.map((event) => event.eventId)); + expect(uniqueEventIds.size).toBeGreaterThanOrEqual(6); + expect( + [...uniqueEventIds].every((eventId) => acceptedEventIds.has(eventId)), + ).toBe(true); + expect(captured.some((event) => event.eventName === "page_engagement")).toBe( + true, + ); + + const state = JSON.parse(fs.readFileSync(statePath, "utf8")) as Record< + string, + unknown + >; + state.browserExpectedEvents = acceptedEventIds.size; + fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, { + mode: 0o600, + }); + const artifact = JSON.parse(fs.readFileSync(artifactPath, "utf8")) as Record< + string, + unknown + > & { assertions?: Record }; + artifact.browser = { + acceptedRequests, + failedRequests, + uniqueEvents: acceptedEventIds.size, + pageViews: captured.filter((event) => event.eventName === "page_view") + .length, + engagementEvents: captured.filter( + (event) => event.eventName === "page_engagement", + ).length, + sameTabReloadPassed: true, + multiTabSessionPassed: true, + activityAt29MinutesPassed: true, + inactivityBoundaryPassed: true, + offlineRetryPassed: true, + unloadPassed: true, + interactionTaskDurationMs, + taskDurationBudgetMs, + }; + artifact.assertions = { + ...(artifact.assertions ?? {}), + deployedBrowserTrackerPassed: true, + browserMainThreadBudgetPassed: true, + }; + fs.writeFileSync(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`); + await context.close(); +}); diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts new file mode 100644 index 00000000000..2d551fe26b8 --- /dev/null +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -0,0 +1,216 @@ +import { createHash, timingSafeEqual } from "node:crypto"; +import { Tinybird } from "@cap/web-backend"; +import { + HttpApi, + HttpApiBuilder, + HttpApiEndpoint, + HttpApiError, + HttpApiGroup, + HttpServerRequest, +} from "@effect/platform"; +import { Effect, Layer, Schema } from "effect"; +import type Stripe from "stripe"; +import { queueServerProductEvent } from "@/lib/analytics/server"; +import type { ServerProductEvent } from "@/lib/analytics/server-event"; +import { subscriptionCheckoutProductEvent } from "@/lib/analytics/stripe-business-events"; +import { apiToHandler } from "@/lib/server"; + +class Api extends HttpApi.make("AnalyticsStagingTestApi").add( + HttpApiGroup.make("stagingTest") + .add( + HttpApiEndpoint.post("run", "/api/analytics/staging-test") + .setPayload( + Schema.Struct({ + scenario: Schema.Literal("business_lifecycle"), + runId: Schema.String, + sha: Schema.String, + }), + ) + .addSuccess( + Schema.Struct({ + accepted: Schema.Number, + uniqueEvents: Schema.Number, + workflowRuns: Schema.Array(Schema.String), + }), + ) + .addError(HttpApiError.BadRequest) + .addError(HttpApiError.Unauthorized) + .addError(HttpApiError.NotFound) + .addError(HttpApiError.ServiceUnavailable), + ) + .add( + HttpApiEndpoint.post("erase", "/api/analytics/staging-test/erase") + .setPayload( + Schema.Struct({ + runId: Schema.String, + sha: Schema.String, + }), + ) + .addSuccess(Schema.Struct({ erased: Schema.Boolean })) + .addError(HttpApiError.BadRequest) + .addError(HttpApiError.Unauthorized) + .addError(HttpApiError.NotFound) + .addError(HttpApiError.ServiceUnavailable), + ), +) {} + +const RequestHeaders = Schema.Struct({ + authorization: Schema.optional(Schema.String), +}); + +const safeEqual = (actual: string | undefined, expected: string) => + Boolean( + actual && + actual.length === expected.length && + timingSafeEqual(Buffer.from(actual), Buffer.from(expected)), + ); + +const boundedRunId = (value: string) => + /^[A-Za-z0-9_-]{8,128}$/.test(value) ? value : undefined; + +const draftSha = (value: string) => /^[0-9a-f]{40}$/.test(value); + +const authorize = (payload: { runId: string; sha: string }) => + Effect.gen(function* () { + if (process.env.VERCEL_ENV !== "preview") { + return yield* Effect.fail(new HttpApiError.NotFound()); + } + const secret = process.env.CAP_ANALYTICS_STAGING_TEST_SECRET; + if (!secret) { + return yield* Effect.fail(new HttpApiError.ServiceUnavailable()); + } + const headers = yield* HttpServerRequest.schemaHeaders(RequestHeaders).pipe( + Effect.mapError(() => new HttpApiError.BadRequest()), + ); + if (!safeEqual(headers.authorization, `Bearer ${secret}`)) { + return yield* Effect.fail(new HttpApiError.Unauthorized()); + } + const runId = boundedRunId(payload.runId); + if ( + !runId || + !draftSha(payload.sha) || + payload.sha !== process.env.VERCEL_GIT_COMMIT_SHA + ) { + return yield* Effect.fail(new HttpApiError.BadRequest()); + } + return runId; + }); + +const ApiLive = HttpApiBuilder.api(Api).pipe( + Layer.provide( + HttpApiBuilder.group(Api, "stagingTest", (handlers) => + Effect.gen(function* () { + const tinybird = yield* Tinybird; + return handlers + .handle("run", ({ payload }) => + Effect.gen(function* () { + const runId = yield* authorize(payload); + const hash = createHash("sha256").update(runId).digest("hex"); + const occurredAt = new Date().toISOString(); + const userId = `staging_user_${hash.slice(0, 20)}`; + const organizationId = `staging_org_${hash.slice(20, 40)}`; + const anonymousId = `staging_anon_${hash.slice(40, 60)}`; + const purchase = subscriptionCheckoutProductEvent({ + eventId: `staging_${hash.slice(0, 24)}`, + occurredAt, + session: { + amount_subtotal: 2_500, + amount_total: 2_500, + currency: "usd", + metadata: { + analyticsAnonymousId: anonymousId, + analyticsIsFirstPurchase: "true", + analyticsOrganizationId: organizationId, + analyticsPriceId: "price_staging_annual", + analyticsQuantity: "1", + analyticsSchemaVersion: "1", + isOnBoarding: "false", + platform: "web", + }, + payment_status: "paid", + total_details: { amount_discount: 0 }, + } as unknown as Stripe.Checkout.Session, + user: { id: userId }, + }); + if (!purchase) { + return yield* Effect.fail( + new HttpApiError.ServiceUnavailable(), + ); + } + const events: ServerProductEvent[] = [ + { + _syntheticRunId: runId, + anonymousId, + eventId: `staging_signup_${hash.slice(0, 24)}`, + eventName: "user_signed_up", + occurredAt, + organizationId, + platform: "web", + userId, + }, + { + _syntheticRunId: runId, + anonymousId, + eventId: `staging_share_${hash.slice(0, 24)}`, + eventName: "share_link_created", + occurredAt, + organizationId, + platform: "server", + properties: { + asset_type: "recording", + recording_mode: "screen", + }, + userId, + }, + { + _syntheticRunId: runId, + anonymousId, + eventId: `staging_checkout_${hash.slice(0, 24)}`, + eventName: "checkout_started", + occurredAt, + organizationId, + platform: "web", + properties: { + is_onboarding: false, + price_id: "price_staging_annual", + quantity: 1, + }, + userId, + }, + { ...purchase, _syntheticRunId: runId }, + { ...purchase, _syntheticRunId: runId }, + ]; + const workflowRuns = yield* Effect.tryPromise({ + try: () => Promise.all(events.map(queueServerProductEvent)), + catch: () => new HttpApiError.ServiceUnavailable(), + }); + return { + accepted: events.length, + uniqueEvents: new Set(events.map((event) => event.eventId)) + .size, + workflowRuns: workflowRuns.map(({ runId: id }) => id), + }; + }), + ) + .handle("erase", ({ payload }) => + Effect.gen(function* () { + const runId = yield* authorize(payload); + const hash = createHash("sha256").update(runId).digest("hex"); + yield* tinybird + .eraseProductAnalytics({ + userId: `synthetic_user_${hash.slice(0, 24)}`, + organizationId: `synthetic_org_${hash.slice(24, 48)}`, + }) + .pipe( + Effect.mapError(() => new HttpApiError.ServiceUnavailable()), + ); + return { erased: true }; + }), + ); + }), + ), + ), +); + +const handler = apiToHandler(ApiLive); +export const POST = handler; diff --git a/package.json b/package.json index fac677d82df..82e81353136 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "analytics:migrate-dub:apply": "dotenv -e .env -- node scripts/analytics/migrate-dub-to-tinybird.js --apply", "analytics:setup": "pnpm analytics:deploy", "analytics:validate": "node scripts/analytics/analytics-cli.js validate", - "analytics:test": "pnpm --filter @cap/analytics test && pnpm --filter @cap/web-backend test && pnpm --filter @cap/mobile test && pnpm --filter @cap/web test --run __tests__/unit/admin-product-analytics.test.ts __tests__/unit/product-analytics-browser-token.test.ts __tests__/unit/product-analytics-delivery.test.ts __tests__/unit/product-analytics-erasure.test.ts __tests__/unit/product-analytics-experiment.test.ts __tests__/unit/product-analytics-queue.test.ts __tests__/unit/product-analytics-scheduler.test.ts __tests__/unit/product-analytics-request.test.ts __tests__/unit/product-analytics-transport.test.ts __tests__/unit/product-analytics-server.test.ts __tests__/unit/guest-checkout-analytics.test.ts __tests__/unit/subscription-analytics-webhook.test.ts __tests__/unit/stripe-analytics-reconciliation.test.ts __tests__/unit/developer-credits-webhook.test.ts && pnpm --dir apps/desktop exec vitest run src/utils/analytics.test.ts src/utils/product-analytics.test.ts && node scripts/analytics/analytics-cli.js test", + "analytics:test": "node scripts/analytics/check-event-contract.js && pnpm --filter @cap/analytics test && pnpm --filter @cap/web-backend test && pnpm --filter @cap/mobile test && pnpm --filter @cap/web test --run __tests__/unit/admin-product-analytics.test.ts __tests__/unit/product-analytics-browser-token.test.ts __tests__/unit/product-analytics-delivery.test.ts __tests__/unit/product-analytics-erasure.test.ts __tests__/unit/product-analytics-experiment.test.ts __tests__/unit/product-analytics-queue.test.ts __tests__/unit/product-analytics-scheduler.test.ts __tests__/unit/product-analytics-request.test.ts __tests__/unit/product-analytics-transport.test.ts __tests__/unit/product-analytics-server.test.ts __tests__/unit/guest-checkout-analytics.test.ts __tests__/unit/subscription-analytics-webhook.test.ts __tests__/unit/stripe-analytics-reconciliation.test.ts __tests__/unit/developer-credits-webhook.test.ts && pnpm --dir apps/desktop exec vitest run src/utils/analytics.test.ts src/utils/product-analytics.test.ts && node scripts/analytics/analytics-cli.js test", "analytics:local": "dotenv -e .env -- node scripts/analytics/analytics-cli.js local", "analytics:local:test": "dotenv -e .env -- node scripts/analytics/analytics-cli.js local-test", "analytics:local:tokens": "dotenv -e .env -- node scripts/analytics/analytics-cli.js local-tokens", diff --git a/packages/analytics/src/staging-fixtures.test.js b/packages/analytics/src/staging-fixtures.test.js new file mode 100644 index 00000000000..44ff2a2d520 --- /dev/null +++ b/packages/analytics/src/staging-fixtures.test.js @@ -0,0 +1,67 @@ +import { describe, expect, it } from "vitest"; +import { + createSyntheticDecisionEvents, + createSyntheticErasureControl, + createSyntheticEvents, + createSyntheticLoadEvents, +} from "../../../scripts/analytics/staging-ci-lib.js"; +import { + getProductEventDefinition, + normalizeAcquisitionChannel, + normalizeProductEventProperties, +} from "./index"; + +const validateRow = (row) => { + const definition = getProductEventDefinition(row.event_name); + const properties = JSON.parse(row.properties); + if (row.schema_version < definition.version) { + expect(row.event_name).toBe("subscription_renewed"); + expect(row.schema_version).toBe(1); + expect(properties).toEqual({ + amount_paid_minor: 1_000, + currency: "gbp", + billing_reason: "subscription_cycle", + }); + return; + } + const normalized = normalizeProductEventProperties( + row.event_name, + properties, + ); + if (normalized === null) { + throw new Error( + `Invalid staging fixture properties for ${row.event_name} in ${row.app_version}: ${row.properties}`, + ); + } + + expect(definition.version).toBe(row.schema_version); + expect(definition.platforms).toContain(row.platform); + expect( + definition.authority === "both" || definition.authority === row.source, + ).toBe(true); + expect(normalized ?? {}).toEqual(properties); + expect( + normalizeAcquisitionChannel(normalized, row.referrer), + `${row.event_name}:${row.event_id}`, + ).toBe(row.channel); +}; + +describe("analytics staging fixtures", () => { + it("uses only registry-valid production event shapes", () => { + const runId = "run_12345678"; + const now = new Date("2026-07-31T10:00:00.000Z"); + const delivery = createSyntheticEvents({ runId, now }); + const control = createSyntheticErasureControl({ runId, now }); + const decisions = createSyntheticDecisionEvents({ runId, now }); + const load = createSyntheticLoadEvents({ runId, count: 100, now }); + + for (const row of [ + ...delivery.rows, + control.row, + ...decisions.rows, + ...load.rows, + ]) { + validateRow(row); + } + }); +}); diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index ad2e216eb9e..a255f061fc7 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -34,14 +34,18 @@ Cloud deployment requires: - `TINYBIRD_URL`: the regional Tinybird API URL. - `TINYBIRD_WORKSPACE_ID`: the production Workspace UUID verified before every cloud check or deployment. -The deployed datafiles create two runtime tokens: +The deployed datafiles create four runtime tokens: - `product_events_ingest`: append-only access to `product_events_v1`. -- `product_events_agent_read`: read-only access to privacy-safe product endpoints and the seven reviewed copy pipes. It has no raw identity datasource scope. +- `product_events_agent_read`: read-only access to privacy-safe product endpoints. It has no raw identity datasource or Copy Pipe scope. +- `product_events_copy_runner`: execution-only access to the eight reviewed Copy Pipes. It cannot query endpoints or raw identity data. +- `product_events_erasure_lookup`: read-only access to `product_events_v1` and `product_events_canonical_v1` for bounded identity erasure. It cannot query decision endpoints, append rows, or execute Copy Pipes. Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. -The staging workflow triggers reviewed Copy Pipes through Tinybird's direct Copy API. This preserves the resource-scoped `product_events_agent_read` token; `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Tinybird does not grant either the resource token or the deployment token access to the resulting Jobs API record, so CI proves terminal completion from canonical, daily, and health state transitions plus a unique zero-valued marker written by every other aggregate Copy. Decision endpoints always exclude those markers. Candidate validation pins the staging deployment parameter, while no-op and post-promotion phases use the already verified live deployment because Tinybird rejects a deployment selector on the Copy service after promotion. +Staging also requires `TINYBIRD_STAGING_SCHEDULER_TOKEN`, a protected token with schedule-control access only to the eight reviewed Copy Pipes. The destructive erasure phase cancels and pauses those schedules before deleting rows, runs each replacement Copy exactly once, proves completion with scoped state transitions or copy-run markers, and resumes every schedule in an always-run recovery step. A failed or ambiguous Copy submission is never retried automatically. + +The staging workflow validates candidate ingestion, duplicate/conflict visibility, live isolation, and every typed endpoint through the exact numeric deployment ID before promotion. Tinybird's Copy service does not reliably route an on-demand Copy mutation to a deployment candidate, so CI refuses that path. After the exact candidate is promoted inside the staging workspace, CI triggers the reviewed Copy Pipes through Tinybird's direct Copy API with the protected `product_events_copy_runner` token. `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Tinybird does not grant either the resource token or the deployment token access to the resulting Jobs API record, so CI proves terminal completion from canonical, daily, and health state transitions plus a unique zero-valued marker written by every other aggregate Copy. A real browser drives the exact-SHA preview through reload, shared-tab, inactivity, SPA retry, and unload behavior. A preview-only authenticated route starts durable server workflows for deduplicated activation and paid-purchase facts. Registry-validated synthetic fixtures separately prove exact anonymous-to-authenticated, guest checkout, cross-device checkout, lifecycle revenue, and public endpoint values while remaining excluded from normal queries. Performance compares shared endpoints with the retained deployment, applies absolute budgets to a newly introduced endpoint that has no honest historical baseline, and validates mixed 1,000-row and 10,000-row traffic, activation, retention, identity, adoption, and revenue corpora before timing them. Before the prior deployment can be deleted, CI switches it live, executes the shared typed data plane, proves the exact-SHA admin client gracefully degrades only the identity endpoint absent from that rollback target, restores the candidate, and repeats the full synthetic business and health assertions. Set `TINYBIRD_AGENT_TOKEN` and `TINYBIRD_URL` for the query command. Set a separate `TINYBIRD_READ_TOKEN` with workspace metadata access when running `pnpm analytics:check`; the append and deployment tokens are intentionally rejected for that task. @@ -56,7 +60,7 @@ The Analytics GitHub workflow runs static tests, Docker Compose validation, a co ## Performance boundaries - `product_events_v1` appends every delivery attempt; the canonical copy deduplicates stable `event_id` values and quarantines conflicting payload hashes. -- Monthly partitions and a 400-day raw, canonical, and decision horizon keep erasure rebuilds complete for every supported dashboard range. +- Monthly partitions and a shared 800-day raw, canonical, and decision horizon support complete year-over-year windows while keeping erasure rebuilds complete for every retained contribution. Health detail retains 90 days. - Common event trends use `product_events_daily_exact`; they do not scan raw events. - Daily counts use unique event states, so retried deliveries cannot inflate the rollup. - Raw health queries require explicit start and end times. @@ -65,4 +69,4 @@ The Analytics GitHub workflow runs static tests, Docker Compose validation, a co Routine commands refuse destructive deployment, workspace-clear, datasource-delete, and datasource-truncate arguments. Destructive recovery is intentionally outside the normal workflow and requires separate review. -Tinybird row deletion currently requires the general `DATASOURCES:CREATE` scope rather than a resource-scoped delete permission. Keep the dedicated erasure operator in a protected environment, give it no read or deployment scope, and expose it only to the reviewed deletion workflow. This provider limitation means the credential can technically mutate other datasources in its workspace even though Cap's workflow accepts only validated identity and synthetic-run conditions. +Tinybird row deletion currently requires the general `DATASOURCES:CREATE` scope rather than a resource-scoped delete permission. Keep the dedicated erasure operator in a protected environment, give it no read or deployment scope, and expose it only to the reviewed deletion workflow. The application uses separate protected credentials for bounded raw/canonical identity lookup, reviewed Copy execution, aggregate marker reads, and Copy schedule cancellation/resumption. This provider limitation means the delete credential can technically mutate other datasources in its workspace even though Cap's workflow accepts only validated identity conditions. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 092759097bf..3dcc329d21c 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -10,19 +10,91 @@ export const COPY_PIPES = [ "snapshot_product_traffic_pages_daily_exact", "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", + "snapshot_product_identity_funnel_exact", "snapshot_product_events_health_hourly", ]; const COPY_MARKER_PIPES = new Set([ + "snapshot_product_events_daily_exact", "snapshot_product_traffic_daily_exact", "snapshot_product_traffic_pages_daily_exact", "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", + "snapshot_product_identity_funnel_exact", + "snapshot_product_events_health_hourly", ]); const SHA_PATTERN = /^[0-9a-f]{40}$/; const SYNTHETIC_RUN_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; const COPY_JOB_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/; const DEPLOYMENT_ID_PATTERN = /^[0-9]+$/; +const EVENT_SCHEMA_VERSIONS = new Map([ + ["purchase_completed", 3], + ["subscription_renewed", 2], + ["trial_converted", 2], + ["subscription_changed", 2], + ["subscription_cancelled", 2], + ["subscription_refunded", 2], + ["subscription_payment_failed", 2], +]); + +const eventSchemaVersion = (eventName) => + EVENT_SCHEMA_VERSIONS.get(eventName) ?? 1; + +const errorMessage = (error) => + error instanceof Error ? error.message : String(error); + +export const applyCopyScheduleAction = async ({ + pipes, + action, + setSchedule, +}) => { + if (!["pause", "resume"].includes(action)) { + throw new Error("Tinybird Copy schedule action must be pause or resume"); + } + const completed = []; + const failures = []; + for (const pipe of pipes) { + try { + await setSchedule(pipe, action); + completed.push(pipe); + } catch (error) { + failures.push({ pipe, error }); + if (action === "pause") break; + } + } + if (failures.length === 0) return completed; + if (action === "pause") { + const compensationFailures = []; + for (const pipe of completed) { + try { + await setSchedule(pipe, "resume"); + } catch (error) { + compensationFailures.push({ pipe, error }); + } + } + const compensation = compensationFailures.length + ? `; resume compensation failed for ${compensationFailures + .map(({ pipe, error }) => `${pipe}: ${errorMessage(error)}`) + .join(", ")}` + : ""; + throw new Error( + `Failed to pause ${failures[0].pipe}: ${errorMessage(failures[0].error)}${compensation}`, + ); + } + throw new Error( + `Failed to resume Copy schedules: ${failures + .map(({ pipe, error }) => `${pipe}: ${errorMessage(error)}`) + .join(", ")}`, + ); +}; + +export const copyScheduleMatchesAction = (value, action) => { + const payload = value?.data ?? value; + const status = String(payload?.schedule?.status ?? "").toLowerCase(); + return action === "pause" + ? status === "paused" + : status === "scheduled" || status === "active"; +}; export const assertExecutionScope = ({ eventName, @@ -430,7 +502,6 @@ export const submitTinybirdCopyJobs = async ({ request, now = () => Date.now(), pipes = COPY_PIPES, - useDeploymentParameter = false, copyRunId = "", assertMutationOwnership, }) => { @@ -452,9 +523,6 @@ export const submitTinybirdCopyJobs = async ({ origin, ); copyUrl.searchParams.set("_mode", "replace"); - if (useDeploymentParameter) { - copyUrl.searchParams.set("__tb__deployment", "staging"); - } if (COPY_MARKER_PIPES.has(pipe)) { if (!copyRunId) { throw new Error(`Tinybird copy marker is required for ${pipe}`); @@ -466,11 +534,12 @@ export const submitTinybirdCopyJobs = async ({ created = await request(copyUrl, { token, method: "POST", - attempts: 3, + attempts: 1, beforeAttempt: assertMutationOwnership, }); } catch (error) { - throw new Error(`Tinybird copy submission failed for ${pipe}`, { + const detail = error instanceof Error ? `: ${error.message}` : ""; + throw new Error(`Tinybird copy submission failed for ${pipe}${detail}`, { cause: error, }); } @@ -579,10 +648,13 @@ export const createSyntheticEvents = ({ runId, now = new Date() }) => { browser: "synthetic", device: "synthetic", os: "synthetic", - channel: "synthetic", + channel: "direct", traffic_class: "synthetic", synthetic_run_id: runId, - properties: JSON.stringify({ test_case: "staging_delivery" }), + properties: JSON.stringify({ + hostname: "preview.cap.so", + is_session_entry: true, + }), }; const duplicateId = `synthetic_duplicate_${runHash.slice(0, 24)}`; const conflictId = `synthetic_conflict_${runHash.slice(0, 24)}`; @@ -598,7 +670,15 @@ export const createSyntheticEvents = ({ runId, now = new Date() }) => { { ...shared, event_id: duplicateId, payload_hash: duplicateHash }, { ...shared, event_id: duplicateId, payload_hash: duplicateHash }, { ...shared, event_id: conflictId, payload_hash: conflictHashA }, - { ...shared, event_id: conflictId, payload_hash: conflictHashB }, + { + ...shared, + event_id: conflictId, + payload_hash: conflictHashB, + properties: JSON.stringify({ + hostname: "preview.cap.so", + is_session_entry: false, + }), + }, ], }; }; @@ -620,21 +700,478 @@ export const createSyntheticErasureControl = ({ runId, now = new Date() }) => { organization_id: `synthetic_control_org_${controlHash.slice(16, 32)}`, app_version: `staging-erasure-control-${controlHash.slice(0, 12)}`, synthetic_run_id: controlRunId, - properties: JSON.stringify({ test_case: "staging_erasure_control" }), + properties: JSON.stringify({ + hostname: fixture.rows[0].hostname, + is_session_entry: true, + }), }, }; }; +export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { + const fixture = createSyntheticEvents({ runId, now }); + const decisionRunId = `${runId}_decisions`; + validateSyntheticRunId(decisionRunId); + const runHash = hashIdentifier(decisionRunId); + const appVersion = `staging-decisions-${runHash.slice(0, 12)}`; + const hostname = `synthetic-${runHash.slice(0, 12)}.preview.cap.so`; + const pathname = `/analytics-synthetic-${runHash.slice(0, 12)}`; + const pageViewId = `synthetic_decision_page_${runHash.slice(0, 20)}`; + const guestAnonymousId = `synthetic_guest_${runHash.slice(0, 20)}`; + const guestSessionId = `synthetic_guest_session_${runHash.slice(0, 16)}`; + const abandonedGuestAnonymousId = `synthetic_abandoned_guest_${runHash.slice(0, 16)}`; + const abandonedGuestSessionId = `synthetic_abandoned_session_${runHash.slice(0, 16)}`; + const sharedAnonymousId = `synthetic_shared_${runHash.slice(0, 20)}`; + const sharedSessionId = `synthetic_shared_session_${runHash.slice(0, 16)}`; + const event = ({ + anonymousId = fixture.anonymousId, + channel = "direct", + device = "desktop", + eventName, + index, + organizationId = fixture.organizationId, + platform, + properties, + referrer = fixture.rows[0].referrer, + schemaVersion, + sessionId = fixture.rows[0].session_id, + source, + userId = fixture.userId, + }) => { + const eventId = + index === 0 + ? pageViewId + : `synthetic_decision_${index}_${runHash.slice(0, 20)}`; + const occurredAt = new Date(now.getTime() + index * 1_000) + .toISOString() + .replace("T", " ") + .replace("Z", ""); + return { + ...fixture.rows[0], + event_id: eventId, + payload_hash: hashIdentifier(`${decisionRunId}:${eventId}`).slice(0, 32), + occurred_at: occurredAt, + received_at: occurredAt, + event_name: eventName, + schema_version: schemaVersion ?? eventSchemaVersion(eventName), + source, + platform, + anonymous_id: anonymousId, + session_id: sessionId, + user_id: userId, + organization_id: organizationId, + app_version: appVersion, + country: "US", + referrer, + device, + browser: "Chrome", + os: "macOS", + channel, + hostname, + pathname, + synthetic_run_id: decisionRunId, + properties: JSON.stringify(properties), + }; + }; + return { + appVersion, + hostname, + pathname, + runId: decisionRunId, + date: now.toISOString().slice(0, 10), + rows: [ + event({ + userId: "", + organizationId: "", + channel: "paid_search", + eventName: "page_view", + index: 0, + platform: "web", + properties: { + hostname, + is_session_entry: true, + session_touch_source: "google", + session_touch_medium: "cpc", + session_touch_campaign: "synthetic-campaign", + session_touch_gclid: "synthetic-click-id", + }, + source: "client", + }), + event({ + userId: "", + organizationId: "", + eventName: "page_engagement", + index: 1, + platform: "web", + properties: { + page_view_id: pageViewId, + engaged_ms: 15_000, + max_scroll_depth: 75, + }, + source: "client", + }), + event({ + eventName: "identity_linked", + index: 2, + platform: "server", + properties: {}, + source: "server", + }), + event({ + eventName: "user_signed_up", + index: 3, + platform: "web", + properties: {}, + source: "server", + }), + event({ + eventName: "share_link_created", + index: 4, + platform: "server", + properties: { + asset_type: "recording", + recording_mode: "screen", + }, + source: "server", + }), + event({ + eventName: "recording_completed", + index: 5, + platform: "desktop", + properties: { + mode: "screen", + status: "success", + duration_secs: 30, + segment_count: 1, + track_failure_count: 0, + }, + source: "client", + }), + event({ + anonymousId: guestAnonymousId, + channel: "paid_search", + eventName: "page_view", + index: 6, + organizationId: "", + platform: "web", + properties: { + hostname, + is_session_entry: true, + session_touch_source: "google", + session_touch_medium: "cpc", + session_touch_campaign: "synthetic-campaign", + session_touch_gclid: "synthetic-guest-click-id", + }, + sessionId: guestSessionId, + source: "client", + userId: "", + }), + event({ + anonymousId: guestAnonymousId, + eventName: "guest_checkout_started", + index: 7, + organizationId: "", + platform: "web", + properties: { + price_id: "price_pro_annual", + quantity: 1, + }, + source: "server", + sessionId: guestSessionId, + userId: "", + }), + event({ + eventName: "checkout_started", + index: 8, + platform: "web", + properties: { + price_id: "price_pro_annual", + quantity: 1, + is_onboarding: false, + }, + source: "server", + }), + event({ + anonymousId: `synthetic_desktop_${runHash.slice(0, 20)}`, + eventName: "checkout_started", + index: 9, + platform: "desktop", + properties: { + price_id: "price_pro_annual", + quantity: 1, + is_onboarding: false, + }, + sessionId: `synthetic_desktop_session_${runHash.slice(0, 16)}`, + source: "server", + }), + event({ + anonymousId: `synthetic_mobile_${runHash.slice(0, 20)}`, + device: "mobile", + eventName: "checkout_started", + index: 10, + platform: "mobile", + properties: { + price_id: "price_pro_annual", + quantity: 1, + is_onboarding: false, + }, + sessionId: `synthetic_mobile_session_${runHash.slice(0, 16)}`, + source: "server", + }), + event({ + eventName: "trial_started", + index: 11, + platform: "web", + properties: { + subscription_status: "trialing", + trial_end_at: 1_900_604_800, + price_id: "price_pro_annual", + quantity: 1, + currency: "gbp", + unit_amount_minor: 2_500, + billing_interval: "year", + billing_interval_count: 1, + is_guest_checkout: false, + is_onboarding: false, + }, + source: "server", + }), + event({ + eventName: "purchase_completed", + index: 12, + platform: "web", + properties: { + payment_status: "paid", + subscription_status: "active", + amount_total_minor: 2_500, + amount_subtotal_minor: 2_500, + discount_amount_minor: 0, + currency: "gbp", + unit_amount_minor: 2_500, + billing_interval: "year", + billing_interval_count: 1, + invite_quota: 1, + price_id: "price_pro_annual", + quantity: 1, + is_first_purchase: true, + is_guest_checkout: false, + is_onboarding: false, + }, + source: "server", + }), + event({ + eventName: "subscription_renewed", + index: 13, + platform: "server", + properties: { + amount_paid_minor: 2_500, + currency: "gbp", + price_id: "price_pro_annual", + billing_reason: "subscription_cycle", + }, + source: "server", + }), + event({ + anonymousId: abandonedGuestAnonymousId, + eventName: "guest_checkout_started", + index: 24, + organizationId: "", + platform: "web", + properties: { + price_id: "price_pro_annual", + quantity: 1, + }, + sessionId: abandonedGuestSessionId, + source: "server", + userId: "", + }), + event({ + eventName: "trial_converted", + index: 14, + platform: "server", + properties: { + previous_status: "trialing", + new_status: "active", + price_id: "price_pro_annual", + }, + source: "server", + }), + event({ + eventName: "subscription_changed", + index: 15, + platform: "server", + properties: { + change_kind: "plan", + previous_price_id: "price_pro_monthly", + new_price_id: "price_pro_annual", + }, + source: "server", + }), + event({ + eventName: "subscription_changed", + index: 16, + platform: "server", + properties: { + change_kind: "seats", + previous_price_id: "price_pro_annual", + new_price_id: "price_pro_annual", + previous_quantity: 1, + new_quantity: 3, + }, + source: "server", + }), + event({ + eventName: "subscription_cancelled", + index: 17, + platform: "server", + properties: { + status: "canceled", + price_id: "price_pro_annual", + ended_at: 1_900_000_000, + cancel_at_period_end: false, + }, + source: "server", + }), + event({ + eventName: "subscription_refunded", + index: 18, + platform: "server", + properties: { + amount_refunded_minor: 500, + currency: "gbp", + price_id: "price_pro_annual", + fully_refunded: false, + }, + source: "server", + }), + event({ + eventName: "subscription_payment_failed", + index: 19, + platform: "server", + properties: { + amount_due_minor: 2_500, + currency: "gbp", + attempt_count: 2, + price_id: "price_pro_annual", + }, + source: "server", + }), + event({ + anonymousId: sharedAnonymousId, + channel: "referral", + eventName: "page_view", + index: 20, + organizationId: "", + platform: "web", + properties: { + hostname, + is_session_entry: true, + session_touch_source: "synthetic-partner", + session_touch_medium: "referral", + }, + referrer: "https://synthetic-partner.example/path", + sessionId: sharedSessionId, + source: "client", + userId: "", + }), + event({ + anonymousId: sharedAnonymousId, + eventName: "identity_linked", + index: 21, + platform: "server", + properties: {}, + sessionId: sharedSessionId, + source: "server", + userId: `synthetic_shared_org_user_${runHash.slice(0, 16)}`, + }), + event({ + anonymousId: guestAnonymousId, + eventName: "purchase_completed", + index: 22, + platform: "web", + properties: { + payment_status: "paid", + subscription_status: "active", + amount_total_minor: 1_500, + amount_subtotal_minor: 1_500, + discount_amount_minor: 0, + currency: "gbp", + unit_amount_minor: 1_500, + billing_interval: "month", + billing_interval_count: 1, + invite_quota: 1, + price_id: "price_guest_monthly", + quantity: 1, + is_first_purchase: true, + is_guest_checkout: true, + is_onboarding: false, + }, + source: "server", + sessionId: guestSessionId, + }), + event({ + eventName: "subscription_renewed", + index: 23, + platform: "server", + properties: { + amount_paid_minor: 1_000, + currency: "gbp", + billing_reason: "subscription_cycle", + }, + schemaVersion: 1, + source: "server", + }), + event({ + eventName: "experiment_exposed", + index: 25, + platform: "web", + properties: { + experiment_id: "synthetic-checkout-copy", + variant: "treatment", + assignment_version: "v1", + }, + source: "client", + }), + event({ + eventName: "analytics_delivery_loss", + index: 26, + platform: "desktop", + properties: { + failure_class: "queue_overflow_unrecoverable", + failed_event_name: "recording_completed", + status: null, + count: 3, + first_sequence: 41, + last_sequence: 43, + first_failed_at_ms: now.getTime(), + last_failed_at_ms: now.getTime() + 2_000, + }, + source: "client", + }), + ], + }; +}; + export const createSyntheticLoadEvents = ({ runId, count, now = new Date(), }) => { - if (!Number.isInteger(count) || count < 100 || count > 10_000) { - throw new Error("Synthetic load size must be between 100 and 10000 rows"); + if ( + !Number.isInteger(count) || + count < 100 || + count > 10_000 || + count % 10 !== 0 + ) { + throw new Error( + "Synthetic load size must be a multiple of 10 between 100 and 10000 rows", + ); } const fixture = createSyntheticEvents({ runId, now }); const runHash = hashIdentifier(runId); + const eventNamespace = runHash + .slice(0, 12) + .replace(/[0-9]/g, (digit) => String.fromCharCode(103 + Number(digit))); const appVersion = `staging-load-${runHash.slice(0, 12)}`; const loadRunId = `${runId}_load`; validateSyntheticRunId(loadRunId); @@ -642,14 +1179,162 @@ export const createSyntheticLoadEvents = ({ appVersion, runId: loadRunId, rows: Array.from({ length: count }, (_, index) => { - const eventId = `synthetic_load_${hashIdentifier(`${runId}:${index}`).slice(0, 24)}`; + const cohort = Math.floor(index / 10); + const eventKind = index % 10; + const eventId = `synthetic_load_${eventNamespace}_${cohort}_${eventKind}`; + const pageViewId = `synthetic_load_${eventNamespace}_${cohort}_0`; + const hostname = `load-${runHash.slice(0, 8)}-${cohort}.preview.cap.so`; + const anonymousId = `synthetic_load_${runHash.slice(0, 8)}_${cohort}`; + const userId = `synthetic_load_user_${runHash.slice(0, 8)}_${cohort}`; + const organizationId = `synthetic_load_org_${runHash.slice(0, 8)}_${cohort}`; + const sessionId = `synthetic_load_session_${runHash.slice(0, 8)}_${cohort}`; + const checkoutPlatform = ["web", "desktop", "mobile"][cohort % 3]; + const occurredAt = new Date(now.getTime() + index * 10) + .toISOString() + .replace("T", " ") + .replace("Z", ""); + const shapes = [ + { + eventName: "page_view", + source: "client", + platform: "web", + properties: { + hostname, + is_session_entry: true, + session_touch_source: `source-${cohort}`, + session_touch_medium: "cpc", + session_touch_campaign: `campaign-${cohort}`, + }, + channel: "paid_other", + }, + { + eventName: "page_engagement", + source: "client", + platform: "web", + properties: { + page_view_id: pageViewId, + engaged_ms: 5_000, + max_scroll_depth: 60, + }, + }, + { + eventName: "identity_linked", + source: "server", + platform: "server", + properties: {}, + }, + { + eventName: "user_signed_up", + source: "server", + platform: "web", + properties: {}, + }, + { + eventName: "share_link_created", + source: "server", + platform: "server", + properties: { + asset_type: "recording", + recording_mode: "screen", + }, + }, + { + eventName: "recording_completed", + source: "client", + platform: "desktop", + properties: { + mode: "screen", + status: "success", + duration_secs: 30, + segment_count: 1, + track_failure_count: 0, + }, + }, + { + eventName: "checkout_started", + source: "server", + platform: checkoutPlatform, + properties: { + price_id: `price_load_${cohort}`, + quantity: 1, + is_onboarding: false, + }, + }, + { + eventName: "trial_started", + source: "server", + platform: checkoutPlatform, + properties: { + subscription_status: "trialing", + price_id: `price_load_${cohort}`, + quantity: 1, + currency: "usd", + unit_amount_minor: 1_000, + billing_interval: "month", + billing_interval_count: 1, + is_guest_checkout: false, + is_onboarding: false, + }, + }, + { + eventName: "purchase_completed", + source: "server", + platform: checkoutPlatform, + properties: { + payment_status: "paid", + subscription_status: "active", + amount_total_minor: 1_000, + amount_subtotal_minor: 1_000, + discount_amount_minor: 0, + currency: "usd", + unit_amount_minor: 1_000, + billing_interval: "month", + billing_interval_count: 1, + invite_quota: 1, + price_id: `price_load_${cohort}`, + quantity: 1, + is_first_purchase: true, + is_guest_checkout: false, + is_onboarding: false, + }, + }, + { + eventName: "subscription_renewed", + source: "server", + platform: "server", + properties: { + amount_paid_minor: 1_000, + currency: "usd", + price_id: `price_load_${cohort}`, + billing_reason: "subscription_cycle", + }, + }, + ]; + const shape = shapes[eventKind]; return { ...fixture.rows[0], event_id: eventId, payload_hash: hashIdentifier(eventId).slice(0, 32), + occurred_at: occurredAt, + received_at: occurredAt, + event_name: shape.eventName, + schema_version: eventSchemaVersion(shape.eventName), + source: shape.source, + platform: shape.platform, + anonymous_id: anonymousId, + session_id: sessionId, + user_id: eventKind < 2 ? "" : userId, + organization_id: eventKind < 2 ? "" : organizationId, app_version: appVersion, + hostname, + pathname: `/analytics-load/${cohort}`, + country: "US", + device: shape.platform === "mobile" ? "mobile" : "desktop", + browser: "Chrome", + os: shape.platform === "mobile" ? "iOS" : "macOS", + channel: shape.channel ?? "direct", synthetic_run_id: loadRunId, - properties: JSON.stringify({ test_case: "staging_load" }), + properties: JSON.stringify(shape.properties), }; }), }; @@ -678,17 +1363,1011 @@ export const normalizeCiAssertions = (payload) => { payloadConflicts: number(row.payload_conflicts), canonicalEvents: number(row.canonical_events), decisionEvents: number(row.decision_events), + decisionRevenueMinor: number(row.decision_revenue_minor), + trafficVisitors: number(row.traffic_visitors), + trafficVisits: number(row.traffic_visits), + trafficPageviews: number(row.traffic_pageviews), + trafficBounces: number(row.traffic_bounces), + trafficDurationMs: number(row.traffic_duration_ms), + pageVisitors: number(row.page_visitors), + pageVisits: number(row.page_visits), + pageviews: number(row.pageviews), + pageLandings: number(row.page_landings), + pageExits: number(row.page_exits), + pageEngagedMs: number(row.page_engaged_ms), + pageScrollDepth: number(row.page_scroll_depth), + activationSignups: number(row.activation_signups), + activatedCreators: number(row.activated_creators), + retentionCreators: number(row.retention_creators), + retentionOrganizations: number(row.retention_organizations), + identityLinkedVisitors: number(row.identity_linked_visitors), + identityLinkedUsers: number(row.identity_linked_users), + identitySignupUsers: number(row.identity_signup_users), + identityOrganizations: number(row.identity_organizations), + identityGuestCheckoutVisitors: number(row.identity_guest_checkout_visitors), + identityGuestPurchasers: number(row.identity_guest_purchasers), + identityAuthenticatedCheckoutUsers: number( + row.identity_authenticated_checkout_users, + ), + identityWebCheckoutUsers: number(row.identity_web_checkout_users), + identityDesktopCheckoutUsers: number(row.identity_desktop_checkout_users), + identityMobileCheckoutUsers: number(row.identity_mobile_checkout_users), + identityCrossDeviceCheckoutUsers: number( + row.identity_cross_device_checkout_users, + ), + identityTrialUsers: number(row.identity_trial_users), + identityPurchasers: number(row.identity_purchasers), + }; +}; + +export const assertSyntheticBusinessDecisions = (assertions) => { + const expected = { + trafficVisitors: 3, + trafficVisits: 3, + trafficPageviews: 3, + trafficBounces: 2, + trafficDurationMs: 15_000, + pageVisitors: 3, + pageVisits: 3, + pageviews: 3, + pageLandings: 3, + pageExits: 3, + pageEngagedMs: 15_000, + pageScrollDepth: 75, + activationSignups: 1, + activatedCreators: 1, + retentionCreators: 1, + retentionOrganizations: 1, + identityLinkedVisitors: 2, + identityLinkedUsers: 2, + identitySignupUsers: 1, + identityOrganizations: 1, + identityGuestCheckoutVisitors: 2, + identityGuestPurchasers: 1, + identityAuthenticatedCheckoutUsers: 1, + identityWebCheckoutUsers: 1, + identityDesktopCheckoutUsers: 1, + identityMobileCheckoutUsers: 1, + identityCrossDeviceCheckoutUsers: 1, + identityTrialUsers: 1, + identityPurchasers: 1, + decisionRevenueMinor: 7_000, + }; + for (const [name, value] of Object.entries(expected)) { + if (assertions[name] !== value) { + throw new Error( + `Synthetic business metric ${name} was ${assertions[name]}, expected ${value}`, + ); + } + } +}; + +export const assertSyntheticLoadDecisions = (assertions, expectedEvents) => { + assertSyntheticLoadHealth(assertions, expectedEvents); + const cohorts = expectedEvents / 10; + const expected = { + canonicalEvents: expectedEvents, + decisionEvents: expectedEvents, + decisionRevenueMinor: cohorts * 2_000, + trafficVisitors: cohorts, + trafficVisits: cohorts, + trafficPageviews: cohorts, + trafficBounces: 0, + trafficDurationMs: cohorts * 5_000, + pageVisitors: cohorts, + pageVisits: cohorts, + pageviews: cohorts, + pageLandings: cohorts, + pageExits: cohorts, + pageEngagedMs: cohorts * 5_000, + pageScrollDepth: cohorts * 60, + activationSignups: cohorts, + activatedCreators: cohorts, + retentionCreators: cohorts, + retentionOrganizations: cohorts, + identityLinkedVisitors: cohorts, + identityLinkedUsers: cohorts, + identitySignupUsers: cohorts, + identityOrganizations: cohorts, + identityGuestCheckoutVisitors: 0, + identityGuestPurchasers: 0, + identityAuthenticatedCheckoutUsers: cohorts, + identityWebCheckoutUsers: Math.ceil(cohorts / 3), + identityDesktopCheckoutUsers: Math.ceil((cohorts - 1) / 3), + identityMobileCheckoutUsers: Math.floor(cohorts / 3), + identityCrossDeviceCheckoutUsers: 0, + identityTrialUsers: cohorts, + identityPurchasers: cohorts, + }; + for (const [name, value] of Object.entries(expected)) { + if (assertions[name] !== value) { + throw new Error( + `Synthetic load metric ${name} was ${assertions[name]}, expected ${value}`, + ); + } + } +}; + +const endpointRows = (payloads, name) => { + const rows = payloads[name]?.data; + if (!Array.isArray(rows)) { + throw new Error(`Synthetic endpoint ${name} returned an invalid payload`); + } + return rows; +}; + +const singleEndpointRow = (payloads, name) => { + const rows = endpointRows(payloads, name); + if (rows.length !== 1) { + throw new Error( + `Synthetic endpoint ${name} returned ${rows.length} rows, expected 1`, + ); + } + return rows[0]; +}; + +const assertEndpointFields = (name, row, expected) => { + for (const [field, value] of Object.entries(expected)) { + const actual = typeof value === "number" ? Number(row[field]) : row[field]; + if (actual !== value) { + throw new Error( + `Synthetic endpoint ${name}.${field} was ${actual}, expected ${value}`, + ); + } + } +}; + +export const assertSyntheticEndpointDecisions = ({ + appVersion, + date, + hostname, + pathname, + payloads, +}) => { + assertEndpointFields( + "product_traffic_overview", + singleEndpointRow(payloads, "product_traffic_overview"), + { + date, + visitors: 3, + visits: 3, + pageviews: 3, + views_per_visit: 1, + bounce_rate: 66.67, + visit_duration_ms: 5_000, + engaged_ms: 15_000, + }, + ); + assertEndpointFields( + "product_traffic_pages", + singleEndpointRow(payloads, "product_traffic_pages"), + { + pathname, + visitors: 3, + visits: 3, + pageviews: 3, + landings: 3, + exits: 3, + time_on_page_ms: 5_000, + average_scroll_depth: 25, + }, + ); + const sourceRows = endpointRows(payloads, "product_traffic_sources"); + if (sourceRows.length !== 2) { + throw new Error( + `Synthetic endpoint product_traffic_sources returned ${sourceRows.length} rows, expected 2`, + ); + } + assertEndpointFields( + "product_traffic_sources", + sourceRows.find((row) => row.channel === "paid_search") ?? {}, + { + channel: "paid_search", + source: "google", + medium: "cpc", + campaign: "synthetic-campaign", + visitors: 2, + visits: 2, + pageviews: 2, + bounce_rate: 50, + }, + ); + assertEndpointFields( + "product_traffic_sources", + sourceRows.find((row) => row.channel === "referral") ?? {}, + { + channel: "referral", + source: "synthetic-partner", + medium: "referral", + campaign: "", + visitors: 1, + visits: 1, + pageviews: 1, + bounce_rate: 100, + }, + ); + assertEndpointFields( + "product_traffic_countries", + singleEndpointRow(payloads, "product_traffic_countries"), + { country: "US", visitors: 3, visits: 3, pageviews: 3 }, + ); + assertEndpointFields( + "product_traffic_technology", + singleEndpointRow(payloads, "product_traffic_technology"), + { + device: "desktop", + browser: "Chrome", + os: "macOS", + visitors: 3, + visits: 3, + pageviews: 3, + }, + ); + assertEndpointFields( + "product_activation", + singleEndpointRow(payloads, "product_activation"), + { + cohort_date: date, + signups: 1, + activated_creators: 1, + activation_rate: 100, + average_time_to_activation_ms: 1_000, + }, + ); + assertEndpointFields( + "product_creator_activity", + singleEndpointRow(payloads, "product_creator_activity"), + { + as_of_date: date, + dau: 1, + wau: 1, + mau: 1, + daily_active_organizations: 1, + new_creators: 1, + returning_creators: 0, + dau_wau_stickiness: 100, + dau_mau_stickiness: 100, + }, + ); + assertEndpointFields( + "product_creator_retention", + singleEndpointRow(payloads, "product_creator_retention"), + { + cohort_date: date, + activity_date: date, + cohort_day: 0, + platform: "all", + creators: 1, + organizations: 1, + }, + ); + assertEndpointFields( + "product_identity_funnel", + singleEndpointRow(payloads, "product_identity_funnel"), + { + linked_visitors: 2, + linked_users: 2, + signup_users: 1, + organizations: 1, + guest_checkout_visitors: 2, + guest_purchasers: 1, + authenticated_checkout_users: 1, + web_checkout_users: 1, + desktop_checkout_users: 1, + mobile_checkout_users: 1, + cross_device_checkout_users: 1, + trial_users: 1, + purchasers: 1, + signup_rate: 50, + purchase_rate: 50, + }, + ); + const eventRows = endpointRows(payloads, "product_events_daily"); + const eventKey = ( + eventName, + platform, + changeKind = "", + planId = "", + channel = "direct", + schemaVersion = eventSchemaVersion(eventName), + ) => + `${eventName}:${platform}:${changeKind}:${planId}:${channel}:${schemaVersion}`; + const expectedEvents = new Map( + [ + { + eventName: "page_view", + source: "client", + platform: "web", + events: 2, + actors: 2, + channel: "paid_search", + users: 0, + organizations: 0, + }, + { + eventName: "page_view", + source: "client", + platform: "web", + channel: "referral", + users: 0, + organizations: 0, + }, + { + eventName: "page_engagement", + source: "client", + platform: "web", + users: 0, + organizations: 0, + }, + { + eventName: "identity_linked", + source: "server", + platform: "server", + events: 2, + actors: 2, + users: 2, + }, + { eventName: "user_signed_up", source: "server", platform: "web" }, + { eventName: "share_link_created", source: "server", platform: "server" }, + { + eventName: "recording_completed", + source: "client", + platform: "desktop", + }, + { + eventName: "guest_checkout_started", + source: "server", + platform: "web", + planId: "price_pro_annual", + events: 2, + actors: 2, + users: 0, + organizations: 0, + quantity: 1, + }, + { + eventName: "checkout_started", + source: "server", + platform: "web", + planId: "price_pro_annual", + quantity: 1, + onboarding: "false", + }, + { + eventName: "checkout_started", + source: "server", + platform: "desktop", + planId: "price_pro_annual", + quantity: 1, + onboarding: "false", + }, + { + eventName: "checkout_started", + source: "server", + platform: "mobile", + planId: "price_pro_annual", + quantity: 1, + onboarding: "false", + }, + { + eventName: "trial_started", + source: "server", + platform: "web", + planId: "price_pro_annual", + subscriptionStatus: "trialing", + currency: "GBP", + billingInterval: "year", + quantity: 1, + guestCheckout: "false", + onboarding: "false", + trialEndAt: 1_900_604_800, + }, + { + eventName: "purchase_completed", + source: "server", + platform: "web", + planId: "price_pro_annual", + paymentStatus: "paid", + subscriptionStatus: "active", + currency: "GBP", + billingInterval: "year", + revenueMinor: 2_500, + quantity: 1, + firstPurchase: "true", + guestCheckout: "false", + onboarding: "false", + }, + { + eventName: "purchase_completed", + source: "server", + platform: "web", + planId: "price_guest_monthly", + paymentStatus: "paid", + subscriptionStatus: "active", + currency: "GBP", + billingInterval: "month", + revenueMinor: 1_500, + quantity: 1, + firstPurchase: "true", + guestCheckout: "true", + onboarding: "false", + }, + { + eventName: "subscription_renewed", + source: "server", + platform: "server", + planId: "price_pro_annual", + currency: "GBP", + revenueMinor: 2_500, + schemaVersion: 2, + }, + { + eventName: "subscription_renewed", + source: "server", + platform: "server", + currency: "GBP", + revenueMinor: 1_000, + schemaVersion: 1, + }, + { + eventName: "trial_converted", + source: "server", + platform: "server", + planId: "price_pro_annual", + subscriptionStatus: "active", + previousStatus: "trialing", + newStatus: "active", + schemaVersion: 2, + }, + { + eventName: "subscription_changed", + source: "server", + platform: "server", + planId: "price_pro_annual", + changeKind: "plan", + previousPlanId: "price_pro_monthly", + schemaVersion: 2, + }, + { + eventName: "subscription_changed", + source: "server", + platform: "server", + planId: "price_pro_annual", + changeKind: "seats", + previousPlanId: "price_pro_annual", + previousQuantity: 1, + newQuantity: 3, + seatDelta: 2, + schemaVersion: 2, + }, + { + eventName: "subscription_cancelled", + source: "server", + platform: "server", + planId: "price_pro_annual", + subscriptionStatus: "canceled", + cancelAtPeriodEnd: "false", + endedAt: 1_900_000_000, + schemaVersion: 2, + }, + { + eventName: "subscription_refunded", + source: "server", + platform: "server", + planId: "price_pro_annual", + currency: "GBP", + revenueMinor: -500, + fullyRefunded: "false", + schemaVersion: 2, + }, + { + eventName: "subscription_payment_failed", + source: "server", + platform: "server", + planId: "price_pro_annual", + currency: "GBP", + amountDueMinor: 2_500, + attemptCount: 2, + schemaVersion: 2, + }, + { + eventName: "experiment_exposed", + source: "client", + platform: "web", + experimentId: "synthetic-checkout-copy", + experimentVariant: "treatment", + assignmentVersion: "v1", + }, + { + eventName: "analytics_delivery_loss", + source: "client", + platform: "desktop", + deliveryLossCount: 3, + }, + ].map((expected) => [ + eventKey( + expected.eventName, + expected.platform, + expected.changeKind, + expected.planId, + expected.channel, + expected.schemaVersion, + ), + expected, + ]), + ); + if (eventRows.length !== expectedEvents.size) { + throw new Error( + `Synthetic endpoint product_events_daily returned ${eventRows.length} rows, expected ${expectedEvents.size}`, + ); + } + for (const row of eventRows) { + const expected = expectedEvents.get( + eventKey( + row.event_name, + row.platform, + row.change_kind, + row.plan_id, + row.channel, + Number(row.schema_version), + ), + ); + if (!expected) { + throw new Error( + `Synthetic endpoint product_events_daily returned unexpected event ${row.event_name}`, + ); + } + assertEndpointFields("product_events_daily", row, { + date, + event_name: expected.eventName, + source: expected.source, + platform: expected.platform, + schema_version: + expected.schemaVersion ?? eventSchemaVersion(expected.eventName), + app_version: appVersion, + hostname, + channel: expected.channel ?? "direct", + events: expected.events ?? 1, + actors: expected.actors ?? 1, + users: expected.users ?? 1, + organizations: expected.organizations ?? 1, + plan_id: expected.planId ?? "", + payment_status: expected.paymentStatus ?? "", + subscription_status: expected.subscriptionStatus ?? "", + currency: expected.currency ?? "", + billing_interval: expected.billingInterval ?? "", + change_kind: expected.changeKind ?? "", + previous_status: expected.previousStatus ?? "", + new_status: expected.newStatus ?? "", + previous_plan_id: expected.previousPlanId ?? "", + quantity: expected.quantity ?? 0, + previous_quantity: expected.previousQuantity ?? 0, + new_quantity: expected.newQuantity ?? 0, + seat_delta: expected.seatDelta ?? 0, + first_purchase: expected.firstPurchase ?? "", + guest_checkout: expected.guestCheckout ?? "", + onboarding: expected.onboarding ?? "", + cancel_at_period_end: expected.cancelAtPeriodEnd ?? "", + fully_refunded: expected.fullyRefunded ?? "", + ended_at: expected.endedAt ?? 0, + trial_end_at: expected.trialEndAt ?? 0, + amount_due_minor: expected.amountDueMinor ?? 0, + attempt_count: expected.attemptCount ?? 0, + experiment_id: expected.experimentId ?? "", + experiment_variant: expected.experimentVariant ?? "", + assignment_version: expected.assignmentVersion ?? "", + delivery_loss_count: expected.deliveryLossCount ?? 0, + revenue_minor: expected.revenueMinor ?? 0, + }); + } + const adoptionRows = endpointRows(payloads, "product_feature_adoption"); + const expectedAdoption = new Map( + [ + ["page_view", 3, 0, 0, 3], + ["page_engagement", 1, 0], + ["identity_linked", 2, 2, 1], + ["user_signed_up", 1, 1], + ["share_link_created", 1, 1], + ["recording_completed", 1, 1], + ["guest_checkout_started", 2, 0, 0, 2], + ["checkout_started", 3, 1], + ["trial_started", 1, 1], + ["purchase_completed", 2, 1], + ["subscription_renewed", 2, 1], + ["trial_converted", 1, 1], + ["subscription_changed", 2, 1], + ["subscription_cancelled", 1, 1], + ["subscription_refunded", 1, 1], + ["subscription_payment_failed", 1, 1], + ["experiment_exposed", 1, 1], + ["analytics_delivery_loss", 1, 1], + ].map( + ([ + eventName, + events, + authenticated, + organizations = authenticated, + actorDays = Math.max(1, authenticated), + ]) => [eventName, { events, authenticated, organizations, actorDays }], + ), + ); + if (adoptionRows.length !== expectedAdoption.size) { + throw new Error( + `Synthetic endpoint product_feature_adoption returned ${adoptionRows.length} rows, expected ${expectedAdoption.size}`, + ); + } + for (const row of adoptionRows) { + const expected = expectedAdoption.get(row.event_name); + if (!expected) { + throw new Error( + `Synthetic endpoint product_feature_adoption returned unexpected event ${row.event_name}`, + ); + } + assertEndpointFields("product_feature_adoption", row, { + event_name: row.event_name, + events: expected.events, + actor_days: expected.actorDays, + user_days: expected.authenticated, + organization_days: expected.organizations, + }); + } + const freshness = singleEndpointRow(payloads, "product_analytics_freshness"); + for (const field of [ + "latest_received_hour", + "product_calculated_at", + "traffic_calculated_at", + "retention_calculated_at", + "identity_calculated_at", + ]) { + if (typeof freshness[field] !== "string" || freshness[field].length === 0) { + throw new Error( + `Synthetic endpoint product_analytics_freshness.${field} is missing`, + ); + } + } +}; + +export const assertRepresentativeEndpointCoverage = ({ + expectedEvents, + payloads, +}) => { + const cohorts = expectedEvents / 10; + const sum = (rows, field) => + rows.reduce((total, row) => total + Number(row[field] ?? 0), 0); + const expectedRows = { + product_traffic_overview: 1, + product_traffic_pages: cohorts, + product_traffic_sources: cohorts, + product_traffic_countries: 1, + product_traffic_technology: 1, + product_activation: 1, + product_creator_activity: 1, + product_creator_retention: 1, + product_identity_funnel: 1, + product_events_daily: Math.min(expectedEvents, 1_000), + product_feature_adoption: 10, + product_analytics_freshness: 1, + }; + for (const [name, count] of Object.entries(expectedRows)) { + const rows = endpointRows(payloads, name); + if (rows.length !== count) { + throw new Error( + `Representative endpoint ${name} returned ${rows.length} rows, expected ${count}`, + ); + } + } + const exactTotals = [ + ["product_traffic_overview", "pageviews", cohorts], + ["product_traffic_pages", "pageviews", cohorts], + ["product_traffic_sources", "pageviews", cohorts], + ["product_activation", "signups", cohorts], + ["product_creator_activity", "dau", cohorts], + ["product_creator_retention", "creators", cohorts], + ["product_identity_funnel", "linked_users", cohorts], + ["product_identity_funnel", "purchasers", cohorts], + ["product_events_daily", "events", expectedEvents], + ["product_events_daily", "revenue_minor", cohorts * 2_000], + ["product_feature_adoption", "events", expectedEvents], + ]; + for (const [name, field, expected] of exactTotals) { + const actual = sum(endpointRows(payloads, name), field); + if (actual !== expected) { + throw new Error( + `Representative endpoint ${name}.${field} totaled ${actual}, expected ${expected}`, + ); + } + } +}; + +export const syntheticMonetizationFilterQueries = ({ + date, + deploymentId, + syntheticRunId, +}) => { + const base = { + start_date: date, + end_date: date, + synthetic_run_id: syntheticRunId, + __tb__deployment: deploymentId, + }; + return [ + { + label: "authenticated_paid_purchase", + parameters: { + ...base, + event_name: "purchase_completed", + payment_status: "paid", + currency: "gbp", + plan_id: "price_pro_annual", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 2_500, + }, + { + label: "guest_paid_purchase", + parameters: { + ...base, + event_name: "purchase_completed", + payment_status: "paid", + currency: "gbp", + plan_id: "price_guest_monthly", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 1_500, + }, + { + label: "unpaid_checkout_not_purchase", + parameters: { + ...base, + event_name: "purchase_completed", + payment_status: "unpaid", + }, + expectedRows: 0, + expectedEvents: 0, + expectedRevenueMinor: 0, + }, + { + label: "trial_without_revenue", + parameters: { + ...base, + event_name: "trial_started", + subscription_status: "trialing", + currency: "gbp", + plan_id: "price_pro_annual", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 0, + }, + { + label: "renewal_revenue", + parameters: { + ...base, + event_name: "subscription_renewed", + currency: "gbp", + plan_id: "price_pro_annual", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 2_500, + expectedFields: { schema_version: 2, plan_id: "price_pro_annual" }, + }, + { + label: "legacy_renewal_without_plan", + parameters: { + ...base, + event_name: "subscription_renewed", + schema_version: 1, + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 1_000, + expectedFields: { schema_version: 1, plan_id: "" }, + }, + { + label: "refund_revenue", + parameters: { + ...base, + event_name: "subscription_refunded", + currency: "gbp", + plan_id: "price_pro_annual", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: -500, + expectedFields: { fully_refunded: "false" }, + }, + { + label: "trial_conversion", + parameters: { + ...base, + event_name: "trial_converted", + plan_id: "price_pro_annual", + previous_status: "trialing", + new_status: "active", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 0, + expectedFields: { subscription_status: "active" }, + }, + { + label: "plan_change", + parameters: { + ...base, + event_name: "subscription_changed", + change_kind: "plan", + plan_id: "price_pro_annual", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 0, + expectedFields: { previous_plan_id: "price_pro_monthly" }, + }, + { + label: "seat_change", + parameters: { + ...base, + event_name: "subscription_changed", + change_kind: "seats", + plan_id: "price_pro_annual", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 0, + expectedFields: { + previous_quantity: 1, + new_quantity: 3, + seat_delta: 2, + }, + }, + { + label: "cancellation", + parameters: { + ...base, + event_name: "subscription_cancelled", + plan_id: "price_pro_annual", + subscription_status: "canceled", + cancel_at_period_end: "false", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 0, + expectedFields: { ended_at: 1_900_000_000 }, + }, + { + label: "payment_failure", + parameters: { + ...base, + event_name: "subscription_payment_failed", + plan_id: "price_pro_annual", + currency: "gbp", + }, + expectedRows: 1, + expectedEvents: 1, + expectedRevenueMinor: 0, + expectedFields: { amount_due_minor: 2_500, attempt_count: 2 }, + }, + ]; +}; + +export const syntheticIdentityFilterQueries = ({ + date, + deploymentId, + syntheticRunId, +}) => { + const base = { + start_date: date, + end_date: date, + synthetic_run_id: syntheticRunId, + __tb__deployment: deploymentId, }; + return [ + { + label: "paid_search_identity", + parameters: { ...base, source: "google" }, + expected: { + linked_visitors: 1, + linked_users: 1, + signup_users: 1, + organizations: 1, + guest_checkout_visitors: 1, + guest_purchasers: 1, + purchasers: 1, + }, + }, + { + label: "referral_identity", + parameters: { ...base, source: "synthetic-partner" }, + expected: { + linked_visitors: 1, + linked_users: 1, + signup_users: 0, + organizations: 0, + guest_checkout_visitors: 0, + guest_purchasers: 0, + purchasers: 0, + }, + }, + { + label: "missing_identity_source", + parameters: { ...base, source: "does-not-exist" }, + expected: { + linked_visitors: 0, + linked_users: 0, + signup_users: 0, + organizations: 0, + guest_checkout_visitors: 0, + guest_purchasers: 0, + purchasers: 0, + }, + }, + ]; +}; + +export const assertSyntheticIdentityFilters = ({ payloads, queries }) => { + for (const query of queries) { + const rows = payloads[query.label]?.data; + if (!Array.isArray(rows) || rows.length !== 1) { + throw new Error( + `Synthetic identity filter ${query.label} did not return one totals row`, + ); + } + assertEndpointFields( + `synthetic identity filter ${query.label}`, + rows[0], + query.expected, + ); + } +}; + +export const assertSyntheticMonetizationFilters = ({ payloads, queries }) => { + for (const query of queries) { + const rows = payloads[query.label]?.data; + if (!Array.isArray(rows) || rows.length !== query.expectedRows) { + throw new Error( + `Synthetic monetization filter ${query.label} returned ${Array.isArray(rows) ? rows.length : "invalid"} rows, expected ${query.expectedRows}`, + ); + } + const events = rows.reduce( + (total, row) => total + Number(row.events ?? 0), + 0, + ); + const revenueMinor = rows.reduce( + (total, row) => total + Number(row.revenue_minor ?? 0), + 0, + ); + if ( + events !== query.expectedEvents || + revenueMinor !== query.expectedRevenueMinor + ) { + throw new Error( + `Synthetic monetization filter ${query.label} returned ${events} events and ${revenueMinor} revenue minor units`, + ); + } + if (query.expectedFields && rows.length === 1) { + assertEndpointFields( + `synthetic monetization filter ${query.label}`, + rows[0], + query.expectedFields, + ); + } + } }; export const normalizeCopyAssertions = (payload) => { const row = payload?.data?.[0] ?? {}; const number = (value) => Number(value ?? 0); return { + decisionMarkers: number(row.decision_markers), trafficMarkers: number(row.traffic_markers), trafficPageMarkers: number(row.traffic_page_markers), activationMarkers: number(row.activation_markers), retentionMarkers: number(row.retention_markers), + identityMarkers: number(row.identity_markers), + healthMarkers: number(row.health_markers), }; }; @@ -727,6 +2406,20 @@ export const assertSyntheticHealth = (health) => { } }; +export const assertSyntheticLoadHealth = (health, expectedEvents) => { + if ( + !Number.isInteger(expectedEvents) || + expectedEvents < 1 || + health.receivedRows < expectedEvents || + health.uniqueEvents !== expectedEvents || + health.uniquePayloads !== expectedEvents || + health.duplicateRows !== health.receivedRows - health.uniquePayloads || + health.payloadConflicts !== 0 + ) { + throw new Error("Synthetic load did not match the accepted event set"); + } +}; + export const percentile = (samples, quantile) => { if (samples.length === 0) { throw new Error("At least one latency sample is required"); @@ -753,6 +2446,7 @@ const DECISION_ENDPOINT_NAMES = [ "product_activation", "product_creator_activity", "product_creator_retention", + "product_identity_funnel", "product_events_daily", "product_feature_adoption", "product_analytics_freshness", @@ -762,8 +2456,12 @@ export const decisionEndpointQueries = ({ startDate, endDate, deploymentId = "", + includeIdentityFunnel = true, + syntheticRunId = "", }) => - DECISION_ENDPOINT_NAMES.map((name) => ({ + DECISION_ENDPOINT_NAMES.filter( + (name) => includeIdentityFunnel || name !== "product_identity_funnel", + ).map((name) => ({ name, parameters: { ...(name === "product_analytics_freshness" @@ -772,6 +2470,9 @@ export const decisionEndpointQueries = ({ ? { as_of_date: endDate } : { start_date: startDate, end_date: endDate }), __tb__deployment: deploymentId, + ...(syntheticRunId && name !== "product_analytics_freshness" + ? { synthetic_run_id: syntheticRunId } + : {}), }, })); @@ -822,10 +2523,14 @@ export const assertWorkflowSafety = (workflow) => { "staging-ci.js discard-deployment", "environment: staging", "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_COPY_TOKEN", + "TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN", + "TINYBIRD_STAGING_SCHEDULER_TOKEN", "TINYBIRD_STAGING_INGEST_TOKEN", "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_CLEANUP_TOKEN", "staging-ci.js run-copies", + "staging-ci.js set-copy-schedules", "probe-preview", "verify-promoted", ]) { diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 561dc4c8eb3..66a6153ba90 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -2,9 +2,20 @@ import fs from "node:fs"; import process from "node:process"; import { + applyCopyScheduleAction, assertExecutionScope, + assertRepresentativeEndpointCoverage, + assertSyntheticBusinessDecisions, assertSyntheticDecisions, + assertSyntheticEndpointDecisions, assertSyntheticHealth, + assertSyntheticIdentityFilters, + assertSyntheticLoadDecisions, + assertSyntheticLoadHealth, + assertSyntheticMonetizationFilters, + COPY_PIPES, + copyScheduleMatchesAction, + createSyntheticDecisionEvents, createSyntheticErasureControl, createSyntheticEvents, createSyntheticLoadEvents, @@ -27,6 +38,8 @@ import { STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, + syntheticIdentityFilterQueries, + syntheticMonetizationFilterQueries, validateSyntheticRunId, validateTinybirdCredentials, } from "./staging-ci-lib.js"; @@ -78,6 +91,9 @@ const writeJson = (filePath, value, mode = 0o644) => { const TINYBIRD_TOKEN_NAMES = [ "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_COPY_TOKEN", + "TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN", + "TINYBIRD_STAGING_SCHEDULER_TOKEN", "TINYBIRD_STAGING_INGEST_TOKEN", "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_CLEANUP_TOKEN", @@ -135,10 +151,12 @@ const request = async ( }; } if (response.status < 500 && response.status !== 429) { - throw new Error( + const error = new Error( `Tinybird request was rejected with HTTP ${response.status}`, { cause: "permanent" }, ); + error.status = response.status; + throw error; } lastError = new Error( `Tinybird request failed with HTTP ${response.status}`, @@ -170,10 +188,14 @@ const healthQuery = async ({ state, deploymentId = "", appVersion }) => { const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_READ_TOKEN", ]); + const previewWindow = + appVersion === state.previewAppVersion && + state.previewStartTime && + state.previewEndTime; return request( tinybirdUrl(origin, "/v0/pipes/product_events_health.json", { - start_time: state.startTime, - end_time: state.endTime, + start_time: previewWindow ? state.previewStartTime : state.startTime, + end_time: previewWindow ? state.previewEndTime : state.endTime, platform: "web", app_version: appVersion ?? state.appVersion, __tb__deployment: deploymentId, @@ -258,6 +280,7 @@ const promoteOwnedDeployment = async () => { const token = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; const initial = await deploymentList({ origin, token }); const plan = resolveExactPromotionPlan(initial.data, deploymentId); + writeOutput("previous_live_id", plan.previousLiveDeploymentId); const promotionDeadline = Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); let lastPromotionError; @@ -301,6 +324,15 @@ const promoteOwnedDeployment = async () => { cause: lastPromotionError, }); } + writeOutput("promoted", "true"); +}; + +const deleteRetiredDeployment = async ({ + origin, + token, + liveDeploymentId, + retiredDeploymentId, +}) => { const deadline = Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); let lastDeletionError; @@ -308,25 +340,24 @@ const promoteOwnedDeployment = async () => { const previous = await exactDeployment({ origin, token, - deploymentId: plan.previousLiveDeploymentId, + deploymentId: retiredDeploymentId, }); const lifecycle = resolveExactDeploymentLifecycle( previous.data, - plan.previousLiveDeploymentId, + retiredDeploymentId, ); if (lifecycle === "deleted") { - writeOutput("promoted", "true"); return; } if (lifecycle === "live") { - throw new Error("The previous Tinybird deployment became live again"); + throw new Error("Refusing to delete a live Tinybird deployment"); } if (lifecycle !== "deleting") { try { await request( tinybirdUrl( origin, - `/v1/deployments/${encodeURIComponent(plan.previousLiveDeploymentId)}`, + `/v1/deployments/${encodeURIComponent(retiredDeploymentId)}`, ), { token, @@ -334,22 +365,20 @@ const promoteOwnedDeployment = async () => { beforeAttempt: async () => { const ownership = await deploymentList({ origin, token }); if ( - resolveOwnedMutationTarget(ownership.data, deploymentId) !== + resolveOwnedMutationTarget(ownership.data, liveDeploymentId) !== "live" ) { - throw new Error( - "The promoted Tinybird deployment is no longer live", - ); + throw new Error("The retained Tinybird deployment is not live"); } const exactPrevious = await exactDeployment({ origin, token, - deploymentId: plan.previousLiveDeploymentId, + deploymentId: retiredDeploymentId, }); if ( resolveExactDeploymentLifecycle( exactPrevious.data, - plan.previousLiveDeploymentId, + retiredDeploymentId, ) === "live" ) { throw new Error( @@ -370,6 +399,380 @@ const promoteOwnedDeployment = async () => { }); }; +const switchLiveDeployment = async ({ + origin, + token, + fromDeploymentId, + toDeploymentId, +}) => { + const before = await deploymentList({ origin, token }); + if ( + resolveOwnedMutationTarget(before.data, fromDeploymentId) !== "live" || + resolveOwnedMutationTarget(before.data, toDeploymentId) !== "staging" + ) { + throw new Error( + "Tinybird live switch did not match the exact deployment pair", + ); + } + let mutationError; + try { + await request( + tinybirdUrl( + origin, + `/v1/deployments/${encodeURIComponent(toDeploymentId)}/set-live`, + ), + { + token, + method: "POST", + beforeAttempt: async () => { + const ownership = await deploymentList({ origin, token }); + if ( + resolveOwnedMutationTarget(ownership.data, fromDeploymentId) !== + "live" || + resolveOwnedMutationTarget(ownership.data, toDeploymentId) !== + "staging" + ) { + throw new Error("Tinybird live switch ownership changed"); + } + }, + }, + ); + } catch (error) { + mutationError = error; + } + const deadline = + Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); + let lastOwnershipError; + while (Date.now() < deadline) { + try { + const current = await deploymentList({ origin, token }); + const toTarget = resolveOwnedMutationTarget(current.data, toDeploymentId); + const fromTarget = resolveOwnedMutationTarget( + current.data, + fromDeploymentId, + ); + if (toTarget === "live" && fromTarget === "staging") return; + if ( + mutationError instanceof Error && + mutationError.cause === "permanent" && + toTarget === "staging" && + fromTarget === "live" + ) { + throw mutationError; + } + } catch (error) { + if (error === mutationError) throw error; + lastOwnershipError = error; + } + await delay(2_000); + } + throw new Error("Timed out reconciling the exact Tinybird live deployment", { + cause: mutationError ?? lastOwnershipError, + }); +}; + +const finalizeOwnedPromotion = async () => { + const deploymentId = option("deployment-id"); + const previousLiveDeploymentId = option("previous-live-id"); + const artifactPath = option("artifact"); + if (deploymentId === previousLiveDeploymentId) { + throw new Error("Tinybird finalization requires distinct deployments"); + } + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + ]); + await deleteRetiredDeployment({ + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + liveDeploymentId: deploymentId, + retiredDeploymentId: previousLiveDeploymentId, + }); + const artifact = readJson(artifactPath); + artifact.promotion = { + deploymentId, + previousLiveDeploymentId, + finalized: true, + verifiedAt: new Date().toISOString(), + }; + writeJson(artifactPath, artifact); + writeOutput("finalized", "true"); +}; + +const drillOwnedRollback = async () => { + const deploymentId = option("deployment-id"); + const previousLiveDeploymentId = option("previous-live-id"); + const state = readJson(option("state")); + if (deploymentId === previousLiveDeploymentId) { + throw new Error("Tinybird rollback drill requires distinct deployments"); + } + if (String(state.deploymentId) !== deploymentId) { + throw new Error( + "Tinybird rollback drill does not match the seeded deployment", + ); + } + const artifactPath = option("artifact"); + const artifact = readJson(artifactPath); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_READ_TOKEN", + ]); + const token = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; + try { + await switchLiveDeployment({ + origin, + token, + fromDeploymentId: deploymentId, + toDeploymentId: previousLiveDeploymentId, + }); + } catch (error) { + let candidateRestored = false; + try { + const ownership = await deploymentList({ origin, token }); + const candidateTarget = resolveOwnedMutationTarget( + ownership.data, + deploymentId, + ); + const previousTarget = resolveOwnedMutationTarget( + ownership.data, + previousLiveDeploymentId, + ); + if (candidateTarget === "live" && previousTarget === "staging") { + candidateRestored = true; + } else if (candidateTarget === "staging" && previousTarget === "live") { + await switchLiveDeployment({ + origin, + token, + fromDeploymentId: previousLiveDeploymentId, + toDeploymentId: deploymentId, + }); + candidateRestored = true; + } + } catch { + candidateRestored = false; + } + if (candidateRestored) { + writeOutput("rollback_target_usable", "false"); + } + artifact.rollbackDrill = { + passed: false, + candidateRestored, + rollbackTargetUsable: false, + verifiedAt: new Date().toISOString(), + }; + writeJson(artifactPath, artifact); + throw error; + } + let rollbackEndpointSuite; + let rollbackProbeError; + let retainedIdentityFunnelAvailable = false; + try { + retainedIdentityFunnelAvailable = await decisionEndpointAvailable({ + deploymentId: previousLiveDeploymentId, + name: "product_identity_funnel", + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + rollbackEndpointSuite = await queryDecisionEndpointSuite({ + deploymentId: previousLiveDeploymentId, + includeIdentityFunnel: retainedIdentityFunnelAvailable, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertDecisionEndpointSuiteReadable(rollbackEndpointSuite.payloads); + } catch (error) { + rollbackProbeError = error; + } + writeOutput("rollback_target_usable", rollbackProbeError ? "false" : "true"); + try { + await switchLiveDeployment({ + origin, + token, + fromDeploymentId: previousLiveDeploymentId, + toDeploymentId: deploymentId, + }); + } catch (error) { + artifact.rollbackDrill = { + passed: false, + candidateRestored: false, + rollbackTargetUsable: !rollbackProbeError, + verifiedAt: new Date().toISOString(), + }; + writeJson(artifactPath, artifact); + throw error; + } + if (rollbackProbeError) { + artifact.rollbackDrill = { + passed: false, + candidateRestored: true, + rollbackTargetUsable: false, + verifiedAt: new Date().toISOString(), + }; + writeJson(artifactPath, artifact); + throw new Error("The prior Tinybird deployment data plane is not usable", { + cause: rollbackProbeError, + }); + } + const restoredBusinessResult = await ciAssertionsQuery({ + state, + deploymentId, + syntheticRunId: state.decisionRunId, + }); + const restoredBusinessAssertions = normalizeCiAssertions( + restoredBusinessResult.data, + ); + assertSyntheticLoadHealth( + restoredBusinessAssertions, + state.decisionEventCount, + ); + if ( + restoredBusinessAssertions.canonicalEvents !== state.decisionEventCount || + restoredBusinessAssertions.decisionEvents !== state.decisionEventCount + ) { + throw new Error( + "Tinybird rollback restoration lost exact decision materialization", + ); + } + assertSyntheticBusinessDecisions(restoredBusinessAssertions); + const restoredEndpointSuite = await queryDecisionEndpointSuite({ + deploymentId, + origin, + state, + syntheticRunId: state.decisionRunId, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertSyntheticEndpointDecisions({ + appVersion: state.decisionAppVersion, + date: state.decisionDate, + hostname: state.decisionHostname, + pathname: state.decisionPathname, + payloads: restoredEndpointSuite.payloads, + }); + const restoredMonetizationFilters = await querySyntheticMonetizationFilters({ + deploymentId, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + const restoredIdentityFilters = await querySyntheticIdentityFilters({ + deploymentId, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + await readAndAssertPhaseHealth({ + state, + phase: "promoted", + deploymentId, + }); + artifact.rollbackDrill = { + passed: true, + dataPlanePassed: true, + candidateRestored: true, + rollbackTargetUsable: true, + rollbackDeploymentId: previousLiveDeploymentId, + restoredDeploymentId: deploymentId, + rollbackEndpointLatencyMs: rollbackEndpointSuite.latencyMs, + restoredEndpointLatencyMs: restoredEndpointSuite.latencyMs, + restoredMonetizationFilterLatencyMs: restoredMonetizationFilters.latencyMs, + restoredIdentityFilterLatencyMs: restoredIdentityFilters.latencyMs, + retainedIdentityFunnelAvailable, + verifiedAt: new Date().toISOString(), + }; + writeJson(artifactPath, artifact); +}; + +const rollbackOwnedPromotion = async () => { + const deploymentId = option("deployment-id"); + const previousLiveDeploymentId = option("previous-live-id"); + const state = readJson(option("state")); + if (deploymentId === previousLiveDeploymentId) { + throw new Error("Tinybird rollback requires distinct deployments"); + } + if (String(state.deploymentId) !== deploymentId) { + throw new Error("Tinybird rollback does not match the seeded deployment"); + } + const artifactPath = options.get("artifact"); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_READ_TOKEN", + ]); + const token = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; + const current = await deploymentList({ origin, token }); + const previousTarget = resolveOwnedMutationTarget( + current.data, + previousLiveDeploymentId, + ); + const rejectedTarget = resolveOwnedMutationTarget(current.data, deploymentId); + if (previousTarget !== "live") { + if (rejectedTarget !== "live" || previousTarget !== "staging") { + throw new Error("Tinybird rollback pair is not recoverable"); + } + await switchLiveDeployment({ + origin, + token, + fromDeploymentId: deploymentId, + toDeploymentId: previousLiveDeploymentId, + }); + } + let rollbackEndpointSuite; + let retainedIdentityFunnelAvailable = false; + try { + retainedIdentityFunnelAvailable = await decisionEndpointAvailable({ + deploymentId: previousLiveDeploymentId, + name: "product_identity_funnel", + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + rollbackEndpointSuite = await queryDecisionEndpointSuite({ + deploymentId: previousLiveDeploymentId, + includeIdentityFunnel: retainedIdentityFunnelAvailable, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertDecisionEndpointSuiteReadable(rollbackEndpointSuite.payloads); + } catch (error) { + const ownership = await deploymentList({ origin, token }); + if ( + resolveOwnedMutationTarget(ownership.data, previousLiveDeploymentId) === + "live" && + resolveOwnedMutationTarget(ownership.data, deploymentId) === "staging" + ) { + await switchLiveDeployment({ + origin, + token, + fromDeploymentId: previousLiveDeploymentId, + toDeploymentId: deploymentId, + }); + } + throw new Error("The Tinybird rollback destination is not usable", { + cause: error, + }); + } + await deleteRetiredDeployment({ + origin, + token, + liveDeploymentId: previousLiveDeploymentId, + retiredDeploymentId: deploymentId, + }); + if (artifactPath && fs.existsSync(artifactPath)) { + const artifact = readJson(artifactPath); + artifact.rollback = { + passed: true, + dataPlanePassed: true, + restoredDeploymentId: previousLiveDeploymentId, + rejectedDeploymentId: deploymentId, + endpointLatencyMs: rollbackEndpointSuite.latencyMs, + retainedIdentityFunnelAvailable, + verifiedAt: new Date().toISOString(), + }; + writeJson(artifactPath, artifact); + } +}; + const discardOwnedDeployment = async () => { const deploymentId = option("deployment-id"); const artifactPath = options.get("artifact"); @@ -443,6 +846,133 @@ const decisionEndpointQuery = async ({ origin, token, name, parameters }) => attempts: 3, }); +const decisionEndpointAvailable = async ({ + deploymentId, + name, + origin, + state, + token, +}) => { + const query = decisionEndpointQueries({ + startDate: state.startTime.slice(0, 10), + endDate: state.endTime.slice(0, 10), + deploymentId, + }).find((candidate) => candidate.name === name); + if (!query) throw new Error(`Unknown decision endpoint ${name}`); + try { + await decisionEndpointQuery({ origin, token, ...query }); + return true; + } catch (error) { + if (error instanceof Error && error.status === 404) return false; + throw error; + } +}; + +const queryDecisionEndpointSuite = async ({ + deploymentId, + includeIdentityFunnel = true, + origin, + state, + syntheticRunId = "", + token, +}) => { + const endDate = syntheticRunId + ? state.decisionDate + : state.endTime.slice(0, 10); + const queries = decisionEndpointQueries({ + startDate: state.startTime.slice(0, 10), + endDate, + deploymentId, + includeIdentityFunnel, + syntheticRunId, + }); + const results = await Promise.all( + queries.map((query) => decisionEndpointQuery({ origin, token, ...query })), + ); + return { + latencyMs: Object.fromEntries( + results.map((result, index) => [queries[index].name, result.latencyMs]), + ), + payloads: Object.fromEntries( + results.map((result, index) => [queries[index].name, result.data]), + ), + }; +}; + +const assertDecisionEndpointSuiteReadable = (payloads) => { + for (const [name, payload] of Object.entries(payloads)) { + if (!Array.isArray(payload?.data)) { + throw new Error(`Tinybird rollback endpoint ${name} was not readable`); + } + } +}; + +const querySyntheticMonetizationFilters = async ({ + deploymentId, + origin, + state, + token, +}) => { + const queries = syntheticMonetizationFilterQueries({ + date: state.decisionDate, + deploymentId, + syntheticRunId: state.decisionRunId, + }); + const results = await Promise.all( + queries.map((query) => + decisionEndpointQuery({ + origin, + token, + name: "product_events_daily", + parameters: query.parameters, + }), + ), + ); + const payloads = Object.fromEntries( + results.map((result, index) => [queries[index].label, result.data]), + ); + assertSyntheticMonetizationFilters({ payloads, queries }); + return { + latencyMs: Object.fromEntries( + results.map((result, index) => [queries[index].label, result.latencyMs]), + ), + payloads, + }; +}; + +const querySyntheticIdentityFilters = async ({ + deploymentId, + origin, + state, + token, +}) => { + const queries = syntheticIdentityFilterQueries({ + date: state.decisionDate, + deploymentId, + syntheticRunId: state.decisionRunId, + }); + const results = await Promise.all( + queries.map((query) => + decisionEndpointQuery({ + origin, + token, + name: "product_identity_funnel", + parameters: query.parameters, + }), + ), + ); + const payloads = Object.fromEntries( + results.map((result, index) => [queries[index].label, result.data]), + ); + assertSyntheticIdentityFilters({ payloads, queries }); + return { + latencyMs: Object.fromEntries( + results.map((result, index) => [queries[index].label, result.latencyMs]), + ), + payloads, + }; +}; + const waitForVercel = async () => { const repository = environment("GITHUB_REPOSITORY"); const sha = environment("EXPECTED_SHA"); @@ -544,7 +1074,7 @@ const previewRequest = async (url, init = {}) => { : {}), ...init.headers, }, - signal: AbortSignal.timeout(20_000), + signal: init.signal ?? AbortSignal.timeout(20_000), }); }; @@ -735,6 +1265,11 @@ const probePreview = async () => { ); } state.previewAcceptedRows = duplicateResponses.length + replayAccepted; + state.previewExpectedEvents = Number(state.browserExpectedEvents ?? 0) + 1; + state.previewStartTime = new Date( + new Date(occurredAt).getTime() - 120_000, + ).toISOString(); + state.previewEndTime = new Date(Date.now() + 300_000).toISOString(); writeJson(statePath, state, 0o600); artifact.previewApi = { bootstrapPassed: true, @@ -764,6 +1299,106 @@ const probePreview = async () => { writeJson(artifactPath, artifact); }; +const probeDurableServerPath = async () => { + const statePath = option("state"); + const artifactPath = option("artifact"); + const state = readJson(statePath); + const artifact = readJson(artifactPath); + const secret = environment("CAP_ANALYTICS_STAGING_TEST_SECRET"); + const serverRunId = validateSyntheticRunId(`${state.runId}_server`); + const url = new URL("/api/analytics/staging-test", artifact.vercel.url); + const body = (sha) => + JSON.stringify({ + scenario: "business_lifecycle", + runId: serverRunId, + sha, + }); + const send = (authorization, sha = artifact.sha) => + previewRequest(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(authorization ? { Authorization: authorization } : {}), + }, + body: body(sha), + }); + const unauthorized = await send(); + if (unauthorized.status !== 401) { + throw new Error( + `The durable staging route accepted missing authorization with HTTP ${unauthorized.status}`, + ); + } + const wrongSha = await send( + `Bearer ${secret}`, + "0000000000000000000000000000000000000000", + ); + if (wrongSha.status !== 400) { + throw new Error( + `The durable staging route accepted a wrong SHA with HTTP ${wrongSha.status}`, + ); + } + const response = await send(`Bearer ${secret}`); + if (!response.ok) { + throw new Error( + `The durable staging route failed with HTTP ${response.status}`, + ); + } + const result = await response.json(); + if ( + Number(result.accepted) !== 5 || + Number(result.uniqueEvents) !== 4 || + !Array.isArray(result.workflowRuns) || + result.workflowRuns.length !== 5 + ) { + throw new Error( + "The durable staging route returned incomplete workflow proof", + ); + } + state.serverRunId = serverRunId; + state.serverExpectedEvents = 4; + state.serverExpectedRows = 5; + writeJson(statePath, state, 0o600); + const visibility = await waitForCopyVisibility({ + label: "Durable exact-SHA server delivery", + read: async () => + normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId: serverRunId, + }) + ).data, + ), + assert: (assertions) => { + if ( + assertions.receivedRows < 5 || + assertions.uniqueEvents !== 4 || + assertions.uniquePayloads !== 4 || + assertions.duplicateRows < 1 || + assertions.payloadConflicts !== 0 + ) { + throw new Error("Durable server events are not fully visible"); + } + }, + }); + artifact.serverDelivery = { + acceptedRows: 5, + uniqueEvents: 4, + duplicateRows: visibility.value.duplicateRows, + workflowRuns: 5, + visibilityMs: visibility.visibilityMs, + unauthorizedRejected: true, + wrongShaRejected: true, + }; + artifact.assertions = { + ...artifact.assertions, + durableServerPathPassed: true, + serverDuplicateDeliveryPassed: true, + }; + writeJson(artifactPath, artifact); +}; + const seed = async () => { const runId = validateSyntheticRunId(option("run-id")); const deploymentId = option("deployment-id"); @@ -771,6 +1406,7 @@ const seed = async () => { const artifactPath = option("artifact"); const sha = environment("EXPECTED_SHA"); const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_COPY_TOKEN", "TINYBIRD_STAGING_INGEST_TOKEN", ]); const startedAt = new Date(); @@ -779,11 +1415,25 @@ const seed = async () => { runId, now: startedAt, }); + const decisionFixture = createSyntheticDecisionEvents({ + runId, + now: startedAt, + }); const loadFixture = createSyntheticLoadEvents({ runId, count: Number(process.env.PERFORMANCE_EVENT_COUNT ?? 1_000), now: startedAt, }); + const largeLoadFixture = createSyntheticLoadEvents({ + runId: `${runId}_large`, + count: Number(process.env.LARGE_PERFORMANCE_EVENT_COUNT ?? 10_000), + now: startedAt, + }); + if (largeLoadFixture.rows.length <= loadFixture.rows.length) { + throw new Error( + "The large performance corpus must exceed the baseline corpus", + ); + } const previewRunId = validateSyntheticRunId(`${runId}_preview`); const previewAppVersion = `staging-preview-${hashIdentifier(runId).slice(0, 12)}`; const state = { @@ -795,6 +1445,22 @@ const seed = async () => { loadAppVersion: loadFixture.appVersion, loadRunId: loadFixture.runId, loadEventCount: loadFixture.rows.length, + largeLoadAppVersion: largeLoadFixture.appVersion, + largeLoadRunId: largeLoadFixture.runId, + largeLoadEventCount: largeLoadFixture.rows.length, + decisionRunId: decisionFixture.runId, + decisionAppVersion: decisionFixture.appVersion, + decisionEventCount: decisionFixture.rows.length, + decisionDate: decisionFixture.date, + decisionHostname: decisionFixture.hostname, + decisionPathname: decisionFixture.pathname, + erasureLinkedAnonymousIds: [ + ...new Set( + decisionFixture.rows + .filter((row) => row.user_id === fixture.userId && row.anonymous_id) + .map((row) => row.anonymous_id), + ), + ].filter((anonymousId) => anonymousId !== fixture.anonymousId), erasureAnonymousId: fixture.anonymousId, erasureOrganizationId: fixture.organizationId, erasureUserId: fixture.userId, @@ -829,6 +1495,11 @@ const seed = async () => { rowsAttempted: 0, rowsAccepted: 0, }, + largeLoad: { + rowsPlanned: largeLoadFixture.rows.length, + rowsAttempted: 0, + rowsAccepted: 0, + }, erasure: { controlRunHash: hashIdentifier(erasureControl.runId), identityHash: hashIdentifier( @@ -837,6 +1508,11 @@ const seed = async () => { controlAttempted: false, controlAccepted: false, }, + decisions: { + rowsPlanned: decisionFixture.rows.length, + rowsAttempted: 0, + rowsAccepted: 0, + }, assertions: { seedAccepted: false }, }; writeJson(artifactPath, artifact); @@ -875,42 +1551,95 @@ const seed = async () => { separateBatchDeliveries.push(await deliver(row, true)); } const deliveries = [...concurrentDeliveries, ...separateBatchDeliveries]; - artifact.load.rowsAttempted = loadFixture.rows.length; - writeJson(artifactPath, artifact); - const loadStartedAt = performance.now(); - const loadDelivery = await request( - tinybirdUrl(origin, "/v0/events", { - name: "product_events_v1", - wait: "true", - __tb__min_deployment: deploymentId, - }), - { - token: tokens.TINYBIRD_STAGING_INGEST_TOKEN, - method: "POST", - body: `${loadFixture.rows.map((row) => JSON.stringify(row)).join("\n")}\n`, - headers: { "Content-Type": "application/x-ndjson" }, - attempts: 4, - }, + const sendLoadFixture = async (fixture, artifactKey) => { + const batchSize = Number(process.env.PERFORMANCE_INGEST_BATCH_SIZE ?? 500); + if (!Number.isInteger(batchSize) || batchSize < 100 || batchSize > 1_000) { + throw new Error("Performance ingestion batch size must be 100 to 1000"); + } + artifact[artifactKey].rowsAttempted = fixture.rows.length; + writeJson(artifactPath, artifact); + const started = performance.now(); + const latencies = []; + let accepted = 0; + let retryAttempts = 0; + for (let offset = 0; offset < fixture.rows.length; offset += batchSize) { + const batch = fixture.rows.slice(offset, offset + batchSize); + const delivery = await request( + tinybirdUrl(origin, "/v0/events", { + name: "product_events_v1", + wait: "true", + __tb__min_deployment: deploymentId, + }), + { + token: tokens.TINYBIRD_STAGING_INGEST_TOKEN, + method: "POST", + body: `${batch.map((row) => JSON.stringify(row)).join("\n")}\n`, + headers: { "Content-Type": "application/x-ndjson" }, + attempts: 1, + }, + ); + latencies.push(delivery.latencyMs); + retryAttempts += delivery.attempt - 1; + accepted += batch.length; + } + const elapsedMs = Math.max(1, Math.round(performance.now() - started)); + artifact[artifactKey] = { + rows: fixture.rows.length, + rowsPlanned: fixture.rows.length, + rowsAttempted: fixture.rows.length, + rowsAccepted: accepted, + batchSize, + batches: latencies.length, + batchLatency: latencySummary(latencies), + errorRate: 0, + retryAttempts, + rowsPerSecond: Math.round((fixture.rows.length * 1_000) / elapsedMs), + }; + }; + await sendLoadFixture(loadFixture, "load"); + await sendLoadFixture(largeLoadFixture, "largeLoad"); + const ingestionBatchP95BudgetMs = Number( + process.env.INGESTION_BATCH_P95_BUDGET_MS ?? 5_000, ); - const loadElapsedMs = Math.max( - 1, - Math.round(performance.now() - loadStartedAt), + const ingestionMinimumRowsPerSecond = Number( + process.env.INGESTION_MINIMUM_ROWS_PER_SECOND ?? 500, ); - artifact.load = { - rows: loadFixture.rows.length, - rowsPlanned: loadFixture.rows.length, - rowsAttempted: loadFixture.rows.length, - rowsAccepted: loadFixture.rows.length, - requestLatencyMs: loadDelivery.latencyMs, - retryAttempts: loadDelivery.attempt - 1, - rowsPerSecond: Math.round( - (loadFixture.rows.length * 1_000) / loadElapsedMs, - ), + artifact.ingestionBudget = { + batchP95Ms: ingestionBatchP95BudgetMs, + minimumRowsPerSecond: ingestionMinimumRowsPerSecond, + passed: + artifact.load.batchLatency.p95Ms <= ingestionBatchP95BudgetMs && + artifact.largeLoad.batchLatency.p95Ms <= ingestionBatchP95BudgetMs && + artifact.load.rowsPerSecond >= ingestionMinimumRowsPerSecond && + artifact.largeLoad.rowsPerSecond >= ingestionMinimumRowsPerSecond && + artifact.load.errorRate === 0 && + artifact.largeLoad.errorRate === 0, }; writeJson(artifactPath, artifact); + if (!artifact.ingestionBudget.passed) { + throw new Error("Synthetic ingestion performance budget failed"); + } artifact.erasure.controlAttempted = true; writeJson(artifactPath, artifact); const erasureControlDelivery = await deliver(erasureControl.row); + artifact.decisions.rowsAttempted = decisionFixture.rows.length; + writeJson(artifactPath, artifact); + const decisionDeliveries = []; + for (const row of decisionFixture.rows) { + decisionDeliveries.push(await deliver(row)); + } + artifact.decisions = { + rowsPlanned: decisionFixture.rows.length, + rowsAttempted: decisionFixture.rows.length, + rowsAccepted: decisionDeliveries.length, + requestLatency: latencySummary( + decisionDeliveries.map((delivery) => delivery.latencyMs), + ), + retryAttempts: decisionDeliveries.reduce( + (total, delivery) => total + delivery.attempts - 1, + 0, + ), + }; artifact.delivery = { rowsPlanned: fixture.rows.length, rowsAttempted: fixture.rows.length, @@ -963,12 +1692,28 @@ const phaseRunExpectations = ({ state, phase }) => { }, { runId: state.loadRunId, - canonicalEvents: ["staged", "promoted"].includes(phase) - ? state.loadEventCount - : 0, - decisionEvents: ["staged", "promoted"].includes(phase) - ? state.loadEventCount - : 0, + canonicalEvents: phase === "cleanup" ? 0 : state.loadEventCount, + decisionEvents: phase === "cleanup" ? 0 : state.loadEventCount, + }, + { + runId: state.largeLoadRunId, + canonicalEvents: phase === "cleanup" ? 0 : state.largeLoadEventCount, + decisionEvents: phase === "cleanup" ? 0 : state.largeLoadEventCount, + }, + { + runId: state.decisionRunId, + canonicalEvents: + phase === "cleanup" + ? 0 + : phase === "erasure" + ? 2 + : state.decisionEventCount, + decisionEvents: + phase === "cleanup" + ? 0 + : phase === "erasure" + ? 2 + : state.decisionEventCount, }, { runId: state.erasureControlRunId, @@ -984,8 +1729,19 @@ const phaseRunExpectations = ({ state, phase }) => { if (state.previewRunId && phase !== "staged") { expectations.push({ runId: state.previewRunId, - canonicalEvents: phase === "cleanup" ? 0 : 1, - decisionEvents: phase === "cleanup" ? 0 : 1, + canonicalEvents: + phase === "cleanup" ? 0 : Number(state.previewExpectedEvents ?? 1), + decisionEvents: + phase === "cleanup" ? 0 : Number(state.previewExpectedEvents ?? 1), + }); + } + if (state.serverRunId && phase !== "staged") { + expectations.push({ + runId: state.serverRunId, + canonicalEvents: + phase === "cleanup" ? 0 : Number(state.serverExpectedEvents), + decisionEvents: + phase === "cleanup" ? 0 : Number(state.serverExpectedEvents), }); } return expectations; @@ -1037,13 +1793,18 @@ const assertSingleHealth = (health) => { }; const readAndAssertPhaseHealth = async ({ state, phase, deploymentId }) => { - const [main, load, control, preview] = await Promise.all([ + const [main, load, largeLoad, control, preview] = await Promise.all([ healthQuery({ state, deploymentId }), healthQuery({ state, deploymentId, appVersion: state.loadAppVersion, }), + healthQuery({ + state, + deploymentId, + appVersion: state.largeLoadAppVersion, + }), healthQuery({ state, deploymentId, @@ -1060,6 +1821,7 @@ const readAndAssertPhaseHealth = async ({ state, phase, deploymentId }) => { const health = { main: normalizeHealth(main.data), load: normalizeHealth(load.data), + largeLoad: normalizeHealth(largeLoad.data), control: normalizeHealth(control.data), preview: preview ? normalizeHealth(preview.data) : null, }; @@ -1073,9 +1835,18 @@ const readAndAssertPhaseHealth = async ({ state, phase, deploymentId }) => { ) { throw new Error("Synthetic load health is incomplete"); } + if ( + health.largeLoad.receivedRows < state.largeLoadEventCount || + health.largeLoad.uniqueEvents !== state.largeLoadEventCount || + health.largeLoad.uniquePayloads !== state.largeLoadEventCount || + health.largeLoad.payloadConflicts !== 0 + ) { + throw new Error("Large synthetic load health is incomplete"); + } } else { assertZeroHealth(health.main); assertZeroHealth(health.load); + assertZeroHealth(health.largeLoad); } if (phase === "cleanup") { assertZeroHealth(health.control); @@ -1096,26 +1867,21 @@ const runCopies = async () => { if (!["staged", "promoted", "erasure", "cleanup"].includes(phase)) { throw new Error("Tinybird copy phase is invalid"); } - if (!["live", "staging"].includes(requestedTarget)) { - throw new Error("Tinybird copy target is invalid"); - } - if (requestedTarget === "staging" && !["staged", "cleanup"].includes(phase)) { - throw new Error("Only staged and cleanup copy phases can target staging"); + if (requestedTarget !== "live") { + throw new Error( + "Tinybird Copy mutations are allowed only after staging promotion", + ); } if (String(state.deploymentId) !== option("deployment-id")) { throw new Error("Tinybird copy deployment does not match the seeded run"); } const { origin, tokens } = tinybirdEnvironment([ - "TINYBIRD_STAGING_READ_TOKEN", + "TINYBIRD_STAGING_COPY_TOKEN", "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); const copyRunId = validateSyntheticRunId(`${state.runId}_${phase}`); const expectations = phaseRunExpectations({ state, phase }); const executeCopies = async (target) => { - const copyToken = - target === "staging" - ? tokens.TINYBIRD_STAGING_DEPLOY_TOKEN - : tokens.TINYBIRD_STAGING_READ_TOKEN; const assertMutationOwnership = async () => { if ( (await ownedMutationTarget({ @@ -1129,11 +1895,10 @@ const runCopies = async () => { }; const canonicalJobs = await submitTinybirdCopyJobs({ origin, - token: copyToken, + token: tokens.TINYBIRD_STAGING_COPY_TOKEN, deploymentId: state.deploymentId, request, pipes: ["snapshot_product_events_canonical_v1"], - useDeploymentParameter: target === "staging", assertMutationOwnership, }); artifact.copyJobs = { @@ -1162,14 +1927,7 @@ const runCopies = async () => { const copySteps = [ { pipe: "snapshot_product_events_daily_exact", - read: () => - readPhaseCiAssertions({ - state, - deploymentId: state.deploymentId, - expectations, - }), - assert: (results) => - assertPhaseCiAssertions(results, ["decisionEvents"]), + marker: "decisionMarkers", }, { pipe: "snapshot_product_traffic_daily_exact", @@ -1187,26 +1945,23 @@ const runCopies = async () => { pipe: "snapshot_product_creator_retention_exact", marker: "retentionMarkers", }, + { + pipe: "snapshot_product_identity_funnel_exact", + marker: "identityMarkers", + }, { pipe: "snapshot_product_events_health_hourly", - read: () => - readAndAssertPhaseHealth({ - state, - phase, - deploymentId: state.deploymentId, - }), - assert: () => undefined, + marker: "healthMarkers", }, ]; for (const copyStep of copySteps) { downstreamJobs.push( ...(await submitTinybirdCopyJobs({ origin, - token: copyToken, + token: tokens.TINYBIRD_STAGING_COPY_TOKEN, deploymentId: state.deploymentId, request, pipes: [copyStep.pipe], - useDeploymentParameter: target === "staging", copyRunId, assertMutationOwnership, })), @@ -1298,6 +2053,235 @@ const runCopies = async () => { throw new Error("Tinybird cleanup copies changed target more than once"); }; +const setCopySchedules = async () => { + const state = readJson(option("state")); + const artifactPath = option("artifact"); + const artifact = readJson(artifactPath); + const action = option("action"); + if (!["pause", "resume"].includes(action)) { + throw new Error("Tinybird Copy schedule action must be pause or resume"); + } + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_SCHEDULER_TOKEN", + ]); + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== "live" + ) { + throw new Error( + "Copy schedules can change only on the owned live deployment", + ); + } + await applyCopyScheduleAction({ + pipes: COPY_PIPES, + action, + setSchedule: async (pipe, scheduleAction) => { + let mutationError; + try { + await request( + tinybirdUrl( + origin, + `/v0/pipes/${encodeURIComponent(pipe)}/copy/${scheduleAction === "pause" ? "cancel" : "resume"}`, + ), + { + token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, + method: "POST", + attempts: 1, + }, + ); + } catch (error) { + mutationError = error; + } + const pipeState = await request( + tinybirdUrl(origin, `/v0/pipes/${encodeURIComponent(pipe)}`), + { + token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, + attempts: 4, + }, + ); + if (copyScheduleMatchesAction(pipeState, scheduleAction)) return; + if (mutationError) throw mutationError; + throw new Error( + `Tinybird did not attest the ${scheduleAction} state for ${pipe}`, + ); + }, + }); + artifact.copySchedule = { + ...(artifact.copySchedule ?? {}), + [action]: { + status: "passed", + deploymentId: String(state.deploymentId), + pipeCount: COPY_PIPES.length, + }, + }; + writeJson(artifactPath, artifact); +}; + +const rawAssertionMetrics = (assertions) => ({ + receivedRows: assertions.receivedRows, + uniqueEvents: assertions.uniqueEvents, + uniquePayloads: assertions.uniquePayloads, + duplicateRows: assertions.duplicateRows, + payloadConflicts: assertions.payloadConflicts, +}); + +const verifyCandidate = async ({ state, artifact, artifactPath }) => { + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_READ_TOKEN", + "TINYBIRD_STAGING_DEPLOY_TOKEN", + ]); + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== "staging" + ) { + throw new Error("Candidate validation lost exact deployment ownership"); + } + const visibility = await waitForCopyVisibility({ + label: "Tinybird candidate raw delivery", + read: async () => + Promise.all( + [ + state.runId, + state.loadRunId, + state.largeLoadRunId, + state.decisionRunId, + state.erasureControlRunId, + ].map(async (syntheticRunId) => + normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId, + }) + ).data, + ), + ), + ), + assert: ([main, load, largeLoad, decisions, control]) => { + assertSyntheticHealth(main); + assertSyntheticLoadHealth(load, state.loadEventCount); + assertSyntheticLoadHealth(largeLoad, state.largeLoadEventCount); + assertSyntheticLoadHealth(decisions, state.decisionEventCount); + assertSingleHealth(control); + }, + }); + const ingestionVisibilityMs = Date.now() - Date.parse(state.startedAt); + const [main, load, largeLoad, decisions, control] = visibility.value; + const liveAssertions = await Promise.all( + [ + state.runId, + state.loadRunId, + state.largeLoadRunId, + state.decisionRunId, + state.erasureControlRunId, + ].map(async (syntheticRunId) => + normalizeCiAssertions( + (await ciAssertionsQuery({ state, syntheticRunId })).data, + ), + ), + ); + if ( + liveAssertions.some((assertions) => + Object.values(assertions).some((value) => value !== 0), + ) + ) { + throw new Error("Candidate-only synthetic events affected live analytics"); + } + const queries = decisionEndpointQueries({ + startDate: state.startTime.slice(0, 10), + endDate: state.endTime.slice(0, 10), + deploymentId: state.deploymentId, + }); + const samples = Object.fromEntries(queries.map(({ name }) => [name, []])); + const fanoutSamples = []; + for (let round = 0; round < 5; round += 1) { + const startedAt = performance.now(); + const results = await Promise.all( + queries.map((query) => + decisionEndpointQuery({ + origin, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + ...query, + }), + ), + ); + fanoutSamples.push(Math.round(performance.now() - startedAt)); + for (let index = 0; index < results.length; index += 1) { + samples[queries[index].name].push(results[index].latencyMs); + } + } + const endpointLatency = Object.fromEntries( + Object.entries(samples).map(([name, values]) => [ + name, + latencySummary(values), + ]), + ); + const dashboardFanoutLatency = latencySummary(fanoutSamples); + const endpointP95BudgetMs = Number( + process.env.ENDPOINT_P95_BUDGET_MS ?? 2_500, + ); + const fanoutP95BudgetMs = Number( + process.env.DASHBOARD_FANOUT_P95_BUDGET_MS ?? 3_500, + ); + const failedEndpoints = Object.entries(endpointLatency) + .filter(([, latency]) => latency.p95Ms > endpointP95BudgetMs) + .map(([name]) => name); + const ingestionSloMs = Number(process.env.INGESTION_SLO_MS ?? 180_000); + artifact.candidateValidation = { + raw: { + main: rawAssertionMetrics(main), + load: rawAssertionMetrics(load), + largeLoad: rawAssertionMetrics(largeLoad), + decisions: rawAssertionMetrics(decisions), + control: rawAssertionMetrics(control), + }, + liveIsolated: true, + polls: visibility.polls, + ingestionVisibilityMs, + endpointLatency, + dashboardFanoutLatency, + }; + artifact.assertions = { + ...artifact.assertions, + candidateRawDeliveryPassed: true, + candidateDuplicateVisible: true, + candidatePayloadConflictVisible: true, + candidateIsolationPassed: true, + candidateEndpointsPassed: + failedEndpoints.length === 0 && + dashboardFanoutLatency.p95Ms <= fanoutP95BudgetMs, + candidateIngestionSloPassed: ingestionVisibilityMs <= ingestionSloMs, + }; + artifact.visibilityMs = ingestionVisibilityMs; + writeJson(artifactPath, artifact); + if (ingestionVisibilityMs > ingestionSloMs) { + throw new Error( + `Candidate events became visible in ${ingestionVisibilityMs}ms, over ${ingestionSloMs}ms`, + ); + } + if ( + failedEndpoints.length > 0 || + dashboardFanoutLatency.p95Ms > fanoutP95BudgetMs + ) { + throw new Error( + `Candidate endpoint budgets failed: ${[ + ...failedEndpoints, + ...(dashboardFanoutLatency.p95Ms <= fanoutP95BudgetMs + ? [] + : ["dashboard_fanout"]), + ].join(", ")}`, + ); + } +}; + const verify = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); @@ -1308,6 +2292,10 @@ const verify = async () => { deploymentId: option("deployment-id"), expectedDeploymentId: String(state.deploymentId), }); + if (target === "staging") { + await verifyCandidate({ state, artifact, artifactPath }); + return; + } const deadline = Date.now() + Number(process.env.INGESTION_SLO_MS ?? 180_000); let result; let health; @@ -1344,37 +2332,32 @@ const verify = async () => { "Synthetic load health did not match the accepted event set", ); } - if (target === "staging") { - const [liveHealthResult, liveLoadResult, liveControlResult, liveDecisions] = - await Promise.all([ - healthQuery({ state }), - healthQuery({ state, appVersion: state.loadAppVersion }), - healthQuery({ state, appVersion: state.erasureControlAppVersion }), - ciAssertionsQuery({ state }), - ]); - for (const liveHealth of [ - normalizeHealth(liveHealthResult.data), - normalizeHealth(liveLoadResult.data), - normalizeHealth(liveControlResult.data), - ]) { - assertZeroHealth(liveHealth); - } - if ( - Object.values(normalizeCiAssertions(liveDecisions.data)).some( - (value) => value !== 0, - ) - ) { - throw new Error( - "Candidate-only synthetic events affected live decisions", - ); - } - artifact.candidateIsolation = { - candidateDeploymentId: state.deploymentId, - candidateVisible: true, - liveVisible: false, - }; - artifact.assertions.candidateIsolationPassed = true; - } + const loadDecisionResult = await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId: state.loadRunId, + }); + const loadDecisionAssertions = normalizeCiAssertions(loadDecisionResult.data); + assertSyntheticLoadDecisions(loadDecisionAssertions, state.loadEventCount); + const largeLoadResult = await healthQuery({ + state, + deploymentId: state.deploymentId, + appVersion: state.largeLoadAppVersion, + }); + const largeLoadHealth = normalizeHealth(largeLoadResult.data); + assertSyntheticLoadHealth(largeLoadHealth, state.largeLoadEventCount); + const largeLoadDecisionResult = await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId: state.largeLoadRunId, + }); + const largeLoadDecisionAssertions = normalizeCiAssertions( + largeLoadDecisionResult.data, + ); + assertSyntheticLoadDecisions( + largeLoadDecisionAssertions, + state.largeLoadEventCount, + ); const samples = [result.latencyMs]; for (let index = 1; index < 20; index += 1) { const sample = await healthQuery({ @@ -1390,23 +2373,91 @@ const verify = async () => { const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_READ_TOKEN", ]); - const decisionQueries = decisionEndpointQueries({ + const baselineDeploymentId = + options.get("baseline-deployment-id") || state.deploymentId; + if (!/^[0-9]+$/.test(baselineDeploymentId)) { + throw new Error( + "Tinybird performance baseline must be a numeric deployment", + ); + } + const retainedIdentityFunnelAvailable = + baselineDeploymentId === state.deploymentId || + (await decisionEndpointAvailable({ + deploymentId: baselineDeploymentId, + name: "product_identity_funnel", + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + })); + const baselineQueries = decisionEndpointQueries({ + startDate: state.startTime.slice(0, 10), + endDate: state.endTime.slice(0, 10), + deploymentId: baselineDeploymentId, + includeIdentityFunnel: retainedIdentityFunnelAvailable, + }); + const measuredQueries = decisionEndpointQueries({ startDate: state.startTime.slice(0, 10), endDate: state.endTime.slice(0, 10), deploymentId: state.deploymentId, }); + const representativeQueries = decisionEndpointQueries({ + startDate: state.startTime.slice(0, 10), + endDate: state.decisionDate, + deploymentId: state.deploymentId, + syntheticRunId: state.loadRunId, + }); + const largeQueries = decisionEndpointQueries({ + startDate: state.startTime.slice(0, 10), + endDate: state.decisionDate, + deploymentId: state.deploymentId, + syntheticRunId: state.largeLoadRunId, + }); + const representativeCoverage = await queryDecisionEndpointSuite({ + deploymentId: state.deploymentId, + origin, + state, + syntheticRunId: state.loadRunId, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertRepresentativeEndpointCoverage({ + expectedEvents: state.loadEventCount, + payloads: representativeCoverage.payloads, + }); + const largeCoverage = await queryDecisionEndpointSuite({ + deploymentId: state.deploymentId, + origin, + state, + syntheticRunId: state.largeLoadRunId, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertRepresentativeEndpointCoverage({ + expectedEvents: state.largeLoadEventCount, + payloads: largeCoverage.payloads, + }); const baselineSamples = Object.fromEntries( - decisionQueries.map(({ name }) => [name, []]), + baselineQueries.map(({ name }) => [name, []]), ); const measuredSamples = Object.fromEntries( - decisionQueries.map(({ name }) => [name, []]), + measuredQueries.map(({ name }) => [name, []]), + ); + const representativeSamples = Object.fromEntries( + representativeQueries.map(({ name }) => [name, []]), + ); + const largeSamples = Object.fromEntries( + largeQueries.map(({ name }) => [name, []]), ); const baselineFanoutSamples = []; const measuredFanoutSamples = []; - const sampleDecisionRound = async (endpointSamples, fanoutSamples) => { + const representativeFanoutSamples = []; + const largeFanoutSamples = []; + const sampleDecisionRound = async ( + queries, + endpointSamples, + fanoutSamples, + ) => { const startedAt = performance.now(); const results = await Promise.all( - decisionQueries.map((query) => + queries.map((query) => decisionEndpointQuery({ origin, token: tokens.TINYBIRD_STAGING_READ_TOKEN, @@ -1416,30 +2467,79 @@ const verify = async () => { ); fanoutSamples.push(Math.round(performance.now() - startedAt)); for (let index = 0; index < results.length; index += 1) { - endpointSamples[decisionQueries[index].name].push( - results[index].latencyMs, - ); + endpointSamples[queries[index].name].push(results[index].latencyMs); } }; - for (let index = 0; index < 5; index += 1) { - await sampleDecisionRound(baselineSamples, baselineFanoutSamples); + for (let index = 0; index < 10; index += 1) { + await sampleDecisionRound( + baselineQueries, + baselineSamples, + baselineFanoutSamples, + ); + } + for (let index = 0; index < 10; index += 1) { + await sampleDecisionRound(largeQueries, largeSamples, largeFanoutSamples); } for (let index = 0; index < 15; index += 1) { - await sampleDecisionRound(measuredSamples, measuredFanoutSamples); + await sampleDecisionRound( + measuredQueries, + measuredSamples, + measuredFanoutSamples, + ); + } + for (let index = 0; index < 10; index += 1) { + await sampleDecisionRound( + representativeQueries, + representativeSamples, + representativeFanoutSamples, + ); } - const regressionFactor = Number(process.env.ENDPOINT_REGRESSION_FACTOR ?? 3); + const regressionFactor = Number(process.env.ENDPOINT_REGRESSION_FACTOR ?? 2); const regressionFloorMs = Number( - process.env.ENDPOINT_REGRESSION_FLOOR_MS ?? 500, + process.env.ENDPOINT_REGRESSION_FLOOR_MS ?? 250, ); const decisionEndpointLatency = Object.fromEntries( - decisionQueries.map(({ name }) => { - const baseline = latencySummary(baselineSamples[name]); + measuredQueries.map(({ name }) => { const measured = latencySummary(measuredSamples[name]); + const representative = latencySummary(representativeSamples[name]); + const large = latencySummary(largeSamples[name]); + const largeBudget = { + absoluteP95Ms: endpointP95BudgetMs, + maximumRepresentativeFactor: 2, + passed: + large.p95Ms <= endpointP95BudgetMs && + large.p95Ms <= Math.max(representative.p95Ms * 2, 250), + }; + if (!baselineSamples[name]) { + return [ + name, + { + baseline: null, + measured, + representative, + large, + budget: { + mode: "new_endpoint_no_baseline", + absoluteP95Ms: endpointP95BudgetMs, + passed: measured.p95Ms <= endpointP95BudgetMs, + }, + representativeBudget: { + mode: "new_endpoint_no_baseline", + absoluteP95Ms: endpointP95BudgetMs, + passed: representative.p95Ms <= endpointP95BudgetMs, + }, + largeBudget, + }, + ]; + } + const baseline = latencySummary(baselineSamples[name]); return [ name, { baseline, measured, + representative, + large, budget: evaluateLatencyBudget({ baseline, measured, @@ -1447,6 +2547,14 @@ const verify = async () => { regressionFactor, regressionFloorMs, }), + representativeBudget: evaluateLatencyBudget({ + baseline, + measured: representative, + absoluteP95Ms: endpointP95BudgetMs, + regressionFactor, + regressionFloorMs, + }), + largeBudget, }, ]; }), @@ -1456,6 +2564,8 @@ const verify = async () => { ); const dashboardBaseline = latencySummary(baselineFanoutSamples); const dashboardMeasured = latencySummary(measuredFanoutSamples); + const dashboardRepresentative = latencySummary(representativeFanoutSamples); + const dashboardLarge = latencySummary(largeFanoutSamples); const dashboardBudget = evaluateLatencyBudget({ baseline: dashboardBaseline, measured: dashboardMeasured, @@ -1463,25 +2573,72 @@ const verify = async () => { regressionFactor, regressionFloorMs, }); + const dashboardRepresentativeBudget = evaluateLatencyBudget({ + baseline: dashboardBaseline, + measured: dashboardRepresentative, + absoluteP95Ms: fanoutP95BudgetMs, + regressionFactor, + regressionFloorMs, + }); + const dashboardLargeBudget = { + absoluteP95Ms: fanoutP95BudgetMs, + maximumRepresentativeFactor: 2, + passed: + dashboardLarge.p95Ms <= fanoutP95BudgetMs && + dashboardLarge.p95Ms <= Math.max(dashboardRepresentative.p95Ms * 2, 500), + }; const failedDecisionEndpoints = Object.entries(decisionEndpointLatency) - .filter(([, value]) => !value.budget.passed) + .filter( + ([, value]) => + !value.budget.passed || + !value.representativeBudget.passed || + !value.largeBudget.passed, + ) .map(([name]) => name); artifact.health = health; artifact.decisionAssertions = decisionAssertions; artifact.load.health = loadHealth; - artifact.visibilityMs = Date.now() - Date.parse(state.startedAt); + artifact.load.decisionAssertions = loadDecisionAssertions; + artifact.largeLoad.health = largeLoadHealth; + artifact.largeLoad.decisionAssertions = largeLoadDecisionAssertions; + const currentVisibilityMs = Date.now() - Date.parse(state.startedAt); + artifact.visibilityMs = Math.min( + artifact.visibilityMs ?? currentVisibilityMs, + currentVisibilityMs, + ); artifact.endpointLatency = endpointLatency; artifact.decisionEndpointLatency = decisionEndpointLatency; + artifact.performanceBaseline = { + deploymentId: baselineDeploymentId, + mode: + baselineDeploymentId === state.deploymentId + ? "same_deployment_noop" + : "retained_deployment", + representativeRows: state.loadEventCount, + largeRows: state.largeLoadEventCount, + newEndpointsWithoutBaseline: retainedIdentityFunnelAvailable + ? [] + : ["product_identity_funnel"], + representativeCoverageLatencyMs: representativeCoverage.latencyMs, + largeCoverageLatencyMs: largeCoverage.latencyMs, + }; artifact.dashboardFanoutLatency = { baseline: dashboardBaseline, measured: dashboardMeasured, + representative: dashboardRepresentative, + large: dashboardLarge, budget: dashboardBudget, + representativeBudget: dashboardRepresentativeBudget, + largeBudget: dashboardLargeBudget, }; const ingestionSloMs = Number(process.env.INGESTION_SLO_MS ?? 180_000); const ingestionSloPassed = artifact.visibilityMs <= ingestionSloMs; const endpointBudgetPassed = endpointLatency.p95Ms <= endpointP95BudgetMs; const decisionEndpointBudgetsPassed = - failedDecisionEndpoints.length === 0 && dashboardBudget.passed; + failedDecisionEndpoints.length === 0 && + dashboardBudget.passed && + dashboardRepresentativeBudget.passed && + dashboardLargeBudget.passed; artifact.budgets = { ingestionVisibilityMs: ingestionSloMs, endpointP95Ms: endpointP95BudgetMs, @@ -1500,6 +2657,7 @@ const verify = async () => { ingestionSloPassed, endpointBudgetPassed, decisionEndpointBudgetsPassed, + representativePerformancePassed: decisionEndpointBudgetsPassed, }; writeJson(artifactPath, artifact); if (!ingestionSloPassed) { @@ -1516,19 +2674,15 @@ const verify = async () => { throw new Error( `Tinybird decision endpoint budgets failed: ${[ ...failedDecisionEndpoints, - ...(dashboardBudget.passed ? [] : ["dashboard_fanout"]), + ...(dashboardBudget.passed && + dashboardRepresentative.p95Ms <= fanoutP95BudgetMs + ? [] + : ["dashboard_fanout"]), ].join(", ")}`, ); } }; -const safeSyntheticIdentifier = (value, name) => { - if (!/^synthetic_[A-Za-z0-9_-]{8,128}$/.test(value)) { - throw new Error(`${name} is not a safe synthetic identifier`); - } - return value; -}; - const deleteProductEventRows = async ({ origin, token, @@ -1581,43 +2735,44 @@ const eraseSyntheticIdentity = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); const artifact = readJson(artifactPath); - const userId = safeSyntheticIdentifier( - state.erasureUserId, - "Synthetic erasure user ID", - ); - const organizationId = safeSyntheticIdentifier( - state.erasureOrganizationId, - "Synthetic erasure organization ID", - ); - const anonymousId = safeSyntheticIdentifier( - state.erasureAnonymousId, - "Synthetic erasure anonymous ID", - ); - const { origin, tokens } = tinybirdEnvironment([ - "TINYBIRD_STAGING_CLEANUP_TOKEN", - "TINYBIRD_STAGING_DEPLOY_TOKEN", - ]); - const assertLiveOwnership = async () => { - if ( - (await ownedMutationTarget({ - state, - origin, - token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, - })) !== "live" - ) { - throw new Error("The owned Tinybird deployment is no longer live"); - } - }; - const rowsAffected = await deleteProductEventRows({ - origin, - token: tokens.TINYBIRD_STAGING_CLEANUP_TOKEN, - condition: `organization_id = '${organizationId}' OR user_id = '${userId}' OR (anonymous_id = '${anonymousId}' AND (user_id = '' OR user_id = '${userId}'))`, - beforeAttempt: assertLiveOwnership, - }); + const secret = environment("CAP_ANALYTICS_STAGING_TEST_SECRET"); + const url = new URL("/api/analytics/staging-test/erase", artifact.vercel.url); + const body = JSON.stringify({ runId: state.runId, sha: artifact.sha }); + const send = (authorization) => + previewRequest(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(authorization ? { Authorization: authorization } : {}), + }, + body, + signal: AbortSignal.timeout(300_000), + }); + const unauthorized = await send(); + if (unauthorized.status !== 401) { + throw new Error( + `The deployed erasure path accepted missing authorization with HTTP ${unauthorized.status}`, + ); + } + const startedAt = performance.now(); + const response = await send(`Bearer ${secret}`); + if (!response.ok) { + throw new Error( + `The deployed erasure path failed with HTTP ${response.status}`, + ); + } + const result = await response.json(); + if (result.erased !== true) { + throw new Error("The deployed erasure path returned incomplete proof"); + } + state.erasureApplicationPath = true; + writeJson(option("state"), state, 0o600); artifact.erasure = { ...artifact.erasure, + applicationPath: true, + unauthorizedRejected: true, + durationMs: Math.round(performance.now() - startedAt), deleteJobCompleted: true, - rowsAffected, }; writeJson(artifactPath, artifact); }; @@ -1626,6 +2781,9 @@ const verifySyntheticIdentityErasure = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); const artifact = readJson(artifactPath); + if (state.erasureApplicationPath !== true) { + throw new Error("The deployed application erasure path did not complete"); + } const erasedHealth = normalizeHealth( (await healthQuery({ state, deploymentId: state.deploymentId })).data, ); @@ -1638,6 +2796,15 @@ const verifySyntheticIdentityErasure = async () => { }) ).data, ); + const erasedLargeLoadHealth = normalizeHealth( + ( + await healthQuery({ + state, + deploymentId: state.deploymentId, + appVersion: state.largeLoadAppVersion, + }) + ).data, + ); const erasedDecisions = normalizeCiAssertions( ( await ciAssertionsQuery({ @@ -1646,6 +2813,15 @@ const verifySyntheticIdentityErasure = async () => { }) ).data, ); + const erasedBusinessDecisions = normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId: state.decisionRunId, + }) + ).data, + ); const previewHealth = normalizeHealth( ( await healthQuery({ @@ -1664,24 +2840,108 @@ const verifySyntheticIdentityErasure = async () => { }) ).data, ); + const serverDecisions = normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId: state.serverRunId, + }) + ).data, + ); if ( Object.values(erasedHealth).some((value) => value !== 0) || - Object.values(erasedLoadHealth).some((value) => value !== 0) || Object.values(erasedDecisions).some((value) => value !== 0) ) { throw new Error( "Synthetic identity erasure left raw-health or decision-facing state", ); } + assertSyntheticLoadHealth(erasedLoadHealth, state.loadEventCount); + assertSyntheticLoadHealth(erasedLargeLoadHealth, state.largeLoadEventCount); + for (const [syntheticRunId, expectedEvents] of [ + [state.loadRunId, state.loadEventCount], + [state.largeLoadRunId, state.largeLoadEventCount], + ]) { + const decisions = normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId, + }) + ).data, + ); + assertSyntheticLoadDecisions(decisions, expectedEvents); + } + const expectedRemainingBusiness = { + receivedRows: 2, + uniqueEvents: 2, + uniquePayloads: 2, + duplicateRows: 0, + payloadConflicts: 0, + canonicalEvents: 2, + decisionEvents: 2, + decisionRevenueMinor: 0, + trafficVisitors: 1, + trafficVisits: 1, + trafficPageviews: 1, + trafficBounces: 1, + trafficDurationMs: 0, + pageVisitors: 1, + pageVisits: 1, + pageviews: 1, + pageLandings: 1, + pageExits: 1, + pageEngagedMs: 0, + pageScrollDepth: 0, + activationSignups: 0, + activatedCreators: 0, + retentionCreators: 0, + retentionOrganizations: 0, + identityLinkedVisitors: 0, + identityLinkedUsers: 0, + identitySignupUsers: 0, + identityOrganizations: 0, + identityGuestCheckoutVisitors: 1, + identityGuestPurchasers: 0, + identityAuthenticatedCheckoutUsers: 0, + identityWebCheckoutUsers: 0, + identityDesktopCheckoutUsers: 0, + identityMobileCheckoutUsers: 0, + identityCrossDeviceCheckoutUsers: 0, + identityTrialUsers: 0, + identityPurchasers: 0, + }; + for (const [name, expected] of Object.entries(expectedRemainingBusiness)) { + if (erasedBusinessDecisions[name] !== expected) { + throw new Error( + `Scoped erasure left ${name}=${erasedBusinessDecisions[name]}, expected ${expected}`, + ); + } + } assertSingleHealth(previewHealth); if ( - previewDecisions.canonicalEvents !== 1 || - previewDecisions.decisionEvents !== 1 + previewDecisions.canonicalEvents !== state.previewExpectedEvents || + previewDecisions.decisionEvents !== state.previewExpectedEvents ) { throw new Error( "Synthetic identity erasure corrupted the unrelated preview control", ); } + if ( + serverDecisions.receivedRows < state.serverExpectedRows || + serverDecisions.uniqueEvents !== state.serverExpectedEvents || + serverDecisions.canonicalEvents !== state.serverExpectedEvents || + serverDecisions.decisionEvents !== state.serverExpectedEvents || + serverDecisions.decisionRevenueMinor !== 2_500 || + serverDecisions.activationSignups !== 1 || + serverDecisions.activatedCreators !== 1 + ) { + throw new Error( + "Synthetic identity erasure corrupted the durable server control", + ); + } const controlHealth = normalizeHealth( ( await healthQuery({ @@ -1705,9 +2965,12 @@ const verifySyntheticIdentityErasure = async () => { ...artifact.erasure, erasedHealth, erasedLoadHealth, + erasedLargeLoadHealth, erasedDecisions, + erasedBusinessDecisions, previewHealth, previewDecisions, + serverDecisions, controlHealth, passed: true, }; @@ -1740,11 +3003,21 @@ const cleanup = async () => { } validateSyntheticRunId(state.runId); validateSyntheticRunId(state.loadRunId); + validateSyntheticRunId(state.largeLoadRunId); + validateSyntheticRunId(state.decisionRunId); validateSyntheticRunId(state.erasureControlRunId); - const runIds = [state.runId, state.loadRunId, state.erasureControlRunId]; + if (state.serverRunId) validateSyntheticRunId(state.serverRunId); + const runIds = [ + state.runId, + state.loadRunId, + state.largeLoadRunId, + state.decisionRunId, + state.erasureControlRunId, + ]; if (state.previewRunId) { runIds.push(validateSyntheticRunId(state.previewRunId)); } + if (state.serverRunId) runIds.push(state.serverRunId); if (target === "staging") { writeOutput("target", target); writeOutput("requires_copies", "false"); @@ -1824,6 +3097,9 @@ const verifyPromoted = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); const artifact = readJson(artifactPath); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_READ_TOKEN", + ]); if ( !state.previewAppVersion || !state.previewAcceptedRows || @@ -1864,25 +3140,189 @@ const verifyPromoted = async () => { ); if ( previewDecisionAssertions.receivedRows < state.previewAcceptedRows || - previewDecisionAssertions.uniqueEvents !== 1 || - previewDecisionAssertions.uniquePayloads !== 1 || + previewDecisionAssertions.uniqueEvents !== state.previewExpectedEvents || + previewDecisionAssertions.uniquePayloads !== state.previewExpectedEvents || previewDecisionAssertions.duplicateRows !== - previewDecisionAssertions.receivedRows - 1 || + previewDecisionAssertions.receivedRows - state.previewExpectedEvents || previewDecisionAssertions.payloadConflicts !== 0 || - previewDecisionAssertions.canonicalEvents !== 1 || - previewDecisionAssertions.decisionEvents !== 1 + previewDecisionAssertions.canonicalEvents !== state.previewExpectedEvents || + previewDecisionAssertions.decisionEvents !== state.previewExpectedEvents ) { throw new Error( "The promoted preview run did not preserve exact retry-deduplicated decisions", ); } + if (!state.serverRunId || state.serverExpectedEvents !== 4) { + throw new Error("The exact-SHA durable server probe did not complete"); + } + const serverDecisionAssertions = normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId: state.serverRunId, + }) + ).data, + ); + if ( + serverDecisionAssertions.receivedRows < state.serverExpectedRows || + serverDecisionAssertions.uniqueEvents !== state.serverExpectedEvents || + serverDecisionAssertions.uniquePayloads !== state.serverExpectedEvents || + serverDecisionAssertions.duplicateRows !== + serverDecisionAssertions.receivedRows - state.serverExpectedEvents || + serverDecisionAssertions.payloadConflicts !== 0 || + serverDecisionAssertions.canonicalEvents !== state.serverExpectedEvents || + serverDecisionAssertions.decisionEvents !== state.serverExpectedEvents || + serverDecisionAssertions.decisionRevenueMinor !== 2_500 || + serverDecisionAssertions.activationSignups !== 1 || + serverDecisionAssertions.activatedCreators !== 1 + ) { + throw new Error( + "The exact-SHA durable server path did not produce deduplicated business decisions", + ); + } + const businessDecisionResult = await ciAssertionsQuery({ + state, + deploymentId: state.deploymentId, + syntheticRunId: state.decisionRunId, + }); + const businessDecisionAssertions = normalizeCiAssertions( + businessDecisionResult.data, + ); + assertSyntheticLoadHealth( + businessDecisionAssertions, + state.decisionEventCount, + ); + if ( + businessDecisionAssertions.canonicalEvents !== state.decisionEventCount || + businessDecisionAssertions.decisionEvents !== state.decisionEventCount + ) { + throw new Error( + "The promoted decision fixture was not exactly deduplicated before materialization", + ); + } + assertSyntheticBusinessDecisions(businessDecisionAssertions); + const businessEndpointSuite = await queryDecisionEndpointSuite({ + deploymentId: state.deploymentId, + origin, + state, + syntheticRunId: state.decisionRunId, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertSyntheticEndpointDecisions({ + appVersion: state.decisionAppVersion, + date: state.decisionDate, + hostname: state.decisionHostname, + pathname: state.decisionPathname, + payloads: businessEndpointSuite.payloads, + }); + const monetizationFilterSuite = await querySyntheticMonetizationFilters({ + deploymentId: state.deploymentId, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + const identityFilterSuite = await querySyntheticIdentityFilters({ + deploymentId: state.deploymentId, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + const syntheticEndpointParameters = { + start_date: state.startTime.slice(0, 10), + end_date: state.endTime.slice(0, 10), + hostname: state.decisionHostname, + __tb__deployment: state.deploymentId, + }; + const [trafficExclusion, pageExclusion] = await Promise.all([ + decisionEndpointQuery({ + origin, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + name: "product_traffic_overview", + parameters: syntheticEndpointParameters, + }), + decisionEndpointQuery({ + origin, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + name: "product_traffic_pages", + parameters: syntheticEndpointParameters, + }), + ]); + const trafficRows = trafficExclusion.data?.data; + const pageRows = pageExclusion.data?.data; + if ( + !Array.isArray(trafficRows) || + !Array.isArray(pageRows) || + trafficRows.length !== 0 || + pageRows.length !== 0 + ) { + throw new Error( + "Synthetic decision rows leaked into normal traffic endpoints", + ); + } artifact.previewApi.health = previewHealth; artifact.previewApi.decisionAssertions = { seed: seedDecisionAssertions, preview: previewDecisionAssertions, }; artifact.previewApi.endpointLatencyMs = previewHealthResult.latencyMs; + artifact.businessDecisions = { + canonicalEvents: businessDecisionAssertions.canonicalEvents, + decisionEvents: businessDecisionAssertions.decisionEvents, + traffic: { + visitors: businessDecisionAssertions.trafficVisitors, + visits: businessDecisionAssertions.trafficVisits, + pageviews: businessDecisionAssertions.trafficPageviews, + bounces: businessDecisionAssertions.trafficBounces, + durationMs: businessDecisionAssertions.trafficDurationMs, + }, + pages: { + visitors: businessDecisionAssertions.pageVisitors, + visits: businessDecisionAssertions.pageVisits, + pageviews: businessDecisionAssertions.pageviews, + landings: businessDecisionAssertions.pageLandings, + exits: businessDecisionAssertions.pageExits, + engagedMs: businessDecisionAssertions.pageEngagedMs, + scrollDepth: businessDecisionAssertions.pageScrollDepth, + }, + activation: { + signups: businessDecisionAssertions.activationSignups, + activatedCreators: businessDecisionAssertions.activatedCreators, + }, + retention: { + creators: businessDecisionAssertions.retentionCreators, + organizations: businessDecisionAssertions.retentionOrganizations, + }, + identity: { + linkedVisitors: businessDecisionAssertions.identityLinkedVisitors, + linkedUsers: businessDecisionAssertions.identityLinkedUsers, + organizations: businessDecisionAssertions.identityOrganizations, + guestCheckoutVisitors: + businessDecisionAssertions.identityGuestCheckoutVisitors, + guestPurchasers: businessDecisionAssertions.identityGuestPurchasers, + purchasers: businessDecisionAssertions.identityPurchasers, + }, + revenueMinor: businessDecisionAssertions.decisionRevenueMinor, + identityFilterLatencyMs: identityFilterSuite.latencyMs, + monetizationFilterLatencyMs: monetizationFilterSuite.latencyMs, + normalTrafficExcluded: true, + endpointLatencyMs: businessEndpointSuite.latencyMs, + exclusionLatencyMs: { + overview: trafficExclusion.latencyMs, + pages: pageExclusion.latencyMs, + }, + }; + artifact.serverDelivery.decisionAssertions = { + canonicalEvents: serverDecisionAssertions.canonicalEvents, + decisionEvents: serverDecisionAssertions.decisionEvents, + duplicateRows: serverDecisionAssertions.duplicateRows, + revenueMinor: serverDecisionAssertions.decisionRevenueMinor, + activationSignups: serverDecisionAssertions.activationSignups, + activatedCreators: serverDecisionAssertions.activatedCreators, + }; artifact.assertions.promotedPreviewDataPassed = true; + artifact.assertions.promotedBusinessDecisionsPassed = true; + artifact.assertions.syntheticDecisionExclusionPassed = true; writeJson(artifactPath, artifact); }; @@ -1914,6 +3354,19 @@ const verifyCleanup = async () => { "Synthetic rows still affect Tinybird decision assertions after cleanup", ); } + const businessDecisionResult = await ciAssertionsQuery({ + state, + deploymentId, + syntheticRunId: state.decisionRunId, + }); + const businessDecisionAssertions = normalizeCiAssertions( + businessDecisionResult.data, + ); + if (Object.values(businessDecisionAssertions).some((value) => value !== 0)) { + throw new Error( + "Synthetic business rows still affect decisions after cleanup", + ); + } const loadResult = await healthQuery({ state, deploymentId, @@ -1925,6 +3378,27 @@ const verifyCleanup = async () => { "Synthetic load rows still affect Tinybird health after cleanup", ); } + for (const syntheticRunId of [state.loadRunId, state.largeLoadRunId]) { + const decisions = normalizeCiAssertions( + (await ciAssertionsQuery({ state, deploymentId, syntheticRunId })).data, + ); + if (Object.values(decisions).some((value) => value !== 0)) { + throw new Error( + "Synthetic load rows still affect Tinybird decisions after cleanup", + ); + } + } + const largeLoadResult = await healthQuery({ + state, + deploymentId, + appVersion: state.largeLoadAppVersion, + }); + const largeLoadHealth = normalizeHealth(largeLoadResult.data); + if (Object.values(largeLoadHealth).some((value) => value !== 0)) { + throw new Error( + "Large synthetic load rows still affect Tinybird health after cleanup", + ); + } const controlResult = await healthQuery({ state, deploymentId, @@ -1962,8 +3436,25 @@ const verifyCleanup = async () => { ); } } + if (state.serverRunId) { + const serverDecisionAssertions = normalizeCiAssertions( + ( + await ciAssertionsQuery({ + state, + deploymentId, + syntheticRunId: state.serverRunId, + }) + ).data, + ); + if (Object.values(serverDecisionAssertions).some((value) => value !== 0)) { + throw new Error( + "Synthetic server rows still affect decisions after cleanup", + ); + } + } artifact.cleanup = { ...artifact.cleanup, + businessDecisionAssertions, passed: true, verifiedAt: new Date().toISOString(), }; @@ -1997,6 +3488,9 @@ const verifyTokenScopes = async () => { const artifact = readJson(artifactPath); const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_INGEST_TOKEN", + "TINYBIRD_STAGING_COPY_TOKEN", + "TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN", + "TINYBIRD_STAGING_SCHEDULER_TOKEN", "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_CLEANUP_TOKEN", ]); @@ -2007,6 +3501,12 @@ const verifyTokenScopes = async () => { }), { token: tokens.TINYBIRD_STAGING_READ_TOKEN }, ); + await request( + tinybirdUrl(origin, "/v0/sql", { + q: "SELECT countIf(user_id != '') AS rows FROM product_events_v1 UNION ALL SELECT countIf(user_id != '') AS rows FROM product_events_canonical_v1", + }), + { token: tokens.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN }, + ); await assertScopeDenied( "The aggregate read token raw identity query", tokenScopeProbe( @@ -2031,6 +3531,45 @@ const verifyTokenScopes = async () => { }, ), ); + await assertScopeDenied( + "The aggregate read token Copy mutation probe", + tokenScopeProbe( + tinybirdUrl( + origin, + "/v0/pipes/snapshot_product_events_canonical_v1/copy", + { _mode: "replace" }, + ), + tokens.TINYBIRD_STAGING_READ_TOKEN, + { method: "POST" }, + ), + ); + await assertScopeDenied( + "The erasure lookup token append probe", + tokenScopeProbe( + tinybirdUrl(origin, "/v0/events", { + name: "product_events_v1", + wait: "true", + }), + tokens.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN, + { + method: "POST", + body: "\n", + headers: { "Content-Type": "application/x-ndjson" }, + }, + ), + ); + await assertScopeDenied( + "The erasure lookup token Copy mutation probe", + tokenScopeProbe( + tinybirdUrl( + origin, + "/v0/pipes/snapshot_product_events_canonical_v1/copy", + { _mode: "replace" }, + ), + tokens.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN, + { method: "POST" }, + ), + ); const ingestProbe = await tokenScopeProbe( tinybirdUrl(origin, "/v0/events", { name: "product_events_v1", @@ -2051,6 +3590,9 @@ const verifyTokenScopes = async () => { for (const [name, token] of [ ["append-only token", tokens.TINYBIRD_STAGING_INGEST_TOKEN], ["cleanup token", tokens.TINYBIRD_STAGING_CLEANUP_TOKEN], + ["copy-runner token", tokens.TINYBIRD_STAGING_COPY_TOKEN], + ["erasure lookup token", tokens.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN], + ["schedule-controller token", tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN], ]) { await assertScopeDenied( `The ${name} aggregate read probe`, @@ -2067,9 +3609,15 @@ const verifyTokenScopes = async () => { aggregateReadPassed: true, rawIdentityReadDenied: true, readTokenAppendDenied: true, + readTokenCopyMutationDenied: true, ingestTokenAppendAuthorized: true, ingestTokenAggregateReadDenied: true, cleanupTokenAggregateReadDenied: true, + copyTokenAggregateReadDenied: true, + erasureLookupRawReadPassed: true, + erasureLookupAppendDenied: true, + erasureLookupCopyMutationDenied: true, + erasureLookupAggregateReadDenied: true, }; artifact.assertions = { ...artifact.assertions, @@ -2091,6 +3639,9 @@ const handlers = { "verify-credentials": async () => tinybirdEnvironment(), "verify-token-scopes": verifyTokenScopes, "promote-deployment": promoteOwnedDeployment, + "drill-rollback": drillOwnedRollback, + "finalize-promotion": finalizeOwnedPromotion, + "rollback-promotion": rollbackOwnedPromotion, "discard-deployment": discardOwnedDeployment, "select-deployment": async () => { const createOutput = readJson(option("create-output")); @@ -2132,8 +3683,10 @@ const handlers = { "wait-vercel": waitForVercel, seed, "run-copies": runCopies, + "set-copy-schedules": setCopySchedules, verify, "probe-preview": probePreview, + "probe-server": probeDurableServerPath, "verify-promoted": verifyPromoted, "erase-synthetic-identity": eraseSyntheticIdentity, "verify-synthetic-identity-erasure": verifySyntheticIdentityErasure, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 9d2867f0b02..4c0c9ce92ab 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -3,10 +3,20 @@ import fs from "node:fs"; import test from "node:test"; import { + applyCopyScheduleAction, assertExecutionScope, + assertRepresentativeEndpointCoverage, + assertSyntheticBusinessDecisions, assertSyntheticDecisions, + assertSyntheticEndpointDecisions, assertSyntheticHealth, + assertSyntheticIdentityFilters, + assertSyntheticLoadDecisions, + assertSyntheticLoadHealth, + assertSyntheticMonetizationFilters, assertWorkflowSafety, + copyScheduleMatchesAction, + createSyntheticDecisionEvents, createSyntheticErasureControl, createSyntheticEvents, createSyntheticLoadEvents, @@ -30,6 +40,8 @@ import { STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, + syntheticIdentityFilterQueries, + syntheticMonetizationFilterQueries, tokenWorkspaceId, validateSyntheticRunId, validateTinybirdCredentials, @@ -39,6 +51,66 @@ const SHA = "1234567890abcdef1234567890abcdef12345678"; const token = (workspaceId = STAGING_WORKSPACE_ID) => `p.${Buffer.from(JSON.stringify({ u: workspaceId })).toString("base64url")}.signature`; +test("schedule pause compensates every schedule already paused", async () => { + const calls = []; + await assert.rejects( + applyCopyScheduleAction({ + pipes: ["one", "two", "three"], + action: "pause", + setSchedule: async (pipe, action) => { + calls.push([pipe, action]); + if (pipe === "two" && action === "pause") { + throw new Error("provider failure"); + } + }, + }), + /Failed to pause two: provider failure/, + ); + assert.deepEqual(calls, [ + ["one", "pause"], + ["two", "pause"], + ["one", "resume"], + ]); +}); + +test("schedule resume attempts every schedule before failing", async () => { + const calls = []; + await assert.rejects( + applyCopyScheduleAction({ + pipes: ["one", "two", "three"], + action: "resume", + setSchedule: async (pipe, action) => { + calls.push([pipe, action]); + if (pipe !== "two") throw new Error(`${pipe} failed`); + }, + }), + /one: one failed, three: three failed/, + ); + assert.deepEqual(calls, [ + ["one", "resume"], + ["two", "resume"], + ["three", "resume"], + ]); +}); + +test("schedule state attestation distinguishes paused from active copies", () => { + assert.equal( + copyScheduleMatchesAction( + { data: { schedule: { status: "paused" } } }, + "pause", + ), + true, + ); + assert.equal( + copyScheduleMatchesAction({ schedule: { status: "scheduled" } }, "resume"), + true, + ); + assert.equal( + copyScheduleMatchesAction({ schedule: { status: "paused" } }, "resume"), + false, + ); +}); + test("execution scope accepts only PR 2003 or manual feature branch runs", () => { assert.doesNotThrow(() => assertExecutionScope({ @@ -455,7 +527,6 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker return responses.shift(); }, now: () => now, - useDeploymentParameter: true, copyRunId: "run_12345678_staged", assertMutationOwnership: async () => { ownershipChecks += 1; @@ -475,7 +546,7 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker }, ]); assert.match(requests[0].url, /_mode=replace/); - assert.match(requests[0].url, /__tb__deployment=staging/); + assert.doesNotMatch(requests[0].url, /__tb__deployment/); assert.doesNotMatch(requests[0].url, /copy_run_id/); assert.equal(requests[0].options.method, "POST"); assert.equal(requests[0].options.token, resourceToken); @@ -513,6 +584,19 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker assertMutationOwnership: async () => undefined, }), ); + await assert.rejects( + submitTinybirdCopyJobs({ + origin: "https://api.us-east.aws.tinybird.co", + token: resourceToken, + deploymentId: "6", + pipes: ["snapshot_product_events_canonical_v1"], + request: async () => { + throw new Error("Tinybird request was rejected with HTTP 403"); + }, + assertMutationOwnership: async () => undefined, + }), + /Tinybird copy submission failed.*HTTP 403/, + ); }); test("synthetic fixtures are deterministic, isolated, and model duplicates and conflicts", () => { @@ -550,6 +634,71 @@ test("synthetic fixtures are deterministic, isolated, and model duplicates and c assert.notEqual(control.row.user_id, fixture.userId); assert.notEqual(control.row.organization_id, fixture.organizationId); assert.equal(control.row.synthetic_run_id, control.runId); + const decisions = createSyntheticDecisionEvents({ + runId, + now: new Date("2026-07-31T10:00:00.000Z"), + }); + assert.equal(decisions.rows.length, 27); + assert.equal(decisions.runId, `${runId}_decisions`); + assert.equal(new Set(decisions.rows.map((row) => row.event_id)).size, 27); + assert.deepEqual( + decisions.rows.map((row) => row.event_name), + [ + "page_view", + "page_engagement", + "identity_linked", + "user_signed_up", + "share_link_created", + "recording_completed", + "page_view", + "guest_checkout_started", + "checkout_started", + "checkout_started", + "checkout_started", + "trial_started", + "purchase_completed", + "subscription_renewed", + "guest_checkout_started", + "trial_converted", + "subscription_changed", + "subscription_changed", + "subscription_cancelled", + "subscription_refunded", + "subscription_payment_failed", + "page_view", + "identity_linked", + "purchase_completed", + "subscription_renewed", + "experiment_exposed", + "analytics_delivery_loss", + ], + ); + assert.equal(decisions.rows[0].user_id, ""); + assert.equal(decisions.rows[1].user_id, ""); + assert.equal(decisions.rows[7].user_id, ""); + assert.equal(decisions.rows[2].anonymous_id, decisions.rows[0].anonymous_id); + assert.notEqual( + decisions.rows[2].anonymous_id, + decisions.rows[23].anonymous_id, + ); + assert.equal(decisions.rows[23].anonymous_id, decisions.rows[7].anonymous_id); + assert.equal( + JSON.parse(decisions.rows[23].properties).is_guest_checkout, + true, + ); + assert.ok( + decisions.rows.every( + (row) => + row.synthetic_run_id === decisions.runId && + row.hostname === decisions.hostname && + row.pathname === decisions.pathname, + ), + ); + assert.match( + decisions.hostname, + /^synthetic-[0-9a-f]{12}\.preview\.cap\.so$/, + ); + assert.match(decisions.pathname, /^\/analytics-synthetic-[0-9a-f]{12}$/); assert.throws(() => createSyntheticLoadEvents({ runId, count: 99 })); }); @@ -580,19 +729,620 @@ test("health normalization and latency percentiles use decision-facing assertion }); }); +test("candidate load assertions require complete unique delivery", () => { + assert.doesNotThrow(() => + assertSyntheticLoadHealth( + { + receivedRows: 1_000, + uniqueEvents: 1_000, + uniquePayloads: 1_000, + duplicateRows: 0, + payloadConflicts: 0, + }, + 1_000, + ), + ); + for (const health of [ + { + receivedRows: 999, + uniqueEvents: 999, + uniquePayloads: 999, + duplicateRows: 0, + payloadConflicts: 0, + }, + { + receivedRows: 1_001, + uniqueEvents: 1_000, + uniquePayloads: 1_000, + duplicateRows: 0, + payloadConflicts: 0, + }, + { + receivedRows: 1_000, + uniqueEvents: 1_000, + uniquePayloads: 1_000, + duplicateRows: 0, + payloadConflicts: 1, + }, + ]) { + assert.throws(() => assertSyntheticLoadHealth(health, 1_000)); + } +}); + +test("business decision assertions require exact materialized metrics", () => { + const assertions = { + trafficVisitors: 3, + trafficVisits: 3, + trafficPageviews: 3, + trafficBounces: 2, + trafficDurationMs: 15_000, + pageVisitors: 3, + pageVisits: 3, + pageviews: 3, + pageLandings: 3, + pageExits: 3, + pageEngagedMs: 15_000, + pageScrollDepth: 75, + activationSignups: 1, + activatedCreators: 1, + retentionCreators: 1, + retentionOrganizations: 1, + identityLinkedVisitors: 2, + identityLinkedUsers: 2, + identitySignupUsers: 1, + identityOrganizations: 1, + identityGuestCheckoutVisitors: 2, + identityGuestPurchasers: 1, + identityAuthenticatedCheckoutUsers: 1, + identityWebCheckoutUsers: 1, + identityDesktopCheckoutUsers: 1, + identityMobileCheckoutUsers: 1, + identityCrossDeviceCheckoutUsers: 1, + identityTrialUsers: 1, + identityPurchasers: 1, + decisionRevenueMinor: 7_000, + }; + assert.doesNotThrow(() => assertSyntheticBusinessDecisions(assertions)); + assert.throws(() => + assertSyntheticBusinessDecisions({ + ...assertions, + activatedCreators: 0, + }), + ); +}); + +test("load decision assertions require a high-cardinality materialization", () => { + const expectedEvents = 1_000; + const assertions = { + receivedRows: expectedEvents, + uniqueEvents: expectedEvents, + uniquePayloads: expectedEvents, + duplicateRows: 0, + payloadConflicts: 0, + canonicalEvents: expectedEvents, + decisionEvents: expectedEvents, + decisionRevenueMinor: 200_000, + trafficVisitors: 100, + trafficVisits: 100, + trafficPageviews: 100, + trafficBounces: 0, + trafficDurationMs: 500_000, + pageVisitors: 100, + pageVisits: 100, + pageviews: 100, + pageLandings: 100, + pageExits: 100, + pageEngagedMs: 500_000, + pageScrollDepth: 6_000, + activationSignups: 100, + activatedCreators: 100, + retentionCreators: 100, + retentionOrganizations: 100, + identityLinkedVisitors: 100, + identityLinkedUsers: 100, + identitySignupUsers: 100, + identityOrganizations: 100, + identityGuestCheckoutVisitors: 0, + identityGuestPurchasers: 0, + identityAuthenticatedCheckoutUsers: 100, + identityWebCheckoutUsers: 34, + identityDesktopCheckoutUsers: 33, + identityMobileCheckoutUsers: 33, + identityCrossDeviceCheckoutUsers: 0, + identityTrialUsers: 100, + identityPurchasers: 100, + }; + assert.doesNotThrow(() => + assertSyntheticLoadDecisions(assertions, expectedEvents), + ); + assert.throws(() => + assertSyntheticLoadDecisions( + { ...assertions, trafficVisits: expectedEvents - 1 }, + expectedEvents, + ), + ); +}); + +test("representative endpoint coverage requires mixed funnel and revenue data", () => { + const row = (data) => ({ data }); + const cohorts = 10; + const repeated = (count, value) => + Array.from({ length: count }, () => ({ ...value })); + const payloads = { + product_traffic_overview: row([{ pageviews: cohorts }]), + product_traffic_pages: row(repeated(cohorts, { pageviews: 1 })), + product_traffic_sources: row(repeated(cohorts, { pageviews: 1 })), + product_traffic_countries: row([{}]), + product_traffic_technology: row([{}]), + product_activation: row([{ signups: cohorts }]), + product_creator_activity: row([{ dau: cohorts }]), + product_creator_retention: row([{ creators: cohorts }]), + product_identity_funnel: row([ + { linked_users: cohorts, purchasers: cohorts }, + ]), + product_events_daily: row([ + ...repeated(99, { events: 1, revenue_minor: 0 }), + { events: 1, revenue_minor: 20_000 }, + ]), + product_feature_adoption: row(repeated(10, { events: 10 })), + product_analytics_freshness: row([{}]), + }; + assert.doesNotThrow(() => + assertRepresentativeEndpointCoverage({ + expectedEvents: 100, + payloads, + }), + ); + assert.throws(() => + assertRepresentativeEndpointCoverage({ + expectedEvents: 100, + payloads: { ...payloads, product_identity_funnel: row([]) }, + }), + ); +}); + +test("synthetic monetization filters prove lifecycle values and legacy coverage", () => { + const queries = syntheticMonetizationFilterQueries({ + date: "2026-07-31", + deploymentId: "7", + syntheticRunId: "run_12345678_decisions", + }); + const payloads = Object.fromEntries( + queries.map((query) => [ + query.label, + { + data: + query.expectedRows === 0 + ? [] + : [ + { + events: query.expectedEvents, + revenue_minor: query.expectedRevenueMinor, + ...query.expectedFields, + }, + ], + }, + ]), + ); + assert.doesNotThrow(() => + assertSyntheticMonetizationFilters({ payloads, queries }), + ); + assert.throws(() => + assertSyntheticMonetizationFilters({ + payloads: { + ...payloads, + renewal_revenue: { + data: [{ events: 1, revenue_minor: 2_499 }], + }, + }, + queries, + }), + ); +}); + +test("synthetic identity filters prove source attribution and empty totals", () => { + const queries = syntheticIdentityFilterQueries({ + date: "2026-07-31", + deploymentId: "7", + syntheticRunId: "run_12345678_decisions", + }); + const payloads = Object.fromEntries( + queries.map((query) => [query.label, { data: [query.expected] }]), + ); + assert.doesNotThrow(() => + assertSyntheticIdentityFilters({ payloads, queries }), + ); + assert.throws(() => + assertSyntheticIdentityFilters({ + payloads: { + ...payloads, + referral_identity: { + data: [{ ...queries[1].expected, organizations: 1 }], + }, + }, + queries, + }), + ); +}); + +test("typed endpoint assertions require exact public response semantics", () => { + const row = (value) => ({ data: [value] }); + const date = "2026-07-31"; + const appVersion = "staging-decisions-123456789abc"; + const hostname = "synthetic-123456789abc.preview.cap.so"; + const pathname = "/analytics-synthetic-123456789abc"; + const event = (eventName, source, platform, overrides = {}) => ({ + date, + event_name: eventName, + schema_version: 1, + source, + platform, + app_version: appVersion, + hostname, + channel: "direct", + plan_id: "", + payment_status: "", + subscription_status: "", + currency: "", + billing_interval: "", + change_kind: "", + previous_status: "", + new_status: "", + previous_plan_id: "", + quantity: 0, + previous_quantity: 0, + new_quantity: 0, + seat_delta: 0, + first_purchase: "", + guest_checkout: "", + onboarding: "", + cancel_at_period_end: "", + fully_refunded: "", + ended_at: 0, + trial_end_at: 0, + amount_due_minor: 0, + attempt_count: 0, + experiment_id: "", + experiment_variant: "", + assignment_version: "", + delivery_loss_count: 0, + events: 1, + actors: 1, + users: 1, + organizations: 1, + revenue_minor: 0, + ...overrides, + }); + const eventShapes = [ + event("page_view", "client", "web", { + events: 2, + actors: 2, + channel: "paid_search", + users: 0, + organizations: 0, + }), + event("page_view", "client", "web", { + channel: "referral", + users: 0, + organizations: 0, + }), + event("page_engagement", "client", "web", { + users: 0, + organizations: 0, + }), + event("identity_linked", "server", "server", { + events: 2, + actors: 2, + users: 2, + }), + event("user_signed_up", "server", "web"), + event("share_link_created", "server", "server"), + event("recording_completed", "client", "desktop"), + event("guest_checkout_started", "server", "web", { + plan_id: "price_pro_annual", + quantity: 1, + events: 2, + actors: 2, + users: 0, + organizations: 0, + }), + event("checkout_started", "server", "web", { + plan_id: "price_pro_annual", + quantity: 1, + onboarding: "false", + }), + event("checkout_started", "server", "desktop", { + plan_id: "price_pro_annual", + quantity: 1, + onboarding: "false", + }), + event("checkout_started", "server", "mobile", { + plan_id: "price_pro_annual", + quantity: 1, + onboarding: "false", + }), + event("trial_started", "server", "web", { + plan_id: "price_pro_annual", + subscription_status: "trialing", + currency: "GBP", + billing_interval: "year", + quantity: 1, + guest_checkout: "false", + onboarding: "false", + trial_end_at: 1_900_604_800, + }), + event("purchase_completed", "server", "web", { + schema_version: 3, + plan_id: "price_pro_annual", + payment_status: "paid", + subscription_status: "active", + currency: "GBP", + billing_interval: "year", + revenue_minor: 2_500, + quantity: 1, + first_purchase: "true", + guest_checkout: "false", + onboarding: "false", + }), + event("purchase_completed", "server", "web", { + schema_version: 3, + plan_id: "price_guest_monthly", + payment_status: "paid", + subscription_status: "active", + currency: "GBP", + billing_interval: "month", + revenue_minor: 1_500, + quantity: 1, + first_purchase: "true", + guest_checkout: "true", + onboarding: "false", + }), + event("subscription_renewed", "server", "server", { + schema_version: 2, + plan_id: "price_pro_annual", + currency: "GBP", + revenue_minor: 2_500, + }), + event("subscription_renewed", "server", "server", { + currency: "GBP", + revenue_minor: 1_000, + }), + event("trial_converted", "server", "server", { + schema_version: 2, + plan_id: "price_pro_annual", + subscription_status: "active", + previous_status: "trialing", + new_status: "active", + }), + event("subscription_changed", "server", "server", { + schema_version: 2, + plan_id: "price_pro_annual", + change_kind: "plan", + previous_plan_id: "price_pro_monthly", + }), + event("subscription_changed", "server", "server", { + schema_version: 2, + plan_id: "price_pro_annual", + change_kind: "seats", + previous_plan_id: "price_pro_annual", + previous_quantity: 1, + new_quantity: 3, + seat_delta: 2, + }), + event("subscription_cancelled", "server", "server", { + schema_version: 2, + plan_id: "price_pro_annual", + subscription_status: "canceled", + cancel_at_period_end: "false", + ended_at: 1_900_000_000, + }), + event("subscription_refunded", "server", "server", { + schema_version: 2, + plan_id: "price_pro_annual", + currency: "GBP", + revenue_minor: -500, + fully_refunded: "false", + }), + event("subscription_payment_failed", "server", "server", { + schema_version: 2, + plan_id: "price_pro_annual", + currency: "GBP", + amount_due_minor: 2_500, + attempt_count: 2, + }), + event("experiment_exposed", "client", "web", { + experiment_id: "synthetic-checkout-copy", + experiment_variant: "treatment", + assignment_version: "v1", + }), + event("analytics_delivery_loss", "client", "desktop", { + delivery_loss_count: 3, + }), + ]; + const adoptionShapes = [ + ["page_view", 3, 3, 0, 0], + ["page_engagement", 1, 1, 0, 0], + ["identity_linked", 2, 2, 2, 1], + ["user_signed_up", 1, 1, 1, 1], + ["share_link_created", 1, 1, 1, 1], + ["recording_completed", 1, 1, 1, 1], + ["guest_checkout_started", 2, 2, 0, 0], + ["checkout_started", 3, 1, 1, 1], + ["trial_started", 1, 1, 1, 1], + ["purchase_completed", 2, 1, 1, 1], + ["subscription_renewed", 2, 1, 1, 1], + ["trial_converted", 1, 1, 1, 1], + ["subscription_changed", 2, 1, 1, 1], + ["subscription_cancelled", 1, 1, 1, 1], + ["subscription_refunded", 1, 1, 1, 1], + ["subscription_payment_failed", 1, 1, 1, 1], + ["experiment_exposed", 1, 1, 1, 1], + ["analytics_delivery_loss", 1, 1, 1, 1], + ]; + const payloads = { + product_traffic_overview: row({ + date, + visitors: 3, + visits: 3, + pageviews: 3, + views_per_visit: 1, + bounce_rate: 66.67, + visit_duration_ms: 5_000, + engaged_ms: 15_000, + }), + product_traffic_pages: row({ + pathname, + visitors: 3, + visits: 3, + pageviews: 3, + landings: 3, + exits: 3, + time_on_page_ms: 5_000, + average_scroll_depth: 25, + }), + product_traffic_sources: { + data: [ + { + channel: "paid_search", + source: "google", + medium: "cpc", + campaign: "synthetic-campaign", + visitors: 2, + visits: 2, + pageviews: 2, + bounce_rate: 50, + }, + { + channel: "referral", + source: "synthetic-partner", + medium: "referral", + campaign: "", + visitors: 1, + visits: 1, + pageviews: 1, + bounce_rate: 100, + }, + ], + }, + product_traffic_countries: row({ + country: "US", + visitors: 3, + visits: 3, + pageviews: 3, + }), + product_traffic_technology: row({ + device: "desktop", + browser: "Chrome", + os: "macOS", + visitors: 3, + visits: 3, + pageviews: 3, + }), + product_activation: row({ + cohort_date: date, + signups: 1, + activated_creators: 1, + activation_rate: 100, + average_time_to_activation_ms: 1_000, + }), + product_creator_activity: row({ + as_of_date: date, + dau: 1, + wau: 1, + mau: 1, + daily_active_organizations: 1, + new_creators: 1, + returning_creators: 0, + dau_wau_stickiness: 100, + dau_mau_stickiness: 100, + }), + product_creator_retention: row({ + cohort_date: date, + activity_date: date, + cohort_day: 0, + platform: "all", + creators: 1, + organizations: 1, + }), + product_identity_funnel: row({ + linked_visitors: 2, + linked_users: 2, + signup_users: 1, + organizations: 1, + guest_checkout_visitors: 2, + guest_purchasers: 1, + authenticated_checkout_users: 1, + web_checkout_users: 1, + desktop_checkout_users: 1, + mobile_checkout_users: 1, + cross_device_checkout_users: 1, + trial_users: 1, + purchasers: 1, + signup_rate: 50, + purchase_rate: 50, + }), + product_events_daily: { + data: eventShapes, + }, + product_feature_adoption: { + data: adoptionShapes.map( + ([eventName, events, actorDays, userDays, organizationDays]) => ({ + event_name: eventName, + events, + actor_days: actorDays, + user_days: userDays, + organization_days: organizationDays, + }), + ), + }, + product_analytics_freshness: row({ + latest_received_hour: "2026-07-31 10:00:00", + product_calculated_at: "2026-07-31 10:01:00", + traffic_calculated_at: "2026-07-31 10:01:00", + retention_calculated_at: "2026-07-31 10:01:00", + identity_calculated_at: "2026-07-31 10:01:00", + }), + }; + const input = { appVersion, date, hostname, pathname, payloads }; + assert.doesNotThrow(() => assertSyntheticEndpointDecisions(input)); + assert.throws(() => + assertSyntheticEndpointDecisions({ + ...input, + payloads: { + ...payloads, + product_activation: row({ + ...payloads.product_activation.data[0], + activated_creators: 0, + }), + }, + }), + ); +}); + test("staging performance covers every typed decision endpoint", () => { const queries = decisionEndpointQueries({ startDate: "2026-07-01", endDate: "2026-07-31", deploymentId: "deployment-1", }); - assert.equal(queries.length, 11); + assert.equal(queries.length, 12); assert.equal(new Set(queries.map(({ name }) => name)).size, queries.length); assert.ok( queries.every( ({ parameters }) => parameters.__tb__deployment === "deployment-1", ), ); + const retainedQueries = decisionEndpointQueries({ + startDate: "2026-07-01", + endDate: "2026-07-31", + deploymentId: "deployment-0", + includeIdentityFunnel: false, + }); + assert.equal(retainedQueries.length, 11); + assert.equal( + retainedQueries.some(({ name }) => name === "product_identity_funnel"), + false, + ); assert.deepEqual( queries.find(({ name }) => name === "product_creator_activity")?.parameters, { as_of_date: "2026-07-31", __tb__deployment: "deployment-1" }, @@ -602,6 +1352,19 @@ test("staging performance covers every typed decision endpoint", () => { ?.parameters, { __tb__deployment: "deployment-1" }, ); + const syntheticQueries = decisionEndpointQueries({ + startDate: "2026-07-01", + endDate: "2026-07-31", + deploymentId: "deployment-1", + syntheticRunId: "run_12345678_load", + }); + assert.ok( + syntheticQueries.every( + ({ name, parameters }) => + name === "product_analytics_freshness" || + parameters.synthetic_run_id === "run_12345678_load", + ), + ); }); test("bundle measurement includes only unique same-origin Next.js scripts", () => { @@ -697,18 +1460,24 @@ test("copy assertion normalization exposes every marker independently", () => { normalizeCopyAssertions({ data: [ { + decision_markers: "1", traffic_markers: "1", traffic_page_markers: "1", activation_markers: "1", retention_markers: "1", + identity_markers: "1", + health_markers: "1", }, ], }), { + decisionMarkers: 1, trafficMarkers: 1, trafficPageMarkers: 1, activationMarkers: 1, retentionMarkers: 1, + identityMarkers: 1, + healthMarkers: 1, }, ); }); @@ -728,21 +1497,58 @@ test("the analytics workflow is statically restricted to staging", () => { assert.equal( workflow.match(/node scripts\/analytics\/staging-ci\.js run-copies/g) ?.length, - 4, + 3, ); assert.equal( workflow.match( /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 11, + 14, ); assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); assert.ok( - workflow.indexOf("Prove synthetic cleanup no longer affects queries") < + workflow.indexOf( + "Prove promoted delivery, business values, and decision deduplication", + ) < workflow.indexOf("Prove exact staging rollback and restoration"), + ); + assert.ok( + workflow.indexOf("Prove exact staging rollback and restoration") < + workflow.indexOf( + "Delete the scoped identity through the exact-SHA application path", + ), + ); + assert.ok( + workflow.indexOf( + "Delete the scoped identity through the exact-SHA application path", + ) < + workflow.indexOf( + "Prove identity erasure and out-of-scope control preservation", + ), + ); + assert.ok( + workflow.indexOf( + "Prove identity erasure and out-of-scope control preservation", + ) < workflow.indexOf( - "Discard an unpromoted staging deployment after cleanup", + "Quiesce scheduled and active Copy jobs before final cleanup", ), ); + assert.match( + workflow, + /Delete the scoped identity through the exact-SHA application path[\s\S]*CAP_ANALYTICS_STAGING_TEST_SECRET[\s\S]*staging-ci\.js erase-synthetic-identity/, + ); + assert.ok( + workflow.indexOf("Prove synthetic cleanup no longer affects queries") < + workflow.indexOf("Resume reviewed Copy schedules"), + ); + assert.ok( + workflow.indexOf("Resume reviewed Copy schedules") < + workflow.indexOf("Finalize the fully verified staging promotion"), + ); + assert.ok( + workflow.indexOf("Finalize the fully verified staging promotion") < + workflow.indexOf("Restore the previous staging deployment on failure"), + ); assert.match(workflow, /steps\.promote\.outcome == 'failure'/); assert.match(workflow, /continue-on-error: true/); assert.match(workflow, /staging-ci\.js resolve-deployment-state/); @@ -756,13 +1562,24 @@ test("the analytics workflow is statically restricted to staging", () => { ); assert.match( workflow, - /Discard an unpromoted staging deployment after cleanup\n {8}id: discard\n {8}if: always\(\) && \(steps\.deployment-state\.outputs\.discard == 'true' \|\| steps\.cleanup\.outputs\.requires_discard == 'true'\)/, + /Discard an unpromoted staging deployment after cleanup\n {8}id: discard\n {8}if: always\(\) && steps\.rollback\.outcome != 'success' && \(steps\.deployment-state\.outputs\.discard == 'true' \|\| steps\.cleanup\.outputs\.requires_discard == 'true'\)/, ); assert.match( workflow, /staging-ci\.js promote-deployment --deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/, ); assert.match(workflow, /staging-ci\.js discard-deployment/); + assert.match(workflow, /staging-ci\.js drill-rollback/); + assert.match( + workflow, + /Prove exact staging rollback and restoration[\s\S]*TINYBIRD_STAGING_READ_TOKEN[\s\S]*--state "\$RUNNER_TEMP\/analytics-staging-state\.json"/, + ); + assert.match(workflow, /staging-ci\.js finalize-promotion/); + assert.match(workflow, /staging-ci\.js rollback-promotion/); + assert.match( + workflow, + /steps\.promote\.outputs\.previous_live_id != '' && steps\.finalize\.outcome != 'success' && steps\.rollback-drill\.outputs\.rollback_target_usable != 'false'/, + ); assert.doesNotMatch( workflow, /tinybird-cloud-cli --cloud deployment promote/, @@ -781,7 +1598,48 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.match( /--target "\$\{\{ steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live' \}\}"/g, )?.length, - 2, + 1, + ); + assert.match( + workflow, + /Rebuild no-op live decision and health copies\n {8}id: staging-copies\n {8}if: steps\.tinybird\.outputs\.needs_promotion != 'true'[\s\S]*--target live/, + ); + assert.ok( + workflow.indexOf("Prove the exact-SHA deployed browser tracker") < + workflow.indexOf( + "Probe the exact-SHA Vercel browser collector and staging rate limit", + ), + ); + assert.ok( + workflow.indexOf( + "Probe the exact-SHA Vercel browser collector and staging rate limit", + ) < workflow.indexOf("Prove exact-SHA durable server delivery"), + ); + assert.ok( + workflow.indexOf("Prove exact-SHA durable server delivery") < + workflow.indexOf("Rebuild promoted decision and health copies"), + ); + assert.match( + workflow, + /Prove exact-SHA durable server delivery[\s\S]*CAP_ANALYTICS_STAGING_TEST_SECRET[\s\S]*staging-ci\.js probe-server/, + ); + assert.ok( + workflow.indexOf("Rebuild promoted decision and health copies") < + workflow.indexOf("Measure populated decision endpoint performance"), + ); + assert.ok( + workflow.indexOf("Measure populated decision endpoint performance") < + workflow.indexOf( + "Prove promoted delivery, business values, and decision deduplication", + ), + ); + assert.match( + workflow, + /Measure populated decision endpoint performance[\s\S]*staging-ci\.js verify[\s\S]*--target live/, + ); + assert.match( + workflow, + /--baseline-deployment-id "\$\{\{ steps\.promote\.outputs\.previous_live_id \|\| steps\.tinybird\.outputs\.id \}\}"/, ); assert.match(workflow, /steps\.seed\.outcome != 'skipped'/); assert.match(workflow, /steps\.cleanup\.outputs\.requires_copies == 'true'/); @@ -822,6 +1680,75 @@ test("the seed persists cleanup state and partial evidence before ingestion", () ); }); +test("rollback drill proves old and restored deployment data planes", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const drillSource = source.slice( + source.indexOf("const drillOwnedRollback"), + source.indexOf("const rollbackOwnedPromotion"), + ); + const recoverySource = source.slice( + source.indexOf("const rollbackOwnedPromotion"), + source.indexOf("const discardOwnedDeployment"), + ); + assert.ok( + (drillSource.match(/await switchLiveDeployment/g)?.length ?? 0) >= 2, + ); + assert.equal( + drillSource.match(/await queryDecisionEndpointSuite/g)?.length, + 2, + ); + assert.match(drillSource, /previousLiveDeploymentId/); + assert.match(drillSource, /assertDecisionEndpointSuiteReadable/); + assert.match(drillSource, /retainedIdentityFunnelAvailable/); + assert.match(drillSource, /decisionEndpointAvailable/); + assert.match(drillSource, /assertSyntheticEndpointDecisions/); + assert.match(drillSource, /assertSyntheticBusinessDecisions/); + assert.match(drillSource, /readAndAssertPhaseHealth/); + assert.match(drillSource, /dataPlanePassed: true/); + assert.match(drillSource, /rollback_target_usable/); + assert.ok( + drillSource.indexOf("rollbackProbeError = error") < + drillSource.lastIndexOf("await switchLiveDeployment"), + ); + assert.match(recoverySource, /await queryDecisionEndpointSuite/); + assert.match(recoverySource, /assertDecisionEndpointSuiteReadable/); + assert.match( + recoverySource, + /The Tinybird rollback destination is not usable/, + ); + assert.ok( + recoverySource.indexOf("await queryDecisionEndpointSuite") < + recoverySource.indexOf("await deleteRetiredDeployment"), + ); +}); + +test("performance compares the retained deployment and a larger synthetic volume", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const verifySource = source.slice( + source.indexOf("const verify = async () =>"), + source.indexOf("const safeSyntheticIdentifier"), + ); + assert.match(verifySource, /options\.get\("baseline-deployment-id"\)/); + assert.match( + verifySource, + /deploymentId: baselineDeploymentId[\s\S]*deploymentId: state\.deploymentId[\s\S]*syntheticRunId: state\.loadRunId/, + ); + assert.match(verifySource, /representativeSamples/); + assert.match(verifySource, /representativeRows: state\.loadEventCount/); + assert.match(verifySource, /representativePerformancePassed/); + assert.match(verifySource, /assertRepresentativeEndpointCoverage/); + assert.match(verifySource, /representativeBudget/); + assert.match(verifySource, /new_endpoint_no_baseline/); + assert.match(verifySource, /retainedIdentityFunnelAvailable/); + assert.match(verifySource, /decisionEndpointAvailable/); +}); + test("synthetic deletion targets the deployment used for ingestion", () => { const source = fs.readFileSync( new URL("../staging-ci.js", import.meta.url), @@ -835,7 +1762,16 @@ test("synthetic deletion targets the deployment used for ingestion", () => { source.indexOf("const deleteProductEventRows"), source.indexOf("const eraseSyntheticIdentity"), ); + const verifyCleanupSource = source.slice( + source.indexOf("const verifyCleanup"), + source.indexOf("const tokenScopeProbe"), + ); assert.doesNotMatch(deleteSource, /__tb__min_deployment/); + assert.match(verifyCleanupSource, /syntheticRunId: state\.decisionRunId/); + assert.match( + verifyCleanupSource, + /Object\.values\(businessDecisionAssertions\)\.some/, + ); assert.equal(deleteSource.match(/\.\.\.deploymentParameters/g)?.length, 2); assert.match( source, @@ -852,10 +1788,8 @@ test("synthetic deletion targets the deployment used for ingestion", () => { assert.match(source, /writeOutput\("requires_copies", "true"\)/); assert.match(source, /writeOutput\("requires_discard", "true"\)/); assert.match(source, /writeOutput\("requires_discard", "false"\)/); - assert.match( - source, - /target === "staging"[\s\S]*tokens\.TINYBIRD_STAGING_DEPLOY_TOKEN[\s\S]*tokens\.TINYBIRD_STAGING_READ_TOKEN/, - ); + assert.match(source, /requestedTarget !== "live"/); + assert.doesNotMatch(source, /useDeploymentParameter/); assert.match( workflow, /steps\.deployment-state\.outputs\.target \|\| \(steps\.tinybird\.outputs\.needs_promotion == 'true' && 'staging' \|\| 'live'\)/, From 80399966a7d89f7b1c0cdd2d96c6cc374dd063c9 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:17:44 +0100 Subject: [PATCH 079/110] ci: compile desktop analytics before staging --- .github/workflows/analytics.yml | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 978c7054e17..1eb23780a42 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -87,10 +87,36 @@ jobs: if: always() run: node scripts/analytics/analytics-cli.js local-stop + validate-desktop: + name: Validate desktop analytics outbox + if: >- + (github.event_name == 'pull_request' && + github.event.pull_request.number == 2003 && + github.event.pull_request.head.ref == 'codex/first-party-analytics') || + (github.event_name == 'workflow_dispatch' && + github.ref == 'refs/heads/codex/first-party-analytics') + runs-on: macos-14 + timeout-minutes: 30 + steps: + - name: Check out exact requested SHA + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 1 + - name: Install Rust 1.88 + uses: dtolnay/rust-toolchain@1.88.0 + - uses: ./.github/actions/setup-rust-cache + with: + target: x86_64-apple-darwin + - name: Compile the desktop analytics path + run: cargo check -p cap-desktop --locked + - name: Prove durable desktop analytics behavior + run: cargo test -p cap-desktop product_analytics::tests --lib --locked + deploy-staging: name: Deploy and prove analytics staging - needs: validate - if: needs.validate.result == 'success' + needs: [validate, validate-desktop] + if: needs.validate.result == 'success' && needs.validate-desktop.result == 'success' runs-on: ubuntu-latest timeout-minutes: 45 environment: staging From 26217fd2f45cb55f50af6158ef20ef03cdcc29bd Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 09:20:50 +0100 Subject: [PATCH 080/110] ci: prepare native deps for analytics Rust gate --- .github/workflows/analytics.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 1eb23780a42..8b7a512dd3e 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -108,6 +108,8 @@ jobs: - uses: ./.github/actions/setup-rust-cache with: target: x86_64-apple-darwin + - name: Prepare pinned native dependencies + run: node scripts/setup.js - name: Compile the desktop analytics path run: cargo check -p cap-desktop --locked - name: Prove durable desktop analytics behavior From af3a0cbfd6e78d91931692a8495195fc364e73d2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:39:48 +0100 Subject: [PATCH 081/110] feat: enforce typed analytics delivery contracts --- apps/cli/src/upload.rs | 2 + .../src-tauri/src/product_analytics.rs | 642 +++++++++++++++--- apps/desktop/src-tauri/src/recording.rs | 34 +- apps/desktop/src/utils/analytics.ts | 11 +- apps/mobile/app/record.tsx | 21 + .../product-analytics-client.test.ts | 2 +- .../mobile/src/analytics/product-analytics.ts | 2 + apps/mobile/src/screens/record.test.tsx | 1 + .../unit/product-analytics-delivery.test.ts | 233 ++++++- .../product-analytics-event-coverage.test.ts | 79 +++ .../unit/product-analytics-platform.test.ts | 28 + .../unit/product-analytics-queue.test.ts | 11 +- .../unit/product-analytics-request.test.ts | 43 ++ .../unit/product-analytics-scheduler.test.ts | 30 +- .../unit/product-analytics-transport.test.ts | 5 + .../app/Layout/ProductAnalyticsPageView.tsx | 36 +- apps/web/app/s/[videoId]/Share.tsx | 48 +- .../[videoId]/_components/CapVideoPlayer.tsx | 3 + .../[videoId]/_components/HLSVideoPlayer.tsx | 3 + .../s/[videoId]/_components/ShareVideo.tsx | 4 + apps/web/app/utils/analytics.ts | 9 +- apps/web/app/utils/product-analytics.ts | 31 +- apps/web/components/pages/HomePage/Header.tsx | 6 +- .../analytics/src/browser-session.test.ts | 29 + packages/analytics/src/browser-session.ts | 8 +- packages/analytics/src/event-registry.ts | 62 +- packages/analytics/src/index.test.ts | 89 +++ packages/analytics/src/index.ts | 69 +- packages/analytics/src/privacy.ts | 10 +- scripts/analytics/check-event-contract.js | 248 ++++++- .../analytics/tests/event-contract.test.js | 148 ++++ 31 files changed, 1732 insertions(+), 215 deletions(-) create mode 100644 apps/web/__tests__/unit/product-analytics-event-coverage.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-platform.test.ts diff --git a/apps/cli/src/upload.rs b/apps/cli/src/upload.rs index fc9d49a6a9e..4665f525dd0 100644 --- a/apps/cli/src/upload.rs +++ b/apps/cli/src/upload.rs @@ -267,6 +267,7 @@ async fn upload_file_with_agent( Method::POST, "/uploads", &json!({ + "initiatingPlatform": "cli", "fileName": file_name, "contentType": "video/mp4", "contentLength": content_length, @@ -402,6 +403,7 @@ async fn create_video( } let mut params: Vec<(&str, String)> = vec![ + ("initiatingPlatform", "cli".to_string()), ("recordingMode", "desktopMP4".to_string()), ("durationInSecs", meta.duration_in_secs.to_string()), ("width", meta.width.to_string()), diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index 6af232a7ec2..4817bc84174 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -8,7 +8,8 @@ use serde_json::{Map, Value}; #[cfg(unix)] use std::os::unix::fs::PermissionsExt; use std::{ - fs, + collections::HashSet, + fs::{self, OpenOptions}, io::Write, path::{Path, PathBuf}, sync::{ @@ -38,6 +39,7 @@ const PRODUCT_EVENT_OUTBOX_STORE_KEY: &str = "product_analytics_outbox_v1"; const PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY: &str = "product_analytics_outbox_recovery_v1"; const PRODUCT_EVENT_OUTBOX_LEGACY_KEY_STORE_KEY: &str = "product_analytics_outbox_fallback_key_v1"; const PRODUCT_EVENT_OUTBOX_KEY_FILE: &str = "product-analytics-outbox-key-v1"; +const PRODUCT_EVENT_OUTBOX_JOURNAL_FILE: &str = "product-analytics-outbox-journal-v1"; const PRODUCT_EVENT_OUTBOX_KEYRING_SERVICE: &str = "so.cap.desktop"; const PRODUCT_EVENT_OUTBOX_KEYRING_USER: &str = "product-analytics-outbox-v1"; const PRODUCT_EVENT_OUTBOX_LEGACY_KEYRING_USER: &str = "product-analytics-outbox-v1-backup"; @@ -79,6 +81,10 @@ pub enum ProductAnalyticsEvent { fragmented: bool, custom_cursor_capture: bool, }, + RecordingStartFailed { + mode: &'static str, + error: String, + }, RecordingCompleted { mode: &'static str, status: &'static str, @@ -229,6 +235,12 @@ fn event_data(event: ProductAnalyticsEvent) -> EventData { data.set("custom_cursor_capture", custom_cursor_capture); data } + ProductAnalyticsEvent::RecordingStartFailed { mode, error } => { + let mut data = EventData::new("recording_start_failed"); + data.set("mode", mode); + data.set("failure_class", classify_failure(&error)); + data + } ProductAnalyticsEvent::RecordingCompleted { mode, status, @@ -290,6 +302,7 @@ fn is_core_product_event(name: &str) -> bool { name, "recording_started" | "recording_completed" + | "recording_start_failed" | "multipart_upload_complete" | "multipart_upload_failed" | "recording_recovery_failed" @@ -302,6 +315,7 @@ fn desktop_client_product_event_name(name: &str) -> Option<&'static str> { "user_signed_in" => Some("user_signed_in"), "user_signed_out" => Some("user_signed_out"), "recording_started" => Some("recording_started"), + "recording_start_failed" => Some("recording_start_failed"), "recording_completed" => Some("recording_completed"), "multipart_upload_complete" => Some("multipart_upload_complete"), "multipart_upload_failed" => Some("multipart_upload_failed"), @@ -410,6 +424,13 @@ struct DeliveryError { status: Option, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ProductEventBatchKind { + LossReport, + Pending, + DeadLetterRetry, +} + #[derive(Clone, Debug, Deserialize, Serialize)] struct ProductEventDeadLetter { event: ProductEvent, @@ -540,6 +561,98 @@ fn file_outbox_key_path(app: &AppHandle) -> Result { .map_err(|_| "outbox_key_directory_unavailable".to_string()) } +fn outbox_journal_path(app: &AppHandle) -> Result { + app.path() + .app_local_data_dir() + .map(|directory| directory.join(PRODUCT_EVENT_OUTBOX_JOURNAL_FILE)) + .map_err(|_| "outbox_journal_directory_unavailable".to_string()) +} + +fn append_event_journal(app: &AppHandle, event: &ProductEvent) -> Result<(), String> { + let path = outbox_journal_path(app)?; + let encryption_key = outbox_encryption_key(app)?; + append_encrypted_event_journal(&path, event, encryption_key).map(|_| ()) +} + +fn append_encrypted_event_journal( + path: &Path, + event: &ProductEvent, + encryption_key: &[u8; 32], +) -> Result { + let parent = path + .parent() + .ok_or_else(|| "outbox_journal_directory_unavailable".to_string())?; + fs::create_dir_all(parent).map_err(|_| "outbox_journal_directory_unavailable".to_string())?; + let encoded = encrypt_outbox_with_key( + &ProductEventOutbox { + pending: vec![event.clone()], + ..ProductEventOutbox::default() + }, + encryption_key, + )?; + let appended_bytes = encoded.len().saturating_add(1); + let mut file = OpenOptions::new() + .create(true) + .append(true) + .open(path) + .map_err(|_| "outbox_journal_write_failed".to_string())?; + #[cfg(unix)] + file.set_permissions(fs::Permissions::from_mode(0o600)) + .map_err(|_| "outbox_journal_write_failed".to_string())?; + file.write_all(encoded.as_bytes()) + .and_then(|()| file.write_all(b"\n")) + .and_then(|()| file.sync_data()) + .map_err(|_| "outbox_journal_write_failed".to_string())?; + Ok(appended_bytes) +} + +fn load_event_journal(app: &AppHandle) -> Result<(ProductEventOutbox, bool), String> { + let path = outbox_journal_path(app)?; + let value = match fs::read_to_string(path) { + Ok(value) => value, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok((ProductEventOutbox::default(), false)); + } + Err(_) => return Err("outbox_journal_read_failed".to_string()), + }; + let mut outbox = ProductEventOutbox::default(); + let mut corrupt = false; + for encoded in value.lines().filter(|line| !line.is_empty()) { + match decrypt_outbox(app, encoded) { + Ok(record) => merge_outbox(&mut outbox, record), + Err(_) => corrupt = true, + } + } + Ok((outbox, corrupt)) +} + +fn truncate_event_journal(app: &AppHandle) -> Result<(), String> { + let path = outbox_journal_path(app)?; + let parent = path + .parent() + .ok_or_else(|| "outbox_journal_directory_unavailable".to_string())?; + fs::create_dir_all(parent).map_err(|_| "outbox_journal_directory_unavailable".to_string())?; + let file = OpenOptions::new() + .create(true) + .write(true) + .truncate(true) + .open(path) + .map_err(|_| "outbox_journal_write_failed".to_string())?; + #[cfg(unix)] + file.set_permissions(fs::Permissions::from_mode(0o600)) + .map_err(|_| "outbox_journal_write_failed".to_string())?; + file.sync_all() + .map_err(|_| "outbox_journal_write_failed".to_string()) +} + +fn delete_event_journal(app: &AppHandle) -> Result<(), String> { + match fs::remove_file(outbox_journal_path(app)?) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(_) => Err("outbox_journal_delete_failed".to_string()), + } +} + fn read_file_outbox_key(path: &Path) -> Result, String> { match fs::read_to_string(path) { Ok(encoded) => decode_outbox_encryption_key(&encoded).map(Some), @@ -818,6 +931,39 @@ fn loss_summary_matches( && summary.session_id == event.session_id } +fn is_capacity_loss_summary(summary: &ProductEventLossSummary) -> bool { + summary.failure_class == "summary_capacity" && summary.failed_event_name == "other" +} + +fn capacity_loss_summary(source: ProductEventLossSummary) -> ProductEventLossSummary { + ProductEventLossSummary { + summary_id: source.summary_id, + failure_class: "summary_capacity".to_string(), + failed_event_name: "other".to_string(), + platform: "desktop".to_string(), + app_version: "mixed".to_string(), + anonymous_id: source.anonymous_id, + session_id: String::new(), + status: None, + count: source.count, + first_sequence: source.first_sequence, + last_sequence: source.last_sequence, + first_failed_at_ms: source.first_failed_at_ms, + last_failed_at_ms: source.last_failed_at_ms, + } +} + +fn aggregate_loss_summary( + fallback: &mut ProductEventLossSummary, + source: &ProductEventLossSummary, +) { + fallback.count = fallback.count.saturating_add(source.count); + fallback.first_sequence = fallback.first_sequence.min(source.first_sequence); + fallback.last_sequence = fallback.last_sequence.max(source.last_sequence); + fallback.first_failed_at_ms = fallback.first_failed_at_ms.min(source.first_failed_at_ms); + fallback.last_failed_at_ms = fallback.last_failed_at_ms.max(source.last_failed_at_ms); +} + fn record_delivery_loss( outbox: &mut ProductEventOutbox, failure_class: &str, @@ -858,12 +1004,11 @@ fn record_delivery_loss( return; } - if let Some(summary) = outbox.loss_summaries.iter_mut().find(|summary| { - summary.failure_class == "summary_capacity" - && summary.failed_event_name == "other" - && summary.platform == event.platform - && summary.app_version == event.app_version - }) { + if let Some(summary) = outbox + .loss_summaries + .iter_mut() + .find(|summary| is_capacity_loss_summary(summary)) + { summary.count = summary.count.saturating_add(1); summary.last_sequence = sequence; summary.last_failed_at_ms = failed_at_ms; @@ -875,10 +1020,10 @@ fn record_delivery_loss( summary_id: Uuid::new_v4().to_string(), failure_class: "summary_capacity".to_string(), failed_event_name: "other".to_string(), - platform: event.platform.clone(), - app_version: event.app_version.clone(), + platform: "desktop".to_string(), + app_version: "mixed".to_string(), anonymous_id: event.anonymous_id.clone(), - session_id: event.session_id.clone(), + session_id: String::new(), status: None, count: 1, first_sequence: sequence, @@ -899,40 +1044,31 @@ fn merge_loss_summary(target: &mut Vec, source: Product } return; } - if target.len() < PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY { + if let Some(fallback) = target + .iter_mut() + .find(|summary| is_capacity_loss_summary(summary)) + { + aggregate_loss_summary(fallback, &source); + return; + } + + let detailed_capacity = PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY.saturating_sub(1); + if target.len() < detailed_capacity && !is_capacity_loss_summary(&source) { target.push(source); return; } - let fallback_index = target.iter().position(|summary| { - summary.failure_class == "summary_capacity" && summary.failed_event_name == "other" - }); - let index = fallback_index.unwrap_or_else(|| target.len().saturating_sub(1)); - let mut fallback = if fallback_index.is_some() { - target.remove(index) + let mut fallback = if target.len() < PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY { + capacity_loss_summary(source) } else { - let displaced = target.remove(index); - ProductEventLossSummary { - summary_id: displaced.summary_id, - failure_class: "summary_capacity".to_string(), - failed_event_name: "other".to_string(), - platform: displaced.platform, - app_version: displaced.app_version, - anonymous_id: displaced.anonymous_id, - session_id: displaced.session_id, - status: None, - count: displaced.count, - first_sequence: displaced.first_sequence, - last_sequence: displaced.last_sequence, - first_failed_at_ms: displaced.first_failed_at_ms, - last_failed_at_ms: displaced.last_failed_at_ms, - } + let displaced = target.remove(target.len().saturating_sub(1)); + let mut fallback = capacity_loss_summary(displaced); + aggregate_loss_summary(&mut fallback, &source); + fallback }; - fallback.count = fallback.count.saturating_add(source.count); - fallback.first_sequence = fallback.first_sequence.min(source.first_sequence); - fallback.last_sequence = fallback.last_sequence.max(source.last_sequence); - fallback.first_failed_at_ms = fallback.first_failed_at_ms.min(source.first_failed_at_ms); - fallback.last_failed_at_ms = fallback.last_failed_at_ms.max(source.last_failed_at_ms); + fallback.platform = "desktop".to_string(); + fallback.app_version = "mixed".to_string(); + fallback.session_id.clear(); target.push(fallback); } @@ -1020,21 +1156,36 @@ fn merge_outbox(target: &mut ProductEventOutbox, source: ProductEventOutbox) { fn persist_outbox(app: &AppHandle, outbox: &mut ProductEventOutbox) -> Result<(), String> { if !PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.load(Ordering::Acquire) { if write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, outbox).is_ok() { - return Ok(()); + return truncate_event_journal(app); } PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); - return write_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY, outbox); + write_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY, outbox)?; + return truncate_event_journal(app); } let Ok(mut restored) = load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY) else { - return write_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY, outbox); + let (journal, corrupt) = load_event_journal(app)?; + merge_outbox(outbox, journal); + if corrupt { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!("Product analytics journal contained an incomplete record"); + } + write_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY, outbox)?; + return truncate_event_journal(app); }; merge_outbox(&mut restored, outbox.clone()); let recovery = load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY)?; merge_outbox(&mut restored, recovery); + let (journal, corrupt) = load_event_journal(app)?; + merge_outbox(&mut restored, journal); + if corrupt { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!("Product analytics journal contained an incomplete record"); + } write_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY, &restored)?; delete_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY)?; delete_legacy_store_outbox_encryption_keys(app)?; + truncate_event_journal(app)?; *outbox = restored; PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(false, Ordering::Release); Ok(()) @@ -1047,31 +1198,25 @@ fn outbox_guard() -> std::sync::MutexGuard<'static, ProductEventOutbox> { .unwrap_or_else(std::sync::PoisonError::into_inner) } -fn persist_current_outbox(app: &AppHandle) -> Result<(), String> { - let mut outbox = outbox_guard(); - if let Err(failure_class) = persist_outbox(app, &mut outbox) { - PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); - warn!( - failure_class, - "Failed to persist encrypted product analytics outbox" - ); - return Err(failure_class); - } - Ok(()) -} - -fn remove_delivered_events(app: &AppHandle, delivered: &[ProductEvent]) { +fn remove_delivered_from_outbox(outbox: &mut ProductEventOutbox, delivered: &[ProductEvent]) { let delivered_ids = delivered .iter() .map(|event| event.event_id.as_str()) .collect::>(); - let mut outbox = outbox_guard(); outbox .pending .retain(|event| !delivered_ids.contains(event.event_id.as_str())); outbox .loss_reports_in_flight .retain(|event| !delivered_ids.contains(event.event_id.as_str())); + outbox + .dead_letters + .retain(|entry| !delivered_ids.contains(entry.event.event_id.as_str())); +} + +fn remove_delivered_events(app: &AppHandle, delivered: &[ProductEvent]) { + let mut outbox = outbox_guard(); + remove_delivered_from_outbox(&mut outbox, delivered); if let Err(failure_class) = persist_outbox(app, &mut outbox) { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( @@ -1081,6 +1226,69 @@ fn remove_delivered_events(app: &AppHandle, delivered: &[ProductEvent]) { } } +fn dead_letter_retry_attempts_guard() -> std::sync::MutexGuard<'static, HashSet> { + PRODUCT_EVENT_DEAD_LETTER_RETRY_ATTEMPTS + .get_or_init(|| Mutex::new(HashSet::new())) + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn select_dead_letter_retry_batch( + outbox: &ProductEventOutbox, + attempted: &mut HashSet, +) -> Vec { + let mut events = Vec::new(); + for entry in &outbox.dead_letters { + if attempted.insert(entry.event.event_id.clone()) { + events.push(entry.event.clone()); + } + if events.len() == PRODUCT_EVENT_BATCH_SIZE { + break; + } + } + events +} + +fn prepare_dead_letter_retry_batch() -> Vec { + let mut attempted = dead_letter_retry_attempts_guard(); + let outbox = outbox_guard(); + select_dead_letter_retry_batch(&outbox, &mut attempted) +} + +fn record_dead_letter_retry_rejection( + app: &AppHandle, + events: &[ProductEvent], + error: &DeliveryError, +) { + let now = chrono::Utc::now().timestamp_millis(); + let mut outbox = outbox_guard(); + record_dead_letter_retry_rejection_in_outbox(&mut outbox, events, error, now); + if let Err(failure_class) = persist_outbox(app, &mut outbox) { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!( + failure_class, + "Failed to persist product analytics dead letter retry state" + ); + } +} + +fn record_dead_letter_retry_rejection_in_outbox( + outbox: &mut ProductEventOutbox, + events: &[ProductEvent], + error: &DeliveryError, + failed_at_ms: i64, +) { + for event in events { + record_delivery_loss( + outbox, + "dead_letter_retry_rejected", + event, + error.status, + failed_at_ms, + ); + } +} + fn prepare_loss_report_batch(app: &AppHandle) -> Result, String> { let mut outbox = outbox_guard(); if outbox.loss_reports_in_flight.is_empty() && !outbox.loss_summaries.is_empty() { @@ -1169,19 +1377,27 @@ async fn run_product_event_worker(app: AppHandle, mut receiver: mpsc::Receiver<( if !live_telemetry_enabled(&app) { break; } - let (events, delivery_loss_batch) = match prepare_loss_report_batch(&app) { - Ok(loss_reports) if !loss_reports.is_empty() => (loss_reports, true), + let (events, batch_kind) = match prepare_loss_report_batch(&app) { + Ok(loss_reports) if !loss_reports.is_empty() => { + (loss_reports, ProductEventBatchKind::LossReport) + } Ok(_) => { let outbox = outbox_guard(); - ( - outbox - .pending - .iter() - .take(PRODUCT_EVENT_BATCH_SIZE) - .cloned() - .collect::>(), - false, - ) + let pending = outbox + .pending + .iter() + .take(PRODUCT_EVENT_BATCH_SIZE) + .cloned() + .collect::>(); + drop(outbox); + if pending.is_empty() { + ( + prepare_dead_letter_retry_batch(), + ProductEventBatchKind::DeadLetterRetry, + ) + } else { + (pending, ProductEventBatchKind::Pending) + } } Err(failure_class) => { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); @@ -1195,6 +1411,9 @@ async fn run_product_event_worker(app: AppHandle, mut receiver: mpsc::Receiver<( if events.is_empty() { break; } + if batch_kind == ProductEventBatchKind::DeadLetterRetry { + PRODUCT_EVENT_RETRIES.fetch_add(events.len() as u64, Ordering::Relaxed); + } match send_product_batch_once(&app, &events).await { Ok(()) => { @@ -1217,13 +1436,25 @@ async fn run_product_event_worker(app: AppHandle, mut receiver: mpsc::Receiver<( tokio::time::sleep(delay).await; } Err(error) => { - if delivery_loss_batch { - warn!( - event_count = events.len(), - status = error.status, - "Product analytics loss report was rejected and remains encrypted for recovery" - ); - break; + match batch_kind { + ProductEventBatchKind::LossReport => { + warn!( + event_count = events.len(), + status = error.status, + "Product analytics loss report was rejected and remains encrypted for recovery" + ); + break; + } + ProductEventBatchKind::DeadLetterRetry => { + record_dead_letter_retry_rejection(&app, &events, &error); + warn!( + event_count = events.len(), + status = error.status, + "Product analytics dead letters remain encrypted after a recovery attempt" + ); + break; + } + ProductEventBatchKind::Pending => {} } warn!( event_count = events.len(), @@ -1278,14 +1509,21 @@ fn insert_product_event( fn enqueue_product_event(app: &AppHandle, event: ProductEvent) -> Result<(), String> { let now = chrono::Utc::now(); - let (should_wake, dead_lettered, dropped) = { + let journal_event = event.clone(); + let (should_wake, dead_lettered, dropped, persistence) = { let mut outbox = outbox_guard(); - insert_product_event( + let (should_wake, dead_lettered, dropped) = insert_product_event( &mut outbox, event, now.timestamp_millis(), &now.to_rfc3339(), - ) + ); + let persistence = if dead_lettered || dropped { + persist_outbox(app, &mut outbox) + } else { + append_event_journal(app, &journal_event).or_else(|_| persist_outbox(app, &mut outbox)) + }; + (should_wake, dead_lettered, dropped, persistence) }; if dead_lettered { PRODUCT_EVENT_DEAD_LETTERS.fetch_add(1, Ordering::Relaxed); @@ -1293,8 +1531,9 @@ fn enqueue_product_event(app: &AppHandle, event: ProductEvent) -> Result<(), Str if dropped { PRODUCT_EVENTS_DROPPED.fetch_add(1, Ordering::Relaxed); } - let persistence = persist_current_outbox(app); - + if persistence.is_err() { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + } let sender = product_event_sender(app); if should_wake && let Err(mpsc::error::TrySendError::Closed(_)) = sender.try_send(()) { @@ -1317,6 +1556,7 @@ fn clear_product_event_outbox(outbox: &mut ProductEventOutbox) { fn purge_stored_product_analytics(app: &AppHandle) -> Result<(), String> { clear_product_event_outbox(&mut outbox_guard()); + dead_letter_retry_attempts_guard().clear(); let store = app .store("store") .map_err(|_| "store_unavailable".to_string())?; @@ -1325,7 +1565,8 @@ fn purge_stored_product_analytics(app: &AppHandle) -> Result<(), String> { store.delete(PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY); store .save() - .map_err(|_| "analytics_opt_out_purge_failed".to_string()) + .map_err(|_| "analytics_opt_out_purge_failed".to_string())?; + delete_event_journal(app) } pub fn init_product_session(app: &AppHandle) { @@ -1353,8 +1594,16 @@ pub fn init_product_session(app: &AppHandle) { Err(err) => warn!("Failed to access store for product analytics session: {err}"), } + let journal = load_event_journal(app); match load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_STORE_KEY) { Ok(mut loaded) => { + if let Ok((journal_outbox, corrupt)) = &journal { + merge_outbox(&mut loaded, journal_outbox.clone()); + if *corrupt { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!("Product analytics journal contained an incomplete record"); + } + } match load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) { Ok(recovery) => { merge_outbox(&mut loaded, recovery); @@ -1362,7 +1611,11 @@ pub fn init_product_session(app: &AppHandle) { .and_then(|()| { delete_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) }) - .and_then(|()| delete_legacy_store_outbox_encryption_keys(app)); + .and_then(|()| delete_legacy_store_outbox_encryption_keys(app)) + .and_then(|()| match journal { + Ok(_) => truncate_event_journal(app), + Err(ref failure_class) => Err(failure_class.clone()), + }); if let Err(failure_class) = consolidation { PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); @@ -1384,6 +1637,7 @@ pub fn init_product_session(app: &AppHandle) { } } let has_pending = !loaded.pending.is_empty() + || !loaded.dead_letters.is_empty() || !loaded.loss_summaries.is_empty() || !loaded.loss_reports_in_flight.is_empty(); *outbox_guard() = loaded; @@ -1395,7 +1649,16 @@ pub fn init_product_session(app: &AppHandle) { PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED.store(true, Ordering::Release); PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); match load_stored_outbox(app, PRODUCT_EVENT_OUTBOX_RECOVERY_STORE_KEY) { - Ok(recovery) => *outbox_guard() = recovery, + Ok(mut recovery) => { + if let Ok((journal_outbox, corrupt)) = &journal { + merge_outbox(&mut recovery, journal_outbox.clone()); + if *corrupt { + PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); + warn!("Product analytics journal contained an incomplete record"); + } + } + *outbox_guard() = recovery; + } Err(recovery_failure_class) => { PRODUCT_EVENT_PERSISTENCE_FAILURES.fetch_add(1, Ordering::Relaxed); warn!( @@ -1411,6 +1674,7 @@ pub fn init_product_session(app: &AppHandle) { let has_pending = { let outbox = outbox_guard(); !outbox.pending.is_empty() + || !outbox.dead_letters.is_empty() || !outbox.loss_summaries.is_empty() || !outbox.loss_reports_in_flight.is_empty() }; @@ -1425,6 +1689,7 @@ static TELEMETRY_ENABLED: AtomicBool = AtomicBool::new(true); static PRODUCT_EVENT_SESSION_ID: OnceLock = OnceLock::new(); static PRODUCT_EVENT_SENDER: OnceLock> = OnceLock::new(); static PRODUCT_EVENT_OUTBOX: OnceLock> = OnceLock::new(); +static PRODUCT_EVENT_DEAD_LETTER_RETRY_ATTEMPTS: OnceLock>> = OnceLock::new(); static PRODUCT_EVENT_OUTBOX_ENCRYPTION_KEY: OnceLock<[u8; 32]> = OnceLock::new(); static PRODUCT_EVENT_OUTBOX_RESTORE_BLOCKED: AtomicBool = AtomicBool::new(false); static PRODUCT_EVENTS_DROPPED: AtomicU64 = AtomicU64::new(0); @@ -1545,6 +1810,7 @@ mod tests { fn core_product_event_catalog_is_intentionally_small() { let included = [ "recording_started", + "recording_start_failed", "recording_completed", "multipart_upload_complete", "multipart_upload_failed", @@ -1580,6 +1846,20 @@ mod tests { })); } + #[test] + fn recording_start_failure_is_bounded_and_excludes_raw_errors() { + let data = event_data(ProductAnalyticsEvent::RecordingStartFailed { + mode: "instant", + error: "/Users/private/recording.cap permission denied".to_string(), + }); + let event = product_event(&data, "install-id".to_string()).unwrap(); + + assert_eq!(event.event_name, "recording_start_failed"); + assert_eq!(event.properties["mode"], "instant"); + assert_eq!(event.properties["failure_class"], "permission"); + assert!(!event.properties.contains_key("error")); + } + #[test] fn product_event_reuses_install_id_and_process_session() { let data = event_data(recording_started()); @@ -1652,6 +1932,89 @@ mod tests { assert_eq!(decrypted.pending[0].event_id, event_id); } + #[test] + fn encrypted_durable_journal_enqueue_meets_latency_and_size_budgets() { + let data = event_data(recording_started()); + let event = product_event(&data, "install-id".to_string()).unwrap(); + let mut pending = Vec::with_capacity(PRODUCT_EVENT_OUTBOX_CAPACITY); + for index in 0..PRODUCT_EVENT_OUTBOX_CAPACITY { + let mut queued = event.clone(); + queued.event_id = format!("event-{index}"); + pending.push(queued); + } + let full_snapshot = encrypt_outbox_with_key( + &ProductEventOutbox { + pending, + ..ProductEventOutbox::default() + }, + &[7_u8; 32], + ) + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("journal"); + let encryption_key = [7_u8; 32]; + let warmup_sample_count = 10; + let measured_sample_count = 100; + let mut failures = 0_usize; + let mut appended_bytes = 0_usize; + for index in 0..warmup_sample_count { + let mut warmup_event = event.clone(); + warmup_event.event_id = format!("warmup-{index}"); + match append_encrypted_event_journal(&path, &warmup_event, &encryption_key) { + Ok(bytes) => appended_bytes = appended_bytes.saturating_add(bytes), + Err(_) => failures = failures.saturating_add(1), + } + } + + let mut timings = Vec::with_capacity(measured_sample_count); + let mut record_sizes = Vec::with_capacity(measured_sample_count); + let measured_started = std::time::Instant::now(); + for index in 0..measured_sample_count { + let mut measured_event = event.clone(); + measured_event.event_id = format!("measured-{index}"); + let started = std::time::Instant::now(); + match append_encrypted_event_journal(&path, &measured_event, &encryption_key) { + Ok(bytes) => { + timings.push(started.elapsed()); + record_sizes.push(bytes); + appended_bytes = appended_bytes.saturating_add(bytes); + } + Err(_) => failures = failures.saturating_add(1), + } + } + let measured_elapsed = measured_started.elapsed(); + timings.sort_unstable(); + let p50 = timings.get(49).copied().unwrap_or(Duration::MAX); + let p95 = timings.get(94).copied().unwrap_or(Duration::MAX); + let p99 = timings.get(98).copied().unwrap_or(Duration::MAX); + let throughput = timings.len() as f64 / measured_elapsed.as_secs_f64(); + let journal_size = usize::try_from(fs::metadata(&path).unwrap().len()).unwrap(); + let max_record_size = record_sizes.iter().copied().max().unwrap_or(usize::MAX); + + eprintln!( + "CAP_ANALYTICS_DESKTOP_PERF={{\"samples\":{},\"failures\":{},\"p50_ms\":{:.3},\"p95_ms\":{:.3},\"p99_ms\":{:.3},\"throughput_events_per_second\":{:.2},\"journal_bytes\":{},\"max_record_bytes\":{},\"full_snapshot_bytes\":{}}}", + timings.len(), + failures, + p50.as_secs_f64() * 1_000.0, + p95.as_secs_f64() * 1_000.0, + p99.as_secs_f64() * 1_000.0, + throughput, + journal_size, + max_record_size, + full_snapshot.len() + ); + + assert_eq!(failures, 0); + assert_eq!(timings.len(), measured_sample_count); + assert_eq!(journal_size, appended_bytes); + assert!(journal_size < full_snapshot.len()); + assert!(max_record_size.saturating_mul(10) < full_snapshot.len()); + assert!(p50 < Duration::from_millis(25)); + assert!(p95 < Duration::from_millis(50)); + assert!(p99 < Duration::from_millis(100)); + assert!(throughput >= 20.0); + } + #[test] fn encrypted_outbox_tries_every_persisted_key_without_replacing_it() { let data = event_data(recording_started()); @@ -1807,6 +2170,81 @@ mod tests { ); } + #[test] + fn dead_letters_retry_once_per_process_and_again_after_restart() { + let mut outbox = ProductEventOutbox::default(); + for index in 0..=PRODUCT_EVENT_BATCH_SIZE { + let mut event = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + event.event_id = format!("dead-letter-{index}"); + outbox.dead_letters.push(ProductEventDeadLetter { + event, + failure_class: "contract_rejected".to_string(), + status: Some(400), + failed_at: "2026-08-01T00:00:00Z".to_string(), + }); + } + let mut attempted = HashSet::new(); + + let first = select_dead_letter_retry_batch(&outbox, &mut attempted); + let second = select_dead_letter_retry_batch(&outbox, &mut attempted); + let exhausted = select_dead_letter_retry_batch(&outbox, &mut attempted); + let restarted = select_dead_letter_retry_batch(&outbox, &mut HashSet::new()); + + assert_eq!(first.len(), PRODUCT_EVENT_BATCH_SIZE); + assert_eq!(second.len(), 1); + assert!(exhausted.is_empty()); + assert_eq!( + restarted + .iter() + .map(|event| event.event_id.as_str()) + .collect::>(), + first + .iter() + .map(|event| event.event_id.as_str()) + .collect::>() + ); + assert_eq!(outbox.dead_letters.len(), PRODUCT_EVENT_BATCH_SIZE + 1); + } + + #[test] + fn dead_letter_recovery_retains_rejections_and_removes_accepted_events() { + let event = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + let mut outbox = ProductEventOutbox { + dead_letters: vec![ProductEventDeadLetter { + event: event.clone(), + failure_class: "contract_rejected".to_string(), + status: Some(400), + failed_at: "2026-08-01T00:00:00Z".to_string(), + }], + ..ProductEventOutbox::default() + }; + let error = DeliveryError { + retryable: false, + status: Some(422), + }; + + record_dead_letter_retry_rejection_in_outbox( + &mut outbox, + std::slice::from_ref(&event), + &error, + 1_785_542_400_000, + ); + + assert_eq!(outbox.dead_letters.len(), 1); + assert_eq!(outbox.loss_summaries.len(), 1); + assert_eq!( + outbox.loss_summaries[0].failure_class, + "dead_letter_retry_rejected" + ); + assert_eq!(outbox.loss_summaries[0].status, Some(422)); + + remove_delivered_from_outbox(&mut outbox, &[event]); + + assert!(outbox.dead_letters.is_empty()); + } + #[test] fn loss_report_payload_and_id_survive_encrypted_restart() { let event = @@ -1837,6 +2275,46 @@ mod tests { ); } + #[test] + fn loss_summary_capacity_preserves_counts_across_app_versions() { + let mut outbox = ProductEventOutbox::default(); + for index in 0..=PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY { + let mut event = + product_event(&event_data(recording_started()), "install-id".to_string()).unwrap(); + event.app_version = format!("version-{index}"); + record_delivery_loss( + &mut outbox, + "queue_overflow_unrecoverable", + &event, + None, + 1_785_542_400_000_i64 + .saturating_add(i64::try_from(index).expect("capacity index fits i64")), + ); + } + + assert_eq!( + outbox.loss_summaries.len(), + PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY + ); + assert_eq!( + outbox + .loss_summaries + .iter() + .map(|summary| summary.count) + .sum::(), + u64::try_from(PRODUCT_EVENT_LOSS_SUMMARY_CAPACITY) + .expect("loss summary capacity fits u64") + + 1 + ); + let fallback = outbox + .loss_summaries + .iter() + .find(|summary| is_capacity_loss_summary(summary)) + .unwrap(); + assert_eq!(fallback.count, 2); + assert_eq!(fallback.app_version, "mixed"); + } + #[test] fn recovery_merge_is_idempotent_for_loss_summaries() { let event = diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index c644a730ec7..6b9ac7b2bbb 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -1146,8 +1146,22 @@ pub enum RecordingEvent { /// event the main window surfaces to the user. The picker overlay that invoked the /// command is often already closed (or being torn down) when the error comes back, so /// an error returned to the caller alone can vanish without a trace. -fn notify_recording_start_failed(app: &AppHandle, error: &str) { +fn capture_recording_start_failed(app: &AppHandle, mode: RecordingMode, error: &str) { + use crate::product_analytics::{ProductAnalyticsEvent, capture_event}; + use crate::recording_telemetry::mode_label; + + capture_event( + app, + ProductAnalyticsEvent::RecordingStartFailed { + mode: mode_label(mode), + error: error.to_string(), + }, + ); +} + +fn notify_recording_start_failed(app: &AppHandle, mode: RecordingMode, error: &str) { error!(%error, "Recording failed to start"); + capture_recording_start_failed(app, mode, error); let _ = RecordingEvent::StartFailed { error: error.to_string(), } @@ -1507,7 +1521,7 @@ pub async fn start_recording( drop(app_state); // Deliberately no clear_pending_recording: the pending/active state that // caused the refusal belongs to another recording and must survive. - notify_recording_start_failed(&app, &error); + notify_recording_start_failed(&app, inputs.mode, &error); return Err(error); } if is_camera_only { @@ -1522,7 +1536,7 @@ pub async fn start_recording( Err(err) => { let error = ($map_err)(err); state_mtx.write().await.clear_pending_recording(); - notify_recording_start_failed(&app, &error); + notify_recording_start_failed(&app, inputs.mode, &error); return Err(error); } } @@ -1535,7 +1549,7 @@ pub async fn start_recording( if let Err(err) = (ShowCapWindow::Camera { centered: true }).show(&app).await { let error = format!("Failed to show centered camera window: {err}"); state_mtx.write().await.clear_pending_recording(); - notify_recording_start_failed(&app, &error); + notify_recording_start_failed(&app, inputs.mode, &error); return Err(error); } } @@ -1580,7 +1594,7 @@ pub async fn start_recording( "Refusing to start recording: disk full" ); state_mtx.write().await.clear_pending_recording(); - notify_recording_start_failed(&app, &error); + notify_recording_start_failed(&app, inputs.mode, &error); return Err(error); } if bytes <= cap_utils::disk_space::LOW_DISK_WARN_BYTES { @@ -1628,7 +1642,7 @@ pub async fn start_recording( let Some(auth) = AuthStore::get(&app).ok().flatten() else { let error = "Please sign in to use instant recording".to_string(); state_mtx.write().await.clear_pending_recording(); - notify_recording_start_failed(&app, &error); + notify_recording_start_failed(&app, inputs.mode, &error); return Err(error); }; let instant_mode_max_resolution = if auth.is_upgraded() { @@ -1663,6 +1677,7 @@ pub async fn start_recording( // invoked us may already be gone — surface it as a start failure too. notify_recording_start_failed( &app, + inputs.mode, "Your session has expired. Please sign in again to use instant recording.", ); return Ok(RecordingAction::InvalidAuthentication); @@ -1671,6 +1686,7 @@ pub async fn start_recording( state_mtx.write().await.clear_pending_recording(); notify_recording_start_failed( &app, + inputs.mode, "Instant recording requires an upgraded plan.", ); return Ok(RecordingAction::UpgradeRequired); @@ -1678,7 +1694,7 @@ pub async fn start_recording( Err(err) => { let error = format!("Could not create the shareable link: {err}"); state_mtx.write().await.clear_pending_recording(); - notify_recording_start_failed(&app, &error); + notify_recording_start_failed(&app, inputs.mode, &error); return Err(error); } }; @@ -1699,7 +1715,7 @@ pub async fn start_recording( RecordingMode::Screenshot => { let error = "Use take_screenshot for screenshots".to_string(); state_mtx.write().await.clear_pending_recording(); - notify_recording_start_failed(&app, &error); + notify_recording_start_failed(&app, inputs.mode, &error); return Err(error); } }; @@ -2085,6 +2101,7 @@ pub async fn start_recording( Ok(Ok(v)) => v, Ok(Err(err)) => { let message = format!("{err:#}"); + capture_recording_start_failed(&app, inputs.mode, &message); handle_spawn_failure( &app, &state_mtx, @@ -2097,6 +2114,7 @@ pub async fn start_recording( Err(panic) => { let panic_msg = panic_message(panic); let message = format!("Failed to spawn recording actor: {panic_msg}"); + capture_recording_start_failed(&app, inputs.mode, &message); handle_spawn_failure( &app, &state_mtx, diff --git a/apps/desktop/src/utils/analytics.ts b/apps/desktop/src/utils/analytics.ts index 10f47ae4314..9a1fb3d0b0e 100644 --- a/apps/desktop/src/utils/analytics.ts +++ b/apps/desktop/src/utils/analytics.ts @@ -1,5 +1,5 @@ import { - type ClientProductEventName, + type ClientProductEventNameForPlatform, isCoreEventName, isServerOnlyEventName, normalizeProductEventProperties, @@ -165,7 +165,7 @@ async function sendProductEventBatch(events: ProductEventInput[]) { async function enqueueProductEvent( eventId: string, - eventName: ClientProductEventName, + eventName: ClientProductEventNameForPlatform<"desktop">, occurredAt: string, properties?: Record, ) { @@ -195,10 +195,9 @@ async function enqueueProductEvent( }); } -export function trackEvent( - eventName: Name, - ...args: ProductEventArguments -) { +export function trackEvent< + Name extends ClientProductEventNameForPlatform<"desktop">, +>(eventName: Name, ...args: ProductEventArguments) { const eventId = uuid(); const occurredAt = new Date().toISOString(); if (!isCoreEventName(eventName) || isServerOnlyEventName(eventName)) return; diff --git a/apps/mobile/app/record.tsx b/apps/mobile/app/record.tsx index e36c7ed8614..21b03bea8af 100644 --- a/apps/mobile/app/record.tsx +++ b/apps/mobile/app/record.tsx @@ -40,6 +40,7 @@ import { } from "react-native-safe-area-context"; import { classifyMobileAnalyticsFailure, + createMobileProductAnalyticsEventId, trackMobileProductEventWithId, } from "@/analytics/product-analytics"; import { apiBaseUrl, useAuth } from "@/auth/AuthContext"; @@ -537,6 +538,7 @@ export default function RecordScreen() { recordingTerminalTracked.current = false; setError(null); setPhase("starting"); + const attemptId = createMobileProductAnalyticsEventId(); let createdId: string | null = null; try { await loadRecordingDurationLimit(); @@ -576,6 +578,15 @@ export default function RecordScreen() { setTeleprompterRestartKey((current) => current + 1); setTeleprompterPlaying(hasScript); } catch (recordingError) { + await trackMobileProductEventWithId( + `mobile:recording_attempt:${attemptId}:failed`, + new Date().toISOString(), + "recording_start_failed", + { + mode: "camera", + failure_class: classifyMobileAnalyticsFailure(recordingError), + }, + ).catch(() => undefined); if (createdId) { await recordingUploads.discardRecording(createdId); } @@ -613,6 +624,7 @@ export default function RecordScreen() { setError(null); setPhase("starting"); screenCompletionStarted.current = false; + const attemptId = createMobileProductAnalyticsEventId(); let createdId: string | null = null; try { const durationLimit = await loadRecordingDurationLimit(); @@ -636,6 +648,15 @@ export default function RecordScreen() { setScreenPrepared(true); setPhase("ready"); } catch (recordingError) { + await trackMobileProductEventWithId( + `mobile:recording_attempt:${attemptId}:failed`, + new Date().toISOString(), + "recording_start_failed", + { + mode: "screen", + failure_class: classifyMobileAnalyticsFailure(recordingError), + }, + ).catch(() => undefined); if (createdId) { await cancelScreenRecording(createdId).catch(() => undefined); await recordingUploads.discardRecording(createdId); diff --git a/apps/mobile/src/analytics/product-analytics-client.test.ts b/apps/mobile/src/analytics/product-analytics-client.test.ts index 5cc0e5d08bf..3eea70dae71 100644 --- a/apps/mobile/src/analytics/product-analytics-client.test.ts +++ b/apps/mobile/src/analytics/product-analytics-client.test.ts @@ -225,7 +225,7 @@ describe("MobileProductAnalyticsClient", () => { credentialScope: "scope_1", baseUrl: "https://cap.so", }); - await harness.client.track("user_signed_in"); + await harness.client.track("user_signed_out"); await harness.client.track("user_signed_out"); const snapshot = await harness.client.snapshot(); expect(snapshot.pending[0]?.event.eventName).toBe("user_signed_out"); diff --git a/apps/mobile/src/analytics/product-analytics.ts b/apps/mobile/src/analytics/product-analytics.ts index 8db359b7c65..4b0ec2c8124 100644 --- a/apps/mobile/src/analytics/product-analytics.ts +++ b/apps/mobile/src/analytics/product-analytics.ts @@ -45,6 +45,8 @@ export const purgeMobileProductAnalytics = (userId: string) => export const flushMobileProductAnalytics = () => client.flush(); +export const createMobileProductAnalyticsEventId = () => Crypto.randomUUID(); + export const trackMobileProductEvent = ( eventName: Name, ...args: ProductEventArguments diff --git a/apps/mobile/src/screens/record.test.tsx b/apps/mobile/src/screens/record.test.tsx index 21bd8db234e..21ebffbdb89 100644 --- a/apps/mobile/src/screens/record.test.tsx +++ b/apps/mobile/src/screens/record.test.tsx @@ -80,6 +80,7 @@ const uploadState = vi.hoisted(() => ({ vi.mock("@/analytics/product-analytics", () => ({ classifyMobileAnalyticsFailure: vi.fn(() => "unknown"), + createMobileProductAnalyticsEventId: vi.fn(() => "attempt_1"), trackMobileProductEventWithId: vi.fn(() => Promise.resolve("event_1")), })); diff --git a/apps/web/__tests__/unit/product-analytics-delivery.test.ts b/apps/web/__tests__/unit/product-analytics-delivery.test.ts index 048d15111a7..b2cd4060958 100644 --- a/apps/web/__tests__/unit/product-analytics-delivery.test.ts +++ b/apps/web/__tests__/unit/product-analytics-delivery.test.ts @@ -1,14 +1,53 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ - identityRows: [[{ id: "user-1" }], [{ id: "org-1" }]] as unknown[][], + identityRows: [[], [{ id: "user-1" }], [{ id: "org-1" }]] as unknown[][], identitySelectIndex: 0, + vercelEnv: "production", + storedRow: { + eventId: "signup:user-1", + lastErrorCode: null as string | null, + payloadHash: "a".repeat(32), + payloadKind: "product_event_row_v1", + payload: { + event_id: "signup:user-1", + event_name: "user_signed_up", + payload_hash: "a".repeat(32), + received_at: "2026-07-12T12:00:00.000Z", + properties: "{}", + user_id: "user-1", + organization_id: "org-1", + synthetic_run_id: "", + }, + }, + deleteSuppressed: vi.fn(), sendProductAnalyticsRows: vi.fn(), + markDeadLetter: vi.fn(), + markDelivered: vi.fn(), + markRetrying: vi.fn(), + acquireIngestionLease: vi.fn(), + releaseIngestionLease: vi.fn(), +})); + +vi.mock("@/lib/analytics/product-event-outbox-state", () => ({ + acquireProductAnalyticsIngestionLease: mocks.acquireIngestionLease, + deleteSuppressedProductAnalyticsOutboxRow: mocks.deleteSuppressed, + markProductAnalyticsOutboxDeadLetter: mocks.markDeadLetter, + markProductAnalyticsOutboxDelivered: mocks.markDelivered, + markProductAnalyticsOutboxRetrying: mocks.markRetrying, + releaseProductAnalyticsIngestionLease: mocks.releaseIngestionLease, })); vi.mock("@cap/database", () => ({ db: () => ({ select: (selection: Record) => { + if ("eventId" in selection) { + return { + from: () => ({ + where: () => ({ limit: async () => [mocks.storedRow] }), + }), + }; + } if ("userId" in selection) { return { from: () => ({ where: () => ({}) }) }; } @@ -41,6 +80,7 @@ vi.mock("@cap/env", () => ({ serverEnv: () => ({ PRODUCT_ANALYTICS_TINYBIRD_HOST: "https://staging.tinybird.test", PRODUCT_ANALYTICS_TINYBIRD_TOKEN: "ingest-token", + VERCEL_ENV: mocks.vercelEnv, }), })); @@ -52,51 +92,202 @@ vi.mock("workflow/api", () => ({ start: vi.fn(), })); -import { deliverProductAnalyticsEventStep } from "@/workflows/deliver-product-analytics-event"; +import { deliverProductAnalyticsRowStep } from "@/workflows/product-analytics-delivery-workflow"; -const event = { - eventId: "signup:user-1", - eventName: "user_signed_up", - occurredAt: "2026-07-12T12:00:00.000Z", - platform: "web", - userId: "user-1", - organizationId: "org-1", -} as const; +const deliveryKey = "00000000-0000-4000-8000-000000000001"; describe("durable product analytics delivery", () => { beforeEach(() => { - mocks.identityRows = [[{ id: "user-1" }], [{ id: "org-1" }]]; + mocks.identityRows = [[], [{ id: "user-1" }], [{ id: "org-1" }]]; mocks.identitySelectIndex = 0; + mocks.vercelEnv = "production"; + mocks.storedRow = { + eventId: "signup:user-1", + lastErrorCode: null, + payloadHash: "a".repeat(32), + payloadKind: "product_event_row_v1", + payload: { + event_id: "signup:user-1", + event_name: "user_signed_up", + payload_hash: "a".repeat(32), + received_at: "2026-07-12T12:00:00.000Z", + properties: "{}", + user_id: "user-1", + organization_id: "org-1", + synthetic_run_id: "", + }, + }; + mocks.deleteSuppressed.mockReset().mockResolvedValue(undefined); mocks.sendProductAnalyticsRows.mockReset().mockResolvedValue(undefined); + mocks.markDeadLetter.mockReset().mockResolvedValue(undefined); + mocks.markDelivered.mockReset().mockResolvedValue(undefined); + mocks.markRetrying.mockReset().mockResolvedValue(undefined); + mocks.acquireIngestionLease + .mockReset() + .mockResolvedValue("ingestion-lease-1"); + mocks.releaseIngestionLease.mockReset().mockResolvedValue(undefined); }); it("suppresses a user that is missing or pending account deletion", async () => { - mocks.identityRows = [[], [{ id: "org-1" }]]; + mocks.identityRows = [[], [], [{ id: "org-1" }]]; - await expect(deliverProductAnalyticsEventStep(event)).resolves.toEqual({ - eventId: event.eventId, - suppressed: true, + await expect(deliverProductAnalyticsRowStep(deliveryKey)).resolves.toEqual({ + status: "suppressed", }); expect(mocks.sendProductAnalyticsRows).not.toHaveBeenCalled(); + expect(mocks.deleteSuppressed).toHaveBeenCalledWith( + deliveryKey, + mocks.storedRow.payloadHash, + ); }); it("suppresses an organization after its tombstone is written", async () => { - mocks.identityRows = [[{ id: "user-1" }], []]; + mocks.identityRows = [[], [{ id: "user-1" }], []]; - await expect(deliverProductAnalyticsEventStep(event)).resolves.toEqual({ - eventId: event.eventId, - suppressed: true, + await expect(deliverProductAnalyticsRowStep(deliveryKey)).resolves.toEqual({ + status: "suppressed", }); expect(mocks.sendProductAnalyticsRows).not.toHaveBeenCalled(); }); it("delivers once when both durable identities remain active", async () => { - await expect(deliverProductAnalyticsEventStep(event)).resolves.toEqual({ - eventId: event.eventId, + await expect(deliverProductAnalyticsRowStep(deliveryKey)).resolves.toEqual({ + status: "delivered", }); expect(mocks.sendProductAnalyticsRows).toHaveBeenCalledTimes(1); expect(mocks.sendProductAnalyticsRows).toHaveBeenCalledWith( expect.objectContaining({ maxAttempts: 1, wait: true }), ); + expect(mocks.markDelivered).toHaveBeenCalledWith( + deliveryKey, + mocks.storedRow.payloadHash, + ); + }); + + it("dead-letters a permanent provider rejection", async () => { + const { ProductAnalyticsError } = await import("@cap/analytics"); + mocks.sendProductAnalyticsRows.mockRejectedValueOnce( + new ProductAnalyticsError({ + cause: new Error("rejected"), + retryable: false, + status: 400, + }), + ); + + await expect(deliverProductAnalyticsRowStep(deliveryKey)).rejects.toThrow( + "permanently rejected", + ); + expect(mocks.markDeadLetter).toHaveBeenCalledWith( + deliveryKey, + mocks.storedRow.payloadHash, + "provider_rejected", + ); + }); + + it("keeps a retryable provider failure observable", async () => { + mocks.sendProductAnalyticsRows.mockRejectedValueOnce( + new Error("temporary"), + ); + + await expect(deliverProductAnalyticsRowStep(deliveryKey)).rejects.toThrow( + "temporarily failed", + ); + expect(mocks.markRetrying).toHaveBeenCalledWith( + deliveryKey, + mocks.storedRow.payloadHash, + ); + }); + + it("retries a preview purchase after a simulated lost acknowledgement", async () => { + mocks.vercelEnv = "preview"; + mocks.identityRows = [[], []]; + mocks.storedRow = { + ...mocks.storedRow, + eventId: "staging_ambiguous_purchase_1", + payload: { + ...mocks.storedRow.payload, + event_id: "staging_ambiguous_purchase_1", + event_name: "purchase_completed", + synthetic_run_id: "run_staging_server", + }, + }; + + await expect(deliverProductAnalyticsRowStep(deliveryKey)).rejects.toThrow( + "acknowledgement lost", + ); + expect(mocks.markRetrying).toHaveBeenCalledWith( + deliveryKey, + mocks.storedRow.payloadHash, + "staging_timeout_after_accept", + ); + + mocks.storedRow.lastErrorCode = "staging_timeout_after_accept"; + await expect(deliverProductAnalyticsRowStep(deliveryKey)).resolves.toEqual({ + status: "delivered", + }); + expect(mocks.sendProductAnalyticsRows).toHaveBeenCalledTimes(2); + }); + + it.each([ + ["429", "staging_provider_429"], + ["503", "staging_provider_503"], + ] as const)( + "retries a preview provider %s response before delivery", + async (status, errorCode) => { + mocks.vercelEnv = "preview"; + mocks.identityRows = [[], []]; + mocks.storedRow = { + ...mocks.storedRow, + eventId: `staging_retry_${status}_event_1`, + payload: { + ...mocks.storedRow.payload, + event_id: `staging_retry_${status}_event_1`, + synthetic_run_id: "run_staging_server", + }, + }; + + await expect(deliverProductAnalyticsRowStep(deliveryKey)).rejects.toThrow( + `returned ${status}`, + ); + expect(mocks.sendProductAnalyticsRows).not.toHaveBeenCalled(); + expect(mocks.markRetrying).toHaveBeenCalledWith( + deliveryKey, + mocks.storedRow.payloadHash, + errorCode, + ); + + mocks.storedRow.lastErrorCode = errorCode; + await expect( + deliverProductAnalyticsRowStep(deliveryKey), + ).resolves.toEqual({ + status: "delivered", + }); + expect(mocks.sendProductAnalyticsRows).toHaveBeenCalledTimes(1); + }, + ); + + it("dead-letters a preview contract rejection without provider delivery", async () => { + mocks.vercelEnv = "preview"; + mocks.identityRows = [[], []]; + mocks.storedRow = { + ...mocks.storedRow, + eventId: "staging_reject_400_event_1", + payload: { + ...mocks.storedRow.payload, + event_id: "staging_reject_400_event_1", + synthetic_run_id: "run_staging_server", + }, + }; + + await expect(deliverProductAnalyticsRowStep(deliveryKey)).rejects.toThrow( + "returned 400", + ); + expect(mocks.markDeadLetter).toHaveBeenCalledWith( + deliveryKey, + mocks.storedRow.payloadHash, + "provider_rejected", + ); + expect(mocks.sendProductAnalyticsRows).not.toHaveBeenCalled(); + expect(mocks.acquireIngestionLease).not.toHaveBeenCalled(); }); }); diff --git a/apps/web/__tests__/unit/product-analytics-event-coverage.test.ts b/apps/web/__tests__/unit/product-analytics-event-coverage.test.ts new file mode 100644 index 00000000000..210fc241ff0 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-event-coverage.test.ts @@ -0,0 +1,79 @@ +import { + EVENT_REGISTRY, + normalizeProductEventInput, + type ProductEventPlatform, +} from "@cap/analytics"; +import { describe, expect, it } from "vitest"; +import { + createServerProductEventRows, + type ServerProductEvent, +} from "@/lib/analytics/server-event"; + +const occurredAt = "2026-07-31T12:00:00.000Z"; + +function runtimeProperties( + properties: Record< + string, + { + type: "boolean" | "number" | "string"; + values?: readonly string[]; + format?: string; + } + >, +) { + return Object.fromEntries( + Object.entries(properties).map(([key, rule]) => { + if (rule.type === "boolean") return [key, true]; + if (rule.type === "number") return [key, 1]; + if (rule.values) return [key, rule.values[0]]; + if (rule.format === "hostname") return [key, "cap.so"]; + if (rule.format === "timestamp") return [key, occurredAt]; + return [key, "bounded_value"]; + }), + ); +} + +describe("product analytics runtime emitter coverage", () => { + it("executes every declared event and platform through an authorized emitter", () => { + const covered = new Set(); + for (const [eventName, definition] of Object.entries(EVENT_REGISTRY)) { + const properties = runtimeProperties(definition.properties); + for (const platform of definition.platforms) { + const key = `${eventName}:${platform}`; + if (definition.authority !== "server" && platform !== "server") { + const event = normalizeProductEventInput({ + eventId: `client:${key}`, + eventName, + occurredAt, + anonymousId: "anonymous-coverage", + sessionId: "session-coverage", + platform, + ...(Object.keys(properties).length > 0 ? { properties } : {}), + }); + expect(event, key).not.toBeNull(); + covered.add(key); + } + if (definition.authority !== "client") { + const rows = createServerProductEventRows({ + eventId: `server:${key}`, + eventName, + occurredAt, + anonymousId: "anonymous-coverage", + platform: platform as ProductEventPlatform, + userId: "user-coverage", + organizationId: "organization-coverage", + ...(Object.keys(properties).length > 0 ? { properties } : {}), + } as ServerProductEvent); + expect(rows, key).toHaveLength(1); + covered.add(key); + } + } + } + + const declared = Object.entries(EVENT_REGISTRY).flatMap( + ([eventName, definition]) => + definition.platforms.map((platform) => `${eventName}:${platform}`), + ); + expect([...covered].sort()).toEqual(declared.sort()); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-platform.test.ts b/apps/web/__tests__/unit/product-analytics-platform.test.ts new file mode 100644 index 00000000000..cf4aff855e6 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-platform.test.ts @@ -0,0 +1,28 @@ +import { describe, expect, it } from "vitest"; +import { videoAnalyticsPlatform } from "@/lib/analytics/video-platform"; + +describe("videoAnalyticsPlatform", () => { + it("preserves an explicit CLI initiating surface", () => { + expect( + videoAnalyticsPlatform({ + metadata: { initiatingPlatform: "cli" }, + source: { type: "webMP4" }, + }), + ).toBe("cli"); + }); + + it("uses bounded server-side fallbacks for historical videos", () => { + expect( + videoAnalyticsPlatform({ + metadata: {}, + source: { type: "desktopSegments" }, + }), + ).toBe("desktop"); + expect( + videoAnalyticsPlatform({ + metadata: {}, + source: { type: "webMP4" }, + }), + ).toBe("server"); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-queue.test.ts b/apps/web/__tests__/unit/product-analytics-queue.test.ts index 80e9a9d8434..14ba6284917 100644 --- a/apps/web/__tests__/unit/product-analytics-queue.test.ts +++ b/apps/web/__tests__/unit/product-analytics-queue.test.ts @@ -20,6 +20,11 @@ const makeEvent = (index: number): ProductEventInput => ({ anonymousId: "anonymous-1", sessionId: "session-1", platform: "web", + properties: { + hostname: "cap.so", + is_session_entry: true, + session_started_at: "2026-07-12T12:00:00.000Z", + }, }); describe("ProductAnalyticsQueue", () => { @@ -249,7 +254,11 @@ describe("ProductAnalyticsQueue", () => { ); abandonedQueue.enqueue({ ...makeEvent(1), - properties: { hostname: "cap.so", is_session_entry: true }, + properties: { + hostname: "cap.so", + is_session_entry: true, + session_started_at: "2026-07-12T12:00:00.000Z", + }, }); void abandonedQueue.flush("unload"); diff --git a/apps/web/__tests__/unit/product-analytics-request.test.ts b/apps/web/__tests__/unit/product-analytics-request.test.ts index 21bb1e9fcfb..7c4e37849be 100644 --- a/apps/web/__tests__/unit/product-analytics-request.test.ts +++ b/apps/web/__tests__/unit/product-analytics-request.test.ts @@ -4,12 +4,14 @@ import { } from "@cap/analytics"; import { describe, expect, it } from "vitest"; import { + classifyAnalyticsTraffic, getProductAnalyticsRateLimitKey, hasExpectedBrowserAnalyticsMetadata, isAllowedAnonymousBrowserProductEvent, isAuthenticatedAnalyticsRequestCandidate, normalizeGeoHeader, normalizeProductEventBatch, + normalizeSyntheticRunId, ProductAnalyticsRateLimiter, shouldRejectUnresolvedAuthenticatedAnalyticsRequest, } from "@/lib/analytics/request"; @@ -26,6 +28,7 @@ const event: ProductEventInput = { properties: { hostname: "cap.so", is_session_entry: true, + session_started_at: "2026-07-12T12:00:00.000Z", }, }; const now = Date.parse("2026-07-12T12:00:01.000Z"); @@ -314,3 +317,43 @@ describe("normalizeGeoHeader", () => { expect(normalizeGeoHeader("unknown")).toBeUndefined(); }); }); + +describe("analytics traffic classification", () => { + const externalRequest = { + userAgent: + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/138.0 Safari/537.36", + vercelEnvironment: "production" as const, + rateLimitKey: `network:${"a".repeat(64)}`, + }; + + it("accepts ordinary production browsers", () => { + expect(classifyAnalyticsTraffic(externalRequest)).toBe("external"); + }); + + it.each([ + ["synthetic", { ...externalRequest, syntheticRunId: "staging_run_123" }], + ["preview", { ...externalRequest, vercelEnvironment: "preview" as const }], + [ + "bot", + { ...externalRequest, userAgent: "Mozilla/5.0 compatible Googlebot/2.1" }, + ], + ["internal", { ...externalRequest, internalIpHashes: "a".repeat(64) }], + ] as const)( + "classifies %s traffic before business events", + (expected, input) => { + expect(classifyAnalyticsTraffic(input)).toBe(expected); + }, + ); + + it("accepts synthetic run identifiers only in preview", () => { + expect(normalizeSyntheticRunId("staging_run_123", "preview")).toBe( + "staging_run_123", + ); + expect( + normalizeSyntheticRunId("staging_run_123", "production"), + ).toBeUndefined(); + expect( + normalizeSyntheticRunId("contains spaces", "preview"), + ).toBeUndefined(); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-scheduler.test.ts b/apps/web/__tests__/unit/product-analytics-scheduler.test.ts index 8ffd86946d5..49fae63d1db 100644 --- a/apps/web/__tests__/unit/product-analytics-scheduler.test.ts +++ b/apps/web/__tests__/unit/product-analytics-scheduler.test.ts @@ -1,14 +1,22 @@ import { afterEach, describe, expect, it, vi } from "vitest"; const mocks = vi.hoisted(() => ({ - start: vi.fn(async () => ({ runId: "run-1" })), + queue: vi.fn(async () => ({ + eventId: "checkout:cs_1", + payloadHash: "hash-1", + payloadConflict: false, + status: "started", + runId: "run-1", + })), })); -vi.mock("workflow/api", () => ({ start: mocks.start })); +vi.mock("@/lib/analytics/product-event-outbox", () => ({ + queueDurableServerProductEvent: mocks.queue, +})); describe("analytics durable enqueue", () => { afterEach(() => { - mocks.start.mockClear(); + mocks.queue.mockClear(); }); it("returns only after the workflow run is durably enqueued", async () => { @@ -22,21 +30,27 @@ describe("analytics durable enqueue", () => { platform: "web", properties: { price_id: "price_1", quantity: 1 }, }), - ).resolves.toEqual({ eventId: "checkout:cs_1", runId: "run-1" }); - expect(mocks.start).toHaveBeenCalledOnce(); + ).resolves.toEqual({ + eventId: "checkout:cs_1", + payloadHash: "hash-1", + payloadConflict: false, + status: "started", + runId: "run-1", + }); + expect(mocks.queue).toHaveBeenCalledOnce(); }); it("surfaces enqueue failure so a critical business request can retry", async () => { - mocks.start.mockRejectedValueOnce(new Error("queue unavailable")); + mocks.queue.mockRejectedValueOnce(new Error("database unavailable")); const { queueServerProductEvent } = await import("@/lib/analytics/server"); await expect( queueServerProductEvent({ eventId: "signup:user-1", eventName: "user_signed_up", occurredAt: "2026-07-12T12:00:00.000Z", - platform: "server", + platform: "web", userId: "user-1", }), - ).rejects.toThrow("queue unavailable"); + ).rejects.toThrow("database unavailable"); }); }); diff --git a/apps/web/__tests__/unit/product-analytics-transport.test.ts b/apps/web/__tests__/unit/product-analytics-transport.test.ts index 2fa585cbe32..cef39b8a8e3 100644 --- a/apps/web/__tests__/unit/product-analytics-transport.test.ts +++ b/apps/web/__tests__/unit/product-analytics-transport.test.ts @@ -14,6 +14,11 @@ const rows = createProductEventRows( occurredAt: "2026-07-12T12:00:00.000Z", anonymousId: "anonymous-1", platform: "web", + properties: { + hostname: "cap.so", + is_session_entry: true, + session_started_at: "2026-07-12T12:00:00.000Z", + }, }, ], { diff --git a/apps/web/app/Layout/ProductAnalyticsPageView.tsx b/apps/web/app/Layout/ProductAnalyticsPageView.tsx index bffc1a8281e..5f2273792ef 100644 --- a/apps/web/app/Layout/ProductAnalyticsPageView.tsx +++ b/apps/web/app/Layout/ProductAnalyticsPageView.tsx @@ -11,8 +11,16 @@ import { touchProductAnalyticsSession, } from "../utils/product-analytics"; -let lastCapturedLocation: string | undefined; +type CapturedPageView = { + location: string; + eventId: string; + sessionId: string; + sessionStartedAt: string; +}; + +let lastCapturedPageView: CapturedPageView | undefined; const SESSION_TOUCH_THROTTLE_MS = 5_000; +const ENGAGEMENT_FLUSH_INTERVAL_MS = 5 * 60 * 1_000; export function ProductAnalyticsPageView() { const pathname = usePathname(); @@ -20,17 +28,16 @@ export function ProductAnalyticsPageView() { const location = `${pathname}?${searchParams.toString()}`; useEffect(() => { - if ( - !pathname || - location === lastCapturedLocation || - !shouldCaptureProductPageView(pathname) - ) { + if (!pathname || !shouldCaptureProductPageView(pathname)) { return; } - lastCapturedLocation = location; - const initialPageView = captureProductPageView(); + const initialPageView = + lastCapturedPageView?.location === location + ? lastCapturedPageView + : captureProductPageView(); if (!initialPageView) return; - let pageView = initialPageView; + let pageView = { ...initialPageView, location }; + lastCapturedPageView = pageView; const initialActivityAt = performance.now(); let activeSince = document.visibilityState === "visible" ? initialActivityAt : undefined; @@ -53,6 +60,7 @@ export function ProductAnalyticsPageView() { captureProductPageEngagement( pageView.eventId, pageView.sessionId, + pageView.sessionStartedAt, pathname, pendingEngagedMs, maxScrollDepth, @@ -79,7 +87,10 @@ export function ProductAnalyticsPageView() { if (!context.isSessionEntry) return; flushEngagement("normal"); const nextPageView = captureProductPageView(context); - if (nextPageView) pageView = nextPageView; + if (nextPageView) { + pageView = { ...nextPageView, location }; + lastCapturedPageView = pageView; + } lastInteractionAt = now; activeSince = now; }; @@ -105,6 +116,10 @@ export function ProductAnalyticsPageView() { window.addEventListener("keydown", handleActivity); window.addEventListener("pagehide", pageHide, { passive: true }); document.addEventListener("visibilitychange", updateVisibility); + const engagementInterval = window.setInterval( + () => flushEngagement("normal"), + ENGAGEMENT_FLUSH_INTERVAL_MS, + ); updateScrollDepth(); return () => { @@ -114,6 +129,7 @@ export function ProductAnalyticsPageView() { window.removeEventListener("keydown", handleActivity); window.removeEventListener("pagehide", pageHide); document.removeEventListener("visibilitychange", updateVisibility); + window.clearInterval(engagementInterval); flushEngagement("normal"); }; }, [location, pathname]); diff --git a/apps/web/app/s/[videoId]/Share.tsx b/apps/web/app/s/[videoId]/Share.tsx index fab166c4728..46335efee79 100644 --- a/apps/web/app/s/[videoId]/Share.tsx +++ b/apps/web/app/s/[videoId]/Share.tsx @@ -21,6 +21,7 @@ import { type VideoStatusResult, } from "@/actions/videos/get-status"; import type { OrganizationSettings } from "@/app/(org)/dashboard/dashboard-data"; +import { touchProductAnalyticsSession } from "@/app/utils/product-analytics"; import { CaptionProvider } from "./_components/CaptionContext"; import { ShareVideo } from "./_components/ShareVideo"; import { Sidebar } from "./_components/Sidebar"; @@ -39,46 +40,13 @@ export type CommentType = typeof commentsSchema.$inferSelect & { sending?: boolean; }; -const SESSION_STORAGE_KEY = "cap_tb_session_id"; -const SESSION_TTL_MS = 30 * 60 * 1000; // 30 minutes -let volatileAnalyticsSessionId: string | undefined; - -const ensureAnalyticsSessionId = () => { - if (typeof window === "undefined") return "anonymous"; - const now = Date.now(); - const newId = - volatileAnalyticsSessionId ?? - (typeof crypto !== "undefined" && "randomUUID" in crypto - ? crypto.randomUUID() - : `${now}-${Math.random().toString(36).slice(2)}`); - volatileAnalyticsSessionId = newId; - try { - const raw = window.localStorage.getItem(SESSION_STORAGE_KEY); - if (raw) { - const parsed = JSON.parse(raw) as { value: string; expiry: number }; - if (parsed?.value && parsed.expiry > now) { - volatileAnalyticsSessionId = parsed.value; - return parsed.value; - } - } - window.localStorage.setItem( - SESSION_STORAGE_KEY, - JSON.stringify({ value: newId, expiry: now + SESSION_TTL_MS }), - ); - return newId; - } catch (error) { - console.warn("Failed to persist analytics session id", error); - return newId; - } -}; - const trackVideoView = (payload: { videoId: string; orgId?: string | null; ownerId?: string | null; }) => { if (typeof window === "undefined") return; - const sessionId = ensureAnalyticsSessionId(); + const sessionId = touchProductAnalyticsSession().sessionId; const screen = window.screen; const body = { videoId: payload.videoId, @@ -102,8 +70,6 @@ const trackVideoView = (payload: { colorDepth: screen.colorDepth, } : undefined, - userAgent: - typeof navigator !== "undefined" ? navigator.userAgent : undefined, occurredAt: new Date().toISOString(), }; @@ -339,11 +305,10 @@ export const Share = ({ [videoStatus], ); - useEffect(() => { - if (viewerId && viewerId === data.owner.id) { - return; - } - + const viewTrackedRef = useRef(false); + const handlePlaybackStarted = useCallback(() => { + if (viewTrackedRef.current || viewerId === data.owner.id) return; + viewTrackedRef.current = true; trackVideoView({ videoId: data.id, orgId: data.orgId, @@ -556,6 +521,7 @@ export const Share = ({ isEditProcessing={isEditProcessing} recordingStopped={recordingStopped} defaultPlaybackSpeed={defaultPlaybackSpeed} + onPlaybackStarted={handlePlaybackStarted} ref={playerRef} /> )} diff --git a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx index efdf063dc72..9500bf028dc 100644 --- a/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx +++ b/apps/web/app/s/[videoId]/_components/CapVideoPlayer.tsx @@ -116,6 +116,7 @@ interface Props { showPlaybackStatusBadge?: boolean; showFloatingVolumeControl?: boolean; onUploadComplete?: () => void; + onPlaybackStarted?: () => void; } export function CapVideoPlayer({ @@ -150,6 +151,7 @@ export function CapVideoPlayer({ showPlaybackStatusBadge = false, showFloatingVolumeControl = false, onUploadComplete, + onPlaybackStarted, }: Props) { const [currentCue, setCurrentCue] = useState(""); const [controlsVisible, setControlsVisible] = useState(false); @@ -661,6 +663,7 @@ export function CapVideoPlayer({ onPlay={() => { setShowPlayButton(false); setHasPlayedOnce(true); + onPlaybackStarted?.(); }} crossOrigin={ resolvedSrc.data.supportsCrossOrigin ? "anonymous" : undefined diff --git a/apps/web/app/s/[videoId]/_components/HLSVideoPlayer.tsx b/apps/web/app/s/[videoId]/_components/HLSVideoPlayer.tsx index a564a6d3005..3c090b2cb8d 100644 --- a/apps/web/app/s/[videoId]/_components/HLSVideoPlayer.tsx +++ b/apps/web/app/s/[videoId]/_components/HLSVideoPlayer.tsx @@ -103,6 +103,7 @@ interface Props { duration?: number | null; defaultPlaybackSpeed?: number; previewMode?: "background"; + onPlaybackStarted?: () => void; } export function HLSVideoPlayer({ @@ -128,6 +129,7 @@ export function HLSVideoPlayer({ duration: fallbackDuration, defaultPlaybackSpeed, previewMode, + onPlaybackStarted, }: Props) { const hlsInstance = useRef(null); const [currentCue, setCurrentCue] = useState(""); @@ -684,6 +686,7 @@ export function HLSVideoPlayer({ setShowPlayButton(false); setHasPlayedOnce(true); hasPlayedOnceRef.current = true; + onPlaybackStarted?.(); }} playsInline autoPlay={autoplay} diff --git a/apps/web/app/s/[videoId]/_components/ShareVideo.tsx b/apps/web/app/s/[videoId]/_components/ShareVideo.tsx index c8d947f360b..ad1608c57af 100644 --- a/apps/web/app/s/[videoId]/_components/ShareVideo.tsx +++ b/apps/web/app/s/[videoId]/_components/ShareVideo.tsx @@ -64,6 +64,7 @@ export const ShareVideo = forwardRef< isEditProcessing: boolean; recordingStopped?: boolean; defaultPlaybackSpeed?: number; + onPlaybackStarted?: () => void; } >( ( @@ -81,6 +82,7 @@ export const ShareVideo = forwardRef< isEditProcessing, recordingStopped = false, defaultPlaybackSpeed, + onPlaybackStarted, }, ref, ) => { @@ -394,6 +396,7 @@ export const ShareVideo = forwardRef< rawFallbackSrc={rawFallbackSrc} duration={data.duration} defaultPlaybackSpeed={defaultPlaybackSpeed} + onPlaybackStarted={onPlaybackStarted} showPlaybackStatusBadge={showPlaybackStatusBadge} disableCaptions={areCaptionsDisabled ?? false} disableCommentStamps={areCommentStampsDisabled ?? false} @@ -428,6 +431,7 @@ export const ShareVideo = forwardRef< videoSrc={videoSrc} duration={data.duration} defaultPlaybackSpeed={defaultPlaybackSpeed} + onPlaybackStarted={onPlaybackStarted} disableCaptions={areCaptionsDisabled ?? false} chaptersSrc={areChaptersDisabled ? "" : chaptersUrl || ""} captionsSrc={areCaptionsDisabled ? "" : subtitleUrl || ""} diff --git a/apps/web/app/utils/analytics.ts b/apps/web/app/utils/analytics.ts index 3001470563f..e9debc1095c 100644 --- a/apps/web/app/utils/analytics.ts +++ b/apps/web/app/utils/analytics.ts @@ -1,14 +1,13 @@ import type { - ClientProductEventName, + ClientProductEventNameForPlatform, ProductEventArguments, ProductEventPropertiesFor, } from "@cap/analytics"; import { captureProductEvent } from "./product-analytics"; -export function trackEvent( - eventName: Name, - ...args: ProductEventArguments -) { +export function trackEvent< + Name extends ClientProductEventNameForPlatform<"web">, +>(eventName: Name, ...args: ProductEventArguments) { captureProductEvent(eventName, ...args); } diff --git a/apps/web/app/utils/product-analytics.ts b/apps/web/app/utils/product-analytics.ts index 08d59838755..f4405bf87c7 100644 --- a/apps/web/app/utils/product-analytics.ts +++ b/apps/web/app/utils/product-analytics.ts @@ -1,6 +1,6 @@ import { type BrowserAnalyticsContext, - type ClientProductEventName, + type ClientProductEventNameForPlatform, isCoreEventName, isServerOnlyEventName, normalizeAnalyticsIdentifier, @@ -353,10 +353,9 @@ let anonymousId: string | undefined; let listenersRegistered = false; let fallbackEventIdCounter = 0; -export function captureProductEvent( - eventName: Name, - ...args: ProductEventArguments -) { +export function captureProductEvent< + Name extends ClientProductEventNameForPlatform<"web">, +>(eventName: Name, ...args: ProductEventArguments) { return enqueueBrowserProductEvent( eventName, args, @@ -367,7 +366,9 @@ export function captureProductEvent( ); } -function enqueueBrowserProductEvent( +function enqueueBrowserProductEvent< + Name extends ClientProductEventNameForPlatform<"web">, +>( eventName: Name, args: ProductEventArguments, context: ReturnType, @@ -411,6 +412,7 @@ function enqueueBrowserProductEvent( export function captureProductPageEngagement( pageViewId: string, sessionId: string, + sessionStartedAt: string, pathname: string, engagedMs: number, maxScrollDepth: number, @@ -420,11 +422,17 @@ export function captureProductPageEngagement( [ { page_view_id: pageViewId, + session_started_at: sessionStartedAt, engaged_ms: Math.max(0, Math.round(engagedMs)), max_scroll_depth: Math.max(0, Math.min(100, maxScrollDepth)), }, ], - { sessionId, isSessionEntry: false, attribution: {} }, + { + sessionId, + sessionStartedAt, + isSessionEntry: false, + attribution: {}, + }, pathname, ); } @@ -439,11 +447,18 @@ export function captureProductPageView( ...context.attribution, hostname: window.location.hostname, is_session_entry: context.isSessionEntry, + session_started_at: context.sessionStartedAt, }, ], context, ); - return eventId ? { eventId, sessionId: context.sessionId } : undefined; + return eventId + ? { + eventId, + sessionId: context.sessionId, + sessionStartedAt: context.sessionStartedAt, + } + : undefined; } export function touchProductAnalyticsSession() { diff --git a/apps/web/components/pages/HomePage/Header.tsx b/apps/web/components/pages/HomePage/Header.tsx index 806ee7f0fb9..a246fdbf944 100644 --- a/apps/web/components/pages/HomePage/Header.tsx +++ b/apps/web/components/pages/HomePage/Header.tsx @@ -2,7 +2,7 @@ "use client"; import type { - ClientProductEventName, + ClientProductEventNameForPlatform, ProductEventArguments, } from "@cap/analytics"; import { Button } from "@cap/ui"; @@ -47,7 +47,9 @@ const HERO_MODE_COLORS = { const TITLE_LEADING = "leading-[2.25rem] md:leading-[3.5rem]"; -const trackHomepageEvent = ( +const trackHomepageEvent = < + Name extends ClientProductEventNameForPlatform<"web">, +>( eventName: Name, ...args: ProductEventArguments ) => { diff --git a/packages/analytics/src/browser-session.test.ts b/packages/analytics/src/browser-session.test.ts index d77900082cc..81344b498b2 100644 --- a/packages/analytics/src/browser-session.test.ts +++ b/packages/analytics/src/browser-session.test.ts @@ -59,6 +59,7 @@ describe("browser analytics sessions", () => { expect(firstTab).toMatchObject({ sessionId: "session-1", + sessionStartedAt: "1970-01-01T00:00:01.000Z", isSessionEntry: true, }); expect(reloadedTab).toMatchObject({ @@ -99,6 +100,34 @@ describe("browser analytics sessions", () => { }); }); + it("converges concurrent tabs on the same next session", () => { + const shared = createStorage(); + resolveBrowserAnalyticsContext({ + storage: shared, + createId: createIds(), + now: 0, + }); + const serialized = shared.getItem("cap_analytics_session_v2"); + if (!serialized) throw new Error("Expected a stored analytics session"); + const firstTab = createStorage(); + const secondTab = createStorage(); + firstTab.setItem("cap_analytics_session_v2", serialized); + secondTab.setItem("cap_analytics_session_v2", serialized); + const first = resolveBrowserAnalyticsContext({ + storage: firstTab, + createId: () => "first-tab-next", + now: PRODUCT_ANALYTICS_SESSION_TIMEOUT_MS + 1, + }); + const second = resolveBrowserAnalyticsContext({ + storage: secondTab, + createId: () => "second-tab-next", + now: PRODUCT_ANALYTICS_SESSION_TIMEOUT_MS + 1, + }); + + expect(first.sessionId).toBe("session-2"); + expect(second.sessionId).toBe(first.sessionId); + }); + it("keeps first and session touch stable while updating last touch", () => { const storage = createStorage(); const createId = createIds(); diff --git a/packages/analytics/src/browser-session.ts b/packages/analytics/src/browser-session.ts index 573526c7faf..60aa4491ea7 100644 --- a/packages/analytics/src/browser-session.ts +++ b/packages/analytics/src/browser-session.ts @@ -30,6 +30,7 @@ export interface AnalyticsTouch { export interface BrowserAnalyticsSession { id: string; + nextId: string; startedAt: number; lastActivityAt: number; sessionTouch?: AnalyticsTouch; @@ -37,6 +38,7 @@ export interface BrowserAnalyticsSession { export interface BrowserAnalyticsContext { sessionId: string; + sessionStartedAt: string; isSessionEntry: boolean; attribution: ProductEventProperties; } @@ -87,7 +89,8 @@ export function resolveBrowserAnalyticsContext(options: { now - existing.lastActivityAt > PRODUCT_ANALYTICS_SESSION_TIMEOUT_MS; const session: BrowserAnalyticsSession = isSessionEntry ? { - id: options.createId(), + id: existing?.nextId ?? options.createId(), + nextId: options.createId(), startedAt: now, lastActivityAt: now, ...(options.touch ? { sessionTouch: options.touch } : {}), @@ -124,6 +127,7 @@ export function resolveBrowserAnalyticsContext(options: { return { sessionId: session.id, + sessionStartedAt: new Date(session.startedAt).toISOString(), isSessionEntry, attribution: { ...touchProperties("first_touch", firstTouch), @@ -138,6 +142,7 @@ function readStoredSession(storage: AnalyticsStorage | undefined) { if (!isRecord(value)) return undefined; if ( typeof value.id !== "string" || + typeof value.nextId !== "string" || typeof value.startedAt !== "number" || typeof value.lastActivityAt !== "number" ) { @@ -146,6 +151,7 @@ function readStoredSession(storage: AnalyticsStorage | undefined) { const sessionTouch = parseTouch(value.sessionTouch); return { id: value.id, + nextId: value.nextId, startedAt: value.startedAt, lastActivityAt: value.lastActivityAt, ...(sessionTouch ? { sessionTouch } : {}), diff --git a/packages/analytics/src/event-registry.ts b/packages/analytics/src/event-registry.ts index 1289007d7ea..2fd445a4d7e 100644 --- a/packages/analytics/src/event-registry.ts +++ b/packages/analytics/src/event-registry.ts @@ -88,6 +88,11 @@ export const EVENT_REGISTRY = { ...optionalAttributionProperties, hostname: { type: "string", required: true, format: "hostname" }, is_session_entry: { type: "boolean", required: true }, + session_started_at: { + type: "string", + required: true, + format: "timestamp", + }, }, }, page_engagement: { @@ -97,6 +102,11 @@ export const EVENT_REGISTRY = { "A bounded summary of foreground engagement for one page view, emitted on route change, hide, or page exit.", properties: { page_view_id: { type: "string", required: true, format: "identifier" }, + session_started_at: { + type: "string", + required: true, + format: "timestamp", + }, engaged_ms: { type: "number", required: true }, max_scroll_depth: { type: "number", required: true }, }, @@ -187,13 +197,16 @@ export const EVENT_REGISTRY = { ...criticalServer, platforms: ["server"], semantic: - "A newly authenticated user was linked to the first-party anonymous browser identity present during signup. Decision metrics use user_id; this event preserves acquisition stitching without changing the authoritative signup fact.", + "A successfully authenticated user was linked to the first-party anonymous browser identity present at authentication. Decision metrics use user_id; this event preserves acquisition stitching without changing authoritative account facts.", properties: {}, }, user_signed_in: { - ...bestEffortClient, - platforms: ["desktop", "mobile"], - semantic: "A native client persisted a valid authenticated session.", + version: 1, + delivery: "critical", + authority: "both", + platforms: ["web", "desktop", "mobile"], + semantic: + "A client persisted a valid authenticated session, or the web authentication provider completed a successful sign-in callback.", properties: {}, }, user_signed_out: { @@ -222,6 +235,18 @@ export const EVENT_REGISTRY = { custom_cursor_capture: { type: "boolean", required: true }, }, }, + recording_start_failed: { + version: 1, + delivery: "critical", + authority: "client", + platforms: ["desktop", "mobile"], + semantic: + "A deliberate recording attempt failed before the recorder crossed its started boundary; the stable event ID is retained across delivery retries.", + properties: { + mode: { type: "string", required: true, format: "category" }, + failure_class: { type: "string", required: true, format: "category" }, + }, + }, recording_completed: { version: 1, delivery: "critical", @@ -301,6 +326,22 @@ export const EVENT_REGISTRY = { }, }, }, + upload_completed: { + ...criticalServer, + platforms: ["cli", "server"], + semantic: + "The authoritative upload state accepted a complete single-part object for processing; platform is the persisted initiating surface.", + properties: {}, + }, + upload_failed: { + ...criticalServer, + platforms: ["cli", "server"], + semantic: + "The authoritative upload state permanently rejected a single-part upload before processing, using a bounded failure class.", + properties: { + failure_class: { type: "string", required: true, format: "category" }, + }, + }, analytics_delivery_loss: { version: 1, delivery: "critical", @@ -434,7 +475,7 @@ export const EVENT_REGISTRY = { }, share_link_created: { ...criticalServer, - platforms: ["desktop", "mobile", "server"], + platforms: ["cli", "desktop", "mobile", "server"], semantic: "The authoritative database committed a new shareable video or screenshot owned by the authenticated creator; platform identifies the initiating client when durably known.", properties: { @@ -876,6 +917,9 @@ type PropertiesFromSchema> = { export type ProductEventPropertiesFor = PropertiesFromSchema<(typeof EVENT_REGISTRY)[Name]["properties"]>; +export type ProductEventPlatformFor = + (typeof EVENT_REGISTRY)[Name]["platforms"][number]; + export type ProductEventArguments = keyof (typeof EVENT_REGISTRY)[Name]["properties"] extends never ? [properties?: undefined] @@ -914,6 +958,14 @@ export type ServerProductEventName = { : Name; }[CoreEventName]; +export type ServerProductEventNameForPlatform< + Platform extends ProductEventPlatform, +> = { + [Name in ServerProductEventName]: Platform extends ProductEventPlatformFor + ? Name + : never; +}[ServerProductEventName]; + export const CORE_EVENT_NAMES = Object.freeze( Object.keys(EVENT_REGISTRY) as CoreEventName[], ); diff --git a/packages/analytics/src/index.test.ts b/packages/analytics/src/index.test.ts index 32db7866266..c74c4ddeaf9 100644 --- a/packages/analytics/src/index.test.ts +++ b/packages/analytics/src/index.test.ts @@ -71,6 +71,8 @@ describe("normalizeProductEventProperties", () => { normalizeProductEventProperties("page_engagement", { engaged_ms: Number.POSITIVE_INFINITY, max_scroll_depth: 50, + page_view_id: "page-1", + session_started_at: "2026-07-12T12:00:00.000Z", }), ).toBeNull(); }); @@ -109,6 +111,7 @@ describe("normalizeProductEventProperties", () => { normalizeProductEventProperties("page_view", { hostname: "cap.so", is_session_entry: true, + session_started_at: "2026-07-12T12:00:00.000Z", first_touch_campaign: campaign, }), ).toBeNull(); @@ -145,6 +148,7 @@ describe("normalizeProductEventProperties", () => { normalizeProductEventProperties("page_view", { hostname: "WWW.Cap.SO", is_session_entry: true, + session_started_at: "2026-07-12T12:00:00.000Z", first_touch_campaign: "Summer launch 2026", first_touch_gclid: "EAIaIQobChMI-safe_click_123", }), @@ -183,6 +187,35 @@ describe("normalizeProductEventInput", () => { expect(normalizeProductEventInput(baseEvent, now)).toEqual(baseEvent); }); + it("requires a valid session start at or before traffic activity", () => { + const pageView = { + eventId: "page-1", + eventName: "page_view", + occurredAt: "2026-07-12T11:59:59.000Z", + anonymousId: "anonymous-1", + sessionId: "session-1", + platform: "web", + properties: { + hostname: "cap.so", + is_session_entry: true, + session_started_at: "2026-07-12T11:30:00.000Z", + }, + } as const; + expect(normalizeProductEventInput(pageView, now)).toEqual(pageView); + expect( + normalizeProductEventInput( + { + ...pageView, + properties: { + ...pageView.properties, + session_started_at: "2026-07-12T12:00:00.000Z", + }, + }, + now, + ), + ).toBeNull(); + }); + it.each([ ["unknown event", { ...baseEvent, eventName: "mouse_moved" }], ["missing id", { ...baseEvent, eventId: "" }], @@ -323,6 +356,25 @@ describe("createProductEventRows", () => { ).toBe("paid_search"); }); + it("does not turn same-site navigation into referral acquisition", () => { + expect(normalizeAcquisitionChannel(undefined, "www.cap.so", "cap.so")).toBe( + "direct", + ); + expect(normalizeAcquisitionChannel(undefined, "cap.so", "app.cap.so")).toBe( + "direct", + ); + expect( + normalizeAcquisitionChannel(undefined, "partner.example", "cap.so"), + ).toBe("referral"); + expect( + normalizeAcquisitionChannel( + { session_touch_source: "partner" }, + "cap.so", + "cap.so", + ), + ).toBe("referral"); + }); + it("fingerprints canonical payloads independent of object key order", () => { expect(createProductEventPayloadHash({ a: 1, b: 2 })).toBe( createProductEventPayloadHash({ b: 2, a: 1 }), @@ -371,4 +423,41 @@ describe("createProductEventRows", () => { '{"payment_status":"paid","subscription_status":"active","is_first_purchase":true,"is_guest_checkout":true,"is_onboarding":false}', }); }); + + it("conflict-hashes every decision and exclusion dimension", () => { + const event = { + eventId: "page-1", + eventName: "page_view" as const, + occurredAt: "2026-07-12T12:00:00.000Z", + anonymousId: "anonymous-1", + sessionId: "session-1", + platform: "web" as const, + pathname: "/pricing", + properties: { + hostname: "cap.so", + is_session_entry: true, + session_started_at: "2026-07-12T12:00:00.000Z", + }, + }; + const baseContext = { + receivedAt: "2026-07-12T12:00:01.000Z", + source: "client" as const, + country: "GB", + hostname: "cap.so", + trafficClass: "external" as const, + }; + const [external] = createProductEventRows([event], baseContext); + const [synthetic] = createProductEventRows([event], { + ...baseContext, + syntheticRunId: "synthetic-run-1", + trafficClass: "synthetic", + }); + const [differentCountry] = createProductEventRows([event], { + ...baseContext, + country: "US", + }); + + expect(external?.payload_hash).not.toBe(synthetic?.payload_hash); + expect(external?.payload_hash).not.toBe(differentCountry?.payload_hash); + }); }); diff --git a/packages/analytics/src/index.ts b/packages/analytics/src/index.ts index 44f9c1a14d8..9e68c8ed1ca 100644 --- a/packages/analytics/src/index.ts +++ b/packages/analytics/src/index.ts @@ -57,9 +57,11 @@ export type { ProductEventAuthority, ProductEventDelivery, ProductEventPlatform, + ProductEventPlatformFor, ProductEventPropertiesFor, ProductEventPropertyField, ServerProductEventName, + ServerProductEventNameForPlatform, } from "./event-registry"; export type ProductEventProperty = string | number | boolean | null; @@ -71,6 +73,22 @@ export type ProductEventProperties = Record; export const PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE = "cap_analytics_anonymous_id"; +export type ProductAnalyticsIdentityKind = + | "anonymous" + | "organization" + | "user"; + +export function productAnalyticsIdentityHash( + kind: ProductAnalyticsIdentityKind, + value: string, +) { + return bytesToHex(sha256(new TextEncoder().encode(`${kind}\0${value}`))); +} + +export function productAnalyticsEventIdHash(eventId: string) { + return bytesToHex(sha256(new TextEncoder().encode(`event\0${eventId}`))); +} + export interface ProductEventInput { eventId: string; eventName: Name; @@ -367,6 +385,16 @@ export function normalizeProductEventInput( hasProperties && isRecord(value.properties) ? value.properties : undefined; const properties = normalizeProductEventProperties(eventName, rawProperties); if (properties === null) return null; + if (eventName === "page_view" || eventName === "page_engagement") { + const sessionStartedAt = (properties as Record | undefined) + ?.session_started_at; + if ( + typeof sessionStartedAt !== "string" || + Date.parse(sessionStartedAt) > Date.parse(occurredAt) + ) { + return null; + } + } return { eventId, @@ -391,6 +419,11 @@ export function createProductEventRows( context: ProductEventContext, ): ProductEventRow[] { return events.map((event) => { + const channel = normalizeAcquisitionChannel( + event.properties as ProductEventProperties | undefined, + event.referrer, + context.hostname, + ); const payload = { event_name: event.eventName, schema_version: getProductEventDefinition(event.eventName).version, @@ -404,13 +437,6 @@ export function createProductEventRows( app_version: event.appVersion ?? "", pathname: event.pathname ?? "", referrer: event.referrer ?? "", - properties: event.properties ? JSON.stringify(event.properties) : "{}", - }; - return { - event_id: event.eventId, - payload_hash: createProductEventPayloadHash(payload), - received_at: context.receivedAt, - ...payload, country: context.country ?? "", region: context.region ?? "", city: context.city ?? "", @@ -418,12 +444,16 @@ export function createProductEventRows( browser: context.browser ?? "", device: context.device ?? "", os: context.os ?? "", - channel: normalizeAcquisitionChannel( - event.properties as ProductEventProperties | undefined, - event.referrer, - ), + channel, traffic_class: context.trafficClass ?? "external", synthetic_run_id: context.syntheticRunId ?? "", + properties: event.properties ? JSON.stringify(event.properties) : "{}", + }; + return { + event_id: event.eventId, + payload_hash: createProductEventPayloadHash(payload), + received_at: context.receivedAt, + ...payload, }; }); } @@ -431,6 +461,7 @@ export function createProductEventRows( export function normalizeAcquisitionChannel( properties: ProductEventProperties | undefined, referrer?: string, + currentHostname?: string, ) { if (properties?.session_touch_gclid) { return "paid_search"; @@ -451,6 +482,22 @@ export function normalizeAcquisitionChannel( const source = String(properties?.session_touch_source ?? "").toLowerCase(); const hostname = referrer?.toLowerCase() ?? ""; if (!source && !hostname) return "direct"; + const normalizedReferrer = hostname.replace(/^www\./, ""); + const normalizedCurrent = (currentHostname?.toLowerCase() ?? "").replace( + /^www\./, + "", + ); + if ( + !source && + !medium && + normalizedReferrer && + normalizedCurrent && + (normalizedReferrer === normalizedCurrent || + normalizedReferrer.endsWith(`.${normalizedCurrent}`) || + normalizedCurrent.endsWith(`.${normalizedReferrer}`)) + ) { + return "direct"; + } if ( /(google|bing|duckduckgo|yahoo|baidu|yandex)/.test(source) || /(google\.|bing\.|duckduckgo\.|search\.yahoo\.|baidu\.|yandex\.)/.test( diff --git a/packages/analytics/src/privacy.ts b/packages/analytics/src/privacy.ts index aa8a322542f..d4d1a0b26e1 100644 --- a/packages/analytics/src/privacy.ts +++ b/packages/analytics/src/privacy.ts @@ -2,7 +2,8 @@ export type AnalyticsStringFormat = | "attribution" | "category" | "hostname" - | "identifier"; + | "identifier" + | "timestamp"; export const PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT = "[PENDING] Account deletion request"; @@ -104,6 +105,13 @@ export function normalizeAnalyticsPropertyString( ? hostname : undefined; } + if (format === "timestamp") { + const timestamp = Date.parse(normalized); + return Number.isFinite(timestamp) && + new Date(timestamp).toISOString() === normalized + ? normalized + : undefined; + } if (containsSensitiveAnalyticsContent(normalized)) return undefined; if (format === "identifier") { return SAFE_IDENTIFIER_PATTERN.test(normalized) ? normalized : undefined; diff --git a/scripts/analytics/check-event-contract.js b/scripts/analytics/check-event-contract.js index ac248a8bf9f..35c73b2f5b2 100644 --- a/scripts/analytics/check-event-contract.js +++ b/scripts/analytics/check-event-contract.js @@ -45,12 +45,38 @@ const WRAPPER_PATHS = new Set([ "apps/mobile/src/analytics/product-analytics.ts", "apps/web/app/utils/analytics.ts", "apps/web/app/utils/product-analytics.ts", + "apps/web/lib/analytics/authentication-events.ts", "apps/web/lib/analytics/server-event.ts", "apps/web/lib/analytics/server.ts", + "apps/web/lib/analytics/product-event-outbox.ts", "apps/web/lib/analytics/stripe-business-events.ts", "apps/web/workflows/deliver-product-analytics-event.ts", ]); +const LOCAL_CAPTURE_FUNCTIONS = new Map([ + [ + "apps/web/workflows/import-loom-video.ts", + new Map([ + ["queueLoomAnalyticsEvent", { kind: "object" }], + ["saveMetadataAndComplete", { kind: "object", argumentIndex: 3 }], + ]), + ], +]); const CAPTURE_MODULES = new Map([ + [ + "@/lib/analytics/authentication-events", + new Map([ + [ + "recordWebAuthenticationSuccess", + { + kind: "helper-emissions", + emissions: [ + { eventName: "user_signed_in", platforms: ["web"] }, + { eventName: "identity_linked", platforms: ["server"] }, + ], + }, + ], + ]), + ], [ "@/app/utils/analytics", new Map([ @@ -83,6 +109,12 @@ const CAPTURE_MODULES = new Map([ "@/lib/analytics/server", new Map([["queueServerProductEvent", { kind: "object" }]]), ], + [ + "@/lib/analytics/product-event-outbox", + new Map([ + ["persistProductAnalyticsEvent", { kind: "object", argumentIndex: 1 }], + ]), + ], [ "@/lib/analytics/business-events", new Map([ @@ -90,6 +122,10 @@ const CAPTURE_MODULES = new Map([ "userSignedUpEvent", { kind: "helper", eventName: "user_signed_up", platforms: ["web"] }, ], + [ + "userSignedInEvent", + { kind: "helper", eventName: "user_signed_in", platforms: ["web"] }, + ], [ "checkoutStartedEvent", { @@ -118,6 +154,22 @@ const CAPTURE_MODULES = new Map([ platformProperty: "platform", }, ], + [ + "uploadCompletedEvent", + { + kind: "helper", + eventName: "upload_completed", + platformProperty: "platform", + }, + ], + [ + "uploadFailedEvent", + { + kind: "helper", + eventName: "upload_failed", + platformProperty: "platform", + }, + ], [ "collaborationActionCreatedEvent", { @@ -236,6 +288,27 @@ const CAPTURE_MODULES = new Map([ "apps/web/lib/analytics/server", new Map([["queueServerProductEvent", { kind: "object" }]]), ], + [ + "apps/web/lib/analytics/authentication-events", + new Map([ + [ + "recordWebAuthenticationSuccess", + { + kind: "helper-emissions", + emissions: [ + { eventName: "user_signed_in", platforms: ["web"] }, + { eventName: "identity_linked", platforms: ["server"] }, + ], + }, + ], + ]), + ], + [ + "apps/web/lib/analytics/product-event-outbox", + new Map([ + ["persistProductAnalyticsEvent", { kind: "object", argumentIndex: 1 }], + ]), + ], [ "apps/web/lib/analytics/business-events", new Map([ @@ -243,6 +316,10 @@ const CAPTURE_MODULES = new Map([ "userSignedUpEvent", { kind: "helper", eventName: "user_signed_up", platforms: ["web"] }, ], + [ + "userSignedInEvent", + { kind: "helper", eventName: "user_signed_in", platforms: ["web"] }, + ], [ "checkoutStartedEvent", { @@ -271,6 +348,22 @@ const CAPTURE_MODULES = new Map([ platformProperty: "platform", }, ], + [ + "uploadCompletedEvent", + { + kind: "helper", + eventName: "upload_completed", + platformProperty: "platform", + }, + ], + [ + "uploadFailedEvent", + { + kind: "helper", + eventName: "upload_failed", + platformProperty: "platform", + }, + ], [ "collaborationActionCreatedEvent", { @@ -510,6 +603,7 @@ function validateStringPropertyRules(sourceFile, file, diagnostics) { "category", "hostname", "identifier", + "timestamp", ]); const visit = (node) => { if (ts.isObjectLiteralExpression(node)) { @@ -662,6 +756,18 @@ export function parseEventRegistry(sourceText, file = DEFAULT_REGISTRY_PATH) { const platforms = platformsProperty ? staticStringArray(platformsProperty.initializer) : undefined; + const propertiesProperty = objectProperty(definition, "properties"); + const propertiesValue = propertiesProperty + ? unwrapExpression(propertiesProperty.initializer) + : undefined; + const propertyKeys = new Set( + propertiesValue && ts.isObjectLiteralExpression(propertiesValue) + ? propertiesValue.properties + .filter((entry) => !ts.isSpreadAssignment(entry)) + .map((entry) => propertyNameText(entry.name)) + .filter(Boolean) + : [], + ); if (!platforms) { diagnostics.push( diagnostic( @@ -678,6 +784,7 @@ export function parseEventRegistry(sourceText, file = DEFAULT_REGISTRY_PATH) { name, file: normalizePath(file), platforms: platforms ?? [], + propertyKeys, ...sourceLocation(sourceFile, property), }); } @@ -698,7 +805,9 @@ function canonicalModuleName(moduleName, file) { } function captureBindings(sourceFile, file) { - const identifiers = new Map(); + const identifiers = new Map( + LOCAL_CAPTURE_FUNCTIONS.get(normalizePath(file)) ?? [], + ); const namespaces = new Map(); for (const statement of sourceFile.statements) { if (!ts.isImportDeclaration(statement)) continue; @@ -859,10 +968,34 @@ function forOfHelperBinding(identifier, call, bindings) { return false; } +function forwardedCaptureParameter(identifier, call, bindings) { + if (!ts.isIdentifier(identifier)) return false; + let current = call.parent; + while (current) { + if ( + ts.isFunctionDeclaration(current) && + current.name && + bindings.identifiers.has(current.name.text) + ) { + const descriptor = bindings.identifiers.get(current.name.text); + const argumentIndex = descriptor.argumentIndex ?? 0; + const parameter = current.parameters[argumentIndex]; + return ( + parameter !== undefined && + ts.isIdentifier(parameter.name) && + parameter.name.text === identifier.text + ); + } + current = current.parent; + } + return false; +} + export function analyzeTypeScriptSource({ sourceText, file, registeredEvents, + eventProperties = new Map(), }) { const scriptKind = file.endsWith(".tsx") ? ts.ScriptKind.TSX @@ -906,10 +1039,50 @@ export function analyzeTypeScriptSource({ ); } }; + const validatePropertyKeys = (eventName, expression) => { + if (!expression) return; + let value = unwrapExpression(expression); + if (ts.isIdentifier(value)) { + const initializer = initializers.get(value.text); + if (!initializer) return; + value = unwrapExpression(initializer); + } + if (!ts.isObjectLiteralExpression(value)) return; + const allowed = eventProperties.get(eventName); + if (!allowed) return; + for (const property of value.properties) { + if (ts.isSpreadAssignment(property)) continue; + const key = propertyNameText(property.name); + if (!key) { + diagnostics.push( + diagnostic( + "dynamic-property-key", + file, + `Analytics event ${eventName} property keys must be static`, + sourceLocation(sourceFile, property), + ), + ); + continue; + } + if (allowed.has(key)) continue; + diagnostics.push( + diagnostic( + "invalid-property-key", + file, + `Analytics event ${eventName} does not declare property ${key}`, + sourceLocation(sourceFile, property), + ), + ); + } + }; const visit = (node) => { if (ts.isCallExpression(node)) { const descriptor = callDescriptor(node.expression, bindings); - if (descriptor?.kind === "helper") { + if (descriptor?.kind === "helper-emissions") { + for (const emission of descriptor.emissions) { + registerEmission(emission.eventName, node, emission.platforms); + } + } else if (descriptor?.kind === "helper") { registerEmission( descriptor.eventName, node, @@ -934,6 +1107,10 @@ export function analyzeTypeScriptSource({ const result = staticEventName(argument); if (result.kind === "static") { registerEmission(result.eventName, argument); + validatePropertyKeys( + result.eventName, + node.arguments[(descriptor.argumentIndex ?? 0) + 1], + ); } else { diagnostics.push( diagnostic( @@ -948,7 +1125,7 @@ export function analyzeTypeScriptSource({ } } } else if (descriptor?.kind === "object") { - const argument = node.arguments[0]; + const argument = node.arguments[descriptor.argumentIndex ?? 0]; if (!argument) { diagnostics.push( diagnostic( @@ -967,13 +1144,25 @@ export function analyzeTypeScriptSource({ result.platforms ?? eventPlatforms(argument, bindings, initializers), ); + const eventObject = unwrapExpression(argument); + if (ts.isObjectLiteralExpression(eventObject)) { + validatePropertyKeys( + result.eventName, + objectProperty(eventObject, "properties")?.initializer, + ); + } } else if (result.kind === "static-set") { for (const eventName of result.eventNames) { registerEmission(eventName, result.node, result.platforms); } } else if ( result.kind !== "non-object" || - !forOfHelperBinding(unwrapExpression(argument), node, bindings) + (!forOfHelperBinding(unwrapExpression(argument), node, bindings) && + !forwardedCaptureParameter( + unwrapExpression(argument), + node, + bindings, + )) ) { const messages = { "non-object": @@ -1194,6 +1383,7 @@ export function analyzeRustNativeContract({ sourceText, file = DEFAULT_NATIVE_PATH, registeredEvents, + eventProperties = new Map(), }) { const tokens = tokenizeRust(sourceText); const diagnostics = []; @@ -1251,6 +1441,37 @@ export function analyzeRustNativeContract({ continue; } const eventName = names[0].value; + const propertyKeys = []; + for (let index = start; index < end - 4; index += 1) { + if (!isTokenSequence(body, index, ["data", ".", "set", "("])) { + continue; + } + const keyToken = body[index + 4]; + if (keyToken?.type !== "string") { + diagnostics.push( + diagnostic( + "native-dynamic-property-key", + file, + `Native event ${eventName} property keys must be static`, + keyToken ?? variantToken, + ), + ); + continue; + } + propertyKeys.push(keyToken.value); + } + const allowedProperties = eventProperties.get(eventName); + for (const propertyKey of propertyKeys) { + if (!allowedProperties || allowedProperties.has(propertyKey)) continue; + diagnostics.push( + diagnostic( + "native-invalid-property-key", + file, + `Native event ${eventName} does not declare property ${propertyKey}`, + names[0], + ), + ); + } const expectedName = pascalToSnake(variant); if (eventName !== expectedName) { diagnostics.push( @@ -1280,6 +1501,7 @@ export function analyzeRustNativeContract({ file: normalizePath(file), line: names[0].line, column: names[0].column, + propertyKeys, }); if (!registeredEvents.has(eventName)) { diagnostics.push( @@ -1477,6 +1699,19 @@ export function findMissingEmitters(registryEvents, emissions) { const emittedPlatforms = new Set( eventEmissions.flatMap((emission) => emission.platforms ?? []), ); + for (const emission of eventEmissions) { + for (const platform of emission.platforms ?? []) { + if (event.platforms.includes(platform)) continue; + diagnostics.push( + diagnostic( + "emitter-platform-not-declared", + emission.file, + `Analytics event ${event.name} is emitted on undeclared platform ${platform}`, + emission, + ), + ); + } + } for (const platform of event.platforms) { if (emittedPlatforms.has(platform)) continue; diagnostics.push( @@ -1513,6 +1748,9 @@ export function runEventContractCheck({ ); const registry = parseEventRegistry(registrySource, registryPath); const registeredEvents = new Set(registry.events.keys()); + const eventProperties = new Map( + [...registry.events].map(([name, event]) => [name, event.propertyKeys]), + ); const diagnostics = [...registry.diagnostics]; const emissions = []; const files = sourceFiles(absoluteRoot); @@ -1529,6 +1767,7 @@ export function runEventContractCheck({ sourceText, file: relativePath, registeredEvents, + eventProperties, }); diagnostics.push(...result.diagnostics); emissions.push(...result.emissions); @@ -1542,6 +1781,7 @@ export function runEventContractCheck({ sourceText: nativeSource, file: nativePath, registeredEvents, + eventProperties, }); diagnostics.push(...native.diagnostics); const usedNativeVariants = new Map(); diff --git a/scripts/analytics/tests/event-contract.test.js b/scripts/analytics/tests/event-contract.test.js index 81986d1b944..0769f5c6dc6 100644 --- a/scripts/analytics/tests/event-contract.test.js +++ b/scripts/analytics/tests/event-contract.test.js @@ -178,6 +178,28 @@ trackEvent(\`tool_\${action}\`); ); }); +test("rejects undeclared and dynamic literal property keys", () => { + const result = analyzeTypeScriptSource({ + file: "caller.ts", + registeredEvents, + eventProperties: new Map([ + ["page_view", new Set(["hostname", "is_session_entry"])], + ]), + sourceText: ` +import { trackEvent } from "@/app/utils/analytics"; +trackEvent("page_view", { + hostname: "cap.so", + invalid: true, + [dynamicKey]: "hidden", +}); +`, + }); + assert.deepEqual( + result.diagnostics.map((entry) => entry.code), + ["invalid-property-key", "dynamic-property-key"], + ); +}); + test("finds inline server event objects and helper-backed emitters", () => { const server = analyzeTypeScriptSource({ file: "apps/web/app/api/webhooks/route.ts", @@ -215,6 +237,104 @@ trackToolInteraction({ tool: "trimmer", action: "loaded" }); assert.deepEqual([...server.diagnostics, ...browser.diagnostics], []); }); +test("finds transactional outbox events after the transaction argument", () => { + const result = analyzeTypeScriptSource({ + file: "apps/web/actions/organization/invite.ts", + registeredEvents: new Set(["organization_invite_sent"]), + sourceText: ` +import { persistProductAnalyticsEvent } from "@/lib/analytics/product-event-outbox"; +await persistProductAnalyticsEvent(tx, { + eventId: "invite:1", + eventName: "organization_invite_sent", + platform: "server", +}); +`, + }); + assert.deepEqual(result.diagnostics, []); + assert.deepEqual( + result.emissions.map(({ eventName, platforms }) => ({ + eventName, + platforms, + })), + [{ eventName: "organization_invite_sent", platforms: ["server"] }], + ); +}); + +test("finds events passed through the bounded Loom workflow wrapper", () => { + const result = analyzeTypeScriptSource({ + file: "apps/web/workflows/import-loom-video.ts", + registeredEvents: new Set(["loom_import_started", "loom_import_completed"]), + sourceText: ` +import { persistProductAnalyticsEvent } from "@/lib/analytics/product-event-outbox"; +async function queueLoomAnalyticsEvent(event: ServerProductEvent) { + return queueDurableServerProductEvent(event); +} +async function saveMetadataAndComplete( + videoId: string, + operationId: string | undefined, + metadata: object, + completedEvent: ServerProductEvent, +) { + await persistProductAnalyticsEvent(tx, completedEvent); +} +await queueLoomAnalyticsEvent({ + eventId: "loom:1", + eventName: "loom_import_started", + platform: "server", +}); +await saveMetadataAndComplete("video-1", undefined, {}, { + eventId: "loom:2", + eventName: "loom_import_completed", + platform: "server", +}); +`, + }); + assert.deepEqual(result.diagnostics, []); + assert.deepEqual( + result.emissions.map(({ eventName, platforms }) => ({ + eventName, + platforms, + })), + [ + { eventName: "loom_import_started", platforms: ["server"] }, + { eventName: "loom_import_completed", platforms: ["server"] }, + ], + ); +}); + +test("rejects emitters on platforms outside the registry contract", () => { + const diagnostics = findMissingEmitters( + new Map([ + [ + "page_view", + { + name: "page_view", + file: "event-registry.ts", + line: 1, + column: 1, + platforms: ["web"], + }, + ], + ]), + [ + { + eventName: "page_view", + file: "desktop.ts", + line: 4, + column: 2, + platforms: ["desktop"], + }, + ], + ); + assert.deepEqual( + diagnostics.map((entry) => entry.code), + [ + "emitter-platform-not-declared", + "registry-event-platform-without-emitter", + ], + ); +}); + test("does not infer a platform from the file when an inline platform is dynamic", () => { const result = analyzeTypeScriptSource({ file: "apps/web/app/api/checkout/route.ts", @@ -459,6 +579,34 @@ fn is_core_product_event(name: &str) -> bool { ]); }); +test("native Rust property keys must match the registry", () => { + const sourceText = ` +enum ProductAnalyticsEvent { RecordingStarted } +fn event_data(event: ProductAnalyticsEvent) { + match event { + ProductAnalyticsEvent::RecordingStarted => { + let mut data = EventData::new("recording_started"); + data.set("mode", "studio"); + data.set("raw_error", "private"); + data + } + } +} +fn is_core_product_event(name: &str) -> bool { + matches!(name, "recording_started") +} +`; + const result = analyzeRustNativeContract({ + sourceText, + registeredEvents: new Set(["recording_started"]), + eventProperties: new Map([["recording_started", new Set(["mode"])]]), + }); + assert.deepEqual( + result.diagnostics.map((entry) => entry.code), + ["native-invalid-property-key"], + ); +}); + test("registry entries without production emitters fail", () => { const diagnostics = findMissingEmitters(registry.events, [ { eventName: "page_view", platforms: ["web"] }, From 5b45cd18dc05221f9fd54a0f8e7c1bedaaff16a4 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:40:03 +0100 Subject: [PATCH 082/110] feat: make analytics delivery and erasure durable --- .../e2e/product-analytics-mysql-e2e.test.ts | 606 ++ .../__tests__/unit/agent-api-contract.test.ts | 3 +- .../deliver-product-analytics-outbox.test.ts | 133 + .../unit/developer-credits-webhook.test.ts | 18 +- .../unit/loom-product-analytics.test.ts | 65 + .../unit/organization-invite-delivery.test.ts | 49 + .../product-analytics-authentication.test.ts | 105 + .../unit/product-analytics-erasure.test.ts | 103 +- .../unit/product-analytics-refresh.test.ts | 174 + .../unit/product-analytics-server.test.ts | 29 +- .../stripe-analytics-reconciliation.test.ts | 251 +- .../subscription-analytics-webhook.test.ts | 13 +- .../unit/update-seat-quantity.test.ts | 46 +- apps/web/actions/organization/send-invites.ts | 113 +- .../organization/update-seat-quantity.ts | 104 +- apps/web/app/api/analytics/track/route.ts | 24 +- apps/web/app/api/auth/[...nextauth]/route.ts | 8 +- .../drain-product-analytics-outbox/route.ts | 33 + .../route.ts | 33 + .../cron/refresh-product-analytics/route.ts | 29 + apps/web/app/api/desktop/[...route]/video.ts | 17 +- apps/web/app/api/events/route.ts | 17 +- apps/web/app/api/invite/accept/route.ts | 74 +- apps/web/app/api/v1/[...route]/route.ts | 73 +- apps/web/app/api/webhooks/stripe/route.ts | 15 +- .../lib/analytics/authentication-events.ts | 58 + apps/web/lib/analytics/business-events.ts | 56 +- .../product-analytics-refresh-state.ts | 115 + .../analytics/product-event-outbox-state.ts | 124 + .../web/lib/analytics/product-event-outbox.ts | 646 ++ apps/web/lib/analytics/server-event.ts | 4 +- apps/web/lib/analytics/server.ts | 13 +- .../lib/analytics/stripe-business-events.ts | 67 +- apps/web/lib/analytics/video-platform.ts | 24 + apps/web/lib/organization-invite-delivery.ts | 306 + apps/web/proxy.ts | 8 + apps/web/vercel.json | 12 + .../deliver-product-analytics-event.ts | 118 +- .../drain-product-analytics-outbox.ts | 12 + apps/web/workflows/import-loom-video.ts | 134 +- .../product-analytics-delivery-workflow.ts | 247 + .../workflows/reconcile-product-analytics.ts | 733 +- .../recover-organization-invite-delivery.ts | 12 + .../workflows/refresh-product-analytics.ts | 180 + .../database/migrations/0040_chief_skreet.sql | 110 + .../migrations/0041_complex_outlaw_kid.sql | 16 + .../migrations/0042_lying_sharon_ventura.sql | 16 + .../migrations/meta/0038_snapshot.json | 8270 ++++++++-------- .../migrations/meta/0039_snapshot.json | 8496 ++++++++--------- .../migrations/meta/0040_snapshot.json | 4740 +++++++++ .../migrations/meta/0041_snapshot.json | 4834 ++++++++++ .../migrations/meta/0042_snapshot.json | 4935 ++++++++++ .../database/migrations/meta/_journal.json | 591 +- packages/database/schema.ts | 237 + packages/database/types/metadata.ts | 3 + .../web-backend/src/Organisations/index.ts | 30 +- .../web-backend/src/ProductAnalytics/index.ts | 289 +- .../ProductAnalyticsErasureLeaseRepo.ts | 517 +- packages/web-backend/src/Tinybird/index.ts | 155 +- packages/web-backend/src/index.ts | 3 + packages/web-domain/src/Agent.ts | 1 + 61 files changed, 28439 insertions(+), 9808 deletions(-) create mode 100644 apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts create mode 100644 apps/web/__tests__/unit/deliver-product-analytics-outbox.test.ts create mode 100644 apps/web/__tests__/unit/loom-product-analytics.test.ts create mode 100644 apps/web/__tests__/unit/organization-invite-delivery.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-authentication.test.ts create mode 100644 apps/web/__tests__/unit/product-analytics-refresh.test.ts create mode 100644 apps/web/app/api/cron/drain-product-analytics-outbox/route.ts create mode 100644 apps/web/app/api/cron/recover-organization-invite-delivery/route.ts create mode 100644 apps/web/app/api/cron/refresh-product-analytics/route.ts create mode 100644 apps/web/lib/analytics/authentication-events.ts create mode 100644 apps/web/lib/analytics/product-analytics-refresh-state.ts create mode 100644 apps/web/lib/analytics/product-event-outbox-state.ts create mode 100644 apps/web/lib/analytics/product-event-outbox.ts create mode 100644 apps/web/lib/analytics/video-platform.ts create mode 100644 apps/web/lib/organization-invite-delivery.ts create mode 100644 apps/web/workflows/drain-product-analytics-outbox.ts create mode 100644 apps/web/workflows/product-analytics-delivery-workflow.ts create mode 100644 apps/web/workflows/recover-organization-invite-delivery.ts create mode 100644 apps/web/workflows/refresh-product-analytics.ts create mode 100644 packages/database/migrations/0040_chief_skreet.sql create mode 100644 packages/database/migrations/0041_complex_outlaw_kid.sql create mode 100644 packages/database/migrations/0042_lying_sharon_ventura.sql create mode 100644 packages/database/migrations/meta/0040_snapshot.json create mode 100644 packages/database/migrations/meta/0041_snapshot.json create mode 100644 packages/database/migrations/meta/0042_snapshot.json diff --git a/apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts b/apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts new file mode 100644 index 00000000000..3c1cd69d755 --- /dev/null +++ b/apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts @@ -0,0 +1,606 @@ +import type { RowDataPacket } from "mysql2"; +import mysql, { type Connection } from "mysql2/promise"; +import { + afterAll, + afterEach, + beforeAll, + describe, + expect, + it, + vi, +} from "vitest"; + +vi.mock("server-only", () => ({})); +vi.mock("workflow/api", () => ({ + start: vi.fn(async () => ({ runId: "analytics-mysql-e2e" })), +})); + +const inviteMocks = vi.hoisted(() => ({ + environment: { + PRODUCT_ANALYTICS_TINYBIRD_HOST: "https://staging.tinybird.test", + PRODUCT_ANALYTICS_TINYBIRD_TOKEN: "staging-ingest-token", + RESEND_API_KEY: "staging-resend-key", + WEB_URL: "https://staging.cap.test", + }, + sendEmail: vi.fn(), +})); + +vi.mock("@cap/database/emails/config", () => ({ + sendEmail: inviteMocks.sendEmail, +})); +vi.mock("@cap/database/emails/organization-invite", () => ({ + OrganizationInvite: vi.fn(() => null), +})); +vi.mock("@cap/env", async (importOriginal) => ({ + ...(await importOriginal()), + serverEnv: () => inviteMocks.environment, +})); + +const enabled = process.env.CAP_PRODUCT_ANALYTICS_MYSQL_E2E === "1"; +const analyticsMysqlE2e = enabled ? describe.sequential : describe.skip; +const databaseName = + process.env.CAP_PRODUCT_ANALYTICS_MYSQL_DATABASE ?? + `cap_product_analytics_e2e_${process.pid}`; +const adminUrl = + process.env.CAP_PRODUCT_ANALYTICS_MYSQL_ADMIN_URL ?? + "mysql://root@127.0.0.1:3306"; +const databaseUrl = `${adminUrl}/${databaseName}`; + +let admin: Connection; + +afterEach(() => { + vi.unstubAllGlobals(); +}); + +const runRepo = async ( + operation: ( + repo: import("@cap/web-backend").ProductAnalyticsErasureLeaseStore, + ) => import("effect").Effect.Effect, +) => { + const { Database, ProductAnalyticsErasureLeaseRepo } = await import( + "@cap/web-backend" + ); + const { Effect } = await import("effect"); + return Effect.runPromise( + Effect.gen(function* () { + const repo = yield* ProductAnalyticsErasureLeaseRepo; + return yield* operation(repo); + }).pipe( + Effect.provide(ProductAnalyticsErasureLeaseRepo.Default), + Effect.provide(Database.Default), + ), + ); +}; + +const appendCollectorRows = async ( + rows: readonly import("@cap/analytics").ProductEventRow[], +) => { + const { Database, ProductAnalytics } = await import("@cap/web-backend"); + const { Effect } = await import("effect"); + return Effect.runPromise( + Effect.gen(function* () { + const analytics = yield* ProductAnalytics; + return yield* analytics.appendWithIdentityFence(rows); + }).pipe( + Effect.provide(ProductAnalytics.Default), + Effect.provide(Database.Default), + ), + ); +}; + +const appendCollectorRowsEither = async ( + rows: readonly import("@cap/analytics").ProductEventRow[], +) => { + const { Database, ProductAnalytics } = await import("@cap/web-backend"); + const { Effect } = await import("effect"); + return Effect.runPromise( + Effect.gen(function* () { + const analytics = yield* ProductAnalytics; + return yield* analytics.appendWithIdentityFence(rows); + }).pipe( + Effect.either, + Effect.provide(ProductAnalytics.Default), + Effect.provide(Database.Default), + ), + ); +}; + +beforeAll(async () => { + if (!enabled) return; + const parsed = new URL(adminUrl); + if ( + parsed.hostname !== "127.0.0.1" || + parsed.port !== "3306" || + !/^[a-z0-9_]+$/.test(databaseName) || + !databaseName.startsWith("cap_product_analytics_e2e_") + ) { + throw new Error("Analytics MySQL E2E requires an isolated local database"); + } + admin = await mysql.createConnection(adminUrl); + await admin.query(`CREATE DATABASE \`${databaseName}\``); + process.env.DATABASE_URL = databaseUrl; + const connection = await mysql.createConnection(databaseUrl); + await connection.query(` + CREATE TABLE product_analytics_erasure_leases ( + name varchar(64) NOT NULL PRIMARY KEY, + ownerId varchar(64), requestId varchar(64), + fencingToken bigint unsigned NOT NULL DEFAULT 0, + leaseExpiresAt timestamp NULL, phase varchar(32) NOT NULL DEFAULT 'idle', + pausedPipes json, userId varchar(255), organizationId varchar(255), + attemptCount int NOT NULL DEFAULT 0, lastErrorCode varchar(64), + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) + `); + await connection.query(` + CREATE TABLE product_analytics_identity_state ( + identityHash varchar(64) NOT NULL PRIMARY KEY, + identityKind varchar(16) NOT NULL, blockedAt timestamp NULL, + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) + `); + await connection.query(` + CREATE TABLE product_analytics_identity_links ( + anonymousIdentityHash varchar(64) NOT NULL, + userIdentityHash varchar(64) NOT NULL, + organizationIdentityHash varchar(64), anonymousId varchar(255) NOT NULL, + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (anonymousIdentityHash, userIdentityHash) + ) + `); + await connection.query(` + CREATE TABLE product_analytics_event_receipts ( + eventIdHash varchar(64) NOT NULL PRIMARY KEY, payloadHash varchar(32) NOT NULL, + anonymousIdentityHash varchar(64), userIdentityHash varchar(64), + organizationIdentityHash varchar(64), conflictCount int NOT NULL DEFAULT 0, + firstSeenAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + lastSeenAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + retainUntil timestamp NOT NULL + ) + `); + await connection.query(` + CREATE TABLE product_analytics_outbox ( + eventId varchar(128) NOT NULL PRIMARY KEY, deliveryKey varchar(36) NOT NULL UNIQUE, + payloadHash varchar(32) NOT NULL, eventName varchar(64) NOT NULL, + payloadKind varchar(32) NOT NULL DEFAULT 'product_event_row_v1', payload json NOT NULL, + anonymousId varchar(255), userId varchar(255), organizationId varchar(255), + status varchar(32) NOT NULL DEFAULT 'pending', attemptCount int NOT NULL DEFAULT 0, + nextAttemptAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, leaseOwnerId varchar(64), + leaseExpiresAt timestamp NULL, workflowRunId varchar(128), + payloadConflict boolean NOT NULL DEFAULT false, lastErrorCode varchar(64), + deliveredAt timestamp NULL, deadLetteredAt timestamp NULL, + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) + `); + await connection.query(` + CREATE TABLE product_analytics_ingestion_leases ( + id varchar(36) NOT NULL PRIMARY KEY, fencingToken bigint unsigned NOT NULL, + expiresAt timestamp NOT NULL, + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `); + await connection.query(` + CREATE TABLE product_analytics_refresh_leases ( + name varchar(64) NOT NULL PRIMARY KEY, ownerId varchar(36), + generation bigint unsigned NOT NULL DEFAULT 0, sourceCutoff timestamp NULL, + leaseExpiresAt timestamp NULL, status varchar(32) NOT NULL DEFAULT 'idle', + lastCompletedAt timestamp NULL, lastErrorCode varchar(64), + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + KEY expiry_idx (leaseExpiresAt), KEY status_idx (status) + ) + `); + await connection.query(` + CREATE TABLE product_analytics_reconciliation_failures ( + sourceHash varchar(64) NOT NULL PRIMARY KEY, sourceType varchar(32) NOT NULL, + errorCode varchar(64) NOT NULL, attemptCount int NOT NULL DEFAULT 1, + firstSeenAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + lastSeenAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + KEY source_type_idx (sourceType, lastSeenAt) + ) + `); + await connection.query(` + CREATE TABLE product_analytics_erasure_requests ( + id varchar(36) NOT NULL PRIMARY KEY, scopeHash varchar(64) NOT NULL UNIQUE, + userId varchar(255), organizationId varchar(255), + status varchar(32) NOT NULL DEFAULT 'pending', attemptCount int NOT NULL DEFAULT 0, + nextAttemptAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, leaseOwnerId varchar(36), + leaseExpiresAt timestamp NULL, lastErrorCode varchar(64), + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP + ) + `); + await connection.query(` + CREATE TABLE organizations ( + id varchar(21) NOT NULL PRIMARY KEY, name varchar(255) NOT NULL, + tombstoneAt timestamp NULL + ) + `); + await connection.query(` + CREATE TABLE organization_invites ( + id varchar(21) NOT NULL PRIMARY KEY, organizationId varchar(21) NOT NULL, + invitedEmail varchar(255) NOT NULL, invitedEmailNormalized varchar(255), + invitedByUserId varchar(21) NOT NULL, role varchar(255) NOT NULL, + status varchar(255) NOT NULL DEFAULT 'pending', + emailDeliveryState varchar(32) NOT NULL DEFAULT 'legacy', + emailDeliveryAttemptCount int NOT NULL DEFAULT 0, + emailDeliveryNextAttemptAt timestamp NULL, emailDeliveryErrorCode varchar(64), + emailDeliveryLeaseOwnerId varchar(36), emailDeliveryLeaseExpiresAt timestamp NULL, + emailProviderMessageId varchar(255), emailSentAt timestamp NULL, + createdAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + updatedAt timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + expiresAt timestamp NULL, + UNIQUE KEY normalized_email_idx (organizationId, invitedEmailNormalized) + ) + `); + await connection.end(); +}); + +afterAll(async () => { + if (!enabled || !admin) return; + if (!databaseName.startsWith("cap_product_analytics_e2e_")) { + throw new Error("Refusing to drop an unscoped database"); + } + await admin.query(`DROP DATABASE \`${databaseName}\``); + await admin.end(); +}); + +analyticsMysqlE2e("product analytics MySQL concurrency", () => { + it("reuses the global lease and increments its fencing token", async () => { + const first = await runRepo((repo) => repo.claimNew({ userId: "user-a" })); + expect(first).not.toBeNull(); + if (!first) return; + expect(await runRepo((repo) => repo.complete(first))).toBe(true); + const second = await runRepo((repo) => repo.claimNew({ userId: "user-b" })); + expect(second?.requestId).not.toBe(first.requestId); + expect(second?.fencingToken).toBe(first.fencingToken + 1); + if (second) + expect(await runRepo((repo) => repo.complete(second))).toBe(true); + }); + + it("allows exactly one concurrent owner", async () => { + const claims = await Promise.all( + Array.from({ length: 8 }, (_, index) => + runRepo((repo) => repo.claimNew({ userId: `concurrent-${index}` })), + ), + ); + const winners = claims.filter((claim) => claim !== null); + expect(winners).toHaveLength(1); + const winner = winners[0]; + if (winner) { + expect(await runRepo((repo) => repo.complete(winner))).toBe(true); + } + }); + + it("admits concurrent ingestion without owning the erasure row lock", async () => { + const { + acquireProductAnalyticsIngestionLease, + releaseProductAnalyticsIngestionLease, + } = await import("@/lib/analytics/product-event-outbox-state"); + const leases = await Promise.all( + Array.from({ length: 32 }, () => acquireProductAnalyticsIngestionLease()), + ); + expect(leases.every((lease) => typeof lease === "string")).toBe(true); + expect(new Set(leases).size).toBe(32); + await Promise.all( + leases.flatMap((lease) => + lease ? [releaseProductAnalyticsIngestionLease(lease)] : [], + ), + ); + }); + + it("allows one refresh owner and fences refresh against erasure", async () => { + const { + acquireProductAnalyticsRefreshLease, + releaseProductAnalyticsRefreshLease, + } = await import("@/lib/analytics/product-analytics-refresh-state"); + const cutoff = new Date("2026-07-31T12:00:00.000Z"); + const claims = await Promise.all( + Array.from({ length: 8 }, () => + acquireProductAnalyticsRefreshLease(cutoff), + ), + ); + const winners = claims.filter((claim): claim is NonNullable => + Boolean(claim), + ); + expect(winners).toHaveLength(1); + expect(winners[0]?.sourceCutoff).toBe(cutoff.toISOString()); + if (!winners[0]) return; + await releaseProductAnalyticsRefreshLease(winners[0].ownerId); + + const verifier = await mysql.createConnection(databaseUrl); + await verifier.query( + "UPDATE product_analytics_erasure_leases SET phase = 'claimed' WHERE name = 'global'", + ); + await expect( + acquireProductAnalyticsRefreshLease(cutoff), + ).resolves.toBeUndefined(); + await verifier.query( + "UPDATE product_analytics_erasure_leases SET phase = 'idle' WHERE name = 'global'", + ); + const [refreshRows] = await verifier.query< + Array< + { + generation: number; + ownerId: string | null; + status: string; + } & RowDataPacket + > + >( + "SELECT ownerId, generation, status FROM product_analytics_refresh_leases WHERE name = 'global'", + ); + const [ingestionRows] = await verifier.query>( + "SELECT id FROM product_analytics_ingestion_leases", + ); + await verifier.end(); + expect(refreshRows[0]).toMatchObject({ + generation: 1, + ownerId: null, + status: "idle", + }); + expect(ingestionRows).toHaveLength(0); + }); + + it("forwards exact retries and rejects event ID payload conflicts", async () => { + const { createProductEventRows, productAnalyticsEventIdHash } = + await import("@cap/analytics"); + const appendRequests: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_input: string | URL | Request, init?: RequestInit) => { + appendRequests.push(String(init?.body ?? "")); + return new Response(null, { status: 202 }); + }), + ); + const [original] = createProductEventRows( + [ + { + anonymousId: "collector-anonymous", + eventId: "collector-retry-alpha", + eventName: "page_view", + occurredAt: "2026-07-31T12:00:00.000Z", + platform: "web", + properties: { + hostname: "cap.test", + is_session_entry: true, + session_started_at: "2026-07-31T12:00:00.000Z", + }, + }, + ], + { + hostname: "cap.test", + receivedAt: "2026-07-31T12:00:01.000Z", + source: "client", + }, + ); + if (!original) throw new Error("Expected a collector fixture row"); + await Promise.all([ + appendCollectorRows([original]), + appendCollectorRows([original]), + ]); + expect(appendRequests).toHaveLength(2); + + const conflicting = { + ...original, + pathname: "/different", + payload_hash: "f".repeat(32), + }; + const conflict = await appendCollectorRowsEither([conflicting]); + expect(conflict._tag).toBe("Left"); + if (conflict._tag === "Left") { + expect(conflict.left).toMatchObject({ retryable: false, status: 409 }); + } + expect(appendRequests).toHaveLength(2); + const verifier = await mysql.createConnection(databaseUrl); + const [receipts] = await verifier.query< + Array< + { + conflictCount: number; + payloadHash: string; + } & RowDataPacket + > + >( + "SELECT payloadHash, conflictCount FROM product_analytics_event_receipts WHERE eventIdHash = ?", + [productAnalyticsEventIdHash(original.event_id)], + ); + expect(receipts[0]).toMatchObject({ + conflictCount: 1, + payloadHash: original.payload_hash, + }); + await verifier.query( + "DELETE FROM product_analytics_event_receipts WHERE eventIdHash = ?", + [productAnalyticsEventIdHash(original.event_id)], + ); + await verifier.end(); + }); + + it("preserves another user's event when an anonymous cookie is shared", async () => { + const { db } = await import("@cap/database"); + const { persistProductAnalyticsEvent } = await import( + "@/lib/analytics/product-event-outbox" + ); + const anonymousId = "shared-cookie-alpha"; + for (const userId of ["shared-user-a", "shared-user-b"]) { + await db().transaction((tx) => + persistProductAnalyticsEvent(tx, { + eventId: `${userId}:signup`, + eventName: "user_signed_up", + occurredAt: new Date().toISOString(), + anonymousId, + platform: "web", + userId, + organizationId: `${userId}:org`, + }), + ); + } + const aliases = await runRepo((repo) => + repo.discardPendingEvents({ userId: "shared-user-a" }, [anonymousId]), + ); + expect(aliases).toEqual([]); + const verifier = await mysql.createConnection(databaseUrl); + const [remaining] = await verifier.query< + Array<{ userId: string } & RowDataPacket> + >("SELECT userId FROM product_analytics_outbox"); + await verifier.end(); + expect(remaining.map(({ userId }) => userId)).toEqual(["shared-user-b"]); + }); + + it("does not spread a deleted user's tombstone to a live organization", async () => { + const { db } = await import("@cap/database"); + const { productAnalyticsIdentityHash } = await import("@cap/web-backend"); + const { persistProductAnalyticsEvent } = await import( + "@/lib/analytics/product-event-outbox" + ); + await runRepo((repo) => + repo.discardPendingEvents({ userId: "deleted-user" }), + ); + const suppressed = await db().transaction((tx) => + persistProductAnalyticsEvent(tx, { + eventId: "deleted-user:late", + eventName: "user_signed_up", + occurredAt: new Date().toISOString(), + anonymousId: "deleted-user:new-alias", + platform: "web", + userId: "deleted-user", + organizationId: "live-organization", + }), + ); + expect(suppressed.status).toBe("suppressed"); + const accepted = await db().transaction((tx) => + persistProductAnalyticsEvent(tx, { + eventId: "live-user:signup", + eventName: "user_signed_up", + occurredAt: new Date().toISOString(), + anonymousId: "live-user:alias", + platform: "web", + userId: "live-user", + organizationId: "live-organization", + }), + ); + expect(accepted.status).not.toBe("suppressed"); + const verifier = await mysql.createConnection(databaseUrl); + const [organizationState] = await verifier.query< + Array<{ blockedAt: Date | null } & RowDataPacket> + >( + "SELECT blockedAt FROM product_analytics_identity_state WHERE identityHash = ?", + [productAnalyticsIdentityHash("organization", "live-organization")], + ); + await verifier.end(); + expect(organizationState[0]?.blockedAt).toBeNull(); + }); + + it("keeps first-payload ownership after delivery payload cleanup", async () => { + const { db } = await import("@cap/database"); + const { getProductAnalyticsOutboxHealth, persistProductAnalyticsEvent } = + await import("@/lib/analytics/product-event-outbox"); + const eventId = "receipt-first-write-alpha"; + const first = await db().transaction((tx) => + persistProductAnalyticsEvent(tx, { + eventId, + eventName: "user_signed_up", + occurredAt: "2026-07-01T12:00:00.000Z", + anonymousId: "receipt-alias", + platform: "web", + userId: "receipt-user", + organizationId: "receipt-org", + }), + ); + const verifier = await mysql.createConnection(databaseUrl); + await verifier.query( + "DELETE FROM product_analytics_outbox WHERE eventId = ?", + [eventId], + ); + await verifier.query( + "UPDATE product_analytics_event_receipts SET firstSeenAt = DATE_SUB(NOW(), INTERVAL 32 DAY), lastSeenAt = DATE_SUB(NOW(), INTERVAL 32 DAY)", + ); + await verifier.end(); + const replay = await db().transaction((tx) => + persistProductAnalyticsEvent(tx, { + eventId, + eventName: "user_signed_up", + occurredAt: "2026-07-01T12:00:01.000Z", + anonymousId: "receipt-alias", + platform: "web", + userId: "receipt-user", + organizationId: "receipt-org", + }), + ); + expect(first.payloadConflict).toBe(false); + expect(replay).toMatchObject({ payloadConflict: true, status: "conflict" }); + const health = await getProductAnalyticsOutboxHealth(); + expect(health.healthy).toBe(false); + expect(health.receiptPayloadConflictEvents).toBe(1); + expect(health.receiptPayloadConflictAttempts).toBe(1); + }); + + it("releases invite locks before provider I/O and fails closed without email configuration", async () => { + const verifier = await mysql.createConnection(databaseUrl); + await verifier.query( + "INSERT INTO organizations (id, name) VALUES ('orginvitealpha', 'Invite Org')", + ); + await verifier.query(` + INSERT INTO organization_invites ( + id, organizationId, invitedEmail, invitedEmailNormalized, + invitedByUserId, role, emailDeliveryState, emailDeliveryNextAttemptAt + ) VALUES ( + 'invitealpha', 'orginvitealpha', 'invitee@example.test', + 'invitee@example.test', 'inviteralpha', 'member', 'pending', NOW() + ) + `); + let releaseProvider: ((value: unknown) => void) | undefined; + inviteMocks.sendEmail.mockImplementationOnce( + () => + new Promise((resolve) => { + releaseProvider = resolve; + }), + ); + const { deliverOrganizationInvite } = await import( + "@/lib/organization-invite-delivery" + ); + const delivery = deliverOrganizationInvite("invitealpha"); + while (!releaseProvider) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + await verifier.beginTransaction(); + await verifier.query("SET SESSION innodb_lock_wait_timeout = 1"); + await verifier.query( + "SELECT id FROM organization_invites WHERE id = 'invitealpha' FOR UPDATE", + ); + await verifier.rollback(); + releaseProvider({ data: { id: "provider-message-alpha" }, error: null }); + await expect(delivery).resolves.toEqual({ status: "sent" }); + + await verifier.query(` + INSERT INTO organization_invites ( + id, organizationId, invitedEmail, invitedEmailNormalized, + invitedByUserId, role, emailDeliveryState, emailDeliveryNextAttemptAt + ) VALUES ( + 'invitebeta', 'orginvitealpha', 'second@example.test', + 'second@example.test', 'inviteralpha', 'member', 'pending', NOW() + ) + `); + inviteMocks.environment.RESEND_API_KEY = ""; + await expect(deliverOrganizationInvite("invitebeta")).resolves.toEqual({ + status: "deferred", + }); + const [deferred] = await verifier.query< + Array< + { + emailDeliveryState: string; + emailDeliveryErrorCode: string; + } & RowDataPacket + > + >( + "SELECT emailDeliveryState, emailDeliveryErrorCode FROM organization_invites WHERE id = 'invitebeta'", + ); + expect(deferred[0]).toMatchObject({ + emailDeliveryState: "pending", + emailDeliveryErrorCode: "provider_send_failed", + }); + inviteMocks.environment.RESEND_API_KEY = "staging-resend-key"; + await verifier.end(); + }); +}); diff --git a/apps/web/__tests__/unit/agent-api-contract.test.ts b/apps/web/__tests__/unit/agent-api-contract.test.ts index 18078ced76b..1ac5ef35eeb 100644 --- a/apps/web/__tests__/unit/agent-api-contract.test.ts +++ b/apps/web/__tests__/unit/agent-api-contract.test.ts @@ -525,7 +525,8 @@ describe("agent API contract", () => { expect(importSource).toContain("agentOperationId: operationId"); expect(importSource).not.toContain("loomDownloadUrl:"); expect(workflow).toContain("claimAgentImport"); - expect(workflow).toContain("completeAgentImport"); + expect(workflow).toContain("saveMetadataAndComplete"); + expect(workflow).toContain("persistProductAnalyticsEvent"); expect(workflow).toContain("failAgentImport"); }); }); diff --git a/apps/web/__tests__/unit/deliver-product-analytics-outbox.test.ts b/apps/web/__tests__/unit/deliver-product-analytics-outbox.test.ts new file mode 100644 index 00000000000..71e7a63a3b2 --- /dev/null +++ b/apps/web/__tests__/unit/deliver-product-analytics-outbox.test.ts @@ -0,0 +1,133 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const outboxSource = readFileSync( + new URL("../../lib/analytics/product-event-outbox.ts", import.meta.url), + "utf8", +); +const stateSource = readFileSync( + new URL("../../lib/analytics/product-event-outbox-state.ts", import.meta.url), + "utf8", +); +const deliverySource = readFileSync( + new URL( + "../../workflows/product-analytics-delivery-workflow.ts", + import.meta.url, + ), + "utf8", +); +const erasureSource = readFileSync( + new URL( + "../../../../packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts", + import.meta.url, + ), + "utf8", +); +const reconciliationSource = readFileSync( + new URL("../../workflows/reconcile-product-analytics.ts", import.meta.url), + "utf8", +); + +describe("product analytics outbox lifecycle", () => { + it("keeps the first payload immutable and exposes a conflicting replay", () => { + expect(outboxSource).toContain( + "productAnalyticsOutbox.payloadHash} <> $" + "{row.payload_hash}", + ); + expect(outboxSource).toContain("'payload_conflict'"); + expect(outboxSource).not.toMatch( + /set:\s*\{[^}]*payloadHash:\s*row\.payload_hash/s, + ); + }); + + it("requeues only a same-hash dead letter after repair", () => { + expect(outboxSource).toContain( + "productAnalyticsOutbox.payloadHash} = $" + "{row.payload_hash}", + ); + expect(outboxSource).toContain( + "productAnalyticsOutbox.deadLetteredAt} IS NOT NULL, 'pending'", + ); + }); + + it("cannot overwrite a terminal result when delivery finishes before start returns", () => { + const workflowStart = outboxSource.indexOf( + "await start(deliverProductAnalyticsRowWorkflow", + ); + const postStartStatus = outboxSource.indexOf( + 'eq(productAnalyticsOutbox.status, "pending")', + workflowStart, + ); + const leaseFence = outboxSource.indexOf( + "productAnalyticsOutbox.leaseOwnerId", + postStartStatus, + ); + + expect(workflowStart).toBeGreaterThan(-1); + expect(postStartStatus).toBeGreaterThan(workflowStart); + expect(leaseFence).toBeGreaterThan(postStartStatus); + expect(stateSource).toContain( + 'inArray(productAnalyticsOutbox.status, ["pending", "workflow_started"])', + ); + }); + + it("keeps identity-bearing payloads out of Workflow history", () => { + expect(outboxSource).toContain( + "await start(deliverProductAnalyticsRowWorkflow, [", + ); + expect(outboxSource).toContain("pending.deliveryKey"); + expect(deliverySource).toContain( + "deliverProductAnalyticsRowWorkflow(deliveryKey: string)", + ); + expect(deliverySource).not.toContain( + "deliverProductAnalyticsRowWorkflow(row: ProductEventRow)", + ); + expect(reconciliationSource).toContain("return { failed, reconciled }"); + expect(reconciliationSource).toContain( + "loadProductAnalyticsReconciliationPageStep", + ); + expect(reconciliationSource).toContain( + "loadStripeAnalyticsReconciliationPageStep", + ); + expect(reconciliationSource).not.toContain("return { events: reconciled"); + }); + + it("terminalizes permanent outcomes and recovers unconfirmed workflows", () => { + expect(deliverySource).toContain( + "markProductAnalyticsOutboxDelivered(deliveryKey, row.payload_hash)", + ); + expect(deliverySource).toContain( + "deleteSuppressedProductAnalyticsOutboxRow", + ); + expect(deliverySource).toContain('"provider_rejected"'); + expect(deliverySource).toContain('"staging_timeout_after_accept"'); + expect(outboxSource).toContain( + 'eq(productAnalyticsOutbox.status, "workflow_started")', + ); + expect(outboxSource).toContain('lastErrorCode: "delivery_unconfirmed"'); + }); + + it("bounds recovery capacity and delivered-row retention", () => { + expect(outboxSource).toContain("const MAX_DRAIN_BATCH_SIZE = 500"); + expect(outboxSource).toContain("const DRAIN_CONCURRENCY = 10"); + expect(outboxSource).toContain( + "const DELIVERED_RETENTION_MS = 31 * 24 * 60 * 60 * 1_000", + ); + expect(outboxSource).toContain("capacityPerDay"); + expect(outboxSource).toContain("oldestPendingAgeSeconds"); + expect(outboxSource).toContain("oldestDeadLetterAgeSeconds"); + expect(outboxSource).toContain("payloadConflict"); + expect(outboxSource).toContain("receiptPayloadConflictAttempts"); + expect(outboxSource).toContain("const CLEANUP_BATCH_SIZE = 1_000"); + }); + + it("fences user, organization and anonymous outbox rows during erasure", () => { + expect(erasureSource).toContain( + "Db.productAnalyticsOutbox.userId, scope.userId", + ); + expect(erasureSource).toContain("Db.productAnalyticsOutbox.organizationId"); + expect(erasureSource).toContain("Db.productAnalyticsOutbox.anonymousId"); + expect(erasureSource).toContain("Db.productAnalyticsIdentityState"); + expect(erasureSource).toContain("delete(Db.productAnalyticsOutbox)"); + expect(outboxSource).toContain('.for("update")'); + expect(outboxSource).toContain('status: "suppressed" as const'); + }); +}); diff --git a/apps/web/__tests__/unit/developer-credits-webhook.test.ts b/apps/web/__tests__/unit/developer-credits-webhook.test.ts index b48513c126d..30297983f97 100644 --- a/apps/web/__tests__/unit/developer-credits-webhook.test.ts +++ b/apps/web/__tests__/unit/developer-credits-webhook.test.ts @@ -55,10 +55,20 @@ vi.mock("@/lib/developer-credits", () => ({ addCreditsToAccount: (...args: unknown[]) => mockAddCredits(...args), })); -vi.mock("@cap/web-domain", () => ({ - Organisation: { OrganisationId: { make: (v: string) => v } }, - User: { UserId: { make: (v: string) => v } }, -})); +vi.mock("@cap/web-domain", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + Organisation: { + ...actual.Organisation, + OrganisationId: { make: (value: string) => value }, + }, + User: { + ...actual.User, + UserId: { make: (value: string) => value }, + }, + }; +}); const mockStripe = { webhooks: { diff --git a/apps/web/__tests__/unit/loom-product-analytics.test.ts b/apps/web/__tests__/unit/loom-product-analytics.test.ts new file mode 100644 index 00000000000..2ba9f413e43 --- /dev/null +++ b/apps/web/__tests__/unit/loom-product-analytics.test.ts @@ -0,0 +1,65 @@ +import { readFileSync } from "node:fs"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + queueDurableServerProductEvent: vi.fn(), +})); + +vi.mock("@/lib/analytics/product-event-outbox", () => ({ + attemptProductAnalyticsOutboxDelivery: vi.fn(), + persistProductAnalyticsEvent: vi.fn(), + queueDurableServerProductEvent: mocks.queueDurableServerProductEvent, +})); + +import { queueLoomAnalyticsEvent } from "@/workflows/import-loom-video"; + +describe("Loom product analytics delivery", () => { + beforeEach(() => { + mocks.queueDurableServerProductEvent.mockReset(); + }); + afterEach(() => { + vi.restoreAllMocks(); + }); + + it("surfaces persistence failure so the durable workflow retries", async () => { + mocks.queueDurableServerProductEvent.mockRejectedValue( + new Error("database unavailable"), + ); + + await expect( + queueLoomAnalyticsEvent({ + eventId: "loom_import:video-1:completed", + eventName: "loom_import_completed", + occurredAt: "2026-08-01T12:00:00.000Z", + platform: "server", + userId: "user-1", + organizationId: "org-1", + properties: { import_mode: "direct", duration_ms: 1_000 }, + }), + ).rejects.toThrow("database unavailable"); + }); + + it("commits completion and its outbox fact in one transaction", () => { + const source = readFileSync( + new URL("../../workflows/import-loom-video.ts", import.meta.url), + "utf8", + ); + const completionFunction = source.indexOf( + "async function saveMetadataAndComplete", + ); + const transaction = source.indexOf( + "await db().transaction", + completionFunction, + ); + const eventPersistence = source.indexOf( + "await persistProductAnalyticsEvent(tx, completedEvent)", + transaction, + ); + const completionDelivery = source.indexOf("await startLoomAnalyticsEvent"); + const businessCatch = source.indexOf("} catch (error)"); + + expect(transaction).toBeGreaterThan(completionFunction); + expect(eventPersistence).toBeGreaterThan(transaction); + expect(completionDelivery).toBeGreaterThan(businessCatch); + }); +}); diff --git a/apps/web/__tests__/unit/organization-invite-delivery.test.ts b/apps/web/__tests__/unit/organization-invite-delivery.test.ts new file mode 100644 index 00000000000..e6e23bb3f22 --- /dev/null +++ b/apps/web/__tests__/unit/organization-invite-delivery.test.ts @@ -0,0 +1,49 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const source = readFileSync( + new URL("../../lib/organization-invite-delivery.ts", import.meta.url), + "utf8", +); + +describe("organization invite delivery recovery", () => { + it("repeats provider delivery with a stable idempotency key", () => { + expect(source).toContain( + "idempotencyKey: `organization-invite:$" + "{claimed.id}`", + ); + expect(source).toContain('emailDeliveryState, "pending"'); + expect(source).toContain("emailDeliveryNextAttemptAt"); + expect(source).toContain('deadLettered ? "dead_letter" : "pending"'); + expect(source).toContain("requeueOrganizationInviteDelivery"); + }); + + it("sends outside the claim transaction and fences durable completion", () => { + const claimTransaction = source.indexOf("await db().transaction"); + const providerSend = source.indexOf("await sendEmail", claimTransaction); + const completionTransaction = source.indexOf( + "const eventId = await db().transaction", + providerSend, + ); + const sentState = source.indexOf( + 'emailDeliveryState: "sent"', + completionTransaction, + ); + const outboxFact = source.indexOf( + "await persistProductAnalyticsEvent(tx", + completionTransaction, + ); + + expect(claimTransaction).toBeGreaterThan(-1); + expect(providerSend).toBeGreaterThan(claimTransaction); + expect(completionTransaction).toBeGreaterThan(providerSend); + expect(sentState).toBeGreaterThan(completionTransaction); + expect(outboxFact).toBeGreaterThan(sentState); + expect(source).toContain( + "await attemptProductAnalyticsOutboxDelivery(eventId)", + ); + expect(source).toContain("isNull(organizations.tombstoneAt)"); + expect(source).toContain("emailDeliveryLeaseOwnerId"); + expect(source).toContain("if (!serverEnv().RESEND_API_KEY)"); + expect(source).toContain('.for("update")'); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-authentication.test.ts b/apps/web/__tests__/unit/product-analytics-authentication.test.ts new file mode 100644 index 00000000000..8251c5c1d35 --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-authentication.test.ts @@ -0,0 +1,105 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + row: { organizationId: "org-1" } as { organizationId: string } | undefined, + cookieValue: "anonymous-1" as string | undefined, + persist: vi.fn(), + queue: vi.fn(), + transaction: vi.fn(), +})); + +vi.mock("@cap/database", () => ({ + db: () => ({ + select: () => ({ + from: () => ({ + where: () => ({ + limit: async () => (mocks.row ? [mocks.row] : []), + }), + }), + }), + transaction: mocks.transaction, + }), +})); + +vi.mock("@cap/database/schema", () => ({ + users: { activeOrganizationId: "activeOrganizationId", id: "id" }, +})); + +vi.mock("drizzle-orm", () => ({ eq: vi.fn(() => "condition") })); + +vi.mock("next/headers", () => ({ + cookies: async () => ({ + get: () => + mocks.cookieValue === undefined + ? undefined + : { value: mocks.cookieValue }, + }), +})); + +vi.mock("@/lib/analytics/product-event-outbox", () => ({ + persistProductAnalyticsEvent: mocks.persist, + queueDurableServerProductEvent: mocks.queue, +})); + +describe("recordWebAuthenticationSuccess", () => { + beforeEach(() => { + mocks.row = { organizationId: "org-1" }; + mocks.cookieValue = "anonymous-1"; + mocks.persist.mockReset().mockResolvedValue({ status: "pending" }); + mocks.queue.mockReset().mockResolvedValue({ status: "delivered" }); + mocks.transaction + .mockReset() + .mockImplementation(async (callback) => callback({ transaction: true })); + }); + + it("durably records sign-in and anonymous identity stitching", async () => { + const { recordWebAuthenticationSuccess } = await import( + "@/lib/analytics/authentication-events" + ); + + await recordWebAuthenticationSuccess("user-1"); + + expect(mocks.persist).toHaveBeenCalledTimes(2); + expect(mocks.queue).toHaveBeenCalledTimes(2); + expect(mocks.persist.mock.calls.map((call) => call[1])).toEqual([ + expect.objectContaining({ + eventName: "user_signed_in", + platform: "web", + userId: "user-1", + organizationId: "org-1", + anonymousId: "anonymous-1", + }), + expect.objectContaining({ + eventName: "identity_linked", + anonymousId: "anonymous-1", + }), + ]); + }); + + it("records authenticated sign-in without inventing an anonymous link", async () => { + mocks.cookieValue = undefined; + const { recordWebAuthenticationSuccess } = await import( + "@/lib/analytics/authentication-events" + ); + + await recordWebAuthenticationSuccess("user-1"); + + expect(mocks.persist).toHaveBeenCalledOnce(); + expect(mocks.queue).toHaveBeenCalledOnce(); + expect(mocks.persist.mock.calls[0]?.[1]).toEqual( + expect.objectContaining({ eventName: "user_signed_in" }), + ); + }); + + it("fails closed before delivery when the durable write fails", async () => { + mocks.persist.mockRejectedValueOnce(new Error("database unavailable")); + const { recordWebAuthenticationSuccess } = await import( + "@/lib/analytics/authentication-events" + ); + + await expect(recordWebAuthenticationSuccess("user-1")).rejects.toThrow( + "database unavailable", + ); + expect(mocks.queue).not.toHaveBeenCalled(); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-erasure.test.ts b/apps/web/__tests__/unit/product-analytics-erasure.test.ts index 9bcb1e92c73..ce5fbe4c4ba 100644 --- a/apps/web/__tests__/unit/product-analytics-erasure.test.ts +++ b/apps/web/__tests__/unit/product-analytics-erasure.test.ts @@ -34,6 +34,18 @@ const tinybirdTestLayer = ( overrides: Partial = {}, ) => { const store = { + enqueueErasureRequest: () => Effect.succeed("request-1"), + claimErasureRequest: () => + Effect.succeed({ + id: "request-1", + ownerId: "request-owner-1", + scope: { organizationId: "organization-1", userId: "user-1" }, + }), + completeErasureRequest: () => Effect.void, + deferErasureRequest: () => Effect.void, + waitForIngestionQuiescence: () => Effect.succeed(undefined), + discardPendingEvents: (_scope, anonymousIds = []) => + Effect.succeed([...anonymousIds]), claimNew: (scope) => Effect.succeed({ ownerId: "owner-1", @@ -116,7 +128,9 @@ describe.sequential("product analytics erasure", () => { data: [ { activation_markers: 1, + attribution_markers: 1, decision_markers: 1, + experiment_markers: 1, health_markers: 1, identity_markers: 1, retention_markers: 1, @@ -126,6 +140,9 @@ describe.sequential("product analytics erasure", () => { ], }); } + if (url.pathname.startsWith("/v0/jobs/")) { + return Response.json({ status: "done" }); + } if (url.pathname.includes("/copy")) { const pipe = url.pathname.split("/").at(-2); return Response.json({ job_id: `${pipe}-job` }); @@ -170,6 +187,8 @@ describe.sequential("product analytics erasure", () => { .map(({ url }) => url.pathname), ).toEqual( [ + "snapshot_product_event_id_states_v2", + "snapshot_product_event_day_states_v2", "snapshot_product_events_canonical_v1", "snapshot_product_events_daily_exact", "snapshot_product_traffic_daily_exact", @@ -177,6 +196,8 @@ describe.sequential("product analytics erasure", () => { "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", "snapshot_product_identity_funnel_exact", + "snapshot_product_attribution_daily_exact", + "snapshot_product_experiment_outcomes_exact", "snapshot_product_events_health_hourly", ].map((pipe) => `/v0/pipes/${pipe}/copy`), ); @@ -185,9 +206,9 @@ describe.sequential("product analytics erasure", () => { )) { expect(request.url.searchParams.get("_mode")).toBe("replace"); } - expect(requests.some(({ url }) => url.pathname.includes("/jobs/"))).toBe( - false, - ); + expect( + requests.filter(({ url }) => url.pathname.startsWith("/v0/jobs/")), + ).toHaveLength(12); expect( requests .filter(({ url }) => url.pathname === "/v0/sql") @@ -278,7 +299,9 @@ describe.sequential("product analytics erasure", () => { data: [ { activation_markers: 1, + attribution_markers: 1, decision_markers: 1, + experiment_markers: 1, health_markers: 1, identity_markers: 1, retention_markers: 1, @@ -288,6 +311,9 @@ describe.sequential("product analytics erasure", () => { ], }); } + if (url.pathname.startsWith("/v0/jobs/")) { + return Response.json({ status: "done" }); + } if (url.pathname.endsWith("/copy")) { return Response.json({ job_id: "copy-job" }); } @@ -309,7 +335,49 @@ describe.sequential("product analytics erasure", () => { requests .filter((url) => url.pathname.endsWith("/copy")) .map((url) => url.pathname), - ).toHaveLength(8); + ).toHaveLength(12); + }); + + it("does not start canonical rebuilding until the event-state copy completes", async () => { + const requests: URL[] = []; + const pausedPipes = new Set(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + requests.push(url); + const scheduleResponse = copyScheduleResponse(url, pausedPipes); + if (scheduleResponse) return scheduleResponse; + if (url.pathname === "/v0/sql") { + return Response.json({ data: [{ matching_rows: 0 }] }); + } + if ( + url.pathname === "/v0/pipes/snapshot_product_event_id_states_v2/copy" + ) { + return Response.json({ job_id: "state-copy-job" }); + } + if (url.pathname === "/v0/jobs/state-copy-job") { + return Response.json({ status: "failed" }); + } + return Response.json({}); + }), + ); + + const error = await Effect.runPromise( + Effect.gen(function* () { + const tinybird = yield* Tinybird; + yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); + }).pipe(Effect.provide(tinybirdTestLayer()), Effect.flip), + ); + + expect(error.message).toContain("ended in failed"); + expect( + requests.some( + (url) => + url.pathname === + "/v0/pipes/snapshot_product_events_canonical_v1/copy", + ), + ).toBe(false); }); it("resumes schedules already paused when a later pause fails", async () => { @@ -345,40 +413,44 @@ describe.sequential("product analytics erasure", () => { .map((url) => url.pathname) .filter((pathname) => /\/copy\/(cancel|resume)$/.test(pathname)), ).toEqual([ + "/v0/pipes/snapshot_product_event_id_states_v2/copy/cancel", + "/v0/pipes/snapshot_product_event_day_states_v2/copy/cancel", "/v0/pipes/snapshot_product_events_canonical_v1/copy/cancel", "/v0/pipes/snapshot_product_events_daily_exact/copy/cancel", "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", "/v0/pipes/snapshot_product_traffic_daily_exact/copy/cancel", + "/v0/pipes/snapshot_product_event_id_states_v2/copy/resume", + "/v0/pipes/snapshot_product_event_day_states_v2/copy/resume", "/v0/pipes/snapshot_product_events_canonical_v1/copy/resume", "/v0/pipes/snapshot_product_events_daily_exact/copy/resume", ]); }); - it("fails visibly without touching Tinybird when another owner holds the lease", async () => { + it("durably queues without touching Tinybird when another owner holds the lease", async () => { const fetch = vi.fn(); + const deferErasureRequest = vi.fn(() => Effect.void); vi.stubGlobal("fetch", fetch); - const error = await Effect.runPromise( + const result = await Effect.runPromise( Effect.gen(function* () { const tinybird = yield* Tinybird; - yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); + return yield* tinybird.eraseProductAnalytics({ + organizationId: "org-1", + }); }).pipe( Effect.provide( tinybirdTestLayer({ claimNew: () => Effect.succeed(null), - claimRecovery: () => Effect.succeed(null), + deferErasureRequest, }), ), - Effect.flip, ), ); - expect(error).toBeInstanceOf(Error); - expect(error.message).toBe( - "Product analytics erasure is already in progress", - ); + expect(result).toEqual({ queued: true, requestId: "request-1" }); + expect(deferErasureRequest).toHaveBeenCalledOnce(); expect(fetch).not.toHaveBeenCalled(); }); @@ -401,7 +473,9 @@ describe.sequential("product analytics erasure", () => { data: [ { activation_markers: 1, + attribution_markers: 1, decision_markers: 1, + experiment_markers: 1, health_markers: 1, identity_markers: 1, retention_markers: 1, @@ -411,6 +485,9 @@ describe.sequential("product analytics erasure", () => { ], }); } + if (url.pathname.startsWith("/v0/jobs/")) { + return Response.json({ status: "done" }); + } if (url.pathname.endsWith("/copy")) { return Response.json({ job_id: "copy-job" }); } diff --git a/apps/web/__tests__/unit/product-analytics-refresh.test.ts b/apps/web/__tests__/unit/product-analytics-refresh.test.ts new file mode 100644 index 00000000000..6e8f905506b --- /dev/null +++ b/apps/web/__tests__/unit/product-analytics-refresh.test.ts @@ -0,0 +1,174 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + acquire: vi.fn(), + release: vi.fn(), + renew: vi.fn(), + start: vi.fn(), +})); + +vi.mock("@cap/env", () => ({ + serverEnv: () => ({ + PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN: "copy-token", + PRODUCT_ANALYTICS_TINYBIRD_HOST: "https://staging.tinybird.test", + PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN: "read-token", + }), +})); + +vi.mock("@/lib/analytics/product-analytics-refresh-state", () => ({ + acquireProductAnalyticsRefreshLease: mocks.acquire, + releaseProductAnalyticsRefreshLease: mocks.release, + renewProductAnalyticsRefreshLease: mocks.renew, +})); + +vi.mock("workflow/api", () => ({ start: mocks.start })); + +const markerPayload = { + activation_markers: 1, + attribution_markers: 1, + decision_markers: 1, + experiment_markers: 1, + identity_markers: 1, + retention_markers: 1, + traffic_markers: 1, + traffic_page_markers: 1, +}; + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + delete process.env.CRON_SECRET; + mocks.acquire.mockReset(); + mocks.release.mockReset(); + mocks.renew.mockReset(); + mocks.start.mockReset(); +}); + +describe("product analytics refresh", () => { + it("runs every decision copy sequentially at one source cutoff", async () => { + const requestedUrls: URL[] = []; + mocks.acquire.mockResolvedValue({ + ownerId: "refresh-owner-1", + sourceCutoff: "2026-07-31T12:00:00.000Z", + }); + mocks.renew.mockResolvedValue(true); + mocks.release.mockResolvedValue(undefined); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + requestedUrls.push(url); + if (url.pathname.endsWith("/copy")) { + return Response.json({ + job_id: `job-${requestedUrls.length.toString().padStart(4, "0")}`, + }); + } + if (url.pathname.startsWith("/v0/jobs/")) { + return Response.json({ status: "done" }); + } + return Response.json({ data: [markerPayload] }); + }), + ); + + const { refreshProductAnalyticsWorkflow } = await import( + "@/workflows/refresh-product-analytics" + ); + const result = await refreshProductAnalyticsWorkflow({ + scheduledAt: "2026-07-31T12:00:00.000Z", + }); + + expect(result).toMatchObject({ + refreshed: true, + sourceCutoff: "2026-07-31T12:00:00.000Z", + }); + expect(result.jobs).toHaveLength(8); + expect(mocks.renew).toHaveBeenCalledTimes(8); + expect(mocks.release).toHaveBeenCalledWith("refresh-owner-1", undefined); + const copyUrls = requestedUrls.filter((url) => + url.pathname.endsWith("/copy"), + ); + expect(copyUrls).toHaveLength(8); + expect( + copyUrls.map((url) => url.searchParams.get("source_cutoff")), + ).toEqual(Array.from({ length: 8 }, () => "2026-07-31T12:00:00.000Z")); + expect( + new Set(copyUrls.map((url) => url.searchParams.get("copy_run_id"))).size, + ).toBe(1); + }); + + it("records a failed lease state when a copy assertion is missing", async () => { + mocks.acquire.mockResolvedValue({ + ownerId: "refresh-owner-2", + sourceCutoff: "2026-07-31T12:00:00.000Z", + }); + mocks.renew.mockResolvedValue(true); + mocks.release.mockResolvedValue(undefined); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + if (url.pathname.endsWith("/copy")) { + return Response.json({ job_id: "job-missing-marker" }); + } + if (url.pathname.startsWith("/v0/jobs/")) { + return Response.json({ status: "done" }); + } + return Response.json({ data: [{}] }); + }), + ); + + const { refreshProductAnalyticsWorkflow } = await import( + "@/workflows/refresh-product-analytics" + ); + await expect( + refreshProductAnalyticsWorkflow({ + scheduledAt: "2026-07-31T12:00:00.000Z", + }), + ).rejects.toThrow("refresh marker was missing"); + expect(mocks.release).toHaveBeenCalledWith( + "refresh-owner-2", + "refresh_failed", + ); + }); + + it("does not touch Tinybird when another refresh owns the lease", async () => { + mocks.acquire.mockResolvedValue(undefined); + const fetchMock = vi.fn(); + vi.stubGlobal("fetch", fetchMock); + const { refreshProductAnalyticsWorkflow } = await import( + "@/workflows/refresh-product-analytics" + ); + + await expect( + refreshProductAnalyticsWorkflow({ + scheduledAt: "2026-07-31T12:00:00.000Z", + }), + ).resolves.toEqual({ refreshed: false, reason: "lease_unavailable" }); + expect(fetchMock).not.toHaveBeenCalled(); + expect(mocks.release).not.toHaveBeenCalled(); + }); + + it("authenticates the cron before starting a durable workflow", async () => { + process.env.CRON_SECRET = "refresh-secret"; + mocks.start.mockResolvedValue({ runId: "refresh-run-1" }); + const { GET } = await import( + "@/app/api/cron/refresh-product-analytics/route" + ); + const unauthorized = await GET( + new Request("https://cap.test/api/cron/refresh-product-analytics"), + ); + expect(unauthorized.status).toBe(401); + + const accepted = await GET( + new Request("https://cap.test/api/cron/refresh-product-analytics", { + headers: { authorization: "Bearer refresh-secret" }, + }), + ); + expect(accepted.status).toBe(200); + expect(await accepted.json()).toEqual({ + accepted: true, + runId: "refresh-run-1", + }); + expect(mocks.start).toHaveBeenCalledOnce(); + }); +}); diff --git a/apps/web/__tests__/unit/product-analytics-server.test.ts b/apps/web/__tests__/unit/product-analytics-server.test.ts index 590624e506d..95576891057 100644 --- a/apps/web/__tests__/unit/product-analytics-server.test.ts +++ b/apps/web/__tests__/unit/product-analytics-server.test.ts @@ -4,6 +4,8 @@ import { firstViewReceivedEvent, identityLinkedEvent, shareLinkCreatedEvent, + uploadCompletedEvent, + uploadFailedEvent, userSignedUpEvent, } from "@/lib/analytics/business-events"; import { createServerProductEventRows } from "@/lib/analytics/server-event"; @@ -192,6 +194,21 @@ describe("server product analytics", () => { organizationId: "org-1", createdAt: "2026-07-31T10:03:00.000Z", }), + uploadCompletedEvent({ + videoId: "video-1", + platform: "cli", + userId: "user-1", + organizationId: "org-1", + createdAt: "2026-07-31T10:04:00.000Z", + }), + uploadFailedEvent({ + videoId: "video-2", + platform: "server", + userId: "user-1", + organizationId: "org-1", + createdAt: "2026-07-31T10:05:00.000Z", + failureClass: "plan", + }), ] as const; for (const fact of facts) { @@ -220,10 +237,20 @@ describe("server product analytics", () => { expect(signup?.anonymous_id).toBe("user:user-1"); expect(link).toMatchObject({ - event_id: "identity_linked:user-1", event_name: "identity_linked", anonymous_id: "anonymous-1", user_id: "user-1", }); + expect(link?.event_id).toMatch(/^identity_linked:user-1:[0-9a-f]{24}$/); + expect( + createServerProductEventRows( + identityLinkedEvent({ + userId: "user-1", + organizationId: "org-1", + anonymousId: "anonymous-2", + createdAt: "2026-07-31T10:00:00.000Z", + }), + )[0]?.event_id, + ).not.toBe(link?.event_id); }); }); diff --git a/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts b/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts index 4021eacf45a..695f6888e06 100644 --- a/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts +++ b/apps/web/__tests__/unit/stripe-analytics-reconciliation.test.ts @@ -5,6 +5,7 @@ const mocks = vi.hoisted(() => ({ invoicesList: vi.fn(), retrieveInvoice: vi.fn(), retrieveSubscription: vi.fn(), + queueDurableServerProductEvent: vi.fn(), users: [] as Array<{ id: string; activeOrganizationId: string; @@ -13,8 +14,12 @@ const mocks = vi.hoisted(() => ({ })); const dbChain = { + delete: vi.fn(), + insert: vi.fn(), + onDuplicateKeyUpdate: vi.fn(), select: vi.fn(), from: vi.fn(), + values: vi.fn(), where: vi.fn(), }; @@ -22,6 +27,10 @@ vi.mock("@cap/database", () => ({ db: () => dbChain })); vi.mock("@cap/database/schema", () => ({ comments: {}, messengerSupportEmails: {}, + productAnalyticsReconciliationFailures: { + attemptCount: "productAnalyticsReconciliationFailures.attemptCount", + sourceHash: "productAnalyticsReconciliationFailures.sourceHash", + }, users: { id: "users.id", activeOrganizationId: "users.activeOrganizationId", @@ -41,18 +50,22 @@ vi.mock("@cap/utils", () => ({ })); vi.mock("drizzle-orm", () => ({ and: (...values: unknown[]) => values, + asc: (value: unknown) => value, eq: (left: unknown, right: unknown) => ({ left, right }), + gt: (left: unknown, right: unknown) => ({ left, right }), gte: (left: unknown, right: unknown) => ({ left, right }), inArray: (left: unknown, right: unknown) => ({ left, right }), isNotNull: (value: unknown) => value, lte: (left: unknown, right: unknown) => ({ left, right }), notInArray: (left: unknown, right: unknown) => ({ left, right }), + or: (...values: unknown[]) => values, + sql: (...values: unknown[]) => values, })); vi.mock("@/lib/account-deletion-request", () => ({ ACCOUNT_DELETION_PENDING_SUBJECT: "pending-deletion", })); -vi.mock("@/workflows/deliver-product-analytics-event", () => ({ - enqueueReconciledProductAnalyticsEventStep: vi.fn(), +vi.mock("@/lib/analytics/product-event-outbox", () => ({ + queueDurableServerProductEvent: mocks.queueDurableServerProductEvent, })); const checkoutSession = { @@ -107,7 +120,11 @@ describe("Stripe analytics reconciliation", () => { stripeCustomerId: "cus_1", }); dbChain.select.mockReturnValue(dbChain); + dbChain.delete.mockReturnValue(dbChain); + dbChain.insert.mockReturnValue(dbChain); dbChain.from.mockReturnValue(dbChain); + dbChain.values.mockReturnValue(dbChain); + dbChain.onDuplicateKeyUpdate.mockResolvedValue(undefined); dbChain.where.mockImplementation(async () => [...mocks.users]); mocks.eventsList.mockImplementation(async ({ type }: { type: string }) => ({ data: @@ -125,6 +142,10 @@ describe("Stripe analytics reconciliation", () => { })); mocks.retrieveSubscription.mockResolvedValue(subscription); mocks.invoicesList.mockResolvedValue({ data: [], has_more: false }); + mocks.queueDurableServerProductEvent.mockResolvedValue({ + deliveryKey: "delivery-key", + status: "started", + }); }); it("rebuilds Stripe checkout facts with the original event identity", async () => { @@ -138,22 +159,24 @@ describe("Stripe analytics reconciliation", () => { }); expect(result).toEqual({ + failed: 0, legacyStripeEventsSkipped: 0, - events: [ - expect.objectContaining({ - eventId: "stripe:evt_checkout:purchase_completed", - eventName: "purchase_completed", - occurredAt: "2025-07-15T00:00:00.000Z", - userId: "user-1", - organizationId: "org-1", - properties: expect.objectContaining({ - amount_total_minor: 2700, - currency: "usd", - is_first_purchase: true, - }), - }), - ], + reconciled: 1, }); + expect(mocks.queueDurableServerProductEvent).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: "stripe:evt_checkout:purchase_completed", + eventName: "purchase_completed", + occurredAt: "2025-07-15T00:00:00.000Z", + userId: "user-1", + organizationId: "org-1", + properties: expect.objectContaining({ + amount_total_minor: 2700, + currency: "usd", + is_first_purchase: true, + }), + }), + ); expect(mocks.eventsList.mock.calls.map(([input]) => input.type)).toEqual([ "checkout.session.created", "checkout.session.completed", @@ -209,23 +232,25 @@ describe("Stripe analytics reconciliation", () => { }); expect(result).toEqual({ + failed: 0, legacyStripeEventsSkipped: 1, - events: [ - expect.objectContaining({ - eventId: "checkout:cs_1", - eventName: "checkout_started", - occurredAt: "2025-07-15T00:00:00.000Z", - organizationId: "org-1", - properties: expect.objectContaining({ - price_id: "price_team", - quantity: 3, - }), - }), - ], + reconciled: 1, }); + expect(mocks.queueDurableServerProductEvent).toHaveBeenCalledWith( + expect.objectContaining({ + eventId: "checkout:cs_1", + eventName: "checkout_started", + occurredAt: "2025-07-15T00:00:00.000Z", + organizationId: "org-1", + properties: expect.objectContaining({ + price_id: "price_team", + quantity: 3, + }), + }), + ); }); - it("fails closed when a paid checkout cannot be tied to a Cap user", async () => { + it("quarantines a paid checkout that cannot be tied to a Cap user", async () => { mocks.users.splice(0); const { loadStripeAnalyticsReconciliationEventsStep } = await import( "@/workflows/reconcile-product-analytics" @@ -236,7 +261,12 @@ describe("Stripe analytics reconciliation", () => { scheduledAt: "2025-07-16T00:00:00.000Z", lookbackHours: 48, }), - ).rejects.toThrow("Stripe checkout has no matching analytics user"); + ).resolves.toEqual({ + failed: 1, + legacyStripeEventsSkipped: 0, + reconciled: 0, + }); + expect(dbChain.insert).toHaveBeenCalled(); }); it("rebuilds a trial from the immutable subscription-created snapshot", async () => { @@ -271,7 +301,12 @@ describe("Stripe analytics reconciliation", () => { lookbackHours: 48, }); - expect(result.events).toEqual([ + expect(result).toEqual({ + failed: 0, + legacyStripeEventsSkipped: 0, + reconciled: 1, + }); + expect(mocks.queueDurableServerProductEvent).toHaveBeenCalledWith( expect.objectContaining({ eventId: "stripe:evt_trial:trial_started", eventName: "trial_started", @@ -279,7 +314,7 @@ describe("Stripe analytics reconciliation", () => { userId: "user-1", organizationId: "org-1", }), - ]); + ); }); it("rebuilds the first settled post-trial invoice as a purchase", async () => { @@ -295,7 +330,16 @@ describe("Stripe analytics reconciliation", () => { subtotal: 3_000, currency: "usd", total_discount_amounts: [{ amount: 300 }], - lines: { data: [{ price: { id: "price_team" } }] }, + subscription_details: { metadata: subscription.metadata }, + lines: { + data: [ + { + quantity: 3, + metadata: subscription.metadata, + price: subscription.items.data[0]?.price, + }, + ], + }, }; mocks.eventsList.mockImplementation(async ({ type }: { type: string }) => ({ data: @@ -320,7 +364,12 @@ describe("Stripe analytics reconciliation", () => { lookbackHours: 48, }); - expect(result.events).toEqual([ + expect(result).toEqual({ + failed: 0, + legacyStripeEventsSkipped: 0, + reconciled: 1, + }); + expect(mocks.queueDurableServerProductEvent).toHaveBeenCalledWith( expect.objectContaining({ eventId: "stripe:evt_first_paid:purchase_completed", eventName: "purchase_completed", @@ -331,6 +380,138 @@ describe("Stripe analytics reconciliation", () => { is_first_purchase: true, }), }), - ]); + ); + }); + + it("reconciles more than 5,000 Stripe facts without a hard row cap", async () => { + const total = 5_101; + mocks.eventsList.mockImplementation( + async ({ + type, + starting_after: startingAfter, + }: { + type: string; + starting_after?: string; + }) => { + if (type !== "checkout.session.created") { + return { data: [], has_more: false }; + } + const offset = startingAfter + ? Number(startingAfter.split("_").at(-1)) + 1 + : 0; + const count = Math.min(100, total - offset); + return { + data: Array.from({ length: count }, (_, index) => { + const sequence = offset + index; + return { + id: `evt_created_${sequence}`, + created: 1_752_537_600, + type, + data: { + object: { + ...checkoutSession, + id: `cs_created_${sequence}`, + }, + }, + }; + }), + has_more: offset + count < total, + }; + }, + ); + const { loadStripeAnalyticsReconciliationEventsStep } = await import( + "@/workflows/reconcile-product-analytics" + ); + + await expect( + loadStripeAnalyticsReconciliationEventsStep({ + scheduledAt: "2025-07-16T00:00:00.000Z", + lookbackHours: 48, + }), + ).resolves.toEqual({ + failed: 0, + legacyStripeEventsSkipped: 0, + reconciled: total, + }); + expect(mocks.queueDurableServerProductEvent).toHaveBeenCalledTimes(total); + }); + + it("resumes from the last durable Stripe page after a later page fails", async () => { + let secondPageAttempts = 0; + mocks.eventsList.mockImplementation( + async ({ + starting_after: startingAfter, + type, + }: { + starting_after?: string; + type: string; + }) => { + if (type !== "checkout.session.completed") { + return { data: [], has_more: false }; + } + if (!startingAfter) { + return { + data: [ + { + id: "evt_page_1", + created: 1_752_537_600, + type, + data: { object: checkoutSession }, + }, + ], + has_more: true, + }; + } + secondPageAttempts += 1; + if (secondPageAttempts === 1) + throw new Error("temporary Stripe outage"); + return { + data: [ + { + id: "evt_page_2", + created: 1_752_537_601, + type, + data: { + object: { ...checkoutSession, id: "cs_page_2" }, + }, + }, + ], + has_more: false, + }; + }, + ); + const { loadStripeAnalyticsReconciliationPageStep } = await import( + "@/workflows/reconcile-product-analytics" + ); + const input = { + scheduledAt: "2025-07-16T00:00:00.000Z", + lookbackHours: 48, + type: "checkout.session.completed" as const, + }; + const firstPage = await loadStripeAnalyticsReconciliationPageStep(input); + expect(firstPage.nextStartingAfter).toBe("evt_page_1"); + await expect( + loadStripeAnalyticsReconciliationPageStep({ + ...input, + startingAfter: firstPage.nextStartingAfter, + }), + ).rejects.toThrow("temporary Stripe outage"); + await expect( + loadStripeAnalyticsReconciliationPageStep({ + ...input, + startingAfter: firstPage.nextStartingAfter, + }), + ).resolves.toEqual({ + failed: 0, + legacyStripeEventsSkipped: 0, + nextStartingAfter: undefined, + reconciled: 1, + }); + expect( + mocks.eventsList.mock.calls.filter( + ([request]) => request.starting_after === "evt_page_1", + ), + ).toHaveLength(2); + expect(mocks.queueDurableServerProductEvent).toHaveBeenCalledTimes(2); }); }); diff --git a/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts b/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts index 64e579505fc..a96dd65cf97 100644 --- a/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts +++ b/apps/web/__tests__/unit/subscription-analytics-webhook.test.ts @@ -118,11 +118,14 @@ const invoice = { subtotal: 3_000, currency: "usd", total_discount_amounts: [{ amount: 300 }], + subscription_details: { metadata: subscription.metadata }, lines: { data: [ { subscription: "sub_1", - price: { id: "price_team" }, + quantity: 3, + metadata: subscription.metadata, + price: subscription.items.data[0]?.price, }, ], }, @@ -415,6 +418,13 @@ describe("Stripe subscription analytics", () => { }); it("counts the first positive post-trial invoice as the purchase", async () => { + mocks.retrieveSubscription.mockResolvedValue({ + ...subscription, + metadata: { + ...subscription.metadata, + analyticsOrganizationId: "org-mutated", + }, + }); mocks.constructEvent.mockReturnValue(event("invoice.paid", invoice)); expect((await POST(request())).status).toBe(200); @@ -433,6 +443,7 @@ describe("Stripe subscription analytics", () => { expect(mocks.product).not.toHaveBeenCalledWith( expect.objectContaining({ eventName: "subscription_renewed" }), ); + expect(mocks.retrieveSubscription).not.toHaveBeenCalled(); }); it("counts a later positive subscription invoice as a renewal", async () => { diff --git a/apps/web/__tests__/unit/update-seat-quantity.test.ts b/apps/web/__tests__/unit/update-seat-quantity.test.ts index 3227a854128..f6e3051a81f 100644 --- a/apps/web/__tests__/unit/update-seat-quantity.test.ts +++ b/apps/web/__tests__/unit/update-seat-quantity.test.ts @@ -8,6 +8,7 @@ const mockDb = { where: vi.fn(), limit: vi.fn(), set: vi.fn(), + transaction: vi.fn(), }; const mockStripe = { @@ -17,10 +18,8 @@ const mockStripe = { }, }; -const queueServerProductEvent = vi.fn(async (event: { eventId: string }) => ({ - eventId: event.eventId, - runId: "run-1", -})); +const persistProductAnalyticsEvent = vi.fn(); +const attemptProductAnalyticsOutboxDelivery = vi.fn(); vi.mock("@cap/database", () => ({ db: () => mockDb, @@ -57,7 +56,10 @@ vi.mock("@cap/utils", () => ({ stripe: () => mockStripe, })); -vi.mock("@/lib/analytics/server", () => ({ queueServerProductEvent })); +vi.mock("@/lib/analytics/product-event-outbox", () => ({ + attemptProductAnalyticsOutboxDelivery, + persistProductAnalyticsEvent, +})); vi.mock("drizzle-orm", () => ({ eq: vi.fn((field: unknown, value: unknown) => ({ field, value })), @@ -81,13 +83,18 @@ function resetMockDb() { mockDb.where.mockReturnValue(mockDb); mockDb.limit.mockResolvedValue([]); mockDb.set.mockReturnValue(mockDb); + mockDb.transaction.mockImplementation( + (callback: (tx: typeof mockDb) => Promise) => callback(mockDb), + ); } function mockSeatLookup({ currentQuantity, + localQuantity = currentQuantity, proSeatsUsed = 1, }: { currentQuantity: number; + localQuantity?: number; proSeatsUsed?: number; }) { mockGetCurrentUser.mockResolvedValue({ @@ -100,7 +107,7 @@ function mockSeatLookup({ { stripeCustomerId: "cus_1", stripeSubscriptionId: "sub_1", - inviteQuota: currentQuantity, + inviteQuota: localQuantity, }, ]); mockDb.where @@ -158,7 +165,8 @@ describe("updateSeatQuantity", () => { proration_behavior: "always_invoice", }); expect(mockDb.set).toHaveBeenCalledWith({ inviteQuota: 2 }); - expect(queueServerProductEvent).toHaveBeenCalledWith( + expect(persistProductAnalyticsEvent).toHaveBeenCalledWith( + mockDb, expect.objectContaining({ eventName: "seat_quantity_changed", occurredAt: expect.any(String), @@ -169,6 +177,9 @@ describe("updateSeatQuantity", () => { }), }), ); + expect(attemptProductAnalyticsOutboxDelivery).toHaveBeenCalledWith( + expect.stringContaining("seat_quantity:sub_1:"), + ); }); it("does not store added seats when Stripe leaves the subscription update pending", async () => { @@ -200,4 +211,25 @@ describe("updateSeatQuantity", () => { proration_behavior: "create_prorations", }); }); + + it("repairs a local write after Stripe already accepted the quantity", async () => { + mockSeatLookup({ currentQuantity: 2, localQuantity: 1 }); + const { updateSeatQuantity } = await import( + "@/actions/organization/update-seat-quantity" + ); + + await updateSeatQuantity("org-1" as never, 2); + + expect(mockStripe.subscriptions.update).not.toHaveBeenCalled(); + expect(mockDb.set).toHaveBeenCalledWith({ inviteQuota: 2 }); + expect(persistProductAnalyticsEvent).toHaveBeenCalledWith( + mockDb, + expect.objectContaining({ + properties: expect.objectContaining({ + previous_quantity: 1, + new_quantity: 2, + }), + }), + ); + }); }); diff --git a/apps/web/actions/organization/send-invites.ts b/apps/web/actions/organization/send-invites.ts index 1cfe49027a2..48fa6cb16d1 100644 --- a/apps/web/actions/organization/send-invites.ts +++ b/apps/web/actions/organization/send-invites.ts @@ -2,8 +2,6 @@ import { db } from "@cap/database"; import { getCurrentUser } from "@cap/database/auth/session"; -import { sendEmail } from "@cap/database/emails/config"; -import { OrganizationInvite } from "@cap/database/emails/organization-invite"; import { nanoId } from "@cap/database/helpers"; import { organizationInvites, @@ -11,11 +9,10 @@ import { organizations, users, } from "@cap/database/schema"; -import { serverEnv } from "@cap/env"; import type { Organisation } from "@cap/web-domain"; -import { and, eq, inArray } from "drizzle-orm"; +import { and, eq, inArray, or, sql } from "drizzle-orm"; import { revalidatePath } from "next/cache"; -import { queueServerProductEvent } from "@/lib/analytics/server"; +import { deliverOrganizationInvite } from "@/lib/organization-invite-delivery"; import { provisionOrganizationInvitee } from "@/lib/organization-provisioning"; import { type AssignableOrganizationRole, @@ -120,7 +117,10 @@ export async function sendOrganizationInvites( .where( and( eq(organizationInvites.organizationId, organizationId), - inArray(organizationInvites.invitedEmail, validEmails), + or( + inArray(organizationInvites.invitedEmailNormalized, validEmails), + inArray(organizationInvites.invitedEmail, validEmails), + ), ), ), tx @@ -158,89 +158,44 @@ export async function sendOrganizationInvites( })); if (records.length > 0) { - await tx.insert(organizationInvites).values( - records.map((r) => ({ - id: r.id, - organizationId: organizationId, - invitedEmail: r.email, - invitedByUserId: user.id, - role: r.role, - createdAt: r.createdAt, - })), - ); + await tx + .insert(organizationInvites) + .values( + records.map((r) => ({ + id: r.id, + organizationId: organizationId, + invitedEmail: r.email, + invitedEmailNormalized: r.email, + invitedByUserId: user.id, + role: r.role, + emailDeliveryState: "pending" as const, + emailDeliveryNextAttemptAt: r.createdAt, + createdAt: r.createdAt, + })), + ) + .onDuplicateKeyUpdate({ + set: { + invitedEmailNormalized: sql`${organizationInvites.invitedEmailNormalized}`, + }, + }); } return records; }); const emailResults = await Promise.allSettled( - inviteRecords.map((record) => { - const inviteUrl = `${serverEnv().WEB_URL}/invite/${record.id}`; - return sendEmail({ - email: record.email, - subject: `Invitation to join ${organization.name} on Cap`, - react: OrganizationInvite({ - email: record.email, - url: inviteUrl, - organizationName: organization.name, - }), - }); - }), + inviteRecords.map((record) => deliverOrganizationInvite(record.id)), ); - const failedInvites = inviteRecords.filter( - (_, i) => emailResults[i]?.status === "rejected", - ); + const failedInvites = inviteRecords.filter((_, i) => { + const result = emailResults[i]; + return ( + result?.status === "rejected" || + (result?.status === "fulfilled" && result.value.status === "deferred") + ); + }); const failedEmails = failedInvites.map((r) => r.email); - if (failedInvites.length > 0) { - try { - await db() - .delete(organizationInvites) - .where( - inArray( - organizationInvites.id, - failedInvites.map((r) => r.id), - ), - ); - } catch (cleanupError) { - console.error( - "Failed to clean up invite records after email delivery failure:", - { - failedInviteIds: failedInvites.map((r) => r.id), - failedEmails, - error: cleanupError, - }, - ); - } - } - - const failedInviteIds = new Set(failedInvites.map((invite) => invite.id)); - const successfulInvites = inviteRecords.filter( - (invite) => !failedInviteIds.has(invite.id), - ); - const firstSuccessfulInvite = successfulInvites[0]; - if (firstSuccessfulInvite) { - await queueServerProductEvent({ - eventId: `organization_invites:${firstSuccessfulInvite.id}`, - eventName: "organization_invite_sent", - occurredAt: firstSuccessfulInvite.createdAt.toISOString(), - platform: "web", - userId: user.id, - organizationId, - properties: { - invite_count: successfulInvites.length, - admin_count: successfulInvites.filter( - (invite) => invite.role === "admin", - ).length, - member_count: successfulInvites.filter( - (invite) => invite.role === "member", - ).length, - delivery: "email", - }, - }); - } - revalidatePath("/dashboard/settings/organization"); return { success: true, failedEmails }; diff --git a/apps/web/actions/organization/update-seat-quantity.ts b/apps/web/actions/organization/update-seat-quantity.ts index f999399d9d3..f120fd8bcfd 100644 --- a/apps/web/actions/organization/update-seat-quantity.ts +++ b/apps/web/actions/organization/update-seat-quantity.ts @@ -11,7 +11,10 @@ import { stripe } from "@cap/utils"; import type { Organisation } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { revalidatePath } from "next/cache"; -import { queueServerProductEvent } from "@/lib/analytics/server"; +import { + attemptProductAnalyticsOutboxDelivery, + persistProductAnalyticsEvent, +} from "@/lib/analytics/product-event-outbox"; import { calculateProSeats } from "@/utils/organization"; async function getOwnerSubscription( @@ -152,10 +155,11 @@ export async function updateSeatQuantity( newQuantity: number, ) { validateQuantity(newQuantity); - const { subscription, subscriptionItem, proSeatsUsed, user } = + const { owner, subscription, subscriptionItem, proSeatsUsed, user } = await getOwnerSubscription(organizationId); const currentQuantity = subscriptionItem.quantity ?? 1; - if (newQuantity === currentQuantity) { + const localQuantity = owner.inviteQuota ?? 1; + if (newQuantity === currentQuantity && newQuantity === localQuantity) { return { success: true, newQuantity }; } @@ -165,24 +169,26 @@ export async function updateSeatQuantity( ); } - const isSeatIncrease = newQuantity > currentQuantity; - const updatedSubscription = await stripe().subscriptions.update( - subscription.id, - { - items: [ - { - id: subscriptionItem.id, - quantity: newQuantity, - }, - ], - proration_behavior: isSeatIncrease - ? "always_invoice" - : "create_prorations", - ...(isSeatIncrease - ? { payment_behavior: "pending_if_incomplete" as const } - : {}), - }, - ); + const previousQuantity = + newQuantity === currentQuantity ? localQuantity : currentQuantity; + const isSeatIncrease = newQuantity > previousQuantity; + const updatedSubscription = + newQuantity === currentQuantity + ? subscription + : await stripe().subscriptions.update(subscription.id, { + items: [ + { + id: subscriptionItem.id, + quantity: newQuantity, + }, + ], + proration_behavior: isSeatIncrease + ? "always_invoice" + : "create_prorations", + ...(isSeatIncrease + ? { payment_behavior: "pending_if_incomplete" as const } + : {}), + }); const changedAt = new Date().toISOString(); if (isSeatIncrease && updatedSubscription.pending_update) { @@ -191,11 +197,36 @@ export async function updateSeatQuantity( ); } + const latestInvoice = + typeof updatedSubscription.latest_invoice === "string" + ? updatedSubscription.latest_invoice + : updatedSubscription.latest_invoice?.id; + const eventId = `seat_quantity:${subscription.id}:${latestInvoice ?? updatedSubscription.current_period_start}:${newQuantity}`; try { - await db() - .update(users) - .set({ inviteQuota: newQuantity }) - .where(eq(users.id, user.id)); + await db().transaction(async (tx) => { + await tx + .update(users) + .set({ inviteQuota: newQuantity }) + .where(eq(users.id, user.id)); + await persistProductAnalyticsEvent(tx, { + eventId, + eventName: "seat_quantity_changed", + occurredAt: changedAt, + platform: "web", + userId: user.id, + organizationId, + properties: { + previous_quantity: previousQuantity, + new_quantity: newQuantity, + quantity_delta: newQuantity - previousQuantity, + direction: isSeatIncrease ? "increase" : "decrease", + price_id: subscriptionItem.price.id, + unit_amount_minor: subscriptionItem.price.unit_amount, + currency: subscriptionItem.price.currency, + billing_interval: subscriptionItem.price.recurring?.interval, + }, + }); + }); } catch (dbError) { console.error( "CRITICAL: Stripe updated to quantity", @@ -210,28 +241,7 @@ export async function updateSeatQuantity( } revalidatePath("/dashboard/settings/organization"); - const latestInvoice = - typeof updatedSubscription.latest_invoice === "string" - ? updatedSubscription.latest_invoice - : updatedSubscription.latest_invoice?.id; - await queueServerProductEvent({ - eventId: `seat_quantity:${subscription.id}:${latestInvoice ?? updatedSubscription.current_period_start}:${newQuantity}`, - eventName: "seat_quantity_changed", - occurredAt: changedAt, - platform: "web", - userId: user.id, - organizationId, - properties: { - previous_quantity: currentQuantity, - new_quantity: newQuantity, - quantity_delta: newQuantity - currentQuantity, - direction: isSeatIncrease ? "increase" : "decrease", - price_id: subscriptionItem.price.id, - unit_amount_minor: subscriptionItem.price.unit_amount, - currency: subscriptionItem.price.currency, - billing_interval: subscriptionItem.price.recurring?.interval, - }, - }); + await attemptProductAnalyticsOutboxDelivery(eventId); return { success: true, newQuantity }; } diff --git a/apps/web/app/api/analytics/track/route.ts b/apps/web/app/api/analytics/track/route.ts index fde38f5713c..ca18dd24517 100644 --- a/apps/web/app/api/analytics/track/route.ts +++ b/apps/web/app/api/analytics/track/route.ts @@ -1,5 +1,6 @@ import { db } from "@cap/database"; import { videos, videoUploads } from "@cap/database/schema"; +import { serverEnv } from "@cap/env"; import { provideOptionalAuth, Tinybird } from "@cap/web-backend"; import { CurrentUser, Video } from "@cap/web-domain"; import { eq } from "drizzle-orm"; @@ -12,7 +13,9 @@ import { firstExternalViewTimestamp, } from "@/lib/analytics/first-view"; import { + classifyAnalyticsTraffic, getProductAnalyticsRateLimitKey, + normalizeSyntheticRunId, ProductAnalyticsRateLimiter, } from "@/lib/analytics/request"; import { queueServerProductEvent } from "@/lib/analytics/server"; @@ -31,7 +34,6 @@ interface TrackPayload { sessionId?: string; pathname?: string; hostname?: string | null; - userAgent?: string; occurredAt?: string; } @@ -113,10 +115,22 @@ export async function POST(request: NextRequest) { ) { return Response.json({ error: "Rate limited" }, { status: 429 }); } - const userAgent = - sanitizeString(request.headers.get("user-agent")) || - sanitizeString(body.userAgent) || - "unknown"; + const userAgent = sanitizeString(request.headers.get("user-agent")) || ""; + const environment = serverEnv(); + const syntheticRunId = normalizeSyntheticRunId( + request.headers.get("x-cap-analytics-test-run") ?? undefined, + environment.VERCEL_ENV, + ); + const trafficClass = classifyAnalyticsTraffic({ + userAgent, + vercelEnvironment: environment.VERCEL_ENV, + syntheticRunId, + rateLimitKey, + internalIpHashes: process.env.PRODUCT_ANALYTICS_INTERNAL_IP_HASHES, + }); + if (trafficClass !== "external") { + return Response.json({ success: true, excluded: trafficClass }); + } const parser = new UAParser(userAgent); const browserName = parser.getBrowser().name ?? "unknown"; const osName = parser.getOS().name ?? "unknown"; diff --git a/apps/web/app/api/auth/[...nextauth]/route.ts b/apps/web/app/api/auth/[...nextauth]/route.ts index e89058ce413..e74823298c5 100644 --- a/apps/web/app/api/auth/[...nextauth]/route.ts +++ b/apps/web/app/api/auth/[...nextauth]/route.ts @@ -1,8 +1,14 @@ import { authOptions } from "@cap/database/auth/auth-options"; import NextAuth from "next-auth"; +import { recordWebAuthenticationSuccess } from "@/lib/analytics/authentication-events"; export const dynamic = "force-dynamic"; -const handler = NextAuth(authOptions()); +const options = authOptions(); +options.events = { + ...options.events, + signIn: ({ user }) => recordWebAuthenticationSuccess(user.id), +}; +const handler = NextAuth(options); export { handler as GET, handler as POST }; diff --git a/apps/web/app/api/cron/drain-product-analytics-outbox/route.ts b/apps/web/app/api/cron/drain-product-analytics-outbox/route.ts new file mode 100644 index 00000000000..ea172bfc271 --- /dev/null +++ b/apps/web/app/api/cron/drain-product-analytics-outbox/route.ts @@ -0,0 +1,33 @@ +import { timingSafeEqual } from "node:crypto"; +import { NextResponse } from "next/server"; +import { start } from "workflow/api"; +import { getProductAnalyticsOutboxHealth } from "@/lib/analytics/product-event-outbox"; +import { drainProductAnalyticsOutboxWorkflow } from "@/workflows/drain-product-analytics-outbox"; + +export const dynamic = "force-dynamic"; + +export async function GET(request: Request) { + const cronSecret = process.env.CRON_SECRET; + if (!cronSecret) { + return NextResponse.json( + { error: "Server misconfiguration" }, + { status: 500 }, + ); + } + + const authorization = request.headers.get("authorization"); + const expected = `Bearer ${cronSecret}`; + if ( + !authorization || + authorization.length !== expected.length || + !timingSafeEqual(Buffer.from(authorization), Buffer.from(expected)) + ) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const [run, health] = await Promise.all([ + start(drainProductAnalyticsOutboxWorkflow, []), + getProductAnalyticsOutboxHealth(), + ]); + return NextResponse.json({ accepted: true, runId: run.runId, health }); +} diff --git a/apps/web/app/api/cron/recover-organization-invite-delivery/route.ts b/apps/web/app/api/cron/recover-organization-invite-delivery/route.ts new file mode 100644 index 00000000000..6b61cebccc2 --- /dev/null +++ b/apps/web/app/api/cron/recover-organization-invite-delivery/route.ts @@ -0,0 +1,33 @@ +import { timingSafeEqual } from "node:crypto"; +import { NextResponse } from "next/server"; +import { start } from "workflow/api"; +import { getOrganizationInviteDeliveryHealth } from "@/lib/organization-invite-delivery"; +import { recoverOrganizationInviteDeliveriesWorkflow } from "@/workflows/recover-organization-invite-delivery"; + +export const dynamic = "force-dynamic"; + +export async function GET(request: Request) { + const cronSecret = process.env.CRON_SECRET; + if (!cronSecret) { + return NextResponse.json( + { error: "Server misconfiguration" }, + { status: 500 }, + ); + } + + const authorization = request.headers.get("authorization"); + const expected = `Bearer ${cronSecret}`; + if ( + !authorization || + authorization.length !== expected.length || + !timingSafeEqual(Buffer.from(authorization), Buffer.from(expected)) + ) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + + const [run, health] = await Promise.all([ + start(recoverOrganizationInviteDeliveriesWorkflow, []), + getOrganizationInviteDeliveryHealth(), + ]); + return NextResponse.json({ accepted: true, runId: run.runId, health }); +} diff --git a/apps/web/app/api/cron/refresh-product-analytics/route.ts b/apps/web/app/api/cron/refresh-product-analytics/route.ts new file mode 100644 index 00000000000..90475f3d6f9 --- /dev/null +++ b/apps/web/app/api/cron/refresh-product-analytics/route.ts @@ -0,0 +1,29 @@ +import { timingSafeEqual } from "node:crypto"; +import { NextResponse } from "next/server"; +import { start } from "workflow/api"; +import { refreshProductAnalyticsWorkflow } from "@/workflows/refresh-product-analytics"; + +export const dynamic = "force-dynamic"; + +export async function GET(request: Request) { + const cronSecret = process.env.CRON_SECRET; + if (!cronSecret) { + return NextResponse.json( + { error: "Server misconfiguration" }, + { status: 500 }, + ); + } + const authorization = request.headers.get("authorization"); + const expected = `Bearer ${cronSecret}`; + if ( + !authorization || + authorization.length !== expected.length || + !timingSafeEqual(Buffer.from(authorization), Buffer.from(expected)) + ) { + return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); + } + const run = await start(refreshProductAnalyticsWorkflow, [ + { scheduledAt: new Date().toISOString() }, + ]); + return NextResponse.json({ accepted: true, runId: run.runId }); +} diff --git a/apps/web/app/api/desktop/[...route]/video.ts b/apps/web/app/api/desktop/[...route]/video.ts index 67ab31b6089..0ca851ac0b7 100644 --- a/apps/web/app/api/desktop/[...route]/video.ts +++ b/apps/web/app/api/desktop/[...route]/video.ts @@ -63,6 +63,9 @@ app.get( zValidator( "query", z.object({ + initiatingPlatform: z + .union([z.literal("cli"), z.literal("desktop")]) + .default("desktop"), recordingMode: z .union([ z.literal("hls"), @@ -88,6 +91,7 @@ app.get( async (c) => { try { const { + initiatingPlatform, recordingMode, isScreenshot, videoId, @@ -230,9 +234,10 @@ app.get( const videoName = name ?? `Cap ${isScreenshot ? "Screenshot" : "Recording"} - ${formattedDate}`; - const metadata: VideoMetadata | undefined = name - ? { sourceName: name } - : undefined; + const metadata: VideoMetadata = { + initiatingPlatform, + ...(name ? { sourceName: name } : {}), + }; const clientSupportsGoogleDriveUpload = hasDesktopFeature( c.req, GOOGLE_DRIVE_UPLOAD_FEATURE, @@ -274,13 +279,15 @@ app.get( width, height, fps, - ...(metadata ? { metadata } : {}), + metadata, }); + const analyticsPlatform = + initiatingPlatform === "cli" ? "cli" : "desktop"; await queueServerProductEvent( shareLinkCreatedEvent({ videoId: idToUse, - platform: "desktop", + platform: analyticsPlatform, userId: user.id, organizationId: videoOrgId, createdAt, diff --git a/apps/web/app/api/events/route.ts b/apps/web/app/api/events/route.ts index e24b709a7a2..f3db66da80f 100644 --- a/apps/web/app/api/events/route.ts +++ b/apps/web/app/api/events/route.ts @@ -227,15 +227,16 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( }); yield* analytics - .append(rows) + .appendWithIdentityFence(rows) .pipe( - Effect.catchTag("ProductAnalyticsError", (error) => - Effect.logWarning( - "Product analytics ingestion failed", - error, - ).pipe( - Effect.andThen( - Effect.fail(new HttpApiError.ServiceUnavailable()), + Effect.catchAll((error) => + Effect.logWarning("Product analytics ingestion failed").pipe( + Effect.zipRight( + Effect.fail( + "retryable" in error && error.retryable === false + ? new HttpApiError.BadRequest() + : new HttpApiError.ServiceUnavailable(), + ), ), ), ), diff --git a/apps/web/app/api/invite/accept/route.ts b/apps/web/app/api/invite/accept/route.ts index 71644b29d15..82bd9353faf 100644 --- a/apps/web/app/api/invite/accept/route.ts +++ b/apps/web/app/api/invite/accept/route.ts @@ -7,12 +7,13 @@ import { organizations, users, } from "@cap/database/schema"; -import { and, eq } from "drizzle-orm"; +import { and, eq, isNull } from "drizzle-orm"; import { type NextRequest, NextResponse } from "next/server"; import { - queueServerProductEvent, - readAnalyticsAnonymousId, -} from "@/lib/analytics/server"; + attemptProductAnalyticsOutboxDelivery, + persistProductAnalyticsEvent, +} from "@/lib/analytics/product-event-outbox"; +import { readAnalyticsAnonymousId } from "@/lib/analytics/server"; import { normalizeAssignableOrganizationRole } from "@/lib/permissions/roles"; import { calculateProSeats, @@ -41,8 +42,7 @@ export async function POST(request: NextRequest) { try { let joinedMemberId: string | undefined; - let joinedOrganizationId: string | undefined; - let joinedRole: string | undefined; + let joinedEventId: string | undefined; let joinedAt: Date | undefined; let assignedProSeat = false; await db().transaction(async (tx) => { @@ -55,6 +55,20 @@ export async function POST(request: NextRequest) { if (!invite) { throw new Error("INVITE_NOT_FOUND"); } + const [org] = await tx + .select({ ownerId: organizations.ownerId }) + .from(organizations) + .where( + and( + eq(organizations.id, invite.organizationId), + isNull(organizations.tombstoneAt), + ), + ) + .limit(1) + .for("update"); + if (!org) { + throw new Error("INVITE_NOT_FOUND"); + } if (user.email.toLowerCase() !== invite.invitedEmail.toLowerCase()) { throw new Error("EMAIL_MISMATCH"); @@ -87,18 +101,10 @@ export async function POST(request: NextRequest) { }); memberId = newId; joinedMemberId = newId; - joinedOrganizationId = invite.organizationId; - joinedRole = role; joinedAt = createdAt; } - const [org] = await tx - .select({ ownerId: organizations.ownerId }) - .from(organizations) - .where(eq(organizations.id, invite.organizationId)) - .limit(1); - - if (org && memberId && !existingMembership) { + if (memberId && !existingMembership) { const [owner] = await tx .select({ id: users.id, @@ -167,27 +173,27 @@ export async function POST(request: NextRequest) { await tx .delete(organizationInvites) .where(eq(organizationInvites.id, inviteId)); + + if (joinedMemberId && joinedAt) { + joinedEventId = `organization_member:${joinedMemberId}:joined`; + await persistProductAnalyticsEvent(tx, { + eventId: joinedEventId, + eventName: "organization_member_joined", + occurredAt: joinedAt.toISOString(), + anonymousId: readAnalyticsAnonymousId(request), + platform: "web", + userId: user.id, + organizationId: invite.organizationId, + properties: { + role: normalizeAssignableOrganizationRole(invite.role) ?? "member", + assigned_pro_seat: assignedProSeat, + }, + }); + } }); - if ( - joinedMemberId && - joinedOrganizationId && - joinedAt && - (joinedRole === "admin" || joinedRole === "member") - ) { - await queueServerProductEvent({ - eventId: `organization_member:${joinedMemberId}:joined`, - eventName: "organization_member_joined", - occurredAt: joinedAt.toISOString(), - anonymousId: readAnalyticsAnonymousId(request), - platform: "web", - userId: user.id, - organizationId: joinedOrganizationId, - properties: { - role: joinedRole, - assigned_pro_seat: assignedProSeat, - }, - }); + if (joinedEventId) { + await attemptProductAnalyticsOutboxDelivery(joinedEventId); } return NextResponse.json({ success: true }); diff --git a/apps/web/app/api/v1/[...route]/route.ts b/apps/web/app/api/v1/[...route]/route.ts index 0a2bae0b39e..737f6b70183 100644 --- a/apps/web/app/api/v1/[...route]/route.ts +++ b/apps/web/app/api/v1/[...route]/route.ts @@ -114,7 +114,12 @@ import { runAgentMutation, updateAgentCap, } from "@/lib/agent-write"; -import { checkoutStartedEvent } from "@/lib/analytics/business-events"; +import { + checkoutStartedEvent, + uploadCompletedEvent, + uploadFailedEvent, +} from "@/lib/analytics/business-events"; +import { persistProductAnalyticsEvent } from "@/lib/analytics/product-event-outbox"; import { queueServerProductEvent } from "@/lib/analytics/server"; import { subscriptionCheckoutAnalyticsMetadata } from "@/lib/analytics/stripe-business-events"; import { hashKey } from "@/lib/developer-key-hash"; @@ -6634,6 +6639,7 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( fps: payload.fps, metadata: { sourceName: payload.fileName, + initiatingPlatform: payload.initiatingPlatform ?? "server", agentUpload: { state: "pending" }, }, createdAt: now, @@ -6952,6 +6958,7 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( .select({ id: Db.videos.id, ownerId: Db.videos.ownerId, + organizationId: Db.videos.orgId, metadata: Db.videos.metadata, }) .from(Db.videos) @@ -6986,12 +6993,18 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( .for("update"); if (!upload) return { state: "not_found" as const }; if (!allowed) { + const rejectedAt = new Date(); + const isCliUpload = + lockedVideo.metadata?.initiatingPlatform === "cli"; await tx .update(Db.videos) .set({ metadata: { ...(lockedVideo.metadata ?? {}), - agentUpload: { state: "rejected" }, + agentUpload: { + state: "rejected", + rejectedAt: rejectedAt.toISOString(), + }, }, }) .where(eq(Db.videos.id, path.id)); @@ -7004,8 +7017,34 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( updatedAt: new Date(), }) .where(eq(Db.videoUploads.videoId, path.id)); + if (isCliUpload) { + await persistProductAnalyticsEvent( + tx, + uploadFailedEvent({ + videoId: path.id, + platform: "cli", + userId: lockedVideo.ownerId, + organizationId: lockedVideo.organizationId, + createdAt: rejectedAt, + failureClass: "plan", + }), + ); + } else { + await persistProductAnalyticsEvent( + tx, + uploadFailedEvent({ + videoId: path.id, + platform: "server", + userId: lockedVideo.ownerId, + organizationId: lockedVideo.organizationId, + createdAt: rejectedAt, + failureClass: "plan", + }), + ); + } return { state: "rejected" as const }; } + const acceptedAt = new Date(); await tx .update(Db.videos) .set({ @@ -7017,6 +7056,7 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( agentUpload: { state: "accepted", rawFileKey: candidateRawFileKey, + acceptedAt: acceptedAt.toISOString(), }, }, }) @@ -7083,6 +7123,8 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( .select({ id: Db.videos.id, ownerId: Db.videos.ownerId, + organizationId: Db.videos.orgId, + createdAt: Db.videos.createdAt, bucketId: Db.videos.bucket, metadata: Db.videos.metadata, }) @@ -7110,6 +7152,33 @@ const AgentManagementHandlersLive = HttpApiBuilder.group( .limit(1) .for("update"); if (!upload) return { state: "not_found" }; + if (video.metadata?.initiatingPlatform === "cli") { + await persistProductAnalyticsEvent( + tx, + uploadCompletedEvent({ + videoId: video.id, + platform: "cli", + userId: video.ownerId, + organizationId: video.organizationId, + createdAt: + video.metadata?.agentUpload?.acceptedAt ?? + video.createdAt, + }), + ); + } else { + await persistProductAnalyticsEvent( + tx, + uploadCompletedEvent({ + videoId: video.id, + platform: "server", + userId: video.ownerId, + organizationId: video.organizationId, + createdAt: + video.metadata?.agentUpload?.acceptedAt ?? + video.createdAt, + }), + ); + } return { state: "success", response: { diff --git a/apps/web/app/api/webhooks/stripe/route.ts b/apps/web/app/api/webhooks/stripe/route.ts index b6769704116..6dd51b97bf7 100644 --- a/apps/web/app/api/webhooks/stripe/route.ts +++ b/apps/web/app/api/webhooks/stripe/route.ts @@ -225,13 +225,13 @@ async function chargeInvoice(charge: Stripe.Charge): Promise { : charge.invoice; } -async function invoiceSubscription(invoice: Stripe.Invoice) { +function invoiceSubscriptionId(invoice: Stripe.Invoice) { if (!invoice.subscription) { throw new Error("Subscription invoice is missing its subscription"); } return typeof invoice.subscription === "string" - ? stripe().subscriptions.retrieve(invoice.subscription) - : invoice.subscription; + ? invoice.subscription + : invoice.subscription.id; } export const POST = async (req: Request) => { @@ -261,18 +261,17 @@ export const POST = async (req: Request) => { if (event.type === "invoice.paid") { const invoice = event.data.object as Stripe.Invoice; if (!invoice.subscription) return NextResponse.json({ received: true }); - const subscription = await invoiceSubscription(invoice); + const subscriptionId = invoiceSubscriptionId(invoice); const dbUser = await findAnalyticsUserForCustomer(invoice.customer); if (!dbUser) return retryableUserResolutionFailure(); const invoicePaidProductEvent = subscriptionInvoicePaidProductEvent({ eventId: event.id, occurredAt: new Date(event.created * 1000).toISOString(), invoice, - subscription, user: dbUser, firstPositivePayment: await isFirstPositiveSubscriptionPayment({ invoice, - subscriptionId: subscription.id, + subscriptionId, listPaidInvoices: (input) => stripe().invoices.list(input), }), }); @@ -283,7 +282,6 @@ export const POST = async (req: Request) => { if (event.type === "invoice.payment_failed") { const invoice = event.data.object as Stripe.Invoice; if (invoice.subscription) { - const subscription = await invoiceSubscription(invoice); const dbUser = await findAnalyticsUserForCustomer(invoice.customer); if (!dbUser) return retryableUserResolutionFailure(); const paymentFailedProductEvent = @@ -291,7 +289,6 @@ export const POST = async (req: Request) => { eventId: event.id, occurredAt: new Date(event.created * 1000).toISOString(), invoice, - subscription, user: dbUser, }); if (paymentFailedProductEvent) @@ -308,7 +305,6 @@ export const POST = async (req: Request) => { const refundedAmount = charge.amount_refunded - previousAmountRefunded; if (charge.invoice && refundedAmount > 0) { const invoice = await chargeInvoice(charge); - const subscription = await invoiceSubscription(invoice); const dbUser = await findAnalyticsUserForCustomer(charge.customer); if (!dbUser) return retryableUserResolutionFailure(); const refundProductEvent = subscriptionRefundedProductEvent({ @@ -316,7 +312,6 @@ export const POST = async (req: Request) => { occurredAt: new Date(event.created * 1000).toISOString(), charge, invoice, - subscription, user: dbUser, refundedAmount, }); diff --git a/apps/web/lib/analytics/authentication-events.ts b/apps/web/lib/analytics/authentication-events.ts new file mode 100644 index 00000000000..e73d56f8c4c --- /dev/null +++ b/apps/web/lib/analytics/authentication-events.ts @@ -0,0 +1,58 @@ +import { randomUUID } from "node:crypto"; +import { PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE } from "@cap/analytics"; +import { db } from "@cap/database"; +import { users } from "@cap/database/schema"; +import { User } from "@cap/web-domain"; +import { eq } from "drizzle-orm"; +import { cookies } from "next/headers"; +import { identityLinkedEvent, userSignedInEvent } from "./business-events"; +import { + persistProductAnalyticsEvent, + queueDurableServerProductEvent, +} from "./product-event-outbox"; +import { normalizeServerIdentifier } from "./server-event"; + +export async function recordWebAuthenticationSuccess(userId: string) { + const normalizedUserId = normalizeServerIdentifier(userId); + if (!normalizedUserId) { + throw new Error("Authenticated user identity is invalid"); + } + const typedUserId = User.UserId.make(normalizedUserId); + const [user] = await db() + .select({ organizationId: users.activeOrganizationId }) + .from(users) + .where(eq(users.id, typedUserId)) + .limit(1); + if (!user) throw new Error("Authenticated user does not exist"); + + const anonymousId = normalizeServerIdentifier( + (await cookies()).get(PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE)?.value, + ); + const createdAt = new Date(); + const events = [ + userSignedInEvent({ + authenticationId: randomUUID(), + userId: normalizedUserId, + organizationId: user.organizationId, + anonymousId, + createdAt, + }), + ...(anonymousId + ? [ + identityLinkedEvent({ + userId: normalizedUserId, + organizationId: user.organizationId, + anonymousId, + createdAt, + }), + ] + : []), + ]; + + await db().transaction(async (tx) => { + for (const event of events) { + await persistProductAnalyticsEvent(tx, event); + } + }); + await Promise.all(events.map(queueDurableServerProductEvent)); +} diff --git a/apps/web/lib/analytics/business-events.ts b/apps/web/lib/analytics/business-events.ts index 63cbc172f04..baaa3b56172 100644 --- a/apps/web/lib/analytics/business-events.ts +++ b/apps/web/lib/analytics/business-events.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import type { ServerProductEvent } from "./server-event"; const occurredAt = (value: Date | string) => @@ -22,7 +23,7 @@ export const identityLinkedEvent = (input: { createdAt: Date | string; }) => ({ - eventId: `identity_linked:${input.userId}`, + eventId: `identity_linked:${input.userId}:${createHash("sha256").update(input.anonymousId).digest("hex").slice(0, 24)}`, eventName: "identity_linked", occurredAt: occurredAt(input.createdAt), anonymousId: input.anonymousId, @@ -31,9 +32,26 @@ export const identityLinkedEvent = (input: { organizationId: input.organizationId ?? undefined, }) satisfies ServerProductEvent; +export const userSignedInEvent = (input: { + authenticationId: string; + userId: string; + organizationId?: string | null; + anonymousId?: string; + createdAt: Date | string; +}) => + ({ + eventId: `signin:${input.authenticationId}`, + eventName: "user_signed_in", + occurredAt: occurredAt(input.createdAt), + anonymousId: input.anonymousId, + platform: "web", + userId: input.userId, + organizationId: input.organizationId ?? undefined, + }) satisfies ServerProductEvent; + export const shareLinkCreatedEvent = (input: { videoId: string; - platform: "desktop" | "mobile" | "server"; + platform: "cli" | "desktop" | "mobile" | "server"; userId: string; organizationId: string; createdAt: Date | string; @@ -53,6 +71,40 @@ export const shareLinkCreatedEvent = (input: { }, }) satisfies ServerProductEvent; +export const uploadCompletedEvent = (input: { + videoId: string; + platform: "cli" | "server"; + userId: string; + organizationId: string; + createdAt: Date | string; +}) => + ({ + eventId: `upload_completed:${input.videoId}`, + eventName: "upload_completed", + occurredAt: occurredAt(input.createdAt), + platform: input.platform, + userId: input.userId, + organizationId: input.organizationId, + }) satisfies ServerProductEvent; + +export const uploadFailedEvent = (input: { + videoId: string; + platform: "cli" | "server"; + userId: string; + organizationId: string; + createdAt: Date | string; + failureClass: string; +}) => + ({ + eventId: `upload_failed:${input.videoId}:${input.failureClass}`, + eventName: "upload_failed", + occurredAt: occurredAt(input.createdAt), + platform: input.platform, + userId: input.userId, + organizationId: input.organizationId, + properties: { failure_class: input.failureClass }, + }) satisfies ServerProductEvent; + export const checkoutStartedEvent = (input: { checkoutId: string; createdAt: Date | string; diff --git a/apps/web/lib/analytics/product-analytics-refresh-state.ts b/apps/web/lib/analytics/product-analytics-refresh-state.ts new file mode 100644 index 00000000000..7a0efaef65c --- /dev/null +++ b/apps/web/lib/analytics/product-analytics-refresh-state.ts @@ -0,0 +1,115 @@ +import { randomUUID } from "node:crypto"; +import { db } from "@cap/database"; +import { + productAnalyticsErasureLeases, + productAnalyticsIngestionLeases, + productAnalyticsRefreshLeases, +} from "@cap/database/schema"; +import { and, eq, isNull, lt, or, sql } from "drizzle-orm"; + +const REFRESH_LEASE_NAME = "global"; +const REFRESH_LEASE_DURATION_MS = 30 * 60 * 1_000; + +const affectedRows = (result: unknown) => + Array.isArray(result) + ? ((result[0] as { affectedRows?: number } | undefined)?.affectedRows ?? 0) + : ((result as { affectedRows?: number }).affectedRows ?? 0); + +export async function acquireProductAnalyticsRefreshLease(sourceCutoff: Date) { + const ownerId = randomUUID(); + const expiresAt = new Date(Date.now() + REFRESH_LEASE_DURATION_MS); + return db().transaction(async (tx) => { + await tx + .insert(productAnalyticsErasureLeases) + .values({ name: "global" }) + .onDuplicateKeyUpdate({ set: { name: "global" } }); + await tx + .insert(productAnalyticsRefreshLeases) + .values({ name: REFRESH_LEASE_NAME }) + .onDuplicateKeyUpdate({ set: { name: REFRESH_LEASE_NAME } }); + const [fence] = await tx + .select({ + fencingToken: productAnalyticsErasureLeases.fencingToken, + phase: productAnalyticsErasureLeases.phase, + }) + .from(productAnalyticsErasureLeases) + .where(eq(productAnalyticsErasureLeases.name, "global")) + .limit(1) + .for("update"); + if (!fence || fence.phase !== "idle") return undefined; + const claimed = await tx + .update(productAnalyticsRefreshLeases) + .set({ + generation: sql`${productAnalyticsRefreshLeases.generation} + 1`, + lastErrorCode: null, + leaseExpiresAt: expiresAt, + ownerId, + sourceCutoff, + status: "running", + }) + .where( + and( + eq(productAnalyticsRefreshLeases.name, REFRESH_LEASE_NAME), + or( + isNull(productAnalyticsRefreshLeases.ownerId), + lt(productAnalyticsRefreshLeases.leaseExpiresAt, new Date()), + ), + ), + ); + if (affectedRows(claimed) === 0) return undefined; + await tx.insert(productAnalyticsIngestionLeases).values({ + id: ownerId, + fencingToken: fence.fencingToken, + expiresAt, + }); + return { ownerId, sourceCutoff: sourceCutoff.toISOString() }; + }); +} + +export async function renewProductAnalyticsRefreshLease(ownerId: string) { + const expiresAt = new Date(Date.now() + REFRESH_LEASE_DURATION_MS); + return db().transaction(async (tx) => { + const refreshed = await tx + .update(productAnalyticsRefreshLeases) + .set({ leaseExpiresAt: expiresAt }) + .where( + and( + eq(productAnalyticsRefreshLeases.name, REFRESH_LEASE_NAME), + eq(productAnalyticsRefreshLeases.ownerId, ownerId), + eq(productAnalyticsRefreshLeases.status, "running"), + ), + ); + if (affectedRows(refreshed) === 0) return false; + const ingestion = await tx + .update(productAnalyticsIngestionLeases) + .set({ expiresAt }) + .where(eq(productAnalyticsIngestionLeases.id, ownerId)); + return affectedRows(ingestion) > 0; + }); +} + +export async function releaseProductAnalyticsRefreshLease( + ownerId: string, + errorCode?: string, +) { + await db().transaction(async (tx) => { + await tx + .delete(productAnalyticsIngestionLeases) + .where(eq(productAnalyticsIngestionLeases.id, ownerId)); + await tx + .update(productAnalyticsRefreshLeases) + .set({ + lastCompletedAt: errorCode ? undefined : new Date(), + lastErrorCode: errorCode ?? null, + leaseExpiresAt: null, + ownerId: null, + status: errorCode ? "failed" : "idle", + }) + .where( + and( + eq(productAnalyticsRefreshLeases.name, REFRESH_LEASE_NAME), + eq(productAnalyticsRefreshLeases.ownerId, ownerId), + ), + ); + }); +} diff --git a/apps/web/lib/analytics/product-event-outbox-state.ts b/apps/web/lib/analytics/product-event-outbox-state.ts new file mode 100644 index 00000000000..73ceaf854b7 --- /dev/null +++ b/apps/web/lib/analytics/product-event-outbox-state.ts @@ -0,0 +1,124 @@ +import { randomUUID } from "node:crypto"; +import { db } from "@cap/database"; +import { + productAnalyticsErasureLeases, + productAnalyticsIngestionLeases, + productAnalyticsOutbox, +} from "@cap/database/schema"; +import { and, eq, inArray } from "drizzle-orm"; + +function terminalEvent(deliveryKey: string, payloadHash: string) { + return and( + eq(productAnalyticsOutbox.deliveryKey, deliveryKey), + eq(productAnalyticsOutbox.payloadHash, payloadHash), + inArray(productAnalyticsOutbox.status, ["pending", "workflow_started"]), + ); +} + +export async function acquireProductAnalyticsIngestionLease() { + await db() + .insert(productAnalyticsErasureLeases) + .values({ name: "global" }) + .onDuplicateKeyUpdate({ set: { name: "global" } }); + const [erasureLease] = await db() + .select({ + fencingToken: productAnalyticsErasureLeases.fencingToken, + phase: productAnalyticsErasureLeases.phase, + }) + .from(productAnalyticsErasureLeases) + .where(eq(productAnalyticsErasureLeases.name, "global")) + .limit(1); + if (!erasureLease || erasureLease.phase !== "idle") return undefined; + const leaseId = randomUUID(); + await db() + .insert(productAnalyticsIngestionLeases) + .values({ + id: leaseId, + fencingToken: erasureLease.fencingToken, + expiresAt: new Date(Date.now() + 5 * 60 * 1_000), + }); + const [confirmed] = await db() + .select({ + fencingToken: productAnalyticsErasureLeases.fencingToken, + phase: productAnalyticsErasureLeases.phase, + }) + .from(productAnalyticsErasureLeases) + .where(eq(productAnalyticsErasureLeases.name, "global")) + .limit(1); + if ( + !confirmed || + confirmed.phase !== "idle" || + confirmed.fencingToken !== erasureLease.fencingToken + ) { + await releaseProductAnalyticsIngestionLease(leaseId); + return undefined; + } + return leaseId; +} + +export async function releaseProductAnalyticsIngestionLease(leaseId: string) { + await db() + .delete(productAnalyticsIngestionLeases) + .where(eq(productAnalyticsIngestionLeases.id, leaseId)); +} + +export async function markProductAnalyticsOutboxDelivered( + deliveryKey: string, + payloadHash: string, + errorCode?: "identity_suppressed", +) { + await db() + .update(productAnalyticsOutbox) + .set({ + status: "delivered", + deliveredAt: new Date(), + lastErrorCode: errorCode ?? null, + leaseOwnerId: null, + leaseExpiresAt: null, + }) + .where(terminalEvent(deliveryKey, payloadHash)); +} + +export async function deleteSuppressedProductAnalyticsOutboxRow( + deliveryKey: string, + payloadHash: string, +) { + await db() + .delete(productAnalyticsOutbox) + .where(terminalEvent(deliveryKey, payloadHash)); +} + +export async function markProductAnalyticsOutboxDeadLetter( + deliveryKey: string, + payloadHash: string, + errorCode: + | "contract_invalid" + | "delivery_not_configured" + | "provider_rejected", +) { + await db() + .update(productAnalyticsOutbox) + .set({ + status: "dead_letter", + deadLetteredAt: new Date(), + lastErrorCode: errorCode, + leaseOwnerId: null, + leaseExpiresAt: null, + }) + .where(terminalEvent(deliveryKey, payloadHash)); +} + +export async function markProductAnalyticsOutboxRetrying( + deliveryKey: string, + payloadHash: string, + errorCode: + | "provider_retryable" + | "staging_provider_429" + | "staging_provider_503" + | "staging_timeout_after_accept" = "provider_retryable", +) { + await db() + .update(productAnalyticsOutbox) + .set({ lastErrorCode: errorCode }) + .where(terminalEvent(deliveryKey, payloadHash)); +} diff --git a/apps/web/lib/analytics/product-event-outbox.ts b/apps/web/lib/analytics/product-event-outbox.ts new file mode 100644 index 00000000000..fab6339e8e1 --- /dev/null +++ b/apps/web/lib/analytics/product-event-outbox.ts @@ -0,0 +1,646 @@ +import { randomUUID } from "node:crypto"; +import { + type ProductEventRow, + productAnalyticsEventIdHash, + productAnalyticsIdentityHash, +} from "@cap/analytics"; +import { db } from "@cap/database"; +import { + productAnalyticsEventReceipts, + productAnalyticsIdentityLinks, + productAnalyticsIdentityState, + productAnalyticsIngestionLeases, + productAnalyticsOutbox, + productAnalyticsReconciliationFailures, +} from "@cap/database/schema"; +import { + and, + asc, + eq, + gt, + inArray, + isNotNull, + isNull, + lt, + lte, + or, + sql, +} from "drizzle-orm"; +import { start } from "workflow/api"; +import { deliverProductAnalyticsRowWorkflow } from "@/workflows/product-analytics-delivery-workflow"; +import { + createServerProductEventRows, + type ServerProductEvent, +} from "./server-event"; + +type DatabaseClient = ReturnType; +export type ProductAnalyticsOutboxTransaction = Parameters< + Parameters[0] +>[0]; + +const MAX_DRAIN_BATCH_SIZE = 500; +const DRAIN_CONCURRENCY = 10; +const MAX_RETRY_DELAY_MS = 60 * 60 * 1_000; +const MAX_OUTBOX_PAYLOAD_BYTES = 16 * 1_024; +const OUTBOX_LEASE_DURATION_MS = 2 * 60 * 1_000; +const DELIVERY_CONFIRMATION_TIMEOUT_MS = 2 * 60 * 60 * 1_000; +const DELIVERED_RETENTION_MS = 31 * 24 * 60 * 60 * 1_000; +const OLDEST_PENDING_SLO_MS = 15 * 60 * 1_000; +const EVENT_RECEIPT_RETENTION_MS = 800 * 24 * 60 * 60 * 1_000; +const CLEANUP_BATCH_SIZE = 1_000; + +const eventIdentityStates = (row: ProductEventRow) => { + return [ + row.anonymous_id + ? { + identityHash: productAnalyticsIdentityHash( + "anonymous", + row.anonymous_id, + ), + identityKind: "anonymous" as const, + } + : undefined, + row.user_id + ? { + identityHash: productAnalyticsIdentityHash("user", row.user_id), + identityKind: "user" as const, + } + : undefined, + row.organization_id + ? { + identityHash: productAnalyticsIdentityHash( + "organization", + row.organization_id, + ), + identityKind: "organization" as const, + } + : undefined, + ] + .filter((entry) => entry !== undefined) + .sort((left, right) => left.identityHash.localeCompare(right.identityHash)); +}; + +function validatedEvent(event: ServerProductEvent) { + const [row] = createServerProductEventRows(event); + if (!row) { + throw new Error("Product analytics event failed contract validation"); + } + return row; +} + +function isStoredProductEventRow( + value: unknown, + eventId: string, +): value is ProductEventRow { + if (!value || typeof value !== "object") return false; + const row = value as Record; + return ( + row.event_id === eventId && + typeof row.payload_hash === "string" && + /^[0-9a-f]{32}$/.test(row.payload_hash) && + typeof row.event_name === "string" && + typeof row.received_at === "string" && + typeof row.properties === "string" + ); +} + +export async function persistProductAnalyticsEvent( + tx: ProductAnalyticsOutboxTransaction, + event: ServerProductEvent, +) { + const row = validatedEvent(event); + const payloadBytes = Buffer.byteLength(JSON.stringify(row), "utf8"); + if (payloadBytes > MAX_OUTBOX_PAYLOAD_BYTES) { + throw new Error("Product analytics outbox payload exceeds its bound"); + } + const identities = eventIdentityStates(row); + if (identities.length > 0) { + await tx + .insert(productAnalyticsIdentityState) + .values(identities) + .onDuplicateKeyUpdate({ + set: { + identityHash: sql`${productAnalyticsIdentityState.identityHash}`, + }, + }); + const blocked = await tx + .select({ + blockedAt: productAnalyticsIdentityState.blockedAt, + identityKind: productAnalyticsIdentityState.identityKind, + }) + .from(productAnalyticsIdentityState) + .where( + inArray( + productAnalyticsIdentityState.identityHash, + identities.map((identity) => identity.identityHash), + ), + ) + .for("update"); + const blockedAt = blocked.find( + (identity) => identity.blockedAt !== null, + )?.blockedAt; + if (blockedAt) { + const blockedPrincipal = blocked.some( + (identity) => + identity.blockedAt !== null && identity.identityKind !== "anonymous", + ); + const anonymousHashes = identities + .filter((identity) => identity.identityKind === "anonymous") + .map((identity) => identity.identityHash); + if (blockedPrincipal && anonymousHashes.length > 0) { + await tx + .update(productAnalyticsIdentityState) + .set({ blockedAt }) + .where( + inArray( + productAnalyticsIdentityState.identityHash, + anonymousHashes, + ), + ); + } + return { + eventId: row.event_id, + payloadHash: row.payload_hash, + payloadConflict: false, + status: "suppressed" as const, + suppressed: true as const, + }; + } + const anonymousIdentity = identities.find( + (identity) => identity.identityKind === "anonymous", + ); + const userIdentity = identities.find( + (identity) => identity.identityKind === "user", + ); + const organizationIdentity = identities.find( + (identity) => identity.identityKind === "organization", + ); + if (anonymousIdentity && userIdentity && row.anonymous_id) { + await tx + .insert(productAnalyticsIdentityLinks) + .values({ + anonymousIdentityHash: anonymousIdentity.identityHash, + userIdentityHash: userIdentity.identityHash, + organizationIdentityHash: organizationIdentity?.identityHash ?? null, + anonymousId: row.anonymous_id, + }) + .onDuplicateKeyUpdate({ + set: { + organizationIdentityHash: + organizationIdentity?.identityHash ?? null, + updatedAt: new Date(), + }, + }); + } + } + const eventIdHash = productAnalyticsEventIdHash(row.event_id); + const identityHashes = Object.fromEntries( + identities.map((identity) => [ + identity.identityKind, + identity.identityHash, + ]), + ) as Partial>; + const now = new Date(); + const retainUntil = new Date(now.getTime() + EVENT_RECEIPT_RETENTION_MS); + await tx + .insert(productAnalyticsEventReceipts) + .values({ + eventIdHash, + payloadHash: row.payload_hash, + anonymousIdentityHash: identityHashes.anonymous ?? null, + userIdentityHash: identityHashes.user ?? null, + organizationIdentityHash: identityHashes.organization ?? null, + retainUntil, + }) + .onDuplicateKeyUpdate({ + set: { + conflictCount: sql`IF(${productAnalyticsEventReceipts.payloadHash} <> ${row.payload_hash}, ${productAnalyticsEventReceipts.conflictCount} + 1, ${productAnalyticsEventReceipts.conflictCount})`, + lastSeenAt: now, + retainUntil: sql`GREATEST(${productAnalyticsEventReceipts.retainUntil}, ${retainUntil})`, + }, + }); + const [receipt] = await tx + .select({ payloadHash: productAnalyticsEventReceipts.payloadHash }) + .from(productAnalyticsEventReceipts) + .where(eq(productAnalyticsEventReceipts.eventIdHash, eventIdHash)) + .limit(1) + .for("update"); + if (!receipt || receipt.payloadHash !== row.payload_hash) { + return { + eventId: row.event_id, + payloadHash: row.payload_hash, + payloadConflict: true, + status: "conflict" as const, + }; + } + const deliveryKey = randomUUID(); + await tx + .insert(productAnalyticsOutbox) + .values({ + eventId: row.event_id, + deliveryKey, + payloadHash: row.payload_hash, + eventName: row.event_name, + payloadKind: "product_event_row_v1", + payload: row, + anonymousId: row.anonymous_id || null, + userId: row.user_id || null, + organizationId: row.organization_id || null, + }) + .onDuplicateKeyUpdate({ + set: { + payloadConflict: sql`IF(${productAnalyticsOutbox.payloadHash} <> ${row.payload_hash}, TRUE, ${productAnalyticsOutbox.payloadConflict})`, + status: sql`IF(${productAnalyticsOutbox.payloadHash} = ${row.payload_hash} AND ${productAnalyticsOutbox.deadLetteredAt} IS NOT NULL, 'pending', ${productAnalyticsOutbox.status})`, + nextAttemptAt: sql`IF(${productAnalyticsOutbox.payloadHash} = ${row.payload_hash} AND ${productAnalyticsOutbox.deadLetteredAt} IS NOT NULL, CURRENT_TIMESTAMP, ${productAnalyticsOutbox.nextAttemptAt})`, + leaseOwnerId: sql`IF(${productAnalyticsOutbox.payloadHash} = ${row.payload_hash} AND ${productAnalyticsOutbox.deadLetteredAt} IS NOT NULL, NULL, ${productAnalyticsOutbox.leaseOwnerId})`, + leaseExpiresAt: sql`IF(${productAnalyticsOutbox.payloadHash} = ${row.payload_hash} AND ${productAnalyticsOutbox.deadLetteredAt} IS NOT NULL, NULL, ${productAnalyticsOutbox.leaseExpiresAt})`, + lastErrorCode: sql`IF(${productAnalyticsOutbox.payloadHash} <> ${row.payload_hash}, 'payload_conflict', IF(${productAnalyticsOutbox.deadLetteredAt} IS NOT NULL, NULL, ${productAnalyticsOutbox.lastErrorCode}))`, + deadLetteredAt: sql`IF(${productAnalyticsOutbox.payloadHash} = ${row.payload_hash}, NULL, ${productAnalyticsOutbox.deadLetteredAt})`, + updatedAt: new Date(), + }, + }); + const [stored] = await tx + .select({ + deliveryKey: productAnalyticsOutbox.deliveryKey, + payloadHash: productAnalyticsOutbox.payloadHash, + status: productAnalyticsOutbox.status, + }) + .from(productAnalyticsOutbox) + .where(eq(productAnalyticsOutbox.eventId, row.event_id)) + .limit(1); + return { + eventId: row.event_id, + deliveryKey: stored?.deliveryKey ?? deliveryKey, + payloadHash: row.payload_hash, + payloadConflict: stored?.payloadHash !== row.payload_hash, + status: stored?.status, + }; +} + +export async function queueDurableServerProductEvent( + event: ServerProductEvent, +) { + const persisted = await db().transaction((tx) => + persistProductAnalyticsEvent(tx, event), + ); + if (persisted.status === "suppressed" || persisted.status === "conflict") { + return persisted; + } + const delivery = await attemptProductAnalyticsOutboxDelivery( + persisted.eventId, + ); + return { ...persisted, ...delivery }; +} + +async function claimPendingProductAnalyticsEvent(eventId: string) { + const leaseOwnerId = randomUUID(); + const now = new Date(); + await db() + .update(productAnalyticsOutbox) + .set({ + leaseOwnerId, + leaseExpiresAt: new Date(now.getTime() + OUTBOX_LEASE_DURATION_MS), + }) + .where( + and( + eq(productAnalyticsOutbox.eventId, eventId), + eq(productAnalyticsOutbox.status, "pending"), + lte(productAnalyticsOutbox.nextAttemptAt, now), + or( + isNull(productAnalyticsOutbox.leaseOwnerId), + isNull(productAnalyticsOutbox.leaseExpiresAt), + lt(productAnalyticsOutbox.leaseExpiresAt, now), + ), + ), + ); + const [claimed] = await db() + .select({ + deliveryKey: productAnalyticsOutbox.deliveryKey, + payload: productAnalyticsOutbox.payload, + payloadKind: productAnalyticsOutbox.payloadKind, + payloadConflict: productAnalyticsOutbox.payloadConflict, + }) + .from(productAnalyticsOutbox) + .where( + and( + eq(productAnalyticsOutbox.eventId, eventId), + eq(productAnalyticsOutbox.leaseOwnerId, leaseOwnerId), + ), + ) + .limit(1); + return claimed ? { ...claimed, leaseOwnerId } : undefined; +} + +async function startPendingProductAnalyticsEvent(eventId: string) { + const pending = await claimPendingProductAnalyticsEvent(eventId); + if (!pending) return { status: "unavailable" as const }; + + if ( + pending.payloadKind !== "product_event_row_v1" || + !isStoredProductEventRow(pending.payload, eventId) + ) { + await db() + .update(productAnalyticsOutbox) + .set({ + status: "dead_letter", + lastErrorCode: "stored_contract_invalid", + deadLetteredAt: new Date(), + leaseOwnerId: null, + leaseExpiresAt: null, + }) + .where( + and( + eq(productAnalyticsOutbox.eventId, eventId), + eq(productAnalyticsOutbox.leaseOwnerId, pending.leaseOwnerId), + ), + ); + return { status: "dead_lettered" as const }; + } + + try { + const run = await start(deliverProductAnalyticsRowWorkflow, [ + pending.deliveryKey, + ]); + await db() + .update(productAnalyticsOutbox) + .set({ + status: "workflow_started", + attemptCount: sql`${productAnalyticsOutbox.attemptCount} + 1`, + nextAttemptAt: new Date(Date.now() + DELIVERY_CONFIRMATION_TIMEOUT_MS), + workflowRunId: run.runId, + lastErrorCode: pending.payloadConflict ? "payload_conflict" : null, + leaseOwnerId: null, + leaseExpiresAt: null, + }) + .where( + and( + eq(productAnalyticsOutbox.eventId, eventId), + eq(productAnalyticsOutbox.status, "pending"), + eq(productAnalyticsOutbox.leaseOwnerId, pending.leaseOwnerId), + ), + ); + return { status: "started" as const, runId: run.runId }; + } catch (error) { + const [row] = await db() + .select({ attemptCount: productAnalyticsOutbox.attemptCount }) + .from(productAnalyticsOutbox) + .where( + and( + eq(productAnalyticsOutbox.eventId, eventId), + eq(productAnalyticsOutbox.leaseOwnerId, pending.leaseOwnerId), + ), + ) + .limit(1); + if (row) { + const attemptCount = row.attemptCount + 1; + const retryDelay = Math.min( + MAX_RETRY_DELAY_MS, + 2 ** Math.min(attemptCount, 10) * 1_000, + ); + await db() + .update(productAnalyticsOutbox) + .set({ + attemptCount, + nextAttemptAt: new Date(Date.now() + retryDelay), + lastErrorCode: "workflow_start_failed", + leaseOwnerId: null, + leaseExpiresAt: null, + }) + .where( + and( + eq(productAnalyticsOutbox.eventId, eventId), + eq(productAnalyticsOutbox.leaseOwnerId, pending.leaseOwnerId), + ), + ); + } + throw error; + } +} + +export async function attemptProductAnalyticsOutboxDelivery(eventId: string) { + try { + return await startPendingProductAnalyticsEvent(eventId); + } catch { + console.error("Product analytics outbox delivery deferred"); + return { status: "deferred" as const }; + } +} + +export async function drainProductAnalyticsOutbox( + limit = MAX_DRAIN_BATCH_SIZE, +) { + const boundedLimit = Math.max(1, Math.min(limit, MAX_DRAIN_BATCH_SIZE)); + const now = new Date(); + await db() + .update(productAnalyticsOutbox) + .set({ + status: "pending", + workflowRunId: null, + lastErrorCode: "delivery_unconfirmed", + leaseOwnerId: null, + leaseExpiresAt: null, + nextAttemptAt: now, + }) + .where( + and( + eq(productAnalyticsOutbox.status, "workflow_started"), + lte(productAnalyticsOutbox.nextAttemptAt, now), + ), + ); + const expiredDelivered = await db() + .select({ eventId: productAnalyticsOutbox.eventId }) + .from(productAnalyticsOutbox) + .where( + and( + eq(productAnalyticsOutbox.status, "delivered"), + isNotNull(productAnalyticsOutbox.deliveredAt), + lt( + productAnalyticsOutbox.deliveredAt, + new Date(now.getTime() - DELIVERED_RETENTION_MS), + ), + ), + ) + .orderBy(asc(productAnalyticsOutbox.deliveredAt)) + .limit(CLEANUP_BATCH_SIZE); + if (expiredDelivered.length > 0) { + await db() + .delete(productAnalyticsOutbox) + .where( + inArray( + productAnalyticsOutbox.eventId, + expiredDelivered.map((row) => row.eventId), + ), + ); + } + const expiredReceipts = await db() + .select({ eventIdHash: productAnalyticsEventReceipts.eventIdHash }) + .from(productAnalyticsEventReceipts) + .where(lt(productAnalyticsEventReceipts.retainUntil, now)) + .orderBy(asc(productAnalyticsEventReceipts.retainUntil)) + .limit(CLEANUP_BATCH_SIZE); + if (expiredReceipts.length > 0) { + await db() + .delete(productAnalyticsEventReceipts) + .where( + inArray( + productAnalyticsEventReceipts.eventIdHash, + expiredReceipts.map((row) => row.eventIdHash), + ), + ); + } + const expiredLeases = await db() + .select({ id: productAnalyticsIngestionLeases.id }) + .from(productAnalyticsIngestionLeases) + .where(lt(productAnalyticsIngestionLeases.expiresAt, now)) + .orderBy(asc(productAnalyticsIngestionLeases.expiresAt)) + .limit(CLEANUP_BATCH_SIZE); + if (expiredLeases.length > 0) { + await db() + .delete(productAnalyticsIngestionLeases) + .where( + inArray( + productAnalyticsIngestionLeases.id, + expiredLeases.map((row) => row.id), + ), + ); + } + const pending = await db() + .select({ eventId: productAnalyticsOutbox.eventId }) + .from(productAnalyticsOutbox) + .where( + and( + eq(productAnalyticsOutbox.status, "pending"), + lte(productAnalyticsOutbox.nextAttemptAt, new Date()), + or( + isNull(productAnalyticsOutbox.leaseOwnerId), + isNull(productAnalyticsOutbox.leaseExpiresAt), + lt(productAnalyticsOutbox.leaseExpiresAt, new Date()), + ), + ), + ) + .orderBy(asc(productAnalyticsOutbox.createdAt)) + .limit(boundedLimit); + + const results: Awaited< + ReturnType + >[] = []; + for (let offset = 0; offset < pending.length; offset += DRAIN_CONCURRENCY) { + const batch = pending.slice(offset, offset + DRAIN_CONCURRENCY); + results.push( + ...(await Promise.all( + batch.map((row) => attemptProductAnalyticsOutboxDelivery(row.eventId)), + )), + ); + } + const started = results.filter( + (result) => result.status === "started", + ).length; + const deferred = results.filter( + (result) => result.status === "deferred", + ).length; + const deadLettered = results.filter( + (result) => result.status === "dead_lettered", + ).length; + return { + attempted: pending.length, + started, + deferred, + deadLettered, + cleanedDelivered: expiredDelivered.length, + cleanedReceipts: expiredReceipts.length, + cleanedLeases: expiredLeases.length, + capacityPerDay: MAX_DRAIN_BATCH_SIZE * 12 * 24, + }; +} + +export async function getProductAnalyticsOutboxHealth() { + const now = new Date(); + const [rows, [receiptConflicts], reconciliationFailures] = await Promise.all([ + db() + .select({ + eventName: productAnalyticsOutbox.eventName, + pending: sql`SUM(IF(${productAnalyticsOutbox.status} = 'pending', 1, 0))`, + due: sql`SUM(IF(${productAnalyticsOutbox.status} = 'pending' AND ${productAnalyticsOutbox.nextAttemptAt} <= ${now}, 1, 0))`, + leased: sql`SUM(IF(${productAnalyticsOutbox.status} = 'pending' AND ${productAnalyticsOutbox.leaseExpiresAt} > ${now}, 1, 0))`, + deadLetter: sql`SUM(IF(${productAnalyticsOutbox.status} = 'dead_letter', 1, 0))`, + payloadConflict: sql`SUM(IF(${productAnalyticsOutbox.payloadConflict}, 1, 0))`, + workflowStarted: sql`SUM(IF(${productAnalyticsOutbox.status} = 'workflow_started', 1, 0))`, + oldestActiveAt: sql< + Date | string | null + >`MIN(${productAnalyticsOutbox.createdAt})`, + oldestDeadLetterAt: sql< + Date | string | null + >`MIN(IF(${productAnalyticsOutbox.status} = 'dead_letter', ${productAnalyticsOutbox.deadLetteredAt}, NULL))`, + }) + .from(productAnalyticsOutbox) + .where( + inArray(productAnalyticsOutbox.status, [ + "pending", + "workflow_started", + "dead_letter", + ]), + ) + .groupBy(productAnalyticsOutbox.eventName) + .orderBy(asc(productAnalyticsOutbox.eventName)), + db() + .select({ + events: sql`COUNT(*)`, + attempts: sql`COALESCE(SUM(${productAnalyticsEventReceipts.conflictCount}), 0)`, + }) + .from(productAnalyticsEventReceipts) + .where(gt(productAnalyticsEventReceipts.conflictCount, 0)), + db() + .select({ + attempts: sql`COALESCE(SUM(${productAnalyticsReconciliationFailures.attemptCount}), 0)`, + events: sql`COUNT(*)`, + sourceType: productAnalyticsReconciliationFailures.sourceType, + }) + .from(productAnalyticsReconciliationFailures) + .groupBy(productAnalyticsReconciliationFailures.sourceType) + .orderBy(asc(productAnalyticsReconciliationFailures.sourceType)), + ]); + const partitions = rows.map((row) => { + const ageSeconds = (value: Date | string | null) => { + if (!value) return 0; + const timestamp = + value instanceof Date ? value.getTime() : Date.parse(value); + return Number.isFinite(timestamp) + ? Math.max(0, Math.floor((now.getTime() - timestamp) / 1_000)) + : 0; + }; + const oldestActiveAgeSeconds = ageSeconds(row.oldestActiveAt); + return { + eventName: row.eventName, + pending: Number(row.pending), + due: Number(row.due), + leased: Number(row.leased), + deadLetter: Number(row.deadLetter), + payloadConflict: Number(row.payloadConflict), + workflowStarted: Number(row.workflowStarted), + oldestPendingAgeSeconds: oldestActiveAgeSeconds, + oldestActiveAgeSeconds, + oldestDeadLetterAgeSeconds: ageSeconds(row.oldestDeadLetterAt), + }; + }); + return { + healthy: + partitions.every( + (row) => + row.deadLetter === 0 && + row.payloadConflict === 0 && + row.oldestActiveAgeSeconds * 1_000 <= OLDEST_PENDING_SLO_MS, + ) && + Number(receiptConflicts?.events ?? 0) === 0 && + reconciliationFailures.length === 0, + oldestPendingSloSeconds: OLDEST_PENDING_SLO_MS / 1_000, + receiptPayloadConflictEvents: Number(receiptConflicts?.events ?? 0), + receiptPayloadConflictAttempts: Number(receiptConflicts?.attempts ?? 0), + reconciliationFailures: reconciliationFailures.map((row) => ({ + attempts: Number(row.attempts), + events: Number(row.events), + sourceType: row.sourceType, + })), + capacityPerDay: MAX_DRAIN_BATCH_SIZE * 12 * 24, + partitions, + }; +} diff --git a/apps/web/lib/analytics/server-event.ts b/apps/web/lib/analytics/server-event.ts index dbb1fa0be1d..01ddd3059e7 100644 --- a/apps/web/lib/analytics/server-event.ts +++ b/apps/web/lib/analytics/server-event.ts @@ -3,7 +3,7 @@ import { normalizeAnalyticsIdentifier, normalizeProductEventProperties, PRODUCT_ANALYTICS_LIMITS, - type ProductEventPlatform, + type ProductEventPlatformFor, type ProductEventPropertyField, type ServerProductEventName, } from "@cap/analytics"; @@ -14,7 +14,7 @@ type ServerProductEventBase = { eventName: Name; occurredAt: string; anonymousId?: string; - platform: ProductEventPlatform; + platform: ProductEventPlatformFor; userId?: string; organizationId?: string; pathname?: string; diff --git a/apps/web/lib/analytics/server.ts b/apps/web/lib/analytics/server.ts index 0b418f16cda..02f33958268 100644 --- a/apps/web/lib/analytics/server.ts +++ b/apps/web/lib/analytics/server.ts @@ -1,7 +1,6 @@ import { PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE } from "@cap/analytics"; import type { NextRequest } from "next/server"; -import { start } from "workflow/api"; -import { deliverProductAnalyticsEventWorkflow } from "@/workflows/deliver-product-analytics-event"; +import { queueDurableServerProductEvent } from "./product-event-outbox"; import { normalizeServerIdentifier, type ServerProductEvent, @@ -10,8 +9,14 @@ import { export { createServerProductEventRows } from "./server-event"; export async function queueServerProductEvent(event: ServerProductEvent) { - const run = await start(deliverProductAnalyticsEventWorkflow, [event]); - return { eventId: event.eventId, runId: run.runId }; + const result = await queueDurableServerProductEvent(event); + return { + ...result, + runId: + "runId" in result && result.runId + ? result.runId + : `outbox:${"deliveryKey" in result ? result.deliveryKey : "suppressed"}`, + }; } export function readAnalyticsAnonymousId(request: NextRequest) { diff --git a/apps/web/lib/analytics/stripe-business-events.ts b/apps/web/lib/analytics/stripe-business-events.ts index f19fa93e621..07f1daed9d1 100644 --- a/apps/web/lib/analytics/stripe-business-events.ts +++ b/apps/web/lib/analytics/stripe-business-events.ts @@ -7,10 +7,8 @@ type AnalyticsUser = { id: string; }; -const subscriptionAnalyticsPlatform = ( - subscription: Stripe.Subscription, -): ProductEventPlatform => { - const platform = subscription.metadata.platform; +const analyticsPlatform = (metadata: Stripe.Metadata): ProductEventPlatform => { + const platform = metadata.platform; return platform === "web" || platform === "desktop" || platform === "mobile" || @@ -30,12 +28,29 @@ const subscriptionPrice = (subscription: Stripe.Subscription) => { return { item, price: item.price }; }; -export function subscriptionInvoicePriceId(invoice: Stripe.Invoice): string { - const priceId = invoice.lines.data.find((line) => line.price)?.price?.id; - if (!priceId) { +const subscriptionInvoiceLine = (invoice: Stripe.Invoice) => { + const line = invoice.lines.data.find((candidate) => candidate.price); + if (!line?.price?.id) { throw new Error("Subscription invoice is missing its Stripe price"); } - return priceId; + return { line, price: line.price }; +}; + +const subscriptionInvoiceMetadata = (invoice: Stripe.Invoice) => + invoice.subscription_details?.metadata ?? + subscriptionInvoiceLine(invoice).line.metadata; + +const validatedInvoiceAnalyticsMetadata = (invoice: Stripe.Invoice) => { + const metadata = subscriptionInvoiceMetadata(invoice); + if (!metadata.analyticsSchemaVersion) return null; + if (metadata.analyticsSchemaVersion !== "1") { + throw new Error("Stripe invoice has an unsupported analytics schema"); + } + return metadata; +}; + +export function subscriptionInvoicePriceId(invoice: Stripe.Invoice): string { + return subscriptionInvoiceLine(invoice).price.id; } export async function isFirstPositiveSubscriptionPayment({ @@ -233,14 +248,12 @@ export function subscriptionInvoicePaidProductEvent({ eventId, occurredAt, invoice, - subscription, user, firstPositivePayment, }: { eventId: string; occurredAt: string; invoice: Stripe.Invoice; - subscription: Stripe.Subscription; user: AnalyticsUser; firstPositivePayment: boolean; }): ServerProductEvent | null { @@ -250,7 +263,9 @@ export function subscriptionInvoicePaidProductEvent({ ) { return null; } - const { item, price } = subscriptionPrice(subscription); + const metadata = validatedInvoiceAnalyticsMetadata(invoice); + if (!metadata) return null; + const { line, price } = subscriptionInvoiceLine(invoice); if (!firstPositivePayment) { return { eventId: `stripe:${eventId}:subscription_renewed`, @@ -258,7 +273,7 @@ export function subscriptionInvoicePaidProductEvent({ occurredAt, platform: "server", userId: user.id, - organizationId: subscriptionOrganizationId(subscription), + organizationId: metadata.analyticsOrganizationId || undefined, properties: { amount_paid_minor: invoice.amount_paid, currency: invoice.currency, @@ -275,13 +290,13 @@ export function subscriptionInvoicePaidProductEvent({ eventId: `stripe:${eventId}:purchase_completed`, eventName: "purchase_completed", occurredAt, - anonymousId: subscription.metadata.analyticsAnonymousId, - platform: subscriptionAnalyticsPlatform(subscription), + anonymousId: metadata.analyticsAnonymousId, + platform: analyticsPlatform(metadata), userId: user.id, - organizationId: subscriptionOrganizationId(subscription), + organizationId: metadata.analyticsOrganizationId || undefined, properties: { payment_status: "paid", - subscription_status: subscription.status, + subscription_status: "active", amount_total_minor: invoice.amount_paid, amount_subtotal_minor: invoice.subtotal, discount_amount_minor: discountAmount ?? null, @@ -289,12 +304,12 @@ export function subscriptionInvoicePaidProductEvent({ unit_amount_minor: price.unit_amount, billing_interval: price.recurring?.interval ?? null, billing_interval_count: price.recurring?.interval_count ?? null, - invite_quota: item.quantity ?? null, + invite_quota: line.quantity ?? null, price_id: price.id, - quantity: item.quantity ?? null, + quantity: line.quantity ?? null, is_first_purchase: true, - is_guest_checkout: subscription.metadata.guestCheckout === "true", - is_onboarding: subscription.metadata.isOnBoarding === "true", + is_guest_checkout: metadata.guestCheckout === "true", + is_onboarding: metadata.isOnBoarding === "true", }, }; } @@ -303,23 +318,23 @@ export function subscriptionPaymentFailedProductEvent({ eventId, occurredAt, invoice, - subscription, user, }: { eventId: string; occurredAt: string; invoice: Stripe.Invoice; - subscription: Stripe.Subscription; user: AnalyticsUser; }): ServerProductEvent | null { if (invoice.amount_due <= 0) return null; + const metadata = validatedInvoiceAnalyticsMetadata(invoice); + if (!metadata) return null; return { eventId: `stripe:${eventId}:subscription_payment_failed`, eventName: "subscription_payment_failed", occurredAt, platform: "server", userId: user.id, - organizationId: subscriptionOrganizationId(subscription), + organizationId: metadata.analyticsOrganizationId || undefined, properties: { amount_due_minor: invoice.amount_due, currency: invoice.currency, @@ -334,7 +349,6 @@ export function subscriptionRefundedProductEvent({ occurredAt, charge, invoice, - subscription, user, refundedAmount, }: { @@ -342,18 +356,19 @@ export function subscriptionRefundedProductEvent({ occurredAt: string; charge: Stripe.Charge; invoice: Stripe.Invoice; - subscription: Stripe.Subscription; user: AnalyticsUser; refundedAmount: number; }): ServerProductEvent | null { if (refundedAmount <= 0) return null; + const metadata = validatedInvoiceAnalyticsMetadata(invoice); + if (!metadata) return null; return { eventId: `stripe:${eventId}:subscription_refunded`, eventName: "subscription_refunded", occurredAt, platform: "server", userId: user.id, - organizationId: subscriptionOrganizationId(subscription), + organizationId: metadata.analyticsOrganizationId || undefined, properties: { amount_refunded_minor: refundedAmount, currency: charge.currency, diff --git a/apps/web/lib/analytics/video-platform.ts b/apps/web/lib/analytics/video-platform.ts new file mode 100644 index 00000000000..451855ac46d --- /dev/null +++ b/apps/web/lib/analytics/video-platform.ts @@ -0,0 +1,24 @@ +export const videoAnalyticsPlatform = (video: { + metadata: unknown; + source: { type: string }; +}) => { + const metadata = + typeof video.metadata === "object" && video.metadata !== null + ? (video.metadata as Record) + : {}; + if (metadata.initiatingPlatform === "cli") return "cli" as const; + if ( + metadata.source === "mobileUpload" || + metadata.source === "mobileCamera" + ) { + return "mobile" as const; + } + if ( + video.source.type === "desktopMP4" || + video.source.type === "desktopSegments" || + video.source.type === "local" + ) { + return "desktop" as const; + } + return "server" as const; +}; diff --git a/apps/web/lib/organization-invite-delivery.ts b/apps/web/lib/organization-invite-delivery.ts new file mode 100644 index 00000000000..bb8431da60d --- /dev/null +++ b/apps/web/lib/organization-invite-delivery.ts @@ -0,0 +1,306 @@ +import { randomUUID } from "node:crypto"; +import { db } from "@cap/database"; +import { sendEmail } from "@cap/database/emails/config"; +import { OrganizationInvite } from "@cap/database/emails/organization-invite"; +import { organizationInvites, organizations } from "@cap/database/schema"; +import { serverEnv } from "@cap/env"; +import { and, asc, eq, gt, isNull, lte, or, sql } from "drizzle-orm"; +import { + attemptProductAnalyticsOutboxDelivery, + persistProductAnalyticsEvent, +} from "./analytics/product-event-outbox"; + +const MAX_DELIVERY_BATCH_SIZE = 25; +const MAX_DELIVERY_RETRY_DELAY_MS = 60 * 60 * 1_000; +const MAX_DELIVERY_ATTEMPTS = 12; +const INVITE_DELIVERY_SLO_MS = 15 * 60 * 1_000; +const INVITE_DELIVERY_LEASE_MS = 5 * 60 * 1_000; + +async function deferOrganizationInviteDelivery( + inviteId: string, + leaseOwnerId: string, +) { + const [invite] = await db() + .select({ + attemptCount: organizationInvites.emailDeliveryAttemptCount, + }) + .from(organizationInvites) + .where( + and( + eq(organizationInvites.id, inviteId), + eq(organizationInvites.emailDeliveryState, "pending"), + eq(organizationInvites.emailDeliveryLeaseOwnerId, leaseOwnerId), + ), + ) + .limit(1); + if (!invite) return; + const attemptCount = invite.attemptCount; + const retryDelay = Math.min( + MAX_DELIVERY_RETRY_DELAY_MS, + 2 ** Math.min(attemptCount, 10) * 1_000, + ); + const deadLettered = attemptCount >= MAX_DELIVERY_ATTEMPTS; + await db() + .update(organizationInvites) + .set({ + emailDeliveryState: deadLettered ? "dead_letter" : "pending", + emailDeliveryAttemptCount: attemptCount, + emailDeliveryNextAttemptAt: deadLettered + ? null + : new Date(Date.now() + retryDelay), + emailDeliveryErrorCode: deadLettered + ? "provider_delivery_dead_letter" + : "provider_send_failed", + emailDeliveryLeaseOwnerId: null, + emailDeliveryLeaseExpiresAt: null, + }) + .where( + and( + eq(organizationInvites.id, inviteId), + eq(organizationInvites.emailDeliveryState, "pending"), + eq(organizationInvites.emailDeliveryLeaseOwnerId, leaseOwnerId), + ), + ); +} + +export async function deliverOrganizationInvite(inviteId: string) { + const leaseOwnerId = randomUUID(); + const claimed = await db().transaction(async (tx) => { + const now = new Date(); + const [invite] = await tx + .select({ + id: organizationInvites.id, + organizationId: organizationInvites.organizationId, + organizationName: organizations.name, + invitedEmail: organizationInvites.invitedEmail, + invitedByUserId: organizationInvites.invitedByUserId, + role: organizationInvites.role, + }) + .from(organizationInvites) + .innerJoin( + organizations, + eq(organizations.id, organizationInvites.organizationId), + ) + .where( + and( + eq(organizationInvites.id, inviteId), + eq(organizationInvites.status, "pending"), + isNull(organizations.tombstoneAt), + eq(organizationInvites.emailDeliveryState, "pending"), + or( + isNull(organizationInvites.expiresAt), + gt(organizationInvites.expiresAt, now), + ), + or( + isNull(organizationInvites.emailDeliveryNextAttemptAt), + lte(organizationInvites.emailDeliveryNextAttemptAt, now), + ), + or( + isNull(organizationInvites.emailDeliveryLeaseOwnerId), + isNull(organizationInvites.emailDeliveryLeaseExpiresAt), + lte(organizationInvites.emailDeliveryLeaseExpiresAt, now), + ), + ), + ) + .limit(1) + .for("update"); + if (!invite) return undefined; + await tx + .update(organizationInvites) + .set({ + emailDeliveryAttemptCount: sql`${organizationInvites.emailDeliveryAttemptCount} + 1`, + emailDeliveryLeaseOwnerId: leaseOwnerId, + emailDeliveryLeaseExpiresAt: new Date( + now.getTime() + INVITE_DELIVERY_LEASE_MS, + ), + }) + .where(eq(organizationInvites.id, inviteId)); + return invite; + }); + if (!claimed) return { status: "not_pending" as const }; + + try { + if (!serverEnv().RESEND_API_KEY) { + throw new Error("Organization invite email provider is not configured"); + } + const result = await sendEmail({ + email: claimed.invitedEmail, + subject: `Invitation to join ${claimed.organizationName} on Cap`, + react: OrganizationInvite({ + email: claimed.invitedEmail, + url: `${serverEnv().WEB_URL}/invite/${claimed.id}`, + organizationName: claimed.organizationName, + }), + idempotencyKey: `organization-invite:${claimed.id}`, + }); + const providerMessageId = result?.data?.id; + if (!result || result.error || !providerMessageId) { + throw new Error("Organization invite provider rejected delivery"); + } + const eventId = await db().transaction(async (tx) => { + const now = new Date(); + const [invite] = await tx + .select({ + id: organizationInvites.id, + organizationId: organizationInvites.organizationId, + organizationName: organizations.name, + invitedEmail: organizationInvites.invitedEmail, + invitedByUserId: organizationInvites.invitedByUserId, + role: organizationInvites.role, + }) + .from(organizationInvites) + .innerJoin( + organizations, + eq(organizations.id, organizationInvites.organizationId), + ) + .where( + and( + eq(organizationInvites.id, inviteId), + eq(organizationInvites.status, "pending"), + isNull(organizations.tombstoneAt), + eq(organizationInvites.emailDeliveryState, "pending"), + eq(organizationInvites.emailDeliveryLeaseOwnerId, leaseOwnerId), + or( + isNull(organizationInvites.expiresAt), + gt(organizationInvites.expiresAt, now), + ), + ), + ) + .limit(1) + .for("update"); + if (!invite) return undefined; + const sentAt = new Date(); + const analyticsEventId = `organization_invite:${inviteId}:sent`; + await tx + .update(organizationInvites) + .set({ + emailDeliveryState: "sent", + emailDeliveryNextAttemptAt: null, + emailDeliveryErrorCode: null, + emailDeliveryLeaseOwnerId: null, + emailDeliveryLeaseExpiresAt: null, + emailProviderMessageId: providerMessageId, + emailSentAt: sentAt, + }) + .where(eq(organizationInvites.id, inviteId)); + await persistProductAnalyticsEvent(tx, { + eventId: analyticsEventId, + eventName: "organization_invite_sent", + occurredAt: sentAt.toISOString(), + platform: "web", + userId: invite.invitedByUserId, + organizationId: invite.organizationId, + properties: { + invite_count: 1, + admin_count: invite.role === "admin" ? 1 : 0, + member_count: invite.role === "admin" ? 0 : 1, + delivery: "email", + }, + }); + return analyticsEventId; + }); + if (!eventId) return { status: "not_pending" as const }; + await attemptProductAnalyticsOutboxDelivery(eventId); + return { status: "sent" as const }; + } catch { + await deferOrganizationInviteDelivery(inviteId, leaseOwnerId); + return { status: "deferred" as const }; + } +} + +export async function recoverOrganizationInviteDeliveries( + limit = MAX_DELIVERY_BATCH_SIZE, +) { + const boundedLimit = Math.max(1, Math.min(limit, MAX_DELIVERY_BATCH_SIZE)); + const pending = await db() + .select({ id: organizationInvites.id }) + .from(organizationInvites) + .innerJoin( + organizations, + eq(organizations.id, organizationInvites.organizationId), + ) + .where( + and( + eq(organizationInvites.emailDeliveryState, "pending"), + eq(organizationInvites.status, "pending"), + isNull(organizations.tombstoneAt), + or( + isNull(organizationInvites.expiresAt), + gt(organizationInvites.expiresAt, new Date()), + ), + or( + isNull(organizationInvites.emailDeliveryNextAttemptAt), + lte(organizationInvites.emailDeliveryNextAttemptAt, new Date()), + ), + or( + isNull(organizationInvites.emailDeliveryLeaseOwnerId), + isNull(organizationInvites.emailDeliveryLeaseExpiresAt), + lte(organizationInvites.emailDeliveryLeaseExpiresAt, new Date()), + ), + ), + ) + .orderBy(asc(organizationInvites.createdAt)) + .limit(boundedLimit); + + let sent = 0; + let deferred = 0; + for (const invite of pending) { + const result = await deliverOrganizationInvite(invite.id); + if (result.status === "sent") sent += 1; + if (result.status === "deferred") deferred += 1; + } + return { attempted: pending.length, sent, deferred }; +} + +export async function getOrganizationInviteDeliveryHealth() { + const now = new Date(); + const [row] = await db() + .select({ + pending: sql`SUM(IF(${organizationInvites.emailDeliveryState} = 'pending', 1, 0))`, + due: sql`SUM(IF(${organizationInvites.emailDeliveryState} = 'pending' AND (${organizationInvites.emailDeliveryNextAttemptAt} IS NULL OR ${organizationInvites.emailDeliveryNextAttemptAt} <= ${now}), 1, 0))`, + deadLetter: sql`SUM(IF(${organizationInvites.emailDeliveryState} = 'dead_letter', 1, 0))`, + oldestPendingAt: sql< + Date | string | null + >`MIN(IF(${organizationInvites.emailDeliveryState} = 'pending', ${organizationInvites.createdAt}, NULL))`, + }) + .from(organizationInvites); + const oldestTimestamp = row?.oldestPendingAt + ? row.oldestPendingAt instanceof Date + ? row.oldestPendingAt.getTime() + : Date.parse(row.oldestPendingAt) + : now.getTime(); + const oldestPendingAgeSeconds = Number.isFinite(oldestTimestamp) + ? Math.max(0, Math.floor((now.getTime() - oldestTimestamp) / 1_000)) + : 0; + const pending = Number(row?.pending ?? 0); + const deadLetter = Number(row?.deadLetter ?? 0); + return { + healthy: + deadLetter === 0 && + oldestPendingAgeSeconds * 1_000 <= INVITE_DELIVERY_SLO_MS, + pending, + due: Number(row?.due ?? 0), + deadLetter, + oldestPendingAgeSeconds, + oldestPendingSloSeconds: INVITE_DELIVERY_SLO_MS / 1_000, + }; +} + +export async function requeueOrganizationInviteDelivery(inviteId: string) { + await db() + .update(organizationInvites) + .set({ + emailDeliveryState: "pending", + emailDeliveryNextAttemptAt: new Date(), + emailDeliveryErrorCode: null, + emailDeliveryAttemptCount: 0, + emailDeliveryLeaseOwnerId: null, + emailDeliveryLeaseExpiresAt: null, + }) + .where( + and( + eq(organizationInvites.id, inviteId), + eq(organizationInvites.emailDeliveryState, "dead_letter"), + ), + ); +} diff --git a/apps/web/proxy.ts b/apps/web/proxy.ts index c5bd0629fcd..10273e81def 100644 --- a/apps/web/proxy.ts +++ b/apps/web/proxy.ts @@ -44,7 +44,15 @@ const nextWithAnalyticsToken = ( const existingAnonymousId = normalizeAnalyticsIdentifier( request.cookies.get(PRODUCT_ANALYTICS_ANONYMOUS_ID_COOKIE)?.value, ); + const stagingRunId = request.headers.get("x-cap-analytics-test-run"); + const stagingAnonymousId = + serverEnv().VERCEL_ENV === "preview" && + stagingRunId && + /^[A-Za-z0-9_-]{8,128}$/.test(stagingRunId) + ? `synthetic_${stagingRunId}` + : undefined; const anonymousId = + stagingAnonymousId ?? existingAnonymousId ?? claims?.anonymousId ?? createProductAnalyticsAnonymousId(); diff --git a/apps/web/vercel.json b/apps/web/vercel.json index fd194c2e332..dae4c97b722 100644 --- a/apps/web/vercel.json +++ b/apps/web/vercel.json @@ -17,9 +17,21 @@ "path": "/api/cron/reconcile-product-analytics", "schedule": "43 4 * * *" }, + { + "path": "/api/cron/refresh-product-analytics", + "schedule": "*/10 * * * *" + }, { "path": "/api/cron/recover-product-analytics-erasure", "schedule": "*/10 * * * *" + }, + { + "path": "/api/cron/drain-product-analytics-outbox", + "schedule": "*/5 * * * *" + }, + { + "path": "/api/cron/recover-organization-invite-delivery", + "schedule": "2-59/5 * * * *" } ], "functions": { diff --git a/apps/web/workflows/deliver-product-analytics-event.ts b/apps/web/workflows/deliver-product-analytics-event.ts index 4f78dc4afb8..8f25c402fb2 100644 --- a/apps/web/workflows/deliver-product-analytics-event.ts +++ b/apps/web/workflows/deliver-product-analytics-event.ts @@ -1,123 +1,11 @@ -import { - PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT, - ProductAnalyticsError, - sendProductAnalyticsRows, -} from "@cap/analytics"; -import { db } from "@cap/database"; -import { - messengerSupportEmails, - organizations, - users, -} from "@cap/database/schema"; -import { serverEnv } from "@cap/env"; -import { Organisation, User } from "@cap/web-domain"; -import { and, eq, isNull, notInArray } from "drizzle-orm"; -import { FatalError } from "workflow"; -import { start } from "workflow/api"; -import { - createServerProductEventRows, - type ServerProductEvent, -} from "@/lib/analytics/server-event"; - -async function isProductAnalyticsIdentitySuppressed(event: ServerProductEvent) { - const pendingDeletionUserIds = db() - .select({ userId: messengerSupportEmails.userId }) - .from(messengerSupportEmails) - .where( - eq( - messengerSupportEmails.subject, - PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT, - ), - ); - const [userRows, organizationRows] = await Promise.all([ - event.userId - ? db() - .select({ id: users.id }) - .from(users) - .where( - and( - eq(users.id, User.UserId.make(event.userId)), - notInArray(users.id, pendingDeletionUserIds), - ), - ) - .limit(1) - : Promise.resolve([{}]), - event.organizationId - ? db() - .select({ id: organizations.id }) - .from(organizations) - .where( - and( - eq( - organizations.id, - Organisation.OrganisationId.make(event.organizationId), - ), - isNull(organizations.tombstoneAt), - ), - ) - .limit(1) - : Promise.resolve([{}]), - ]); - return userRows.length === 0 || organizationRows.length === 0; -} - -export async function deliverProductAnalyticsEventStep( - event: ServerProductEvent, -) { - "use step"; - if (await isProductAnalyticsIdentitySuppressed(event)) { - return { eventId: event.eventId, suppressed: true as const }; - } - - const rows = createServerProductEventRows(event); - if (rows.length !== 1) { - throw new FatalError("Product analytics event failed contract validation"); - } - const env = serverEnv(); - const host = env.PRODUCT_ANALYTICS_TINYBIRD_HOST; - const token = env.PRODUCT_ANALYTICS_TINYBIRD_TOKEN; - if (!host || !token) { - throw new FatalError("Product analytics delivery is not configured"); - } - - try { - await sendProductAnalyticsRows({ - host, - token, - rows, - wait: true, - maxAttempts: 1, - }); - } catch (error) { - if (error instanceof ProductAnalyticsError && !error.retryable) { - throw new FatalError( - `Product analytics permanently rejected event with status ${error.status ?? "unknown"}`, - ); - } - throw new Error("Product analytics delivery temporarily failed"); - } - - return { eventId: event.eventId }; -} -deliverProductAnalyticsEventStep.maxRetries = 8; +import { queueDurableServerProductEvent } from "@/lib/analytics/product-event-outbox"; +import type { ServerProductEvent } from "@/lib/analytics/server-event"; export async function enqueueProductAnalyticsEventStep( event: ServerProductEvent, ) { - "use step"; - - const run = await start(deliverProductAnalyticsEventWorkflow, [event]); - return { eventId: event.eventId, runId: run.runId }; + return queueDurableServerProductEvent(event); } -enqueueProductAnalyticsEventStep.maxRetries = 8; export const enqueueReconciledProductAnalyticsEventStep = enqueueProductAnalyticsEventStep; - -export async function deliverProductAnalyticsEventWorkflow( - event: ServerProductEvent, -) { - "use workflow"; - - return deliverProductAnalyticsEventStep(event); -} diff --git a/apps/web/workflows/drain-product-analytics-outbox.ts b/apps/web/workflows/drain-product-analytics-outbox.ts new file mode 100644 index 00000000000..c51c321d0e9 --- /dev/null +++ b/apps/web/workflows/drain-product-analytics-outbox.ts @@ -0,0 +1,12 @@ +import { drainProductAnalyticsOutbox } from "@/lib/analytics/product-event-outbox"; + +async function drainProductAnalyticsOutboxStep() { + "use step"; + return drainProductAnalyticsOutbox(); +} +drainProductAnalyticsOutboxStep.maxRetries = 4; + +export async function drainProductAnalyticsOutboxWorkflow() { + "use workflow"; + return drainProductAnalyticsOutboxStep(); +} diff --git a/apps/web/workflows/import-loom-video.ts b/apps/web/workflows/import-loom-video.ts index 38f024fcc85..cab15a55b8c 100644 --- a/apps/web/workflows/import-loom-video.ts +++ b/apps/web/workflows/import-loom-video.ts @@ -7,8 +7,13 @@ import { Video } from "@cap/web-domain"; import { eq } from "drizzle-orm"; import { Effect } from "effect"; import { FatalError } from "workflow"; +import { + attemptProductAnalyticsOutboxDelivery, + persistProductAnalyticsEvent, + queueDurableServerProductEvent, +} from "@/lib/analytics/product-event-outbox"; +import type { ServerProductEvent } from "@/lib/analytics/server-event"; import { runWorkflowPromise } from "@/lib/workflow-runtime"; -import { enqueueProductAnalyticsEventStep } from "@/workflows/deliver-product-analytics-event"; interface ImportLoomPayload { videoId: string; @@ -171,25 +176,6 @@ async function claimAgentImport(operationId: string | undefined) { }); } -async function completeAgentImport( - operationId: string | undefined, - videoId: string, -) { - "use step"; - - if (!operationId) return; - const now = new Date(); - await db() - .update(agentApiOperations) - .set({ - state: "succeeded", - result: { videoId }, - updatedAt: now, - completedAt: now, - }) - .where(eq(agentApiOperations.id, operationId)); -} - async function failAgentImport( operationId: string | undefined, message: string, @@ -210,6 +196,17 @@ async function failAgentImport( .where(eq(agentApiOperations.id, operationId)); } +export async function queueLoomAnalyticsEvent(event: ServerProductEvent) { + "use step"; + return queueDurableServerProductEvent(event); +} +queueLoomAnalyticsEvent.maxRetries = 8; + +async function startLoomAnalyticsEvent(eventId: string) { + "use step"; + return attemptProductAnalyticsOutboxDelivery(eventId); +} + export async function importLoomVideoWorkflow( payload: ImportLoomPayload, ): Promise { @@ -221,7 +218,7 @@ export async function importLoomVideoWorkflow( const startedAt = new Date().toISOString(); const analyticsContext = await getLoomAnalyticsContext(payload.videoId); const importMode = payload.agentOperationId ? "agent" : "direct"; - await enqueueProductAnalyticsEventStep({ + await queueLoomAnalyticsEvent({ eventId: `loom_import:${payload.videoId}:started`, eventName: "loom_import_started", occurredAt: startedAt, @@ -230,36 +227,34 @@ export async function importLoomVideoWorkflow( organizationId: analyticsContext?.organizationId, properties: { import_mode: importMode }, }); + let result: MediaServerProcessResult; try { const processingInput = await downloadLoomToS3(payload); - const result = await processVideoOnMediaServer(payload, processingInput); - - await saveMetadataAndComplete(payload.videoId, result.metadata); - await completeAgentImport(payload.agentOperationId, payload.videoId); - await enqueueProductAnalyticsEventStep({ - eventId: `loom_import:${payload.videoId}:completed`, - eventName: "loom_import_completed", - occurredAt: new Date().toISOString(), - platform: "server", - userId: analyticsContext?.ownerId ?? payload.userId, - organizationId: analyticsContext?.organizationId, - properties: { - import_mode: importMode, - duration_ms: Date.now() - Date.parse(startedAt), + result = await processVideoOnMediaServer(payload, processingInput); + const completedAt = new Date().toISOString(); + await saveMetadataAndComplete( + payload.videoId, + payload.agentOperationId, + result.metadata, + { + eventId: `loom_import:${payload.videoId}:completed`, + eventName: "loom_import_completed", + occurredAt: completedAt, + platform: "server", + userId: analyticsContext?.ownerId ?? payload.userId, + organizationId: analyticsContext?.organizationId, + properties: { + import_mode: importMode, + duration_ms: Date.parse(completedAt) - Date.parse(startedAt), + }, }, - }); - - return { - success: true, - message: "Loom video imported successfully", - metadata: result.metadata, - }; + ); } catch (error) { const errorMessage = error instanceof Error ? error.message : String(error); await setProcessingError(payload.videoId, errorMessage); await failAgentImport(payload.agentOperationId, errorMessage); - await enqueueProductAnalyticsEventStep({ + await queueLoomAnalyticsEvent({ eventId: `loom_import:${payload.videoId}:failed`, eventName: "loom_import_failed", occurredAt: new Date().toISOString(), @@ -273,6 +268,14 @@ export async function importLoomVideoWorkflow( }); throw new FatalError(errorMessage); } + + await startLoomAnalyticsEvent(`loom_import:${payload.videoId}:completed`); + + return { + success: true, + message: "Loom video imported successfully", + metadata: result.metadata, + }; } async function getLoomAnalyticsContext(videoId: string) { @@ -704,25 +707,42 @@ async function waitForProcessingCompletion( async function saveMetadataAndComplete( videoId: string, + agentOperationId: string | undefined, metadata: { duration: number; width: number; height: number; fps: number }, + completedEvent: ServerProductEvent<"loom_import_completed">, ): Promise { "use step"; - await db() - .update(videos) - .set({ - width: metadata.width, - height: metadata.height, - fps: metadata.fps, - ...(getValidDuration(metadata.duration) === undefined - ? {} - : { duration: metadata.duration }), - }) - .where(eq(videos.id, videoId as Video.VideoId)); + await db().transaction(async (tx) => { + await tx + .update(videos) + .set({ + width: metadata.width, + height: metadata.height, + fps: metadata.fps, + ...(getValidDuration(metadata.duration) === undefined + ? {} + : { duration: metadata.duration }), + }) + .where(eq(videos.id, videoId as Video.VideoId)); - await db() - .delete(videoUploads) - .where(eq(videoUploads.videoId, videoId as Video.VideoId)); + await tx + .delete(videoUploads) + .where(eq(videoUploads.videoId, videoId as Video.VideoId)); + if (agentOperationId) { + const now = new Date(); + await tx + .update(agentApiOperations) + .set({ + state: "succeeded", + result: { videoId }, + updatedAt: now, + completedAt: now, + }) + .where(eq(agentApiOperations.id, agentOperationId)); + } + await persistProductAnalyticsEvent(tx, completedEvent); + }); } async function setProcessingError( diff --git a/apps/web/workflows/product-analytics-delivery-workflow.ts b/apps/web/workflows/product-analytics-delivery-workflow.ts new file mode 100644 index 00000000000..b18adcd6dd2 --- /dev/null +++ b/apps/web/workflows/product-analytics-delivery-workflow.ts @@ -0,0 +1,247 @@ +import { + PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT, + ProductAnalyticsError, + type ProductEventRow, + productAnalyticsIdentityHash, + sendProductAnalyticsRows, +} from "@cap/analytics"; +import { db } from "@cap/database"; +import { + messengerSupportEmails, + organizations, + productAnalyticsIdentityState, + productAnalyticsOutbox, + users, +} from "@cap/database/schema"; +import { serverEnv } from "@cap/env"; +import { Organisation, User } from "@cap/web-domain"; +import { and, eq, inArray, isNotNull, isNull, notInArray } from "drizzle-orm"; +import { FatalError } from "workflow"; +import { + acquireProductAnalyticsIngestionLease, + deleteSuppressedProductAnalyticsOutboxRow, + markProductAnalyticsOutboxDeadLetter, + markProductAnalyticsOutboxDelivered, + markProductAnalyticsOutboxRetrying, + releaseProductAnalyticsIngestionLease, +} from "@/lib/analytics/product-event-outbox-state"; + +function isStoredProductEventRow( + value: unknown, + eventId: string, +): value is ProductEventRow { + if (!value || typeof value !== "object") return false; + const row = value as Record; + return ( + row.event_id === eventId && + typeof row.payload_hash === "string" && + /^[0-9a-f]{32}$/.test(row.payload_hash) && + typeof row.event_name === "string" && + typeof row.received_at === "string" && + typeof row.properties === "string" + ); +} + +async function isProductAnalyticsIdentitySuppressed(row: ProductEventRow) { + const identityHashes = [ + row.anonymous_id + ? productAnalyticsIdentityHash("anonymous", row.anonymous_id) + : undefined, + row.user_id ? productAnalyticsIdentityHash("user", row.user_id) : undefined, + row.organization_id + ? productAnalyticsIdentityHash("organization", row.organization_id) + : undefined, + ].filter((identityHash) => identityHash !== undefined); + if (identityHashes.length > 0) { + const [blocked] = await db() + .select({ identityHash: productAnalyticsIdentityState.identityHash }) + .from(productAnalyticsIdentityState) + .where( + and( + inArray(productAnalyticsIdentityState.identityHash, identityHashes), + isNotNull(productAnalyticsIdentityState.blockedAt), + ), + ) + .limit(1); + if (blocked) return true; + } + if (row.synthetic_run_id) return false; + const pendingDeletionUserIds = db() + .select({ userId: messengerSupportEmails.userId }) + .from(messengerSupportEmails) + .where( + eq( + messengerSupportEmails.subject, + PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT, + ), + ); + const [userRows, organizationRows] = await Promise.all([ + row.user_id + ? db() + .select({ id: users.id }) + .from(users) + .where( + and( + eq(users.id, User.UserId.make(row.user_id)), + notInArray(users.id, pendingDeletionUserIds), + ), + ) + .limit(1) + : Promise.resolve([{}]), + row.organization_id + ? db() + .select({ id: organizations.id }) + .from(organizations) + .where( + and( + eq( + organizations.id, + Organisation.OrganisationId.make(row.organization_id), + ), + isNull(organizations.tombstoneAt), + ), + ) + .limit(1) + : Promise.resolve([{}]), + ]); + return userRows.length === 0 || organizationRows.length === 0; +} + +async function deliverProductAnalyticsRow( + deliveryKey: string, + row: ProductEventRow, + lastErrorCode: string | null, +) { + if (await isProductAnalyticsIdentitySuppressed(row)) { + await deleteSuppressedProductAnalyticsOutboxRow( + deliveryKey, + row.payload_hash, + ); + return { status: "suppressed" as const }; + } + const env = serverEnv(); + const host = env.PRODUCT_ANALYTICS_TINYBIRD_HOST; + const token = env.PRODUCT_ANALYTICS_TINYBIRD_TOKEN; + if (!host || !token) { + await markProductAnalyticsOutboxDeadLetter( + deliveryKey, + row.payload_hash, + "delivery_not_configured", + ); + throw new FatalError("Product analytics delivery is not configured"); + } + const stagingFault = + env.VERCEL_ENV === "preview" && + row.synthetic_run_id.endsWith("_server") && + row.event_id.startsWith("staging_"); + const retryFault = row.event_id.startsWith("staging_retry_429_") + ? { + code: "staging_provider_429" as const, + message: "Product analytics staging provider returned 429", + } + : row.event_id.startsWith("staging_retry_503_") + ? { + code: "staging_provider_503" as const, + message: "Product analytics staging provider returned 503", + } + : undefined; + if (stagingFault && retryFault && lastErrorCode !== retryFault.code) { + await markProductAnalyticsOutboxRetrying( + deliveryKey, + row.payload_hash, + retryFault.code, + ); + throw new Error(retryFault.message); + } + if (stagingFault && row.event_id.startsWith("staging_reject_400_")) { + await markProductAnalyticsOutboxDeadLetter( + deliveryKey, + row.payload_hash, + "provider_rejected", + ); + throw new FatalError("Product analytics staging provider returned 400"); + } + const ingestionLeaseId = await acquireProductAnalyticsIngestionLease(); + if (!ingestionLeaseId) { + await markProductAnalyticsOutboxRetrying(deliveryKey, row.payload_hash); + throw new Error("Product analytics erasure is in progress"); + } + + try { + await sendProductAnalyticsRows({ + host, + token, + rows: [row], + wait: true, + maxAttempts: 1, + }); + } catch (error) { + if (error instanceof ProductAnalyticsError && !error.retryable) { + await markProductAnalyticsOutboxDeadLetter( + deliveryKey, + row.payload_hash, + "provider_rejected", + ); + throw new FatalError( + `Product analytics permanently rejected event with status ${error.status ?? "unknown"}`, + ); + } + await markProductAnalyticsOutboxRetrying(deliveryKey, row.payload_hash); + throw new Error("Product analytics delivery temporarily failed"); + } finally { + await releaseProductAnalyticsIngestionLease(ingestionLeaseId); + } + if ( + stagingFault && + row.event_name === "purchase_completed" && + row.event_id.startsWith("staging_ambiguous_") && + lastErrorCode !== "staging_timeout_after_accept" + ) { + await markProductAnalyticsOutboxRetrying( + deliveryKey, + row.payload_hash, + "staging_timeout_after_accept", + ); + throw new Error("Product analytics staging delivery acknowledgement lost"); + } + await markProductAnalyticsOutboxDelivered(deliveryKey, row.payload_hash); + return { status: "delivered" as const }; +} + +export async function deliverProductAnalyticsRowStep(deliveryKey: string) { + "use step"; + const [stored] = await db() + .select({ + eventId: productAnalyticsOutbox.eventId, + payload: productAnalyticsOutbox.payload, + payloadHash: productAnalyticsOutbox.payloadHash, + payloadKind: productAnalyticsOutbox.payloadKind, + lastErrorCode: productAnalyticsOutbox.lastErrorCode, + }) + .from(productAnalyticsOutbox) + .where(eq(productAnalyticsOutbox.deliveryKey, deliveryKey)) + .limit(1); + if (!stored) return { status: "discarded" as const }; + if ( + stored.payloadKind !== "product_event_row_v1" || + !isStoredProductEventRow(stored.payload, stored.eventId) + ) { + await markProductAnalyticsOutboxDeadLetter( + deliveryKey, + stored.payloadHash, + "contract_invalid", + ); + throw new FatalError("Stored product analytics event is invalid"); + } + return deliverProductAnalyticsRow( + deliveryKey, + stored.payload, + stored.lastErrorCode, + ); +} +deliverProductAnalyticsRowStep.maxRetries = 8; + +export async function deliverProductAnalyticsRowWorkflow(deliveryKey: string) { + "use workflow"; + return deliverProductAnalyticsRowStep(deliveryKey); +} diff --git a/apps/web/workflows/reconcile-product-analytics.ts b/apps/web/workflows/reconcile-product-analytics.ts index 355b41c3198..75e31387a04 100644 --- a/apps/web/workflows/reconcile-product-analytics.ts +++ b/apps/web/workflows/reconcile-product-analytics.ts @@ -1,13 +1,27 @@ +import { createHash } from "node:crypto"; import { PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT } from "@cap/analytics"; import { db } from "@cap/database"; import { comments, messengerSupportEmails, + productAnalyticsReconciliationFailures, users, videos, } from "@cap/database/schema"; import { stripe } from "@cap/utils"; -import { and, eq, gte, inArray, isNotNull, lte, notInArray } from "drizzle-orm"; +import { + and, + asc, + eq, + gt, + gte, + inArray, + isNotNull, + lte, + notInArray, + or, + sql, +} from "drizzle-orm"; import type Stripe from "stripe"; import { checkoutStartedEvent, @@ -17,6 +31,7 @@ import { shareLinkCreatedEvent, userSignedUpEvent, } from "@/lib/analytics/business-events"; +import { queueDurableServerProductEvent } from "@/lib/analytics/product-event-outbox"; import { isFirstPositiveSubscriptionPayment, isSettledSubscriptionPurchase, @@ -29,9 +44,11 @@ import { subscriptionTrialConvertedProductEvent, subscriptionTrialStartedProductEvent, } from "@/lib/analytics/stripe-business-events"; -import { enqueueReconciledProductAnalyticsEventStep } from "./deliver-product-analytics-event"; +import { videoAnalyticsPlatform } from "@/lib/analytics/video-platform"; -const RECONCILIATION_ROW_LIMIT = 5_000; +const RECONCILIATION_PAGE_SIZE = 250; +const RECONCILIATION_ENQUEUE_CONCURRENCY = 10; +const MAX_RECONCILIATION_PAGES_PER_SOURCE = 10_000; const STRIPE_ANALYTICS_EVENT_TYPES = [ "checkout.session.created", "checkout.session.completed", @@ -49,30 +66,6 @@ const checkoutQuantity = (session: Stripe.Checkout.Session) => { return Number.isSafeInteger(quantity) && quantity > 0 ? quantity : undefined; }; -const videoAnalyticsPlatform = (video: { - metadata: unknown; - source: { type: string }; -}) => { - const metadata = - typeof video.metadata === "object" && video.metadata !== null - ? (video.metadata as Record) - : {}; - if ( - metadata.source === "mobileUpload" || - metadata.source === "mobileCamera" - ) { - return "mobile" as const; - } - if ( - video.source.type === "desktopMP4" || - video.source.type === "desktopSegments" || - video.source.type === "local" - ) { - return "desktop" as const; - } - return "server" as const; -}; - function reconciliationWindow(scheduledAt: string, lookbackHours: number) { const end = new Date(scheduledAt); if ( @@ -89,16 +82,122 @@ function reconciliationWindow(scheduledAt: string, lookbackHours: number) { }; } -export async function loadProductAnalyticsReconciliationEventsStep({ +type DatabaseReconciliationSource = + | "comment" + | "first_view" + | "signup" + | "video"; + +type ReconciliationSourceType = + | `database_${DatabaseReconciliationSource}` + | "stripe_event"; + +type ReconciliationCandidate = { + event: () => Parameters[0]; + sourceId: string; + sourceType: ReconciliationSourceType; +}; + +const reconciliationSourceHash = ( + sourceType: ReconciliationSourceType, + sourceId: string, +) => + createHash("sha256") + .update(`reconciliation\0${sourceType}\0${sourceId}`) + .digest("hex"); + +async function recordReconciliationFailure({ + errorCode, + sourceHash, + sourceType, +}: { + errorCode: "event_invalid" | "queue_failed"; + sourceHash: string; + sourceType: ReconciliationSourceType; +}) { + await db() + .insert(productAnalyticsReconciliationFailures) + .values({ errorCode, sourceHash, sourceType }) + .onDuplicateKeyUpdate({ + set: { + attemptCount: sql`${productAnalyticsReconciliationFailures.attemptCount} + 1`, + errorCode, + lastSeenAt: new Date(), + }, + }); +} + +async function reconcileCandidate(candidate: ReconciliationCandidate) { + const sourceHash = reconciliationSourceHash( + candidate.sourceType, + candidate.sourceId, + ); + let event: Parameters[0]; + try { + event = candidate.event(); + } catch { + await recordReconciliationFailure({ + errorCode: "event_invalid", + sourceHash, + sourceType: candidate.sourceType, + }); + return false; + } + try { + await queueDurableServerProductEvent(event); + await db() + .delete(productAnalyticsReconciliationFailures) + .where(eq(productAnalyticsReconciliationFailures.sourceHash, sourceHash)); + return true; + } catch { + await recordReconciliationFailure({ + errorCode: "queue_failed", + sourceHash, + sourceType: candidate.sourceType, + }); + return false; + } +} + +async function reconcileCandidates(candidates: ReconciliationCandidate[]) { + let reconciled = 0; + let failed = 0; + for ( + let offset = 0; + offset < candidates.length; + offset += RECONCILIATION_ENQUEUE_CONCURRENCY + ) { + const results = await Promise.all( + candidates + .slice(offset, offset + RECONCILIATION_ENQUEUE_CONCURRENCY) + .map(reconcileCandidate), + ); + reconciled += results.filter(Boolean).length; + failed += results.filter((result) => !result).length; + } + return { failed, reconciled }; +} + +type ReconciliationCursor = { at: string; id: string }; + +export async function loadProductAnalyticsReconciliationPageStep({ scheduledAt, lookbackHours, + source, + cursor, }: { scheduledAt: string; lookbackHours: number; + source: DatabaseReconciliationSource; + cursor?: ReconciliationCursor; }) { "use step"; const { end, start } = reconciliationWindow(scheduledAt, lookbackHours); + const cursorAt = cursor ? new Date(cursor.at) : undefined; + if (cursorAt && !Number.isFinite(cursorAt.getTime())) { + throw new Error("Invalid product analytics reconciliation cursor"); + } const pendingDeletionUserIds = db() .select({ userId: messengerSupportEmails.userId }) .from(messengerSupportEmails) @@ -111,169 +210,257 @@ export async function loadProductAnalyticsReconciliationEventsStep({ isNotNull(messengerSupportEmails.userId), ), ); - const [recentUsers, recentVideos, recentComments, recentFirstViews] = - await Promise.all([ - db() - .select({ - id: users.id, - organizationId: users.activeOrganizationId, - createdAt: users.created_at, - }) - .from(users) - .where( - and( - gte(users.created_at, start), - lte(users.created_at, end), - notInArray(users.id, pendingDeletionUserIds), - ), - ) - .limit(RECONCILIATION_ROW_LIMIT + 1), - db() - .select({ - id: videos.id, - userId: videos.ownerId, - organizationId: videos.orgId, - createdAt: videos.createdAt, - isScreenshot: videos.isScreenshot, - source: videos.source, - metadata: videos.metadata, - }) - .from(videos) - .where( - and( - gte(videos.createdAt, start), - lte(videos.createdAt, end), - notInArray(videos.ownerId, pendingDeletionUserIds), - ), - ) - .limit(RECONCILIATION_ROW_LIMIT + 1), - db() - .select({ - id: comments.id, - authorId: comments.authorId, - organizationId: videos.orgId, - createdAt: comments.createdAt, - type: comments.type, - parentCommentId: comments.parentCommentId, - }) - .from(comments) - .leftJoin(videos, eq(comments.videoId, videos.id)) - .where( - and( - gte(comments.createdAt, start), - lte(comments.createdAt, end), - notInArray(comments.authorId, pendingDeletionUserIds), - ), - ) - .limit(RECONCILIATION_ROW_LIMIT + 1), - db() - .select({ - id: videos.id, - userId: videos.ownerId, - organizationId: videos.orgId, - firstExternalViewAt: videos.firstExternalViewAt, - }) - .from(videos) - .where( - and( - isNotNull(videos.firstExternalViewAt), - gte(videos.firstExternalViewAt, start), - lte(videos.firstExternalViewAt, end), - notInArray(videos.ownerId, pendingDeletionUserIds), - ), - ) - .limit(RECONCILIATION_ROW_LIMIT + 1), - ]); - if ( - recentUsers.length > RECONCILIATION_ROW_LIMIT || - recentVideos.length > RECONCILIATION_ROW_LIMIT || - recentComments.length > RECONCILIATION_ROW_LIMIT || - recentFirstViews.length > RECONCILIATION_ROW_LIMIT - ) { - throw new Error("Product analytics reconciliation row limit exceeded"); + if (source === "signup") { + const page = await db() + .select({ id: users.id, createdAt: users.created_at }) + .from(users) + .where( + and( + gte(users.created_at, start), + lte(users.created_at, end), + notInArray(users.id, pendingDeletionUserIds), + cursor && cursorAt + ? or( + gt(users.created_at, cursorAt), + and( + eq(users.created_at, cursorAt), + gt(users.id, cursor.id as (typeof users.$inferSelect)["id"]), + ), + ) + : undefined, + ), + ) + .orderBy(asc(users.created_at), asc(users.id)) + .limit(RECONCILIATION_PAGE_SIZE); + const result = await reconcileCandidates( + page.map((user) => ({ + event: () => + userSignedUpEvent({ userId: user.id, createdAt: user.createdAt }), + sourceId: user.id, + sourceType: "database_signup", + })), + ); + const last = page.at(-1); + return { + ...result, + nextCursor: + last && page.length === RECONCILIATION_PAGE_SIZE + ? { at: last.createdAt.toISOString(), id: last.id } + : undefined, + }; } - - return [ - ...recentUsers.map((user) => - userSignedUpEvent({ - userId: user.id, - createdAt: user.createdAt, - }), - ), - ...recentVideos.map((video) => - shareLinkCreatedEvent({ - videoId: video.id, - platform: videoAnalyticsPlatform(video), - userId: video.userId, - organizationId: video.organizationId, - createdAt: video.createdAt, - isScreenshot: video.isScreenshot, - sourceType: video.source.type, - }), - ), - ...recentComments.map((comment) => - collaborationActionCreatedEvent({ - commentId: comment.id, - userId: comment.authorId, - organizationId: comment.organizationId, - createdAt: comment.createdAt, - action: comment.parentCommentId - ? "reply" - : comment.type === "emoji" - ? "reaction" - : "comment", - }), - ), - ...recentFirstViews.map((video) => { - if (!video.firstExternalViewAt) { - throw new Error("First-view reconciliation timestamp is missing"); - } - return firstViewReceivedEvent({ - videoId: video.id, - userId: video.userId, - organizationId: video.organizationId, - createdAt: video.firstExternalViewAt, - }); - }), - ]; + if (source === "video") { + const page = await db() + .select({ + id: videos.id, + userId: videos.ownerId, + organizationId: videos.orgId, + createdAt: videos.createdAt, + isScreenshot: videos.isScreenshot, + source: videos.source, + metadata: videos.metadata, + }) + .from(videos) + .where( + and( + gte(videos.createdAt, start), + lte(videos.createdAt, end), + notInArray(videos.ownerId, pendingDeletionUserIds), + cursor && cursorAt + ? or( + gt(videos.createdAt, cursorAt), + and( + eq(videos.createdAt, cursorAt), + gt( + videos.id, + cursor.id as (typeof videos.$inferSelect)["id"], + ), + ), + ) + : undefined, + ), + ) + .orderBy(asc(videos.createdAt), asc(videos.id)) + .limit(RECONCILIATION_PAGE_SIZE); + const result = await reconcileCandidates( + page.map((video) => ({ + event: () => { + const platform = videoAnalyticsPlatform(video); + const analyticsPlatform = + platform === "cli" + ? "cli" + : platform === "desktop" + ? "desktop" + : platform === "mobile" + ? "mobile" + : "server"; + return shareLinkCreatedEvent({ + videoId: video.id, + platform: analyticsPlatform, + userId: video.userId, + organizationId: video.organizationId, + createdAt: video.createdAt, + isScreenshot: video.isScreenshot, + sourceType: video.source.type, + }); + }, + sourceId: video.id, + sourceType: "database_video", + })), + ); + const last = page.at(-1); + return { + ...result, + nextCursor: + last && page.length === RECONCILIATION_PAGE_SIZE + ? { at: last.createdAt.toISOString(), id: last.id } + : undefined, + }; + } + if (source === "comment") { + const page = await db() + .select({ + id: comments.id, + authorId: comments.authorId, + organizationId: videos.orgId, + createdAt: comments.createdAt, + type: comments.type, + parentCommentId: comments.parentCommentId, + }) + .from(comments) + .leftJoin(videos, eq(comments.videoId, videos.id)) + .where( + and( + gte(comments.createdAt, start), + lte(comments.createdAt, end), + notInArray(comments.authorId, pendingDeletionUserIds), + cursor && cursorAt + ? or( + gt(comments.createdAt, cursorAt), + and( + eq(comments.createdAt, cursorAt), + gt( + comments.id, + cursor.id as (typeof comments.$inferSelect)["id"], + ), + ), + ) + : undefined, + ), + ) + .orderBy(asc(comments.createdAt), asc(comments.id)) + .limit(RECONCILIATION_PAGE_SIZE); + const result = await reconcileCandidates( + page.map((comment) => ({ + event: () => + collaborationActionCreatedEvent({ + commentId: comment.id, + userId: comment.authorId, + organizationId: comment.organizationId, + createdAt: comment.createdAt, + action: comment.parentCommentId + ? "reply" + : comment.type === "emoji" + ? "reaction" + : "comment", + }), + sourceId: comment.id, + sourceType: "database_comment", + })), + ); + const last = page.at(-1); + return { + ...result, + nextCursor: + last && page.length === RECONCILIATION_PAGE_SIZE + ? { at: last.createdAt.toISOString(), id: last.id } + : undefined, + }; + } + const page = await db() + .select({ + id: videos.id, + userId: videos.ownerId, + organizationId: videos.orgId, + firstExternalViewAt: videos.firstExternalViewAt, + }) + .from(videos) + .where( + and( + isNotNull(videos.firstExternalViewAt), + gte(videos.firstExternalViewAt, start), + lte(videos.firstExternalViewAt, end), + notInArray(videos.ownerId, pendingDeletionUserIds), + cursor && cursorAt + ? or( + gt(videos.firstExternalViewAt, cursorAt), + and( + eq(videos.firstExternalViewAt, cursorAt), + gt(videos.id, cursor.id as (typeof videos.$inferSelect)["id"]), + ), + ) + : undefined, + ), + ) + .orderBy(asc(videos.firstExternalViewAt), asc(videos.id)) + .limit(RECONCILIATION_PAGE_SIZE); + const result = await reconcileCandidates( + page.map((video) => ({ + event: () => { + if (!video.firstExternalViewAt) { + throw new Error("First-view reconciliation timestamp is missing"); + } + return firstViewReceivedEvent({ + videoId: video.id, + userId: video.userId, + organizationId: video.organizationId, + createdAt: video.firstExternalViewAt, + }); + }, + sourceId: video.id, + sourceType: "database_first_view", + })), + ); + const last = page.at(-1); + return { + ...result, + nextCursor: + last?.firstExternalViewAt && page.length === RECONCILIATION_PAGE_SIZE + ? { at: last.firstExternalViewAt.toISOString(), id: last.id } + : undefined, + }; } -loadProductAnalyticsReconciliationEventsStep.maxRetries = 4; +loadProductAnalyticsReconciliationPageStep.maxRetries = 4; -export async function loadStripeAnalyticsReconciliationEventsStep({ - scheduledAt, - lookbackHours, -}: { +export async function loadProductAnalyticsReconciliationEventsStep(input: { scheduledAt: string; lookbackHours: number; }) { - "use step"; - - const { end, start } = reconciliationWindow(scheduledAt, lookbackHours); - const stripeEvents: Stripe.Event[] = []; - for (const type of STRIPE_ANALYTICS_EVENT_TYPES) { - let startingAfter: string | undefined; - for (;;) { - const page = await stripe().events.list({ - type, - created: { - gte: Math.floor(start.getTime() / 1_000), - lte: Math.floor(end.getTime() / 1_000), - }, - limit: 100, - ...(startingAfter ? { starting_after: startingAfter } : {}), - }); - stripeEvents.push(...page.data); - if (stripeEvents.length > RECONCILIATION_ROW_LIMIT) { - throw new Error("Stripe analytics reconciliation row limit exceeded"); - } - if (!page.has_more || page.data.length === 0) break; - startingAfter = page.data.at(-1)?.id; - if (!startingAfter) { - throw new Error("Stripe analytics reconciliation pagination failed"); + let reconciled = 0; + let failed = 0; + for (const source of ["signup", "video", "comment", "first_view"] as const) { + let cursor: ReconciliationCursor | undefined; + for (let pageNumber = 0; ; pageNumber += 1) { + if (pageNumber >= MAX_RECONCILIATION_PAGES_PER_SOURCE) { + throw new Error("Product analytics reconciliation page limit exceeded"); } + const page = await loadProductAnalyticsReconciliationPageStep({ + ...input, + source, + cursor, + }); + reconciled += page.reconciled; + failed += page.failed; + if (!page.nextCursor) break; + cursor = page.nextCursor; } } + return { failed, reconciled }; +} +async function reconcileStripeAnalyticsEventBatch( + stripeEvents: Stripe.Event[], +) { const sessions = stripeEvents .filter(({ type }) => type.startsWith("checkout.session.")) .map((event) => ({ @@ -332,23 +519,6 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ ); const reconciled = []; let legacyStripeEventsSkipped = 0; - const subscriptionsById = new Map( - subscriptionEvents.map(({ subscription }) => [ - subscription.id, - subscription, - ]), - ); - const getSubscription = async ( - value: string | Stripe.Subscription | null, - ) => { - if (!value) throw new Error("Stripe event is missing a subscription"); - if (typeof value !== "string") return value; - const existing = subscriptionsById.get(value); - if (existing) return existing; - const subscription = await stripe().subscriptions.retrieve(value); - subscriptionsById.set(value, subscription); - return subscription; - }; const getUser = ( customer: string | Stripe.Customer | Stripe.DeletedCustomer | null, ) => { @@ -427,7 +597,10 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ } for (const { event, invoice } of invoiceEvents) { if (!invoice.subscription) continue; - const subscription = await getSubscription(invoice.subscription); + const subscriptionId = + typeof invoice.subscription === "string" + ? invoice.subscription + : invoice.subscription.id; const user = getUser(invoice.customer); const occurredAt = new Date(event.created * 1_000).toISOString(); if (event.type === "invoice.paid") { @@ -435,11 +608,10 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ eventId: event.id, occurredAt, invoice, - subscription, user, firstPositivePayment: await isFirstPositiveSubscriptionPayment({ invoice, - subscriptionId: subscription.id, + subscriptionId, listPaidInvoices: (input) => stripe().invoices.list(input), }), }); @@ -450,7 +622,6 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ eventId: event.id, occurredAt, invoice, - subscription, user, }); if (productEvent) reconciled.push(productEvent); @@ -462,7 +633,6 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ ? await stripe().invoices.retrieve(charge.invoice) : charge.invoice; if (!invoice.subscription) continue; - const subscription = await getSubscription(invoice.subscription); const previous = event.data.previous_attributes as | Partial | undefined; @@ -471,7 +641,6 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ occurredAt: new Date(event.created * 1_000).toISOString(), charge, invoice, - subscription, user: getUser(charge.customer), refundedAmount: charge.amount_refunded - (previous?.amount_refunded ?? 0), }); @@ -566,9 +735,109 @@ export async function loadStripeAnalyticsReconciliationEventsStep({ }); if (productEvent) reconciled.push(productEvent); } - return { events: reconciled, legacyStripeEventsSkipped }; + const delivery = await reconcileCandidates( + reconciled.map((event) => ({ + event: () => event, + sourceId: event.eventId, + sourceType: "stripe_event", + })), + ); + return { + failed: delivery.failed, + reconciled: delivery.reconciled, + legacyStripeEventsSkipped, + }; +} + +export async function loadStripeAnalyticsReconciliationPageStep({ + scheduledAt, + lookbackHours, + type, + startingAfter, +}: { + scheduledAt: string; + lookbackHours: number; + type: (typeof STRIPE_ANALYTICS_EVENT_TYPES)[number]; + startingAfter?: string; +}) { + "use step"; + + const { end, start } = reconciliationWindow(scheduledAt, lookbackHours); + let reconciled = 0; + let failed = 0; + let legacyStripeEventsSkipped = 0; + const page = await stripe().events.list({ + type, + created: { + gte: Math.floor(start.getTime() / 1_000), + lte: Math.floor(end.getTime() / 1_000), + }, + limit: 100, + ...(startingAfter ? { starting_after: startingAfter } : {}), + }); + for (const event of page.data) { + try { + const batch = await reconcileStripeAnalyticsEventBatch([event]); + await db() + .delete(productAnalyticsReconciliationFailures) + .where( + eq( + productAnalyticsReconciliationFailures.sourceHash, + reconciliationSourceHash("stripe_event", event.id), + ), + ); + reconciled += batch.reconciled; + failed += batch.failed; + legacyStripeEventsSkipped += batch.legacyStripeEventsSkipped; + } catch { + await recordReconciliationFailure({ + errorCode: "event_invalid", + sourceHash: reconciliationSourceHash("stripe_event", event.id), + sourceType: "stripe_event", + }); + failed += 1; + } + } + const nextStartingAfter = page.has_more ? page.data.at(-1)?.id : undefined; + if (page.has_more && !nextStartingAfter) { + throw new Error("Stripe analytics reconciliation pagination failed"); + } + return { + failed, + legacyStripeEventsSkipped, + nextStartingAfter, + reconciled, + }; +} +loadStripeAnalyticsReconciliationPageStep.maxRetries = 4; + +export async function loadStripeAnalyticsReconciliationEventsStep(input: { + scheduledAt: string; + lookbackHours: number; +}) { + let reconciled = 0; + let failed = 0; + let legacyStripeEventsSkipped = 0; + for (const type of STRIPE_ANALYTICS_EVENT_TYPES) { + let startingAfter: string | undefined; + for (let pageNumber = 0; ; pageNumber += 1) { + if (pageNumber >= MAX_RECONCILIATION_PAGES_PER_SOURCE) { + throw new Error("Stripe analytics reconciliation page limit exceeded"); + } + const page = await loadStripeAnalyticsReconciliationPageStep({ + ...input, + type, + startingAfter, + }); + reconciled += page.reconciled; + failed += page.failed; + legacyStripeEventsSkipped += page.legacyStripeEventsSkipped; + if (!page.nextStartingAfter) break; + startingAfter = page.nextStartingAfter; + } + } + return { failed, legacyStripeEventsSkipped, reconciled }; } -loadStripeAnalyticsReconciliationEventsStep.maxRetries = 4; export async function reconcileProductAnalyticsWorkflow(input: { scheduledAt: string; @@ -576,15 +845,51 @@ export async function reconcileProductAnalyticsWorkflow(input: { }) { "use workflow"; - const events = await loadProductAnalyticsReconciliationEventsStep(input); - const stripeReconciliation = - await loadStripeAnalyticsReconciliationEventsStep(input); - for (const event of [...events, ...stripeReconciliation.events]) { - await enqueueReconciledProductAnalyticsEventStep(event); + let databaseReconciled = 0; + let databaseFailed = 0; + for (const source of ["signup", "video", "comment", "first_view"] as const) { + let cursor: ReconciliationCursor | undefined; + for (let pageNumber = 0; ; pageNumber += 1) { + if (pageNumber >= MAX_RECONCILIATION_PAGES_PER_SOURCE) { + throw new Error("Product analytics reconciliation page limit exceeded"); + } + const page = await loadProductAnalyticsReconciliationPageStep({ + ...input, + source, + cursor, + }); + databaseReconciled += page.reconciled; + databaseFailed += page.failed; + if (!page.nextCursor) break; + cursor = page.nextCursor; + } + } + let stripeReconciled = 0; + let stripeFailed = 0; + let legacyStripeEventsSkipped = 0; + for (const type of STRIPE_ANALYTICS_EVENT_TYPES) { + let startingAfter: string | undefined; + for (let pageNumber = 0; ; pageNumber += 1) { + if (pageNumber >= MAX_RECONCILIATION_PAGES_PER_SOURCE) { + throw new Error("Stripe analytics reconciliation page limit exceeded"); + } + const page = await loadStripeAnalyticsReconciliationPageStep({ + ...input, + type, + startingAfter, + }); + stripeReconciled += page.reconciled; + stripeFailed += page.failed; + legacyStripeEventsSkipped += page.legacyStripeEventsSkipped; + if (!page.nextStartingAfter) break; + startingAfter = page.nextStartingAfter; + } } return { - databaseReconciled: events.length, - stripeReconciled: stripeReconciliation.events.length, - legacyStripeEventsSkipped: stripeReconciliation.legacyStripeEventsSkipped, + databaseFailed, + databaseReconciled, + stripeFailed, + stripeReconciled, + legacyStripeEventsSkipped, }; } diff --git a/apps/web/workflows/recover-organization-invite-delivery.ts b/apps/web/workflows/recover-organization-invite-delivery.ts new file mode 100644 index 00000000000..308677b9014 --- /dev/null +++ b/apps/web/workflows/recover-organization-invite-delivery.ts @@ -0,0 +1,12 @@ +import { recoverOrganizationInviteDeliveries } from "@/lib/organization-invite-delivery"; + +async function recoverOrganizationInviteDeliveriesStep() { + "use step"; + return recoverOrganizationInviteDeliveries(); +} +recoverOrganizationInviteDeliveriesStep.maxRetries = 4; + +export async function recoverOrganizationInviteDeliveriesWorkflow() { + "use workflow"; + return recoverOrganizationInviteDeliveriesStep(); +} diff --git a/apps/web/workflows/refresh-product-analytics.ts b/apps/web/workflows/refresh-product-analytics.ts new file mode 100644 index 00000000000..28da7e3c109 --- /dev/null +++ b/apps/web/workflows/refresh-product-analytics.ts @@ -0,0 +1,180 @@ +import { createHash } from "node:crypto"; +import { serverEnv } from "@cap/env"; +import { FatalError } from "workflow"; +import { + acquireProductAnalyticsRefreshLease, + releaseProductAnalyticsRefreshLease, + renewProductAnalyticsRefreshLease, +} from "@/lib/analytics/product-analytics-refresh-state"; + +const REFRESH_COPIES = [ + ["snapshot_product_events_daily_exact", "decision_markers"], + ["snapshot_product_traffic_daily_exact", "traffic_markers"], + ["snapshot_product_traffic_pages_daily_exact", "traffic_page_markers"], + ["snapshot_product_activation_daily_exact", "activation_markers"], + ["snapshot_product_creator_retention_exact", "retention_markers"], + ["snapshot_product_identity_funnel_exact", "identity_markers"], + ["snapshot_product_attribution_daily_exact", "attribution_markers"], + ["snapshot_product_experiment_outcomes_exact", "experiment_markers"], +] as const; + +type TinybirdJobResponse = { + id?: unknown; + job_id?: unknown; + status?: unknown; + state?: unknown; + job?: { id?: unknown; status?: unknown; state?: unknown }; +}; + +const request = async (url: URL, token: string, init?: RequestInit) => { + const response = await fetch(url, { + ...init, + headers: { + Accept: "application/json", + Authorization: `Bearer ${token}`, + ...(init?.headers ?? {}), + }, + signal: AbortSignal.timeout(20_000), + }); + if (!response.ok) { + throw new Error( + `Tinybird refresh request failed with HTTP ${response.status}`, + ); + } + const body = await response.text(); + return body ? (JSON.parse(body) as T) : ({} as T); +}; + +const jobId = (response: TinybirdJobResponse) => { + const value = response.job_id ?? response.job?.id ?? response.id; + return typeof value === "string" && /^[A-Za-z0-9_-]{8,128}$/.test(value) + ? value + : undefined; +}; + +const jobStatus = (response: TinybirdJobResponse) => + String( + response.status ?? + response.state ?? + response.job?.status ?? + response.job?.state ?? + "", + ).toLowerCase(); + +export async function acquireProductAnalyticsRefreshStep(sourceCutoff: string) { + "use step"; + const parsed = new Date(sourceCutoff); + if (!Number.isFinite(parsed.getTime())) { + throw new FatalError("Product analytics refresh cutoff is invalid"); + } + return acquireProductAnalyticsRefreshLease(parsed); +} + +export async function runProductAnalyticsCopyStep( + ownerId: string, + sourceCutoff: string, + copyRunId: string, + pipe: (typeof REFRESH_COPIES)[number][0], + marker: (typeof REFRESH_COPIES)[number][1], +) { + "use step"; + if (!(await renewProductAnalyticsRefreshLease(ownerId))) { + throw new FatalError("Product analytics refresh lease was lost"); + } + const env = serverEnv(); + const host = env.PRODUCT_ANALYTICS_TINYBIRD_HOST; + const copyToken = env.PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN; + const readToken = env.PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN; + if (!host || !copyToken || !readToken) { + throw new FatalError("Product analytics refresh is not configured"); + } + const origin = new URL(host); + if (origin.protocol !== "https:") { + throw new FatalError("Product analytics refresh host must use HTTPS"); + } + const copyUrl = new URL(`/v0/pipes/${encodeURIComponent(pipe)}/copy`, origin); + copyUrl.searchParams.set("_mode", "replace"); + copyUrl.searchParams.set("copy_max_threads", "2"); + copyUrl.searchParams.set("copy_run_id", copyRunId); + copyUrl.searchParams.set("source_cutoff", sourceCutoff); + const created = await request(copyUrl, copyToken, { + method: "POST", + }); + const id = jobId(created); + if (!id) throw new Error("Tinybird refresh did not return a job ID"); + const deadline = Date.now() + 15 * 60 * 1_000; + let lastLeaseRenewal = Date.now(); + while (Date.now() < deadline) { + const job = await request( + new URL(`/v0/jobs/${encodeURIComponent(id)}`, origin), + copyToken, + ); + const status = jobStatus(job); + if (["done", "success", "finished", "completed"].includes(status)) break; + if (["failed", "error", "cancelled", "canceled"].includes(status)) { + throw new Error(`Tinybird refresh job ended in ${status}`); + } + if (Date.now() - lastLeaseRenewal >= 60_000) { + if (!(await renewProductAnalyticsRefreshLease(ownerId))) { + throw new FatalError("Product analytics refresh lease was lost"); + } + lastLeaseRenewal = Date.now(); + } + await new Promise((resolve) => setTimeout(resolve, 2_000)); + } + if (Date.now() >= deadline) { + throw new Error("Tinybird refresh job exceeded its time budget"); + } + const markerUrl = new URL( + "/v0/pipes/product_analytics_copy_assertions.json", + origin, + ); + markerUrl.searchParams.set("copy_run_id", copyRunId); + const markerResponse = await request<{ + data?: Array>; + }>(markerUrl, readToken); + if (Number(markerResponse.data?.[0]?.[marker] ?? 0) !== 1) { + throw new Error(`Tinybird refresh marker was missing for ${pipe}`); + } + return { jobId: id, marker, pipe }; +} + +export async function releaseProductAnalyticsRefreshStep( + ownerId: string, + errorCode?: string, +) { + "use step"; + await releaseProductAnalyticsRefreshLease(ownerId, errorCode); +} + +export async function refreshProductAnalyticsWorkflow(input: { + scheduledAt: string; +}) { + "use workflow"; + const sourceCutoff = new Date(input.scheduledAt).toISOString(); + const lease = await acquireProductAnalyticsRefreshStep(sourceCutoff); + if (!lease) return { refreshed: false as const, reason: "lease_unavailable" }; + const copyRunId = `refresh_${createHash("sha256") + .update(sourceCutoff) + .digest("hex") + .slice(0, 24)}`; + try { + const jobs = []; + for (const [pipe, marker] of REFRESH_COPIES) { + jobs.push( + await runProductAnalyticsCopyStep( + lease.ownerId, + lease.sourceCutoff, + copyRunId, + pipe, + marker, + ), + ); + } + await releaseProductAnalyticsRefreshStep(lease.ownerId); + return { copyRunId, jobs, refreshed: true as const, sourceCutoff }; + } catch (error) { + await releaseProductAnalyticsRefreshStep(lease.ownerId, "refresh_failed"); + throw error; + } +} diff --git a/packages/database/migrations/0040_chief_skreet.sql b/packages/database/migrations/0040_chief_skreet.sql new file mode 100644 index 00000000000..e6fd05e62d5 --- /dev/null +++ b/packages/database/migrations/0040_chief_skreet.sql @@ -0,0 +1,110 @@ +CREATE TABLE `product_analytics_erasure_requests` ( + `id` varchar(36) NOT NULL, + `scopeHash` varchar(64) NOT NULL, + `userId` varchar(255), + `organizationId` varchar(255), + `status` varchar(32) NOT NULL DEFAULT 'pending', + `attemptCount` int NOT NULL DEFAULT 0, + `nextAttemptAt` timestamp NOT NULL DEFAULT (now()), + `leaseOwnerId` varchar(36), + `leaseExpiresAt` timestamp, + `lastErrorCode` varchar(64), + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `product_analytics_erasure_requests_id` PRIMARY KEY(`id`), + CONSTRAINT `scope_hash_idx` UNIQUE(`scopeHash`) +); +--> statement-breakpoint +CREATE TABLE `product_analytics_event_receipts` ( + `eventIdHash` varchar(64) NOT NULL, + `payloadHash` varchar(32) NOT NULL, + `anonymousIdentityHash` varchar(64), + `userIdentityHash` varchar(64), + `organizationIdentityHash` varchar(64), + `conflictCount` int NOT NULL DEFAULT 0, + `firstSeenAt` timestamp NOT NULL DEFAULT (now()), + `lastSeenAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + `retainUntil` timestamp NOT NULL, + CONSTRAINT `product_analytics_event_receipts_eventIdHash` PRIMARY KEY(`eventIdHash`) +); +--> statement-breakpoint +CREATE TABLE `product_analytics_identity_links` ( + `anonymousIdentityHash` varchar(64) NOT NULL, + `userIdentityHash` varchar(64) NOT NULL, + `organizationIdentityHash` varchar(64), + `anonymousId` varchar(255) NOT NULL, + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `identity_link_pk` PRIMARY KEY(`anonymousIdentityHash`,`userIdentityHash`) +); +--> statement-breakpoint +CREATE TABLE `product_analytics_identity_state` ( + `identityHash` varchar(64) NOT NULL, + `identityKind` varchar(16) NOT NULL, + `blockedAt` timestamp, + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `product_analytics_identity_state_identityHash` PRIMARY KEY(`identityHash`) +); +--> statement-breakpoint +CREATE TABLE `product_analytics_ingestion_leases` ( + `id` varchar(36) NOT NULL, + `expiresAt` timestamp NOT NULL, + `createdAt` timestamp NOT NULL DEFAULT (now()), + CONSTRAINT `product_analytics_ingestion_leases_id` PRIMARY KEY(`id`) +); +--> statement-breakpoint +CREATE TABLE `product_analytics_outbox` ( + `eventId` varchar(128) NOT NULL, + `deliveryKey` varchar(36) NOT NULL, + `payloadHash` varchar(32) NOT NULL, + `eventName` varchar(64) NOT NULL, + `payloadKind` varchar(32) NOT NULL DEFAULT 'product_event_row_v1', + `payload` json NOT NULL, + `anonymousId` varchar(255), + `userId` varchar(255), + `organizationId` varchar(255), + `status` varchar(32) NOT NULL DEFAULT 'pending', + `attemptCount` int NOT NULL DEFAULT 0, + `nextAttemptAt` timestamp NOT NULL DEFAULT (now()), + `leaseOwnerId` varchar(64), + `leaseExpiresAt` timestamp, + `workflowRunId` varchar(128), + `payloadConflict` boolean NOT NULL DEFAULT false, + `lastErrorCode` varchar(64), + `deliveredAt` timestamp, + `deadLetteredAt` timestamp, + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `product_analytics_outbox_eventId` PRIMARY KEY(`eventId`), + CONSTRAINT `delivery_key_idx` UNIQUE(`deliveryKey`) +); +--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `invitedEmailNormalized` varchar(255);--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailDeliveryState` varchar(32) DEFAULT 'legacy' NOT NULL;--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailDeliveryAttemptCount` int DEFAULT 0 NOT NULL;--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailDeliveryNextAttemptAt` timestamp;--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailDeliveryErrorCode` varchar(64);--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailDeliveryLeaseOwnerId` varchar(36);--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailDeliveryLeaseExpiresAt` timestamp;--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailProviderMessageId` varchar(255);--> statement-breakpoint +ALTER TABLE `organization_invites` ADD `emailSentAt` timestamp;--> statement-breakpoint +ALTER TABLE `organization_invites` ADD CONSTRAINT `normalized_email_idx` UNIQUE(`organizationId`,`invitedEmailNormalized`);--> statement-breakpoint +CREATE INDEX `queue_idx` ON `product_analytics_erasure_requests` (`status`,`nextAttemptAt`,`createdAt`);--> statement-breakpoint +CREATE INDEX `lease_idx` ON `product_analytics_erasure_requests` (`leaseExpiresAt`);--> statement-breakpoint +CREATE INDEX `anonymous_identity_idx` ON `product_analytics_event_receipts` (`anonymousIdentityHash`);--> statement-breakpoint +CREATE INDEX `user_identity_idx` ON `product_analytics_event_receipts` (`userIdentityHash`);--> statement-breakpoint +CREATE INDEX `organization_identity_idx` ON `product_analytics_event_receipts` (`organizationIdentityHash`);--> statement-breakpoint +CREATE INDEX `retention_idx` ON `product_analytics_event_receipts` (`retainUntil`);--> statement-breakpoint +CREATE INDEX `conflict_idx` ON `product_analytics_event_receipts` (`conflictCount`);--> statement-breakpoint +CREATE INDEX `user_identity_idx` ON `product_analytics_identity_links` (`userIdentityHash`);--> statement-breakpoint +CREATE INDEX `organization_identity_idx` ON `product_analytics_identity_links` (`organizationIdentityHash`);--> statement-breakpoint +CREATE INDEX `blocked_idx` ON `product_analytics_identity_state` (`blockedAt`);--> statement-breakpoint +CREATE INDEX `expiry_idx` ON `product_analytics_ingestion_leases` (`expiresAt`);--> statement-breakpoint +CREATE INDEX `delivery_idx` ON `product_analytics_outbox` (`status`,`nextAttemptAt`,`createdAt`);--> statement-breakpoint +CREATE INDEX `lease_idx` ON `product_analytics_outbox` (`leaseExpiresAt`);--> statement-breakpoint +CREATE INDEX `retention_idx` ON `product_analytics_outbox` (`status`,`deliveredAt`);--> statement-breakpoint +CREATE INDEX `user_id_idx` ON `product_analytics_outbox` (`userId`);--> statement-breakpoint +CREATE INDEX `organization_id_idx` ON `product_analytics_outbox` (`organizationId`);--> statement-breakpoint +CREATE INDEX `anonymous_id_idx` ON `product_analytics_outbox` (`anonymousId`);--> statement-breakpoint +CREATE INDEX `email_delivery_idx` ON `organization_invites` (`emailDeliveryState`,`emailDeliveryNextAttemptAt`); \ No newline at end of file diff --git a/packages/database/migrations/0041_complex_outlaw_kid.sql b/packages/database/migrations/0041_complex_outlaw_kid.sql new file mode 100644 index 00000000000..9123ee0767c --- /dev/null +++ b/packages/database/migrations/0041_complex_outlaw_kid.sql @@ -0,0 +1,16 @@ +CREATE TABLE `product_analytics_reconciliation_failures` ( + `sourceHash` varchar(64) NOT NULL, + `sourceType` varchar(32) NOT NULL, + `errorCode` varchar(64) NOT NULL, + `attemptCount` int NOT NULL DEFAULT 1, + `firstSeenAt` timestamp NOT NULL DEFAULT (now()), + `lastSeenAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `product_analytics_reconciliation_failures_sourceHash` PRIMARY KEY(`sourceHash`) +); +--> statement-breakpoint +ALTER TABLE `product_analytics_ingestion_leases` ADD `fencingToken` bigint unsigned NOT NULL;--> statement-breakpoint +CREATE INDEX `source_type_idx` ON `product_analytics_reconciliation_failures` (`sourceType`,`lastSeenAt`);--> statement-breakpoint +CREATE INDEX `analytics_reconciliation_idx` ON `comments` (`createdAt`,`id`);--> statement-breakpoint +CREATE INDEX `analytics_reconciliation_idx` ON `users` (`created_at`,`id`);--> statement-breakpoint +CREATE INDEX `analytics_created_at_idx` ON `videos` (`createdAt`,`id`);--> statement-breakpoint +CREATE INDEX `analytics_first_view_at_idx` ON `videos` (`firstExternalViewAt`,`id`); \ No newline at end of file diff --git a/packages/database/migrations/0042_lying_sharon_ventura.sql b/packages/database/migrations/0042_lying_sharon_ventura.sql new file mode 100644 index 00000000000..b13dd335236 --- /dev/null +++ b/packages/database/migrations/0042_lying_sharon_ventura.sql @@ -0,0 +1,16 @@ +CREATE TABLE `product_analytics_refresh_leases` ( + `name` varchar(64) NOT NULL, + `ownerId` varchar(36), + `generation` bigint unsigned NOT NULL DEFAULT 0, + `sourceCutoff` timestamp, + `leaseExpiresAt` timestamp, + `status` varchar(32) NOT NULL DEFAULT 'idle', + `lastCompletedAt` timestamp, + `lastErrorCode` varchar(64), + `createdAt` timestamp NOT NULL DEFAULT (now()), + `updatedAt` timestamp NOT NULL DEFAULT (now()) ON UPDATE CURRENT_TIMESTAMP, + CONSTRAINT `product_analytics_refresh_leases_name` PRIMARY KEY(`name`) +); +--> statement-breakpoint +CREATE INDEX `expiry_idx` ON `product_analytics_refresh_leases` (`leaseExpiresAt`);--> statement-breakpoint +CREATE INDEX `status_idx` ON `product_analytics_refresh_leases` (`status`); \ No newline at end of file diff --git a/packages/database/migrations/meta/0038_snapshot.json b/packages/database/migrations/meta/0038_snapshot.json index fadf6b52fcb..dd095a86836 100644 --- a/packages/database/migrations/meta/0038_snapshot.json +++ b/packages/database/migrations/meta/0038_snapshot.json @@ -1,4327 +1,3945 @@ { - "version": "5", - "dialect": "mysql", - "id": "794035aa-b67e-4430-8034-d3f6210b5262", - "prevId": "76d6a0f0-dc80-439a-9632-7d808d45e548", - "tables": { - "accounts": { - "name": "accounts", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerAccountId": { - "name": "providerAccountId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "access_token": { - "name": "access_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "expires_in": { - "name": "expires_in", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "id_token": { - "name": "id_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "refresh_token": { - "name": "refresh_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "refresh_token_expires_in": { - "name": "refresh_token_expires_in", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "scope": { - "name": "scope", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "token_type": { - "name": "token_type", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "tempColumn": { - "name": "tempColumn", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "user_id_idx": { - "name": "user_id_idx", - "columns": [ - "userId" - ], - "isUnique": false - }, - "provider_account_id_idx": { - "name": "provider_account_id_idx", - "columns": [ - "providerAccountId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "accounts_id": { - "name": "accounts_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_authorization_codes": { - "name": "agent_api_authorization_codes", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "codeHash": { - "name": "codeHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "codeChallenge": { - "name": "codeChallenge", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "redirectUri": { - "name": "redirectUri", - "type": "varchar(512)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "scopes": { - "name": "scopes", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "consumedAt": { - "name": "consumedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "code_hash_idx": { - "name": "code_hash_idx", - "columns": [ - "codeHash" - ], - "isUnique": true - }, - "expires_at_idx": { - "name": "expires_at_idx", - "columns": [ - "expiresAt" - ], - "isUnique": false - }, - "user_created_at_idx": { - "name": "user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_authorization_codes_id": { - "name": "agent_api_authorization_codes_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_idempotency": { - "name": "agent_api_idempotency", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "operation": { - "name": "operation", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyHash": { - "name": "keyHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "requestHash": { - "name": "requestHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "state": { - "name": "state", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "statusCode": { - "name": "statusCode", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "response": { - "name": "response", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "user_operation_key_idx": { - "name": "user_operation_key_idx", - "columns": [ - "userId", - "operation", - "keyHash" - ], - "isUnique": true - }, - "expires_at_idx": { - "name": "expires_at_idx", - "columns": [ - "expiresAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_idempotency_id": { - "name": "agent_api_idempotency_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_keys": { - "name": "agent_api_keys", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "tokenHash": { - "name": "tokenHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(100)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Cap CLI'" - }, - "scopes": { - "name": "scopes", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "revokedAt": { - "name": "revokedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "lastUsedAt": { - "name": "lastUsedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "token_hash_idx": { - "name": "token_hash_idx", - "columns": [ - "tokenHash" - ], - "isUnique": true - }, - "user_created_at_idx": { - "name": "user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - }, - "expires_at_idx": { - "name": "expires_at_idx", - "columns": [ - "expiresAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_keys_id": { - "name": "agent_api_keys_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_operations": { - "name": "agent_api_operations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "kind": { - "name": "kind", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "resourceId": { - "name": "resourceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "resultResourceId": { - "name": "resultResourceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "state": { - "name": "state", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'queued'" - }, - "payload": { - "name": "payload", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "result": { - "name": "result", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "errorCode": { - "name": "errorCode", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "errorMessage": { - "name": "errorMessage", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "completedAt": { - "name": "completedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "user_created_at_idx": { - "name": "user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - }, - "state_updated_at_idx": { - "name": "state_updated_at_idx", - "columns": [ - "state", - "updatedAt" - ], - "isUnique": false - }, - "resource_id_idx": { - "name": "resource_id_idx", - "columns": [ - "resourceId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_operations_id": { - "name": "agent_api_operations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "auth_api_keys": { - "name": "auth_api_keys", - "columns": { - "id": { - "name": "id", - "type": "varchar(36)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "source": { - "name": "source", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'unknown'" - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "user_id_created_at_idx": { - "name": "user_id_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "auth_api_keys_id": { - "name": "auth_api_keys_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "comments": { - "name": "comments", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(6)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "timestamp": { - "name": "timestamp", - "type": "float", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "authorId": { - "name": "authorId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "parentCommentId": { - "name": "parentCommentId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "video_type_created_idx": { - "name": "video_type_created_idx", - "columns": [ - "videoId", - "type", - "createdAt", - "id" - ], - "isUnique": false - }, - "author_id_idx": { - "name": "author_id_idx", - "columns": [ - "authorId" - ], - "isUnique": false - }, - "parent_comment_id_idx": { - "name": "parent_comment_id_idx", - "columns": [ - "parentCommentId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "comments_id": { - "name": "comments_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_api_keys": { - "name": "developer_api_keys", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyType": { - "name": "keyType", - "type": "varchar(8)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyPrefix": { - "name": "keyPrefix", - "type": "varchar(12)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyHash": { - "name": "keyHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "encryptedKey": { - "name": "encryptedKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "lastUsedAt": { - "name": "lastUsedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "revokedAt": { - "name": "revokedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "key_hash_idx": { - "name": "key_hash_idx", - "columns": [ - "keyHash" - ], - "isUnique": true - }, - "app_key_type_idx": { - "name": "app_key_type_idx", - "columns": [ - "appId", - "keyType" - ], - "isUnique": false - } - }, - "foreignKeys": { - "developer_api_keys_appId_developer_apps_id_fk": { - "name": "developer_api_keys_appId_developer_apps_id_fk", - "tableFrom": "developer_api_keys", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_api_keys_id": { - "name": "developer_api_keys_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_app_domains": { - "name": "developer_app_domains", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "domain": { - "name": "domain", - "type": "varchar(253)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": { - "developer_app_domains_appId_developer_apps_id_fk": { - "name": "developer_app_domains_appId_developer_apps_id_fk", - "tableFrom": "developer_app_domains", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_app_domains_id": { - "name": "developer_app_domains_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": { - "app_domain_unique": { - "name": "app_domain_unique", - "columns": [ - "appId", - "domain" - ] - } - }, - "checkConstraint": {} - }, - "developer_apps": { - "name": "developer_apps", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "environment": { - "name": "environment", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "logoUrl": { - "name": "logoUrl", - "type": "varchar(1024)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "deletedAt": { - "name": "deletedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "owner_deleted_idx": { - "name": "owner_deleted_idx", - "columns": [ - "ownerId", - "deletedAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "developer_apps_id": { - "name": "developer_apps_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_credit_accounts": { - "name": "developer_credit_accounts", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "balanceMicroCredits": { - "name": "balanceMicroCredits", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "stripeCustomerId": { - "name": "stripeCustomerId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripePaymentMethodId": { - "name": "stripePaymentMethodId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "autoTopUpEnabled": { - "name": "autoTopUpEnabled", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "autoTopUpThresholdMicroCredits": { - "name": "autoTopUpThresholdMicroCredits", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "autoTopUpAmountCents": { - "name": "autoTopUpAmountCents", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "app_id_unique": { - "name": "app_id_unique", - "columns": [ - "appId" - ], - "isUnique": true - } - }, - "foreignKeys": { - "developer_credit_accounts_appId_developer_apps_id_fk": { - "name": "developer_credit_accounts_appId_developer_apps_id_fk", - "tableFrom": "developer_credit_accounts", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_credit_accounts_id": { - "name": "developer_credit_accounts_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_credit_transactions": { - "name": "developer_credit_transactions", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "accountId": { - "name": "accountId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "amountMicroCredits": { - "name": "amountMicroCredits", - "type": "bigint", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "balanceAfterMicroCredits": { - "name": "balanceAfterMicroCredits", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "referenceId": { - "name": "referenceId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "referenceType": { - "name": "referenceType", - "type": "varchar(32)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "account_type_created_idx": { - "name": "account_type_created_idx", - "columns": [ - "accountId", - "type", - "createdAt" - ], - "isUnique": false - }, - "account_ref_dedup_idx": { - "name": "account_ref_dedup_idx", - "columns": [ - "accountId", - "referenceId", - "referenceType" - ], - "isUnique": false - } - }, - "foreignKeys": { - "dev_credit_txn_account_fk": { - "name": "dev_credit_txn_account_fk", - "tableFrom": "developer_credit_transactions", - "tableTo": "developer_credit_accounts", - "columnsFrom": [ - "accountId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_credit_transactions_id": { - "name": "developer_credit_transactions_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_daily_storage_snapshots": { - "name": "developer_daily_storage_snapshots", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "snapshotDate": { - "name": "snapshotDate", - "type": "varchar(10)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "totalDurationMinutes": { - "name": "totalDurationMinutes", - "type": "float", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "videoCount": { - "name": "videoCount", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "microCreditsCharged": { - "name": "microCreditsCharged", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "processedAt": { - "name": "processedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": { - "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { - "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", - "tableFrom": "developer_daily_storage_snapshots", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_daily_storage_snapshots_id": { - "name": "developer_daily_storage_snapshots_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": { - "app_date_unique": { - "name": "app_date_unique", - "columns": [ - "appId", - "snapshotDate" - ] - } - }, - "checkConstraint": {} - }, - "developer_videos": { - "name": "developer_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "externalUserId": { - "name": "externalUserId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Untitled'" - }, - "duration": { - "name": "duration", - "type": "float", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "width": { - "name": "width", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "height": { - "name": "height", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "fps": { - "name": "fps", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "s3Key": { - "name": "s3Key", - "type": "varchar(512)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "transcriptionStatus": { - "name": "transcriptionStatus", - "type": "varchar(16)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "deletedAt": { - "name": "deletedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "app_created_idx": { - "name": "app_created_idx", - "columns": [ - "appId", - "createdAt" - ], - "isUnique": false - }, - "app_user_idx": { - "name": "app_user_idx", - "columns": [ - "appId", - "externalUserId" - ], - "isUnique": false - }, - "app_deleted_idx": { - "name": "app_deleted_idx", - "columns": [ - "appId", - "deletedAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "developer_videos_appId_developer_apps_id_fk": { - "name": "developer_videos_appId_developer_apps_id_fk", - "tableFrom": "developer_videos", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_videos_id": { - "name": "developer_videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "folders": { - "name": "folders", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'normal'" - }, - "public": { - "name": "public", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdById": { - "name": "createdById", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "parentId": { - "name": "parentId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "created_by_id_idx": { - "name": "created_by_id_idx", - "columns": [ - "createdById" - ], - "isUnique": false - }, - "parent_id_idx": { - "name": "parent_id_idx", - "columns": [ - "parentId" - ], - "isUnique": false - }, - "space_id_idx": { - "name": "space_id_idx", - "columns": [ - "spaceId" - ], - "isUnique": false - }, - "public_parent_id_idx": { - "name": "public_parent_id_idx", - "columns": [ - "public", - "parentId" - ], - "isUnique": false - }, - "public_space_parent_id_idx": { - "name": "public_space_parent_id_idx", - "columns": [ - "public", - "spaceId", - "parentId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "folders_id": { - "name": "folders_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "imported_videos": { - "name": "imported_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "orgId": { - "name": "orgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "source": { - "name": "source", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "source_id": { - "name": "source_id", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": { - "imported_videos_orgId_source_source_id_pk": { - "name": "imported_videos_orgId_source_source_id_pk", - "columns": [ - "orgId", - "source", - "source_id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "integration_installations": { - "name": "integration_installations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "externalId": { - "name": "externalId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "displayName": { - "name": "displayName", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "installedByUserId": { - "name": "installedByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "encryptedCredentials": { - "name": "encryptedCredentials", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "provider_external_id_idx": { - "name": "provider_external_id_idx", - "columns": [ - "provider", - "externalId" - ], - "isUnique": true - }, - "organization_provider_display_name_idx": { - "name": "organization_provider_display_name_idx", - "columns": [ - "organizationId", - "provider", - "displayName" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "integration_installations_id": { - "name": "integration_installations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "messenger_conversations": { - "name": "messenger_conversations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "agent": { - "name": "agent", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "mode": { - "name": "mode", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'agent'" - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "anonymousId": { - "name": "anonymousId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "takeoverByUserId": { - "name": "takeoverByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "takeoverAt": { - "name": "takeoverAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "lastMessageAt": { - "name": "lastMessageAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "user_last_message_idx": { - "name": "user_last_message_idx", - "columns": [ - "userId", - "lastMessageAt" - ], - "isUnique": false - }, - "anonymous_last_message_idx": { - "name": "anonymous_last_message_idx", - "columns": [ - "anonymousId", - "lastMessageAt" - ], - "isUnique": false - }, - "mode_last_message_idx": { - "name": "mode_last_message_idx", - "columns": [ - "mode", - "lastMessageAt" - ], - "isUnique": false - }, - "updated_at_idx": { - "name": "updated_at_idx", - "columns": [ - "updatedAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "messenger_conversations_id": { - "name": "messenger_conversations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "messenger_messages": { - "name": "messenger_messages", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "conversationId": { - "name": "conversationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "anonymousId": { - "name": "anonymousId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "conversation_created_at_idx": { - "name": "conversation_created_at_idx", - "columns": [ - "conversationId", - "createdAt" - ], - "isUnique": false - }, - "role_created_at_idx": { - "name": "role_created_at_idx", - "columns": [ - "role", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "messenger_messages_conversationId_messenger_conversations_id_fk": { - "name": "messenger_messages_conversationId_messenger_conversations_id_fk", - "tableFrom": "messenger_messages", - "tableTo": "messenger_conversations", - "columnsFrom": [ - "conversationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "messenger_messages_id": { - "name": "messenger_messages_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "messenger_support_emails": { - "name": "messenger_support_emails", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "conversationId": { - "name": "conversationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userEmail": { - "name": "userEmail", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "subject": { - "name": "subject", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "message": { - "name": "message", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "support_email_user_created_at_idx": { - "name": "support_email_user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - }, - "support_email_conversation_created_at_idx": { - "name": "support_email_conversation_created_at_idx", - "columns": [ - "conversationId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "support_email_conversation_fk": { - "name": "support_email_conversation_fk", - "tableFrom": "messenger_support_emails", - "tableTo": "messenger_conversations", - "columnsFrom": [ - "conversationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "messenger_support_emails_id": { - "name": "messenger_support_emails_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "notifications": { - "name": "notifications", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "orgId": { - "name": "orgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "recipientId": { - "name": "recipientId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "data": { - "name": "data", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(50)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "dedupKey": { - "name": "dedupKey", - "type": "varchar(128)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "readAt": { - "name": "readAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "org_id_idx": { - "name": "org_id_idx", - "columns": [ - "orgId" - ], - "isUnique": false - }, - "type_idx": { - "name": "type_idx", - "columns": [ - "type" - ], - "isUnique": false - }, - "read_at_idx": { - "name": "read_at_idx", - "columns": [ - "readAt" - ], - "isUnique": false - }, - "created_at_idx": { - "name": "created_at_idx", - "columns": [ - "createdAt" - ], - "isUnique": false - }, - "recipient_read_idx": { - "name": "recipient_read_idx", - "columns": [ - "recipientId", - "readAt" - ], - "isUnique": false - }, - "recipient_created_idx": { - "name": "recipient_created_idx", - "columns": [ - "recipientId", - "createdAt" - ], - "isUnique": false - }, - "dedup_key_idx": { - "name": "dedup_key_idx", - "columns": [ - "dedupKey" - ], - "isUnique": true - }, - "type_recipient_created_idx": { - "name": "type_recipient_created_idx", - "columns": [ - "type", - "recipientId", - "createdAt" - ], - "isUnique": false - }, - "type_recipient_video_created_idx": { - "name": "type_recipient_video_created_idx", - "columns": [ - "type", - "recipientId", - "videoId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "notifications_id": { - "name": "notifications_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "organization_invites": { - "name": "organization_invites", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "invitedEmail": { - "name": "invitedEmail", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "invitedByUserId": { - "name": "invitedByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "invited_email_idx": { - "name": "invited_email_idx", - "columns": [ - "invitedEmail" - ], - "isUnique": false - }, - "invited_by_user_id_idx": { - "name": "invited_by_user_id_idx", - "columns": [ - "invitedByUserId" - ], - "isUnique": false - }, - "status_idx": { - "name": "status_idx", - "columns": [ - "status" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "organization_invites_id": { - "name": "organization_invites_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "organization_members": { - "name": "organization_members", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "hasProSeat": { - "name": "hasProSeat", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "user_id_organization_id_idx": { - "name": "user_id_organization_id_idx", - "columns": [ - "userId", - "organizationId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "organization_members_id": { - "name": "organization_members_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "organizations": { - "name": "organizations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "tombstoneAt": { - "name": "tombstoneAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "allowedEmailDomain": { - "name": "allowedEmailDomain", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "customDomain": { - "name": "customDomain", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "domainVerified": { - "name": "domainVerified", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "iconUrl": { - "name": "iconUrl", - "type": "varchar(1024)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "shareableLinkIconUrl": { - "name": "shareableLinkIconUrl", - "type": "varchar(1024)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "workosOrganizationId": { - "name": "workosOrganizationId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "workosConnectionId": { - "name": "workosConnectionId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "owner_id_tombstone_idx": { - "name": "owner_id_tombstone_idx", - "columns": [ - "ownerId", - "tombstoneAt" - ], - "isUnique": false - }, - "custom_domain_idx": { - "name": "custom_domain_idx", - "columns": [ - "customDomain" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "organizations_id": { - "name": "organizations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "s3_buckets": { - "name": "s3_buckets", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "region": { - "name": "region", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "endpoint": { - "name": "endpoint", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "bucketName": { - "name": "bucketName", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "accessKeyId": { - "name": "accessKeyId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "secretAccessKey": { - "name": "secretAccessKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "('aws')" - }, - "active": { - "name": "active", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": true - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "owner_organization_idx": { - "name": "owner_organization_idx", - "columns": [ - "ownerId", - "organizationId" - ], - "isUnique": false - }, - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "organization_active_idx": { - "name": "organization_active_idx", - "columns": [ - "organizationId", - "active", - "updatedAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "s3_buckets_id": { - "name": "s3_buckets_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "sessions": { - "name": "sessions", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sessionToken": { - "name": "sessionToken", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "datetime", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "session_token_idx": { - "name": "session_token_idx", - "columns": [ - "sessionToken" - ], - "isUnique": true - }, - "user_id_idx": { - "name": "user_id_idx", - "columns": [ - "userId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "sessions_id": { - "name": "sessions_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "shared_videos": { - "name": "shared_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "folderId": { - "name": "folderId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sharedByUserId": { - "name": "sharedByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sharedAt": { - "name": "sharedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "folder_id_idx": { - "name": "folder_id_idx", - "columns": [ - "folderId" - ], - "isUnique": false - }, - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "shared_by_user_id_idx": { - "name": "shared_by_user_id_idx", - "columns": [ - "sharedByUserId" - ], - "isUnique": false - }, - "video_id_organization_id_idx": { - "name": "video_id_organization_id_idx", - "columns": [ - "videoId", - "organizationId" - ], - "isUnique": false - }, - "video_id_folder_id_idx": { - "name": "video_id_folder_id_idx", - "columns": [ - "videoId", - "folderId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "shared_videos_id": { - "name": "shared_videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "space_members": { - "name": "space_members", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'member'" - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "user_id_idx": { - "name": "user_id_idx", - "columns": [ - "userId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "space_members_id": { - "name": "space_members_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": { - "space_id_user_id_unique": { - "name": "space_id_user_id_unique", - "columns": [ - "spaceId", - "userId" - ] - } - }, - "checkConstraint": {} - }, - "space_videos": { - "name": "space_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "folderId": { - "name": "folderId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "addedById": { - "name": "addedById", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "addedAt": { - "name": "addedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "folder_id_idx": { - "name": "folder_id_idx", - "columns": [ - "folderId" - ], - "isUnique": false - }, - "video_id_idx": { - "name": "video_id_idx", - "columns": [ - "videoId" - ], - "isUnique": false - }, - "added_by_id_idx": { - "name": "added_by_id_idx", - "columns": [ - "addedById" - ], - "isUnique": false - }, - "space_id_video_id_idx": { - "name": "space_id_video_id_idx", - "columns": [ - "spaceId", - "videoId" - ], - "isUnique": false - }, - "space_id_folder_id_idx": { - "name": "space_id_folder_id_idx", - "columns": [ - "spaceId", - "folderId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "space_videos_id": { - "name": "space_videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "spaces": { - "name": "spaces", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "primary": { - "name": "primary", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdById": { - "name": "createdById", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "iconUrl": { - "name": "iconUrl", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "description": { - "name": "description", - "type": "varchar(1000)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "privacy": { - "name": "privacy", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Private'" - }, - "public": { - "name": "public", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "created_by_id_idx": { - "name": "created_by_id_idx", - "columns": [ - "createdById" - ], - "isUnique": false - }, - "public_organization_id_idx": { - "name": "public_organization_id_idx", - "columns": [ - "public", - "organizationId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "spaces_id": { - "name": "spaces_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "storage_integrations": { - "name": "storage_integrations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "displayName": { - "name": "displayName", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'active'" - }, - "active": { - "name": "active", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "encryptedConfig": { - "name": "encryptedConfig", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "googleDriveAccessToken": { - "name": "googleDriveAccessToken", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveAccessTokenExpiresAt": { - "name": "googleDriveAccessTokenExpiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveTokenRefreshLeaseId": { - "name": "googleDriveTokenRefreshLeaseId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveTokenRefreshLeaseExpiresAt": { - "name": "googleDriveTokenRefreshLeaseExpiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveStorageQuotaCache": { - "name": "googleDriveStorageQuotaCache", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "owner_provider_idx": { - "name": "owner_provider_idx", - "columns": [ - "ownerId", - "provider" - ], - "isUnique": false - }, - "owner_active_idx": { - "name": "owner_active_idx", - "columns": [ - "ownerId", - "active" - ], - "isUnique": false - }, - "organization_provider_idx": { - "name": "organization_provider_idx", - "columns": [ - "organizationId", - "provider" - ], - "isUnique": false - }, - "organization_active_idx": { - "name": "organization_active_idx", - "columns": [ - "organizationId", - "active", - "status" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "storage_integrations_id": { - "name": "storage_integrations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "storage_objects": { - "name": "storage_objects", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "integrationId": { - "name": "integrationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "objectKey": { - "name": "objectKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "objectKeyHash": { - "name": "objectKeyHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerObjectId": { - "name": "providerObjectId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "uploadSessionUrl": { - "name": "uploadSessionUrl", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "uploadStatus": { - "name": "uploadStatus", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "contentType": { - "name": "contentType", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "contentLength": { - "name": "contentLength", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "integration_key_hash_idx": { - "name": "integration_key_hash_idx", - "columns": [ - "integrationId", - "objectKeyHash" - ], - "isUnique": true - }, - "integration_status_idx": { - "name": "integration_status_idx", - "columns": [ - "integrationId", - "uploadStatus" - ], - "isUnique": false - }, - "video_id_idx": { - "name": "video_id_idx", - "columns": [ - "videoId" - ], - "isUnique": false - }, - "owner_id_idx": { - "name": "owner_id_idx", - "columns": [ - "ownerId" - ], - "isUnique": false - } - }, - "foreignKeys": { - "storage_objects_integrationId_storage_integrations_id_fk": { - "name": "storage_objects_integrationId_storage_integrations_id_fk", - "tableFrom": "storage_objects", - "tableTo": "storage_integrations", - "columnsFrom": [ - "integrationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "restrict", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "storage_objects_id": { - "name": "storage_objects_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "users": { - "name": "users", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "lastName": { - "name": "lastName", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "email": { - "name": "email", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "emailVerified": { - "name": "emailVerified", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "image": { - "name": "image", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeCustomerId": { - "name": "stripeCustomerId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeSubscriptionId": { - "name": "stripeSubscriptionId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "thirdPartyStripeSubscriptionId": { - "name": "thirdPartyStripeSubscriptionId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeSubscriptionStatus": { - "name": "stripeSubscriptionStatus", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeSubscriptionPriceId": { - "name": "stripeSubscriptionPriceId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "preferences": { - "name": "preferences", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "('null')" - }, - "activeOrganizationId": { - "name": "activeOrganizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "onboardingSteps": { - "name": "onboardingSteps", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "onboarding_completed_at": { - "name": "onboarding_completed_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "customBucket": { - "name": "customBucket", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "inviteQuota": { - "name": "inviteQuota", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "defaultOrgId": { - "name": "defaultOrgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "authSessionVersion": { - "name": "authSessionVersion", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - } - }, - "indexes": { - "email_idx": { - "name": "email_idx", - "columns": [ - "email" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "users_id": { - "name": "users_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "verification_tokens": { - "name": "verification_tokens", - "columns": { - "identifier": { - "name": "identifier", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "token": { - "name": "token", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "datetime", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": { - "verification_tokens_identifier": { - "name": "verification_tokens_identifier", - "columns": [ - "identifier" - ] - } - }, - "uniqueConstraints": { - "verification_tokens_token_unique": { - "name": "verification_tokens_token_unique", - "columns": [ - "token" - ] - } - }, - "checkConstraint": {} - }, - "video_edits": { - "name": "video_edits", - "columns": { - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sourceKey": { - "name": "sourceKey", - "type": "varchar(512)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "editSpec": { - "name": "editSpec", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": { - "video_edits_videoId_videos_id_fk": { - "name": "video_edits_videoId_videos_id_fk", - "tableFrom": "video_edits", - "tableTo": "videos", - "columnsFrom": [ - "videoId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "video_edits_videoId": { - "name": "video_edits_videoId", - "columns": [ - "videoId" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "video_uploads": { - "name": "video_uploads", - "columns": { - "video_id": { - "name": "video_id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "uploaded": { - "name": "uploaded", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "total": { - "name": "total", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "started_at": { - "name": "started_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "mode": { - "name": "mode", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "phase": { - "name": "phase", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'uploading'" - }, - "processing_progress": { - "name": "processing_progress", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "processing_message": { - "name": "processing_message", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "processing_error": { - "name": "processing_error", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "raw_file_key": { - "name": "raw_file_key", - "type": "varchar(512)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "phase_updated_at_video_id_idx": { - "name": "phase_updated_at_video_id_idx", - "columns": [ - "phase", - "updated_at", - "video_id" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "video_uploads_video_id": { - "name": "video_uploads_video_id", - "columns": [ - "video_id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "videos": { - "name": "videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "orgId": { - "name": "orgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'My Video'" - }, - "bucket": { - "name": "bucket", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "storageIntegrationId": { - "name": "storageIntegrationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "duration": { - "name": "duration", - "type": "float", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "width": { - "name": "width", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "height": { - "name": "height", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "fps": { - "name": "fps", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "public": { - "name": "public", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": true - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "transcriptionStatus": { - "name": "transcriptionStatus", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "source": { - "name": "source", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "('{\"type\":\"MediaConvert\"}')" - }, - "folderId": { - "name": "folderId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "effectiveCreatedAt": { - "name": "effectiveCreatedAt", - "type": "datetime", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "generated": { - "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", - "type": "stored" - } - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "xStreamInfo": { - "name": "xStreamInfo", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "firstExternalViewAt": { - "name": "firstExternalViewAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "firstViewEmailSentAt": { - "name": "firstViewEmailSentAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "isScreenshot": { - "name": "isScreenshot", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "awsRegion": { - "name": "awsRegion", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "awsBucket": { - "name": "awsBucket", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "videoStartTime": { - "name": "videoStartTime", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "audioStartTime": { - "name": "audioStartTime", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "jobId": { - "name": "jobId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "jobStatus": { - "name": "jobStatus", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "skipProcessing": { - "name": "skipProcessing", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - } - }, - "indexes": { - "owner_id_idx": { - "name": "owner_id_idx", - "columns": [ - "ownerId" - ], - "isUnique": false - }, - "is_public_idx": { - "name": "is_public_idx", - "columns": [ - "public" - ], - "isUnique": false - }, - "folder_id_idx": { - "name": "folder_id_idx", - "columns": [ - "folderId" - ], - "isUnique": false - }, - "storage_integration_id_idx": { - "name": "storage_integration_id_idx", - "columns": [ - "storageIntegrationId" - ], - "isUnique": false - }, - "first_external_view_at_idx": { - "name": "first_external_view_at_idx", - "columns": [ - "firstExternalViewAt" - ], - "isUnique": false - }, - "org_owner_folder_idx": { - "name": "org_owner_folder_idx", - "columns": [ - "orgId", - "ownerId", - "folderId" - ], - "isUnique": false - }, - "org_effective_created_idx": { - "name": "org_effective_created_idx", - "columns": [ - "orgId", - "effectiveCreatedAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "videos_storageIntegrationId_storage_integrations_id_fk": { - "name": "videos_storageIntegrationId_storage_integrations_id_fk", - "tableFrom": "videos", - "tableTo": "storage_integrations", - "columnsFrom": [ - "storageIntegrationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "restrict", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "videos_id": { - "name": "videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - } - }, - "views": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "tables": {}, - "indexes": {} - } -} \ No newline at end of file + "version": "5", + "dialect": "mysql", + "id": "794035aa-b67e-4430-8034-d3f6210b5262", + "prevId": "76d6a0f0-dc80-439a-9632-7d808d45e548", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_in": { + "name": "expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_in": { + "name": "refresh_token_expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "tempColumn": { + "name": "tempColumn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "provider_account_id_idx": { + "name": "provider_account_id_idx", + "columns": ["providerAccountId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accounts_id": { + "name": "accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_authorization_codes": { + "name": "agent_api_authorization_codes", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeHash": { + "name": "codeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeChallenge": { + "name": "codeChallenge", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirectUri": { + "name": "redirectUri", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consumedAt": { + "name": "consumedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "code_hash_idx": { + "name": "code_hash_idx", + "columns": ["codeHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_authorization_codes_id": { + "name": "agent_api_authorization_codes_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_idempotency": { + "name": "agent_api_idempotency", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestHash": { + "name": "requestHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "statusCode": { + "name": "statusCode", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response": { + "name": "response", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_operation_key_idx": { + "name": "user_operation_key_idx", + "columns": ["userId", "operation", "keyHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_idempotency_id": { + "name": "agent_api_idempotency_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_keys": { + "name": "agent_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tokenHash": { + "name": "tokenHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Cap CLI'" + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "token_hash_idx": { + "name": "token_hash_idx", + "columns": ["tokenHash"], + "isUnique": true + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_keys_id": { + "name": "agent_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_operations": { + "name": "agent_api_operations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resourceId": { + "name": "resourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resultResourceId": { + "name": "resultResourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "state_updated_at_idx": { + "name": "state_updated_at_idx", + "columns": ["state", "updatedAt"], + "isUnique": false + }, + "resource_id_idx": { + "name": "resource_id_idx", + "columns": ["resourceId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_operations_id": { + "name": "agent_api_operations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "auth_api_keys": { + "name": "auth_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_id_created_at_idx": { + "name": "user_id_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "auth_api_keys_id": { + "name": "auth_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "comments": { + "name": "comments", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authorId": { + "name": "authorId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "parentCommentId": { + "name": "parentCommentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "video_type_created_idx": { + "name": "video_type_created_idx", + "columns": ["videoId", "type", "createdAt", "id"], + "isUnique": false + }, + "author_id_idx": { + "name": "author_id_idx", + "columns": ["authorId"], + "isUnique": false + }, + "parent_comment_id_idx": { + "name": "parent_comment_id_idx", + "columns": ["parentCommentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "comments_id": { + "name": "comments_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_api_keys": { + "name": "developer_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyType": { + "name": "keyType", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyPrefix": { + "name": "keyPrefix", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedKey": { + "name": "encryptedKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "key_hash_idx": { + "name": "key_hash_idx", + "columns": ["keyHash"], + "isUnique": true + }, + "app_key_type_idx": { + "name": "app_key_type_idx", + "columns": ["appId", "keyType"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_api_keys_appId_developer_apps_id_fk": { + "name": "developer_api_keys_appId_developer_apps_id_fk", + "tableFrom": "developer_api_keys", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_api_keys_id": { + "name": "developer_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_app_domains": { + "name": "developer_app_domains", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_app_domains_appId_developer_apps_id_fk": { + "name": "developer_app_domains_appId_developer_apps_id_fk", + "tableFrom": "developer_app_domains", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_app_domains_id": { + "name": "developer_app_domains_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_domain_unique": { + "name": "app_domain_unique", + "columns": ["appId", "domain"] + } + }, + "checkConstraint": {} + }, + "developer_apps": { + "name": "developer_apps", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logoUrl": { + "name": "logoUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_deleted_idx": { + "name": "owner_deleted_idx", + "columns": ["ownerId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "developer_apps_id": { + "name": "developer_apps_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_accounts": { + "name": "developer_credit_accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceMicroCredits": { + "name": "balanceMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripePaymentMethodId": { + "name": "stripePaymentMethodId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoTopUpEnabled": { + "name": "autoTopUpEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "autoTopUpThresholdMicroCredits": { + "name": "autoTopUpThresholdMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoTopUpAmountCents": { + "name": "autoTopUpAmountCents", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_id_unique": { + "name": "app_id_unique", + "columns": ["appId"], + "isUnique": true + } + }, + "foreignKeys": { + "developer_credit_accounts_appId_developer_apps_id_fk": { + "name": "developer_credit_accounts_appId_developer_apps_id_fk", + "tableFrom": "developer_credit_accounts", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_accounts_id": { + "name": "developer_credit_accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_transactions": { + "name": "developer_credit_transactions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accountId": { + "name": "accountId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amountMicroCredits": { + "name": "amountMicroCredits", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceAfterMicroCredits": { + "name": "balanceAfterMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referenceType": { + "name": "referenceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "account_type_created_idx": { + "name": "account_type_created_idx", + "columns": ["accountId", "type", "createdAt"], + "isUnique": false + }, + "account_ref_dedup_idx": { + "name": "account_ref_dedup_idx", + "columns": ["accountId", "referenceId", "referenceType"], + "isUnique": false + } + }, + "foreignKeys": { + "dev_credit_txn_account_fk": { + "name": "dev_credit_txn_account_fk", + "tableFrom": "developer_credit_transactions", + "tableTo": "developer_credit_accounts", + "columnsFrom": ["accountId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_transactions_id": { + "name": "developer_credit_transactions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_daily_storage_snapshots": { + "name": "developer_daily_storage_snapshots", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshotDate": { + "name": "snapshotDate", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalDurationMinutes": { + "name": "totalDurationMinutes", + "type": "float", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "videoCount": { + "name": "videoCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "microCreditsCharged": { + "name": "microCreditsCharged", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { + "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", + "tableFrom": "developer_daily_storage_snapshots", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_daily_storage_snapshots_id": { + "name": "developer_daily_storage_snapshots_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_date_unique": { + "name": "app_date_unique", + "columns": ["appId", "snapshotDate"] + } + }, + "checkConstraint": {} + }, + "developer_videos": { + "name": "developer_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalUserId": { + "name": "externalUserId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3Key": { + "name": "s3Key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_created_idx": { + "name": "app_created_idx", + "columns": ["appId", "createdAt"], + "isUnique": false + }, + "app_user_idx": { + "name": "app_user_idx", + "columns": ["appId", "externalUserId"], + "isUnique": false + }, + "app_deleted_idx": { + "name": "app_deleted_idx", + "columns": ["appId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_videos_appId_developer_apps_id_fk": { + "name": "developer_videos_appId_developer_apps_id_fk", + "tableFrom": "developer_videos", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_videos_id": { + "name": "developer_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "folders": { + "name": "folders", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "parent_id_idx": { + "name": "parent_id_idx", + "columns": ["parentId"], + "isUnique": false + }, + "space_id_idx": { + "name": "space_id_idx", + "columns": ["spaceId"], + "isUnique": false + }, + "public_parent_id_idx": { + "name": "public_parent_id_idx", + "columns": ["public", "parentId"], + "isUnique": false + }, + "public_space_parent_id_idx": { + "name": "public_space_parent_id_idx", + "columns": ["public", "spaceId", "parentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "folders_id": { + "name": "folders_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "imported_videos": { + "name": "imported_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "imported_videos_orgId_source_source_id_pk": { + "name": "imported_videos_orgId_source_source_id_pk", + "columns": ["orgId", "source", "source_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "integration_installations": { + "name": "integration_installations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalId": { + "name": "externalId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "installedByUserId": { + "name": "installedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedCredentials": { + "name": "encryptedCredentials", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "provider_external_id_idx": { + "name": "provider_external_id_idx", + "columns": ["provider", "externalId"], + "isUnique": true + }, + "organization_provider_display_name_idx": { + "name": "organization_provider_display_name_idx", + "columns": ["organizationId", "provider", "displayName"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "integration_installations_id": { + "name": "integration_installations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_conversations": { + "name": "messenger_conversations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverByUserId": { + "name": "takeoverByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverAt": { + "name": "takeoverAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastMessageAt": { + "name": "lastMessageAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_last_message_idx": { + "name": "user_last_message_idx", + "columns": ["userId", "lastMessageAt"], + "isUnique": false + }, + "anonymous_last_message_idx": { + "name": "anonymous_last_message_idx", + "columns": ["anonymousId", "lastMessageAt"], + "isUnique": false + }, + "mode_last_message_idx": { + "name": "mode_last_message_idx", + "columns": ["mode", "lastMessageAt"], + "isUnique": false + }, + "updated_at_idx": { + "name": "updated_at_idx", + "columns": ["updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "messenger_conversations_id": { + "name": "messenger_conversations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_messages": { + "name": "messenger_messages", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "conversation_created_at_idx": { + "name": "conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + }, + "role_created_at_idx": { + "name": "role_created_at_idx", + "columns": ["role", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "messenger_messages_conversationId_messenger_conversations_id_fk": { + "name": "messenger_messages_conversationId_messenger_conversations_id_fk", + "tableFrom": "messenger_messages", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_messages_id": { + "name": "messenger_messages_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_support_emails": { + "name": "messenger_support_emails", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "support_email_user_created_at_idx": { + "name": "support_email_user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "support_email_conversation_created_at_idx": { + "name": "support_email_conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "support_email_conversation_fk": { + "name": "support_email_conversation_fk", + "tableFrom": "messenger_support_emails", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_support_emails_id": { + "name": "messenger_support_emails_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipientId": { + "name": "recipientId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "org_id_idx": { + "name": "org_id_idx", + "columns": ["orgId"], + "isUnique": false + }, + "type_idx": { + "name": "type_idx", + "columns": ["type"], + "isUnique": false + }, + "read_at_idx": { + "name": "read_at_idx", + "columns": ["readAt"], + "isUnique": false + }, + "created_at_idx": { + "name": "created_at_idx", + "columns": ["createdAt"], + "isUnique": false + }, + "recipient_read_idx": { + "name": "recipient_read_idx", + "columns": ["recipientId", "readAt"], + "isUnique": false + }, + "recipient_created_idx": { + "name": "recipient_created_idx", + "columns": ["recipientId", "createdAt"], + "isUnique": false + }, + "dedup_key_idx": { + "name": "dedup_key_idx", + "columns": ["dedupKey"], + "isUnique": true + }, + "type_recipient_created_idx": { + "name": "type_recipient_created_idx", + "columns": ["type", "recipientId", "createdAt"], + "isUnique": false + }, + "type_recipient_video_created_idx": { + "name": "type_recipient_video_created_idx", + "columns": ["type", "recipientId", "videoId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "notifications_id": { + "name": "notifications_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_invites": { + "name": "organization_invites", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmail": { + "name": "invitedEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedByUserId": { + "name": "invitedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "invited_email_idx": { + "name": "invited_email_idx", + "columns": ["invitedEmail"], + "isUnique": false + }, + "invited_by_user_id_idx": { + "name": "invited_by_user_id_idx", + "columns": ["invitedByUserId"], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": ["status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_invites_id": { + "name": "organization_invites_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hasProSeat": { + "name": "hasProSeat", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "user_id_organization_id_idx": { + "name": "user_id_organization_id_idx", + "columns": ["userId", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_members_id": { + "name": "organization_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tombstoneAt": { + "name": "tombstoneAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowedEmailDomain": { + "name": "allowedEmailDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customDomain": { + "name": "customDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domainVerified": { + "name": "domainVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "shareableLinkIconUrl": { + "name": "shareableLinkIconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "workosOrganizationId": { + "name": "workosOrganizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workosConnectionId": { + "name": "workosConnectionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "owner_id_tombstone_idx": { + "name": "owner_id_tombstone_idx", + "columns": ["ownerId", "tombstoneAt"], + "isUnique": false + }, + "custom_domain_idx": { + "name": "custom_domain_idx", + "columns": ["customDomain"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organizations_id": { + "name": "organizations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "s3_buckets": { + "name": "s3_buckets", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucketName": { + "name": "bucketName", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessKeyId": { + "name": "accessKeyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretAccessKey": { + "name": "secretAccessKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('aws')" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_organization_idx": { + "name": "owner_organization_idx", + "columns": ["ownerId", "organizationId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "s3_buckets_id": { + "name": "s3_buckets_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "session_token_idx": { + "name": "session_token_idx", + "columns": ["sessionToken"], + "isUnique": true + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sessions_id": { + "name": "sessions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "shared_videos": { + "name": "shared_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedByUserId": { + "name": "sharedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedAt": { + "name": "sharedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "shared_by_user_id_idx": { + "name": "shared_by_user_id_idx", + "columns": ["sharedByUserId"], + "isUnique": false + }, + "video_id_organization_id_idx": { + "name": "video_id_organization_id_idx", + "columns": ["videoId", "organizationId"], + "isUnique": false + }, + "video_id_folder_id_idx": { + "name": "video_id_folder_id_idx", + "columns": ["videoId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "shared_videos_id": { + "name": "shared_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_members_id": { + "name": "space_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "space_id_user_id_unique": { + "name": "space_id_user_id_unique", + "columns": ["spaceId", "userId"] + } + }, + "checkConstraint": {} + }, + "space_videos": { + "name": "space_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedById": { + "name": "addedById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "added_by_id_idx": { + "name": "added_by_id_idx", + "columns": ["addedById"], + "isUnique": false + }, + "space_id_video_id_idx": { + "name": "space_id_video_id_idx", + "columns": ["spaceId", "videoId"], + "isUnique": false + }, + "space_id_folder_id_idx": { + "name": "space_id_folder_id_idx", + "columns": ["spaceId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_videos_id": { + "name": "space_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "primary": { + "name": "primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "privacy": { + "name": "privacy", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Private'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "public_organization_id_idx": { + "name": "public_organization_id_idx", + "columns": ["public", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "spaces_id": { + "name": "spaces_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_integrations": { + "name": "storage_integrations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "encryptedConfig": { + "name": "encryptedConfig", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "googleDriveAccessToken": { + "name": "googleDriveAccessToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveAccessTokenExpiresAt": { + "name": "googleDriveAccessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseId": { + "name": "googleDriveTokenRefreshLeaseId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseExpiresAt": { + "name": "googleDriveTokenRefreshLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveStorageQuotaCache": { + "name": "googleDriveStorageQuotaCache", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_provider_idx": { + "name": "owner_provider_idx", + "columns": ["ownerId", "provider"], + "isUnique": false + }, + "owner_active_idx": { + "name": "owner_active_idx", + "columns": ["ownerId", "active"], + "isUnique": false + }, + "organization_provider_idx": { + "name": "organization_provider_idx", + "columns": ["organizationId", "provider"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "storage_integrations_id": { + "name": "storage_integrations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_objects": { + "name": "storage_objects", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrationId": { + "name": "integrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "objectKey": { + "name": "objectKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "objectKeyHash": { + "name": "objectKeyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerObjectId": { + "name": "providerObjectId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploadSessionUrl": { + "name": "uploadSessionUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploadStatus": { + "name": "uploadStatus", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "contentType": { + "name": "contentType", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentLength": { + "name": "contentLength", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "integration_key_hash_idx": { + "name": "integration_key_hash_idx", + "columns": ["integrationId", "objectKeyHash"], + "isUnique": true + }, + "integration_status_idx": { + "name": "integration_status_idx", + "columns": ["integrationId", "uploadStatus"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + } + }, + "foreignKeys": { + "storage_objects_integrationId_storage_integrations_id_fk": { + "name": "storage_objects_integrationId_storage_integrations_id_fk", + "tableFrom": "storage_objects", + "tableTo": "storage_integrations", + "columnsFrom": ["integrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "storage_objects_id": { + "name": "storage_objects_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastName": { + "name": "lastName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionId": { + "name": "stripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thirdPartyStripeSubscriptionId": { + "name": "thirdPartyStripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionStatus": { + "name": "stripeSubscriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionPriceId": { + "name": "stripeSubscriptionPriceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferences": { + "name": "preferences", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "('null')" + }, + "activeOrganizationId": { + "name": "activeOrganizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "onboardingSteps": { + "name": "onboardingSteps", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customBucket": { + "name": "customBucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "inviteQuota": { + "name": "inviteQuota", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "defaultOrgId": { + "name": "defaultOrgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authSessionVersion": { + "name": "authSessionVersion", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "email_idx": { + "name": "email_idx", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "identifier": { + "name": "identifier", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier": { + "name": "verification_tokens_identifier", + "columns": ["identifier"] + } + }, + "uniqueConstraints": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": ["token"] + } + }, + "checkConstraint": {} + }, + "video_edits": { + "name": "video_edits", + "columns": { + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceKey": { + "name": "sourceKey", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "editSpec": { + "name": "editSpec", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "video_edits_videoId_videos_id_fk": { + "name": "video_edits_videoId_videos_id_fk", + "tableFrom": "video_edits", + "tableTo": "videos", + "columnsFrom": ["videoId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "video_edits_videoId": { + "name": "video_edits_videoId", + "columns": ["videoId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "video_uploads": { + "name": "video_uploads", + "columns": { + "video_id": { + "name": "video_id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded": { + "name": "uploaded", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "mode": { + "name": "mode", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'uploading'" + }, + "processing_progress": { + "name": "processing_progress", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processing_message": { + "name": "processing_message", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_file_key": { + "name": "raw_file_key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "phase_updated_at_video_id_idx": { + "name": "phase_updated_at_video_id_idx", + "columns": ["phase", "updated_at", "video_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "video_uploads_video_id": { + "name": "video_uploads_video_id", + "columns": ["video_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'My Video'" + }, + "bucket": { + "name": "bucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageIntegrationId": { + "name": "storageIntegrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('{\"type\":\"MediaConvert\"}')" + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "effectiveCreatedAt": { + "name": "effectiveCreatedAt", + "type": "datetime", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "generated": { + "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", + "type": "stored" + } + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "xStreamInfo": { + "name": "xStreamInfo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstExternalViewAt": { + "name": "firstExternalViewAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstViewEmailSentAt": { + "name": "firstViewEmailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isScreenshot": { + "name": "isScreenshot", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "awsRegion": { + "name": "awsRegion", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "awsBucket": { + "name": "awsBucket", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoStartTime": { + "name": "videoStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audioStartTime": { + "name": "audioStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobId": { + "name": "jobId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobStatus": { + "name": "jobStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipProcessing": { + "name": "skipProcessing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + }, + "is_public_idx": { + "name": "is_public_idx", + "columns": ["public"], + "isUnique": false + }, + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "storage_integration_id_idx": { + "name": "storage_integration_id_idx", + "columns": ["storageIntegrationId"], + "isUnique": false + }, + "first_external_view_at_idx": { + "name": "first_external_view_at_idx", + "columns": ["firstExternalViewAt"], + "isUnique": false + }, + "org_owner_folder_idx": { + "name": "org_owner_folder_idx", + "columns": ["orgId", "ownerId", "folderId"], + "isUnique": false + }, + "org_effective_created_idx": { + "name": "org_effective_created_idx", + "columns": ["orgId", "effectiveCreatedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "videos_storageIntegrationId_storage_integrations_id_fk": { + "name": "videos_storageIntegrationId_storage_integrations_id_fk", + "tableFrom": "videos", + "tableTo": "storage_integrations", + "columnsFrom": ["storageIntegrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "videos_id": { + "name": "videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} diff --git a/packages/database/migrations/meta/0039_snapshot.json b/packages/database/migrations/meta/0039_snapshot.json index 369efc9972e..d6faa593658 100644 --- a/packages/database/migrations/meta/0039_snapshot.json +++ b/packages/database/migrations/meta/0039_snapshot.json @@ -1,4441 +1,4057 @@ { - "version": "5", - "dialect": "mysql", - "id": "f7a1a6b4-42da-4813-9462-a09d30766433", - "prevId": "794035aa-b67e-4430-8034-d3f6210b5262", - "tables": { - "accounts": { - "name": "accounts", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerAccountId": { - "name": "providerAccountId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "access_token": { - "name": "access_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "expires_in": { - "name": "expires_in", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "id_token": { - "name": "id_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "refresh_token": { - "name": "refresh_token", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "refresh_token_expires_in": { - "name": "refresh_token_expires_in", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "scope": { - "name": "scope", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "token_type": { - "name": "token_type", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "tempColumn": { - "name": "tempColumn", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "user_id_idx": { - "name": "user_id_idx", - "columns": [ - "userId" - ], - "isUnique": false - }, - "provider_account_id_idx": { - "name": "provider_account_id_idx", - "columns": [ - "providerAccountId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "accounts_id": { - "name": "accounts_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_authorization_codes": { - "name": "agent_api_authorization_codes", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "codeHash": { - "name": "codeHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "codeChallenge": { - "name": "codeChallenge", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "redirectUri": { - "name": "redirectUri", - "type": "varchar(512)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "scopes": { - "name": "scopes", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "consumedAt": { - "name": "consumedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "code_hash_idx": { - "name": "code_hash_idx", - "columns": [ - "codeHash" - ], - "isUnique": true - }, - "expires_at_idx": { - "name": "expires_at_idx", - "columns": [ - "expiresAt" - ], - "isUnique": false - }, - "user_created_at_idx": { - "name": "user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_authorization_codes_id": { - "name": "agent_api_authorization_codes_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_idempotency": { - "name": "agent_api_idempotency", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "operation": { - "name": "operation", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyHash": { - "name": "keyHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "requestHash": { - "name": "requestHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "state": { - "name": "state", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "statusCode": { - "name": "statusCode", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "response": { - "name": "response", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "user_operation_key_idx": { - "name": "user_operation_key_idx", - "columns": [ - "userId", - "operation", - "keyHash" - ], - "isUnique": true - }, - "expires_at_idx": { - "name": "expires_at_idx", - "columns": [ - "expiresAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_idempotency_id": { - "name": "agent_api_idempotency_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_keys": { - "name": "agent_api_keys", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "tokenHash": { - "name": "tokenHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(100)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Cap CLI'" - }, - "scopes": { - "name": "scopes", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "revokedAt": { - "name": "revokedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "lastUsedAt": { - "name": "lastUsedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "token_hash_idx": { - "name": "token_hash_idx", - "columns": [ - "tokenHash" - ], - "isUnique": true - }, - "user_created_at_idx": { - "name": "user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - }, - "expires_at_idx": { - "name": "expires_at_idx", - "columns": [ - "expiresAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_keys_id": { - "name": "agent_api_keys_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "agent_api_operations": { - "name": "agent_api_operations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "kind": { - "name": "kind", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "resourceId": { - "name": "resourceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "resultResourceId": { - "name": "resultResourceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "state": { - "name": "state", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'queued'" - }, - "payload": { - "name": "payload", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "result": { - "name": "result", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "errorCode": { - "name": "errorCode", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "errorMessage": { - "name": "errorMessage", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "completedAt": { - "name": "completedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "user_created_at_idx": { - "name": "user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - }, - "state_updated_at_idx": { - "name": "state_updated_at_idx", - "columns": [ - "state", - "updatedAt" - ], - "isUnique": false - }, - "resource_id_idx": { - "name": "resource_id_idx", - "columns": [ - "resourceId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "agent_api_operations_id": { - "name": "agent_api_operations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "auth_api_keys": { - "name": "auth_api_keys", - "columns": { - "id": { - "name": "id", - "type": "varchar(36)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "source": { - "name": "source", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'unknown'" - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "user_id_created_at_idx": { - "name": "user_id_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "auth_api_keys_id": { - "name": "auth_api_keys_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "comments": { - "name": "comments", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(6)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "timestamp": { - "name": "timestamp", - "type": "float", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "authorId": { - "name": "authorId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "parentCommentId": { - "name": "parentCommentId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "video_type_created_idx": { - "name": "video_type_created_idx", - "columns": [ - "videoId", - "type", - "createdAt", - "id" - ], - "isUnique": false - }, - "author_id_idx": { - "name": "author_id_idx", - "columns": [ - "authorId" - ], - "isUnique": false - }, - "parent_comment_id_idx": { - "name": "parent_comment_id_idx", - "columns": [ - "parentCommentId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "comments_id": { - "name": "comments_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_api_keys": { - "name": "developer_api_keys", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyType": { - "name": "keyType", - "type": "varchar(8)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyPrefix": { - "name": "keyPrefix", - "type": "varchar(12)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "keyHash": { - "name": "keyHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "encryptedKey": { - "name": "encryptedKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "lastUsedAt": { - "name": "lastUsedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "revokedAt": { - "name": "revokedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "key_hash_idx": { - "name": "key_hash_idx", - "columns": [ - "keyHash" - ], - "isUnique": true - }, - "app_key_type_idx": { - "name": "app_key_type_idx", - "columns": [ - "appId", - "keyType" - ], - "isUnique": false - } - }, - "foreignKeys": { - "developer_api_keys_appId_developer_apps_id_fk": { - "name": "developer_api_keys_appId_developer_apps_id_fk", - "tableFrom": "developer_api_keys", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_api_keys_id": { - "name": "developer_api_keys_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_app_domains": { - "name": "developer_app_domains", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "domain": { - "name": "domain", - "type": "varchar(253)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": { - "developer_app_domains_appId_developer_apps_id_fk": { - "name": "developer_app_domains_appId_developer_apps_id_fk", - "tableFrom": "developer_app_domains", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_app_domains_id": { - "name": "developer_app_domains_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": { - "app_domain_unique": { - "name": "app_domain_unique", - "columns": [ - "appId", - "domain" - ] - } - }, - "checkConstraint": {} - }, - "developer_apps": { - "name": "developer_apps", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "environment": { - "name": "environment", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "logoUrl": { - "name": "logoUrl", - "type": "varchar(1024)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "deletedAt": { - "name": "deletedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "owner_deleted_idx": { - "name": "owner_deleted_idx", - "columns": [ - "ownerId", - "deletedAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "developer_apps_id": { - "name": "developer_apps_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_credit_accounts": { - "name": "developer_credit_accounts", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "balanceMicroCredits": { - "name": "balanceMicroCredits", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "stripeCustomerId": { - "name": "stripeCustomerId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripePaymentMethodId": { - "name": "stripePaymentMethodId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "autoTopUpEnabled": { - "name": "autoTopUpEnabled", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "autoTopUpThresholdMicroCredits": { - "name": "autoTopUpThresholdMicroCredits", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "autoTopUpAmountCents": { - "name": "autoTopUpAmountCents", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "app_id_unique": { - "name": "app_id_unique", - "columns": [ - "appId" - ], - "isUnique": true - } - }, - "foreignKeys": { - "developer_credit_accounts_appId_developer_apps_id_fk": { - "name": "developer_credit_accounts_appId_developer_apps_id_fk", - "tableFrom": "developer_credit_accounts", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_credit_accounts_id": { - "name": "developer_credit_accounts_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_credit_transactions": { - "name": "developer_credit_transactions", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "accountId": { - "name": "accountId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "amountMicroCredits": { - "name": "amountMicroCredits", - "type": "bigint", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "balanceAfterMicroCredits": { - "name": "balanceAfterMicroCredits", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "referenceId": { - "name": "referenceId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "referenceType": { - "name": "referenceType", - "type": "varchar(32)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "account_type_created_idx": { - "name": "account_type_created_idx", - "columns": [ - "accountId", - "type", - "createdAt" - ], - "isUnique": false - }, - "account_ref_dedup_idx": { - "name": "account_ref_dedup_idx", - "columns": [ - "accountId", - "referenceId", - "referenceType" - ], - "isUnique": false - } - }, - "foreignKeys": { - "dev_credit_txn_account_fk": { - "name": "dev_credit_txn_account_fk", - "tableFrom": "developer_credit_transactions", - "tableTo": "developer_credit_accounts", - "columnsFrom": [ - "accountId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_credit_transactions_id": { - "name": "developer_credit_transactions_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "developer_daily_storage_snapshots": { - "name": "developer_daily_storage_snapshots", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "snapshotDate": { - "name": "snapshotDate", - "type": "varchar(10)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "totalDurationMinutes": { - "name": "totalDurationMinutes", - "type": "float", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "videoCount": { - "name": "videoCount", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "microCreditsCharged": { - "name": "microCreditsCharged", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "processedAt": { - "name": "processedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": { - "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { - "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", - "tableFrom": "developer_daily_storage_snapshots", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_daily_storage_snapshots_id": { - "name": "developer_daily_storage_snapshots_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": { - "app_date_unique": { - "name": "app_date_unique", - "columns": [ - "appId", - "snapshotDate" - ] - } - }, - "checkConstraint": {} - }, - "developer_videos": { - "name": "developer_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "appId": { - "name": "appId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "externalUserId": { - "name": "externalUserId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Untitled'" - }, - "duration": { - "name": "duration", - "type": "float", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "width": { - "name": "width", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "height": { - "name": "height", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "fps": { - "name": "fps", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "s3Key": { - "name": "s3Key", - "type": "varchar(512)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "transcriptionStatus": { - "name": "transcriptionStatus", - "type": "varchar(16)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "deletedAt": { - "name": "deletedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "app_created_idx": { - "name": "app_created_idx", - "columns": [ - "appId", - "createdAt" - ], - "isUnique": false - }, - "app_user_idx": { - "name": "app_user_idx", - "columns": [ - "appId", - "externalUserId" - ], - "isUnique": false - }, - "app_deleted_idx": { - "name": "app_deleted_idx", - "columns": [ - "appId", - "deletedAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "developer_videos_appId_developer_apps_id_fk": { - "name": "developer_videos_appId_developer_apps_id_fk", - "tableFrom": "developer_videos", - "tableTo": "developer_apps", - "columnsFrom": [ - "appId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "developer_videos_id": { - "name": "developer_videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "folders": { - "name": "folders", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'normal'" - }, - "public": { - "name": "public", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdById": { - "name": "createdById", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "parentId": { - "name": "parentId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "created_by_id_idx": { - "name": "created_by_id_idx", - "columns": [ - "createdById" - ], - "isUnique": false - }, - "parent_id_idx": { - "name": "parent_id_idx", - "columns": [ - "parentId" - ], - "isUnique": false - }, - "space_id_idx": { - "name": "space_id_idx", - "columns": [ - "spaceId" - ], - "isUnique": false - }, - "public_parent_id_idx": { - "name": "public_parent_id_idx", - "columns": [ - "public", - "parentId" - ], - "isUnique": false - }, - "public_space_parent_id_idx": { - "name": "public_space_parent_id_idx", - "columns": [ - "public", - "spaceId", - "parentId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "folders_id": { - "name": "folders_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "imported_videos": { - "name": "imported_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "orgId": { - "name": "orgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "source": { - "name": "source", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "source_id": { - "name": "source_id", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": { - "imported_videos_orgId_source_source_id_pk": { - "name": "imported_videos_orgId_source_source_id_pk", - "columns": [ - "orgId", - "source", - "source_id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "integration_installations": { - "name": "integration_installations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "externalId": { - "name": "externalId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "displayName": { - "name": "displayName", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "installedByUserId": { - "name": "installedByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "encryptedCredentials": { - "name": "encryptedCredentials", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "provider_external_id_idx": { - "name": "provider_external_id_idx", - "columns": [ - "provider", - "externalId" - ], - "isUnique": true - }, - "organization_provider_display_name_idx": { - "name": "organization_provider_display_name_idx", - "columns": [ - "organizationId", - "provider", - "displayName" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "integration_installations_id": { - "name": "integration_installations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "messenger_conversations": { - "name": "messenger_conversations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "agent": { - "name": "agent", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "mode": { - "name": "mode", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'agent'" - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "anonymousId": { - "name": "anonymousId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "takeoverByUserId": { - "name": "takeoverByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "takeoverAt": { - "name": "takeoverAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "lastMessageAt": { - "name": "lastMessageAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "user_last_message_idx": { - "name": "user_last_message_idx", - "columns": [ - "userId", - "lastMessageAt" - ], - "isUnique": false - }, - "anonymous_last_message_idx": { - "name": "anonymous_last_message_idx", - "columns": [ - "anonymousId", - "lastMessageAt" - ], - "isUnique": false - }, - "mode_last_message_idx": { - "name": "mode_last_message_idx", - "columns": [ - "mode", - "lastMessageAt" - ], - "isUnique": false - }, - "updated_at_idx": { - "name": "updated_at_idx", - "columns": [ - "updatedAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "messenger_conversations_id": { - "name": "messenger_conversations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "messenger_messages": { - "name": "messenger_messages", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "conversationId": { - "name": "conversationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "content": { - "name": "content", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "anonymousId": { - "name": "anonymousId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "conversation_created_at_idx": { - "name": "conversation_created_at_idx", - "columns": [ - "conversationId", - "createdAt" - ], - "isUnique": false - }, - "role_created_at_idx": { - "name": "role_created_at_idx", - "columns": [ - "role", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "messenger_messages_conversationId_messenger_conversations_id_fk": { - "name": "messenger_messages_conversationId_messenger_conversations_id_fk", - "tableFrom": "messenger_messages", - "tableTo": "messenger_conversations", - "columnsFrom": [ - "conversationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "messenger_messages_id": { - "name": "messenger_messages_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "messenger_support_emails": { - "name": "messenger_support_emails", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "conversationId": { - "name": "conversationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userEmail": { - "name": "userEmail", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "subject": { - "name": "subject", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "message": { - "name": "message", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "support_email_user_created_at_idx": { - "name": "support_email_user_created_at_idx", - "columns": [ - "userId", - "createdAt" - ], - "isUnique": false - }, - "support_email_conversation_created_at_idx": { - "name": "support_email_conversation_created_at_idx", - "columns": [ - "conversationId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "support_email_conversation_fk": { - "name": "support_email_conversation_fk", - "tableFrom": "messenger_support_emails", - "tableTo": "messenger_conversations", - "columnsFrom": [ - "conversationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "messenger_support_emails_id": { - "name": "messenger_support_emails_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "notifications": { - "name": "notifications", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "orgId": { - "name": "orgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "recipientId": { - "name": "recipientId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "type": { - "name": "type", - "type": "varchar(16)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "data": { - "name": "data", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(50)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "dedupKey": { - "name": "dedupKey", - "type": "varchar(128)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "readAt": { - "name": "readAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "org_id_idx": { - "name": "org_id_idx", - "columns": [ - "orgId" - ], - "isUnique": false - }, - "type_idx": { - "name": "type_idx", - "columns": [ - "type" - ], - "isUnique": false - }, - "read_at_idx": { - "name": "read_at_idx", - "columns": [ - "readAt" - ], - "isUnique": false - }, - "created_at_idx": { - "name": "created_at_idx", - "columns": [ - "createdAt" - ], - "isUnique": false - }, - "recipient_read_idx": { - "name": "recipient_read_idx", - "columns": [ - "recipientId", - "readAt" - ], - "isUnique": false - }, - "recipient_created_idx": { - "name": "recipient_created_idx", - "columns": [ - "recipientId", - "createdAt" - ], - "isUnique": false - }, - "dedup_key_idx": { - "name": "dedup_key_idx", - "columns": [ - "dedupKey" - ], - "isUnique": true - }, - "type_recipient_created_idx": { - "name": "type_recipient_created_idx", - "columns": [ - "type", - "recipientId", - "createdAt" - ], - "isUnique": false - }, - "type_recipient_video_created_idx": { - "name": "type_recipient_video_created_idx", - "columns": [ - "type", - "recipientId", - "videoId", - "createdAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "notifications_id": { - "name": "notifications_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "organization_invites": { - "name": "organization_invites", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "invitedEmail": { - "name": "invitedEmail", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "invitedByUserId": { - "name": "invitedByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "expiresAt": { - "name": "expiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "invited_email_idx": { - "name": "invited_email_idx", - "columns": [ - "invitedEmail" - ], - "isUnique": false - }, - "invited_by_user_id_idx": { - "name": "invited_by_user_id_idx", - "columns": [ - "invitedByUserId" - ], - "isUnique": false - }, - "status_idx": { - "name": "status_idx", - "columns": [ - "status" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "organization_invites_id": { - "name": "organization_invites_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "organization_members": { - "name": "organization_members", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "hasProSeat": { - "name": "hasProSeat", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "user_id_organization_id_idx": { - "name": "user_id_organization_id_idx", - "columns": [ - "userId", - "organizationId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "organization_members_id": { - "name": "organization_members_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "organizations": { - "name": "organizations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "tombstoneAt": { - "name": "tombstoneAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "allowedEmailDomain": { - "name": "allowedEmailDomain", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "customDomain": { - "name": "customDomain", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "domainVerified": { - "name": "domainVerified", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "iconUrl": { - "name": "iconUrl", - "type": "varchar(1024)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "shareableLinkIconUrl": { - "name": "shareableLinkIconUrl", - "type": "varchar(1024)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "workosOrganizationId": { - "name": "workosOrganizationId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "workosConnectionId": { - "name": "workosConnectionId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "owner_id_tombstone_idx": { - "name": "owner_id_tombstone_idx", - "columns": [ - "ownerId", - "tombstoneAt" - ], - "isUnique": false - }, - "custom_domain_idx": { - "name": "custom_domain_idx", - "columns": [ - "customDomain" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "organizations_id": { - "name": "organizations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "product_analytics_erasure_leases": { - "name": "product_analytics_erasure_leases", - "columns": { - "name": { - "name": "name", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "requestId": { - "name": "requestId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "fencingToken": { - "name": "fencingToken", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "leaseExpiresAt": { - "name": "leaseExpiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "phase": { - "name": "phase", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'idle'" - }, - "pausedPipes": { - "name": "pausedPipes", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "attemptCount": { - "name": "attemptCount", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "lastErrorCode": { - "name": "lastErrorCode", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": { - "product_analytics_erasure_leases_name": { - "name": "product_analytics_erasure_leases_name", - "columns": [ - "name" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "s3_buckets": { - "name": "s3_buckets", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "region": { - "name": "region", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "endpoint": { - "name": "endpoint", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "bucketName": { - "name": "bucketName", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "accessKeyId": { - "name": "accessKeyId", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "secretAccessKey": { - "name": "secretAccessKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "('aws')" - }, - "active": { - "name": "active", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": true - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "owner_organization_idx": { - "name": "owner_organization_idx", - "columns": [ - "ownerId", - "organizationId" - ], - "isUnique": false - }, - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "organization_active_idx": { - "name": "organization_active_idx", - "columns": [ - "organizationId", - "active", - "updatedAt" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "s3_buckets_id": { - "name": "s3_buckets_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "sessions": { - "name": "sessions", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sessionToken": { - "name": "sessionToken", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "datetime", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "session_token_idx": { - "name": "session_token_idx", - "columns": [ - "sessionToken" - ], - "isUnique": true - }, - "user_id_idx": { - "name": "user_id_idx", - "columns": [ - "userId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "sessions_id": { - "name": "sessions_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "shared_videos": { - "name": "shared_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "folderId": { - "name": "folderId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sharedByUserId": { - "name": "sharedByUserId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sharedAt": { - "name": "sharedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "folder_id_idx": { - "name": "folder_id_idx", - "columns": [ - "folderId" - ], - "isUnique": false - }, - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "shared_by_user_id_idx": { - "name": "shared_by_user_id_idx", - "columns": [ - "sharedByUserId" - ], - "isUnique": false - }, - "video_id_organization_id_idx": { - "name": "video_id_organization_id_idx", - "columns": [ - "videoId", - "organizationId" - ], - "isUnique": false - }, - "video_id_folder_id_idx": { - "name": "video_id_folder_id_idx", - "columns": [ - "videoId", - "folderId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "shared_videos_id": { - "name": "shared_videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "space_members": { - "name": "space_members", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "userId": { - "name": "userId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "role": { - "name": "role", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'member'" - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "user_id_idx": { - "name": "user_id_idx", - "columns": [ - "userId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "space_members_id": { - "name": "space_members_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": { - "space_id_user_id_unique": { - "name": "space_id_user_id_unique", - "columns": [ - "spaceId", - "userId" - ] - } - }, - "checkConstraint": {} - }, - "space_videos": { - "name": "space_videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "spaceId": { - "name": "spaceId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "folderId": { - "name": "folderId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "addedById": { - "name": "addedById", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "addedAt": { - "name": "addedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - } - }, - "indexes": { - "folder_id_idx": { - "name": "folder_id_idx", - "columns": [ - "folderId" - ], - "isUnique": false - }, - "video_id_idx": { - "name": "video_id_idx", - "columns": [ - "videoId" - ], - "isUnique": false - }, - "added_by_id_idx": { - "name": "added_by_id_idx", - "columns": [ - "addedById" - ], - "isUnique": false - }, - "space_id_video_id_idx": { - "name": "space_id_video_id_idx", - "columns": [ - "spaceId", - "videoId" - ], - "isUnique": false - }, - "space_id_folder_id_idx": { - "name": "space_id_folder_id_idx", - "columns": [ - "spaceId", - "folderId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "space_videos_id": { - "name": "space_videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "spaces": { - "name": "spaces", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "primary": { - "name": "primary", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdById": { - "name": "createdById", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "iconUrl": { - "name": "iconUrl", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "description": { - "name": "description", - "type": "varchar(1000)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "privacy": { - "name": "privacy", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'Private'" - }, - "public": { - "name": "public", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - } - }, - "indexes": { - "organization_id_idx": { - "name": "organization_id_idx", - "columns": [ - "organizationId" - ], - "isUnique": false - }, - "created_by_id_idx": { - "name": "created_by_id_idx", - "columns": [ - "createdById" - ], - "isUnique": false - }, - "public_organization_id_idx": { - "name": "public_organization_id_idx", - "columns": [ - "public", - "organizationId" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "spaces_id": { - "name": "spaces_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "storage_integrations": { - "name": "storage_integrations", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "organizationId": { - "name": "organizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "provider": { - "name": "provider", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "displayName": { - "name": "displayName", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "status": { - "name": "status", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'active'" - }, - "active": { - "name": "active", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "encryptedConfig": { - "name": "encryptedConfig", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "googleDriveAccessToken": { - "name": "googleDriveAccessToken", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveAccessTokenExpiresAt": { - "name": "googleDriveAccessTokenExpiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveTokenRefreshLeaseId": { - "name": "googleDriveTokenRefreshLeaseId", - "type": "varchar(64)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveTokenRefreshLeaseExpiresAt": { - "name": "googleDriveTokenRefreshLeaseExpiresAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "googleDriveStorageQuotaCache": { - "name": "googleDriveStorageQuotaCache", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "owner_provider_idx": { - "name": "owner_provider_idx", - "columns": [ - "ownerId", - "provider" - ], - "isUnique": false - }, - "owner_active_idx": { - "name": "owner_active_idx", - "columns": [ - "ownerId", - "active" - ], - "isUnique": false - }, - "organization_provider_idx": { - "name": "organization_provider_idx", - "columns": [ - "organizationId", - "provider" - ], - "isUnique": false - }, - "organization_active_idx": { - "name": "organization_active_idx", - "columns": [ - "organizationId", - "active", - "status" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "storage_integrations_id": { - "name": "storage_integrations_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "storage_objects": { - "name": "storage_objects", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "integrationId": { - "name": "integrationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "objectKey": { - "name": "objectKey", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "objectKeyHash": { - "name": "objectKeyHash", - "type": "varchar(64)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "providerObjectId": { - "name": "providerObjectId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "uploadSessionUrl": { - "name": "uploadSessionUrl", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "uploadStatus": { - "name": "uploadStatus", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'pending'" - }, - "contentType": { - "name": "contentType", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "contentLength": { - "name": "contentLength", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": { - "integration_key_hash_idx": { - "name": "integration_key_hash_idx", - "columns": [ - "integrationId", - "objectKeyHash" - ], - "isUnique": true - }, - "integration_status_idx": { - "name": "integration_status_idx", - "columns": [ - "integrationId", - "uploadStatus" - ], - "isUnique": false - }, - "video_id_idx": { - "name": "video_id_idx", - "columns": [ - "videoId" - ], - "isUnique": false - }, - "owner_id_idx": { - "name": "owner_id_idx", - "columns": [ - "ownerId" - ], - "isUnique": false - } - }, - "foreignKeys": { - "storage_objects_integrationId_storage_integrations_id_fk": { - "name": "storage_objects_integrationId_storage_integrations_id_fk", - "tableFrom": "storage_objects", - "tableTo": "storage_integrations", - "columnsFrom": [ - "integrationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "restrict", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "storage_objects_id": { - "name": "storage_objects_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "users": { - "name": "users", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "lastName": { - "name": "lastName", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "email": { - "name": "email", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "emailVerified": { - "name": "emailVerified", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "image": { - "name": "image", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeCustomerId": { - "name": "stripeCustomerId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeSubscriptionId": { - "name": "stripeSubscriptionId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "thirdPartyStripeSubscriptionId": { - "name": "thirdPartyStripeSubscriptionId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeSubscriptionStatus": { - "name": "stripeSubscriptionStatus", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "stripeSubscriptionPriceId": { - "name": "stripeSubscriptionPriceId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "preferences": { - "name": "preferences", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "('null')" - }, - "activeOrganizationId": { - "name": "activeOrganizationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "onboardingSteps": { - "name": "onboardingSteps", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "onboarding_completed_at": { - "name": "onboarding_completed_at", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "customBucket": { - "name": "customBucket", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "inviteQuota": { - "name": "inviteQuota", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "defaultOrgId": { - "name": "defaultOrgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "authSessionVersion": { - "name": "authSessionVersion", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - } - }, - "indexes": { - "email_idx": { - "name": "email_idx", - "columns": [ - "email" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "users_id": { - "name": "users_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "verification_tokens": { - "name": "verification_tokens", - "columns": { - "identifier": { - "name": "identifier", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "token": { - "name": "token", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "expires": { - "name": "expires", - "type": "datetime", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": { - "verification_tokens_identifier": { - "name": "verification_tokens_identifier", - "columns": [ - "identifier" - ] - } - }, - "uniqueConstraints": { - "verification_tokens_token_unique": { - "name": "verification_tokens_token_unique", - "columns": [ - "token" - ] - } - }, - "checkConstraint": {} - }, - "video_edits": { - "name": "video_edits", - "columns": { - "videoId": { - "name": "videoId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "sourceKey": { - "name": "sourceKey", - "type": "varchar(512)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "editSpec": { - "name": "editSpec", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - } - }, - "indexes": {}, - "foreignKeys": { - "video_edits_videoId_videos_id_fk": { - "name": "video_edits_videoId_videos_id_fk", - "tableFrom": "video_edits", - "tableTo": "videos", - "columnsFrom": [ - "videoId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "cascade", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "video_edits_videoId": { - "name": "video_edits_videoId", - "columns": [ - "videoId" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "video_uploads": { - "name": "video_uploads", - "columns": { - "video_id": { - "name": "video_id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "uploaded": { - "name": "uploaded", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "total": { - "name": "total", - "type": "bigint unsigned", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "started_at": { - "name": "started_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "updated_at": { - "name": "updated_at", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "mode": { - "name": "mode", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "phase": { - "name": "phase", - "type": "varchar(32)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'uploading'" - }, - "processing_progress": { - "name": "processing_progress", - "type": "int", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "processing_message": { - "name": "processing_message", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "processing_error": { - "name": "processing_error", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "raw_file_key": { - "name": "raw_file_key", - "type": "varchar(512)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - } - }, - "indexes": { - "phase_updated_at_video_id_idx": { - "name": "phase_updated_at_video_id_idx", - "columns": [ - "phase", - "updated_at", - "video_id" - ], - "isUnique": false - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": { - "video_uploads_video_id": { - "name": "video_uploads_video_id", - "columns": [ - "video_id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - }, - "videos": { - "name": "videos", - "columns": { - "id": { - "name": "id", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "ownerId": { - "name": "ownerId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "orgId": { - "name": "orgId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "name": { - "name": "name", - "type": "varchar(255)", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'My Video'" - }, - "bucket": { - "name": "bucket", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "storageIntegrationId": { - "name": "storageIntegrationId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "duration": { - "name": "duration", - "type": "float", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "width": { - "name": "width", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "height": { - "name": "height", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "fps": { - "name": "fps", - "type": "int", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "metadata": { - "name": "metadata", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "public": { - "name": "public", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": true - }, - "settings": { - "name": "settings", - "type": "json", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "transcriptionStatus": { - "name": "transcriptionStatus", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "source": { - "name": "source", - "type": "json", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "('{\"type\":\"MediaConvert\"}')" - }, - "folderId": { - "name": "folderId", - "type": "varchar(15)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "(now())" - }, - "effectiveCreatedAt": { - "name": "effectiveCreatedAt", - "type": "datetime", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "generated": { - "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", - "type": "stored" - } - }, - "updatedAt": { - "name": "updatedAt", - "type": "timestamp", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "onUpdate": true, - "default": "(now())" - }, - "password": { - "name": "password", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "xStreamInfo": { - "name": "xStreamInfo", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "firstExternalViewAt": { - "name": "firstExternalViewAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "firstViewEmailSentAt": { - "name": "firstViewEmailSentAt", - "type": "timestamp", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "isScreenshot": { - "name": "isScreenshot", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - }, - "awsRegion": { - "name": "awsRegion", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "awsBucket": { - "name": "awsBucket", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "videoStartTime": { - "name": "videoStartTime", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "audioStartTime": { - "name": "audioStartTime", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "jobId": { - "name": "jobId", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "jobStatus": { - "name": "jobStatus", - "type": "varchar(255)", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "skipProcessing": { - "name": "skipProcessing", - "type": "boolean", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": false - } - }, - "indexes": { - "owner_id_idx": { - "name": "owner_id_idx", - "columns": [ - "ownerId" - ], - "isUnique": false - }, - "is_public_idx": { - "name": "is_public_idx", - "columns": [ - "public" - ], - "isUnique": false - }, - "folder_id_idx": { - "name": "folder_id_idx", - "columns": [ - "folderId" - ], - "isUnique": false - }, - "storage_integration_id_idx": { - "name": "storage_integration_id_idx", - "columns": [ - "storageIntegrationId" - ], - "isUnique": false - }, - "first_external_view_at_idx": { - "name": "first_external_view_at_idx", - "columns": [ - "firstExternalViewAt" - ], - "isUnique": false - }, - "org_owner_folder_idx": { - "name": "org_owner_folder_idx", - "columns": [ - "orgId", - "ownerId", - "folderId" - ], - "isUnique": false - }, - "org_effective_created_idx": { - "name": "org_effective_created_idx", - "columns": [ - "orgId", - "effectiveCreatedAt" - ], - "isUnique": false - } - }, - "foreignKeys": { - "videos_storageIntegrationId_storage_integrations_id_fk": { - "name": "videos_storageIntegrationId_storage_integrations_id_fk", - "tableFrom": "videos", - "tableTo": "storage_integrations", - "columnsFrom": [ - "storageIntegrationId" - ], - "columnsTo": [ - "id" - ], - "onDelete": "restrict", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": { - "videos_id": { - "name": "videos_id", - "columns": [ - "id" - ] - } - }, - "uniqueConstraints": {}, - "checkConstraint": {} - } - }, - "views": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "tables": {}, - "indexes": {} - } -} \ No newline at end of file + "version": "5", + "dialect": "mysql", + "id": "f7a1a6b4-42da-4813-9462-a09d30766433", + "prevId": "794035aa-b67e-4430-8034-d3f6210b5262", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_in": { + "name": "expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_in": { + "name": "refresh_token_expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "tempColumn": { + "name": "tempColumn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "provider_account_id_idx": { + "name": "provider_account_id_idx", + "columns": ["providerAccountId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accounts_id": { + "name": "accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_authorization_codes": { + "name": "agent_api_authorization_codes", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeHash": { + "name": "codeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeChallenge": { + "name": "codeChallenge", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirectUri": { + "name": "redirectUri", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consumedAt": { + "name": "consumedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "code_hash_idx": { + "name": "code_hash_idx", + "columns": ["codeHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_authorization_codes_id": { + "name": "agent_api_authorization_codes_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_idempotency": { + "name": "agent_api_idempotency", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestHash": { + "name": "requestHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "statusCode": { + "name": "statusCode", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response": { + "name": "response", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_operation_key_idx": { + "name": "user_operation_key_idx", + "columns": ["userId", "operation", "keyHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_idempotency_id": { + "name": "agent_api_idempotency_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_keys": { + "name": "agent_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tokenHash": { + "name": "tokenHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Cap CLI'" + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "token_hash_idx": { + "name": "token_hash_idx", + "columns": ["tokenHash"], + "isUnique": true + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_keys_id": { + "name": "agent_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_operations": { + "name": "agent_api_operations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resourceId": { + "name": "resourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resultResourceId": { + "name": "resultResourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "state_updated_at_idx": { + "name": "state_updated_at_idx", + "columns": ["state", "updatedAt"], + "isUnique": false + }, + "resource_id_idx": { + "name": "resource_id_idx", + "columns": ["resourceId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_operations_id": { + "name": "agent_api_operations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "auth_api_keys": { + "name": "auth_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_id_created_at_idx": { + "name": "user_id_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "auth_api_keys_id": { + "name": "auth_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "comments": { + "name": "comments", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authorId": { + "name": "authorId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "parentCommentId": { + "name": "parentCommentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "video_type_created_idx": { + "name": "video_type_created_idx", + "columns": ["videoId", "type", "createdAt", "id"], + "isUnique": false + }, + "author_id_idx": { + "name": "author_id_idx", + "columns": ["authorId"], + "isUnique": false + }, + "parent_comment_id_idx": { + "name": "parent_comment_id_idx", + "columns": ["parentCommentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "comments_id": { + "name": "comments_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_api_keys": { + "name": "developer_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyType": { + "name": "keyType", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyPrefix": { + "name": "keyPrefix", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedKey": { + "name": "encryptedKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "key_hash_idx": { + "name": "key_hash_idx", + "columns": ["keyHash"], + "isUnique": true + }, + "app_key_type_idx": { + "name": "app_key_type_idx", + "columns": ["appId", "keyType"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_api_keys_appId_developer_apps_id_fk": { + "name": "developer_api_keys_appId_developer_apps_id_fk", + "tableFrom": "developer_api_keys", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_api_keys_id": { + "name": "developer_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_app_domains": { + "name": "developer_app_domains", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_app_domains_appId_developer_apps_id_fk": { + "name": "developer_app_domains_appId_developer_apps_id_fk", + "tableFrom": "developer_app_domains", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_app_domains_id": { + "name": "developer_app_domains_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_domain_unique": { + "name": "app_domain_unique", + "columns": ["appId", "domain"] + } + }, + "checkConstraint": {} + }, + "developer_apps": { + "name": "developer_apps", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logoUrl": { + "name": "logoUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_deleted_idx": { + "name": "owner_deleted_idx", + "columns": ["ownerId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "developer_apps_id": { + "name": "developer_apps_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_accounts": { + "name": "developer_credit_accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceMicroCredits": { + "name": "balanceMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripePaymentMethodId": { + "name": "stripePaymentMethodId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoTopUpEnabled": { + "name": "autoTopUpEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "autoTopUpThresholdMicroCredits": { + "name": "autoTopUpThresholdMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoTopUpAmountCents": { + "name": "autoTopUpAmountCents", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_id_unique": { + "name": "app_id_unique", + "columns": ["appId"], + "isUnique": true + } + }, + "foreignKeys": { + "developer_credit_accounts_appId_developer_apps_id_fk": { + "name": "developer_credit_accounts_appId_developer_apps_id_fk", + "tableFrom": "developer_credit_accounts", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_accounts_id": { + "name": "developer_credit_accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_transactions": { + "name": "developer_credit_transactions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accountId": { + "name": "accountId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amountMicroCredits": { + "name": "amountMicroCredits", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceAfterMicroCredits": { + "name": "balanceAfterMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referenceType": { + "name": "referenceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "account_type_created_idx": { + "name": "account_type_created_idx", + "columns": ["accountId", "type", "createdAt"], + "isUnique": false + }, + "account_ref_dedup_idx": { + "name": "account_ref_dedup_idx", + "columns": ["accountId", "referenceId", "referenceType"], + "isUnique": false + } + }, + "foreignKeys": { + "dev_credit_txn_account_fk": { + "name": "dev_credit_txn_account_fk", + "tableFrom": "developer_credit_transactions", + "tableTo": "developer_credit_accounts", + "columnsFrom": ["accountId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_transactions_id": { + "name": "developer_credit_transactions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_daily_storage_snapshots": { + "name": "developer_daily_storage_snapshots", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshotDate": { + "name": "snapshotDate", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalDurationMinutes": { + "name": "totalDurationMinutes", + "type": "float", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "videoCount": { + "name": "videoCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "microCreditsCharged": { + "name": "microCreditsCharged", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { + "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", + "tableFrom": "developer_daily_storage_snapshots", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_daily_storage_snapshots_id": { + "name": "developer_daily_storage_snapshots_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_date_unique": { + "name": "app_date_unique", + "columns": ["appId", "snapshotDate"] + } + }, + "checkConstraint": {} + }, + "developer_videos": { + "name": "developer_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalUserId": { + "name": "externalUserId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3Key": { + "name": "s3Key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_created_idx": { + "name": "app_created_idx", + "columns": ["appId", "createdAt"], + "isUnique": false + }, + "app_user_idx": { + "name": "app_user_idx", + "columns": ["appId", "externalUserId"], + "isUnique": false + }, + "app_deleted_idx": { + "name": "app_deleted_idx", + "columns": ["appId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_videos_appId_developer_apps_id_fk": { + "name": "developer_videos_appId_developer_apps_id_fk", + "tableFrom": "developer_videos", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_videos_id": { + "name": "developer_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "folders": { + "name": "folders", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "parent_id_idx": { + "name": "parent_id_idx", + "columns": ["parentId"], + "isUnique": false + }, + "space_id_idx": { + "name": "space_id_idx", + "columns": ["spaceId"], + "isUnique": false + }, + "public_parent_id_idx": { + "name": "public_parent_id_idx", + "columns": ["public", "parentId"], + "isUnique": false + }, + "public_space_parent_id_idx": { + "name": "public_space_parent_id_idx", + "columns": ["public", "spaceId", "parentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "folders_id": { + "name": "folders_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "imported_videos": { + "name": "imported_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "imported_videos_orgId_source_source_id_pk": { + "name": "imported_videos_orgId_source_source_id_pk", + "columns": ["orgId", "source", "source_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "integration_installations": { + "name": "integration_installations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalId": { + "name": "externalId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "installedByUserId": { + "name": "installedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedCredentials": { + "name": "encryptedCredentials", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "provider_external_id_idx": { + "name": "provider_external_id_idx", + "columns": ["provider", "externalId"], + "isUnique": true + }, + "organization_provider_display_name_idx": { + "name": "organization_provider_display_name_idx", + "columns": ["organizationId", "provider", "displayName"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "integration_installations_id": { + "name": "integration_installations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_conversations": { + "name": "messenger_conversations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverByUserId": { + "name": "takeoverByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverAt": { + "name": "takeoverAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastMessageAt": { + "name": "lastMessageAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_last_message_idx": { + "name": "user_last_message_idx", + "columns": ["userId", "lastMessageAt"], + "isUnique": false + }, + "anonymous_last_message_idx": { + "name": "anonymous_last_message_idx", + "columns": ["anonymousId", "lastMessageAt"], + "isUnique": false + }, + "mode_last_message_idx": { + "name": "mode_last_message_idx", + "columns": ["mode", "lastMessageAt"], + "isUnique": false + }, + "updated_at_idx": { + "name": "updated_at_idx", + "columns": ["updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "messenger_conversations_id": { + "name": "messenger_conversations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_messages": { + "name": "messenger_messages", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "conversation_created_at_idx": { + "name": "conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + }, + "role_created_at_idx": { + "name": "role_created_at_idx", + "columns": ["role", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "messenger_messages_conversationId_messenger_conversations_id_fk": { + "name": "messenger_messages_conversationId_messenger_conversations_id_fk", + "tableFrom": "messenger_messages", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_messages_id": { + "name": "messenger_messages_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_support_emails": { + "name": "messenger_support_emails", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "support_email_user_created_at_idx": { + "name": "support_email_user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "support_email_conversation_created_at_idx": { + "name": "support_email_conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "support_email_conversation_fk": { + "name": "support_email_conversation_fk", + "tableFrom": "messenger_support_emails", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_support_emails_id": { + "name": "messenger_support_emails_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipientId": { + "name": "recipientId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "org_id_idx": { + "name": "org_id_idx", + "columns": ["orgId"], + "isUnique": false + }, + "type_idx": { + "name": "type_idx", + "columns": ["type"], + "isUnique": false + }, + "read_at_idx": { + "name": "read_at_idx", + "columns": ["readAt"], + "isUnique": false + }, + "created_at_idx": { + "name": "created_at_idx", + "columns": ["createdAt"], + "isUnique": false + }, + "recipient_read_idx": { + "name": "recipient_read_idx", + "columns": ["recipientId", "readAt"], + "isUnique": false + }, + "recipient_created_idx": { + "name": "recipient_created_idx", + "columns": ["recipientId", "createdAt"], + "isUnique": false + }, + "dedup_key_idx": { + "name": "dedup_key_idx", + "columns": ["dedupKey"], + "isUnique": true + }, + "type_recipient_created_idx": { + "name": "type_recipient_created_idx", + "columns": ["type", "recipientId", "createdAt"], + "isUnique": false + }, + "type_recipient_video_created_idx": { + "name": "type_recipient_video_created_idx", + "columns": ["type", "recipientId", "videoId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "notifications_id": { + "name": "notifications_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_invites": { + "name": "organization_invites", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmail": { + "name": "invitedEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedByUserId": { + "name": "invitedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "invited_email_idx": { + "name": "invited_email_idx", + "columns": ["invitedEmail"], + "isUnique": false + }, + "invited_by_user_id_idx": { + "name": "invited_by_user_id_idx", + "columns": ["invitedByUserId"], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": ["status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_invites_id": { + "name": "organization_invites_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hasProSeat": { + "name": "hasProSeat", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "user_id_organization_id_idx": { + "name": "user_id_organization_id_idx", + "columns": ["userId", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_members_id": { + "name": "organization_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tombstoneAt": { + "name": "tombstoneAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowedEmailDomain": { + "name": "allowedEmailDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customDomain": { + "name": "customDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domainVerified": { + "name": "domainVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "shareableLinkIconUrl": { + "name": "shareableLinkIconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "workosOrganizationId": { + "name": "workosOrganizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workosConnectionId": { + "name": "workosConnectionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "owner_id_tombstone_idx": { + "name": "owner_id_tombstone_idx", + "columns": ["ownerId", "tombstoneAt"], + "isUnique": false + }, + "custom_domain_idx": { + "name": "custom_domain_idx", + "columns": ["customDomain"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organizations_id": { + "name": "organizations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_leases": { + "name": "product_analytics_erasure_leases", + "columns": { + "name": { + "name": "name", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requestId": { + "name": "requestId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fencingToken": { + "name": "fencingToken", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "pausedPipes": { + "name": "pausedPipes", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_leases_name": { + "name": "product_analytics_erasure_leases_name", + "columns": ["name"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "s3_buckets": { + "name": "s3_buckets", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucketName": { + "name": "bucketName", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessKeyId": { + "name": "accessKeyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretAccessKey": { + "name": "secretAccessKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('aws')" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_organization_idx": { + "name": "owner_organization_idx", + "columns": ["ownerId", "organizationId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "s3_buckets_id": { + "name": "s3_buckets_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "session_token_idx": { + "name": "session_token_idx", + "columns": ["sessionToken"], + "isUnique": true + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sessions_id": { + "name": "sessions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "shared_videos": { + "name": "shared_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedByUserId": { + "name": "sharedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedAt": { + "name": "sharedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "shared_by_user_id_idx": { + "name": "shared_by_user_id_idx", + "columns": ["sharedByUserId"], + "isUnique": false + }, + "video_id_organization_id_idx": { + "name": "video_id_organization_id_idx", + "columns": ["videoId", "organizationId"], + "isUnique": false + }, + "video_id_folder_id_idx": { + "name": "video_id_folder_id_idx", + "columns": ["videoId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "shared_videos_id": { + "name": "shared_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_members_id": { + "name": "space_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "space_id_user_id_unique": { + "name": "space_id_user_id_unique", + "columns": ["spaceId", "userId"] + } + }, + "checkConstraint": {} + }, + "space_videos": { + "name": "space_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedById": { + "name": "addedById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "added_by_id_idx": { + "name": "added_by_id_idx", + "columns": ["addedById"], + "isUnique": false + }, + "space_id_video_id_idx": { + "name": "space_id_video_id_idx", + "columns": ["spaceId", "videoId"], + "isUnique": false + }, + "space_id_folder_id_idx": { + "name": "space_id_folder_id_idx", + "columns": ["spaceId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_videos_id": { + "name": "space_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "primary": { + "name": "primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "privacy": { + "name": "privacy", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Private'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "public_organization_id_idx": { + "name": "public_organization_id_idx", + "columns": ["public", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "spaces_id": { + "name": "spaces_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_integrations": { + "name": "storage_integrations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "encryptedConfig": { + "name": "encryptedConfig", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "googleDriveAccessToken": { + "name": "googleDriveAccessToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveAccessTokenExpiresAt": { + "name": "googleDriveAccessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseId": { + "name": "googleDriveTokenRefreshLeaseId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseExpiresAt": { + "name": "googleDriveTokenRefreshLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveStorageQuotaCache": { + "name": "googleDriveStorageQuotaCache", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_provider_idx": { + "name": "owner_provider_idx", + "columns": ["ownerId", "provider"], + "isUnique": false + }, + "owner_active_idx": { + "name": "owner_active_idx", + "columns": ["ownerId", "active"], + "isUnique": false + }, + "organization_provider_idx": { + "name": "organization_provider_idx", + "columns": ["organizationId", "provider"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "storage_integrations_id": { + "name": "storage_integrations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_objects": { + "name": "storage_objects", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrationId": { + "name": "integrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "objectKey": { + "name": "objectKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "objectKeyHash": { + "name": "objectKeyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerObjectId": { + "name": "providerObjectId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploadSessionUrl": { + "name": "uploadSessionUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploadStatus": { + "name": "uploadStatus", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "contentType": { + "name": "contentType", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentLength": { + "name": "contentLength", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "integration_key_hash_idx": { + "name": "integration_key_hash_idx", + "columns": ["integrationId", "objectKeyHash"], + "isUnique": true + }, + "integration_status_idx": { + "name": "integration_status_idx", + "columns": ["integrationId", "uploadStatus"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + } + }, + "foreignKeys": { + "storage_objects_integrationId_storage_integrations_id_fk": { + "name": "storage_objects_integrationId_storage_integrations_id_fk", + "tableFrom": "storage_objects", + "tableTo": "storage_integrations", + "columnsFrom": ["integrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "storage_objects_id": { + "name": "storage_objects_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastName": { + "name": "lastName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionId": { + "name": "stripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thirdPartyStripeSubscriptionId": { + "name": "thirdPartyStripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionStatus": { + "name": "stripeSubscriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionPriceId": { + "name": "stripeSubscriptionPriceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferences": { + "name": "preferences", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "('null')" + }, + "activeOrganizationId": { + "name": "activeOrganizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "onboardingSteps": { + "name": "onboardingSteps", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customBucket": { + "name": "customBucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "inviteQuota": { + "name": "inviteQuota", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "defaultOrgId": { + "name": "defaultOrgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authSessionVersion": { + "name": "authSessionVersion", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "email_idx": { + "name": "email_idx", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "identifier": { + "name": "identifier", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier": { + "name": "verification_tokens_identifier", + "columns": ["identifier"] + } + }, + "uniqueConstraints": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": ["token"] + } + }, + "checkConstraint": {} + }, + "video_edits": { + "name": "video_edits", + "columns": { + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceKey": { + "name": "sourceKey", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "editSpec": { + "name": "editSpec", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "video_edits_videoId_videos_id_fk": { + "name": "video_edits_videoId_videos_id_fk", + "tableFrom": "video_edits", + "tableTo": "videos", + "columnsFrom": ["videoId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "video_edits_videoId": { + "name": "video_edits_videoId", + "columns": ["videoId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "video_uploads": { + "name": "video_uploads", + "columns": { + "video_id": { + "name": "video_id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded": { + "name": "uploaded", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "mode": { + "name": "mode", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'uploading'" + }, + "processing_progress": { + "name": "processing_progress", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processing_message": { + "name": "processing_message", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_file_key": { + "name": "raw_file_key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "phase_updated_at_video_id_idx": { + "name": "phase_updated_at_video_id_idx", + "columns": ["phase", "updated_at", "video_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "video_uploads_video_id": { + "name": "video_uploads_video_id", + "columns": ["video_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'My Video'" + }, + "bucket": { + "name": "bucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageIntegrationId": { + "name": "storageIntegrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('{\"type\":\"MediaConvert\"}')" + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "effectiveCreatedAt": { + "name": "effectiveCreatedAt", + "type": "datetime", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "generated": { + "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", + "type": "stored" + } + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "xStreamInfo": { + "name": "xStreamInfo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstExternalViewAt": { + "name": "firstExternalViewAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstViewEmailSentAt": { + "name": "firstViewEmailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isScreenshot": { + "name": "isScreenshot", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "awsRegion": { + "name": "awsRegion", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "awsBucket": { + "name": "awsBucket", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoStartTime": { + "name": "videoStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audioStartTime": { + "name": "audioStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobId": { + "name": "jobId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobStatus": { + "name": "jobStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipProcessing": { + "name": "skipProcessing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + }, + "is_public_idx": { + "name": "is_public_idx", + "columns": ["public"], + "isUnique": false + }, + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "storage_integration_id_idx": { + "name": "storage_integration_id_idx", + "columns": ["storageIntegrationId"], + "isUnique": false + }, + "first_external_view_at_idx": { + "name": "first_external_view_at_idx", + "columns": ["firstExternalViewAt"], + "isUnique": false + }, + "org_owner_folder_idx": { + "name": "org_owner_folder_idx", + "columns": ["orgId", "ownerId", "folderId"], + "isUnique": false + }, + "org_effective_created_idx": { + "name": "org_effective_created_idx", + "columns": ["orgId", "effectiveCreatedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "videos_storageIntegrationId_storage_integrations_id_fk": { + "name": "videos_storageIntegrationId_storage_integrations_id_fk", + "tableFrom": "videos", + "tableTo": "storage_integrations", + "columnsFrom": ["storageIntegrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "videos_id": { + "name": "videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} diff --git a/packages/database/migrations/meta/0040_snapshot.json b/packages/database/migrations/meta/0040_snapshot.json new file mode 100644 index 00000000000..8ec34947e39 --- /dev/null +++ b/packages/database/migrations/meta/0040_snapshot.json @@ -0,0 +1,4740 @@ +{ + "version": "5", + "dialect": "mysql", + "id": "e7dd8548-04d7-4482-9b35-a23dba999c86", + "prevId": "f7a1a6b4-42da-4813-9462-a09d30766433", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_in": { + "name": "expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_in": { + "name": "refresh_token_expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "tempColumn": { + "name": "tempColumn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "provider_account_id_idx": { + "name": "provider_account_id_idx", + "columns": ["providerAccountId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accounts_id": { + "name": "accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_authorization_codes": { + "name": "agent_api_authorization_codes", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeHash": { + "name": "codeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeChallenge": { + "name": "codeChallenge", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirectUri": { + "name": "redirectUri", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consumedAt": { + "name": "consumedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "code_hash_idx": { + "name": "code_hash_idx", + "columns": ["codeHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_authorization_codes_id": { + "name": "agent_api_authorization_codes_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_idempotency": { + "name": "agent_api_idempotency", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestHash": { + "name": "requestHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "statusCode": { + "name": "statusCode", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response": { + "name": "response", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_operation_key_idx": { + "name": "user_operation_key_idx", + "columns": ["userId", "operation", "keyHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_idempotency_id": { + "name": "agent_api_idempotency_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_keys": { + "name": "agent_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tokenHash": { + "name": "tokenHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Cap CLI'" + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "token_hash_idx": { + "name": "token_hash_idx", + "columns": ["tokenHash"], + "isUnique": true + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_keys_id": { + "name": "agent_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_operations": { + "name": "agent_api_operations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resourceId": { + "name": "resourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resultResourceId": { + "name": "resultResourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "state_updated_at_idx": { + "name": "state_updated_at_idx", + "columns": ["state", "updatedAt"], + "isUnique": false + }, + "resource_id_idx": { + "name": "resource_id_idx", + "columns": ["resourceId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_operations_id": { + "name": "agent_api_operations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "auth_api_keys": { + "name": "auth_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_id_created_at_idx": { + "name": "user_id_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "auth_api_keys_id": { + "name": "auth_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "comments": { + "name": "comments", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authorId": { + "name": "authorId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "parentCommentId": { + "name": "parentCommentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "video_type_created_idx": { + "name": "video_type_created_idx", + "columns": ["videoId", "type", "createdAt", "id"], + "isUnique": false + }, + "author_id_idx": { + "name": "author_id_idx", + "columns": ["authorId"], + "isUnique": false + }, + "parent_comment_id_idx": { + "name": "parent_comment_id_idx", + "columns": ["parentCommentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "comments_id": { + "name": "comments_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_api_keys": { + "name": "developer_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyType": { + "name": "keyType", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyPrefix": { + "name": "keyPrefix", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedKey": { + "name": "encryptedKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "key_hash_idx": { + "name": "key_hash_idx", + "columns": ["keyHash"], + "isUnique": true + }, + "app_key_type_idx": { + "name": "app_key_type_idx", + "columns": ["appId", "keyType"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_api_keys_appId_developer_apps_id_fk": { + "name": "developer_api_keys_appId_developer_apps_id_fk", + "tableFrom": "developer_api_keys", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_api_keys_id": { + "name": "developer_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_app_domains": { + "name": "developer_app_domains", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_app_domains_appId_developer_apps_id_fk": { + "name": "developer_app_domains_appId_developer_apps_id_fk", + "tableFrom": "developer_app_domains", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_app_domains_id": { + "name": "developer_app_domains_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_domain_unique": { + "name": "app_domain_unique", + "columns": ["appId", "domain"] + } + }, + "checkConstraint": {} + }, + "developer_apps": { + "name": "developer_apps", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logoUrl": { + "name": "logoUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_deleted_idx": { + "name": "owner_deleted_idx", + "columns": ["ownerId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "developer_apps_id": { + "name": "developer_apps_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_accounts": { + "name": "developer_credit_accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceMicroCredits": { + "name": "balanceMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripePaymentMethodId": { + "name": "stripePaymentMethodId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoTopUpEnabled": { + "name": "autoTopUpEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "autoTopUpThresholdMicroCredits": { + "name": "autoTopUpThresholdMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoTopUpAmountCents": { + "name": "autoTopUpAmountCents", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_id_unique": { + "name": "app_id_unique", + "columns": ["appId"], + "isUnique": true + } + }, + "foreignKeys": { + "developer_credit_accounts_appId_developer_apps_id_fk": { + "name": "developer_credit_accounts_appId_developer_apps_id_fk", + "tableFrom": "developer_credit_accounts", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_accounts_id": { + "name": "developer_credit_accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_transactions": { + "name": "developer_credit_transactions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accountId": { + "name": "accountId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amountMicroCredits": { + "name": "amountMicroCredits", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceAfterMicroCredits": { + "name": "balanceAfterMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referenceType": { + "name": "referenceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "account_type_created_idx": { + "name": "account_type_created_idx", + "columns": ["accountId", "type", "createdAt"], + "isUnique": false + }, + "account_ref_dedup_idx": { + "name": "account_ref_dedup_idx", + "columns": ["accountId", "referenceId", "referenceType"], + "isUnique": false + } + }, + "foreignKeys": { + "dev_credit_txn_account_fk": { + "name": "dev_credit_txn_account_fk", + "tableFrom": "developer_credit_transactions", + "tableTo": "developer_credit_accounts", + "columnsFrom": ["accountId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_transactions_id": { + "name": "developer_credit_transactions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_daily_storage_snapshots": { + "name": "developer_daily_storage_snapshots", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshotDate": { + "name": "snapshotDate", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalDurationMinutes": { + "name": "totalDurationMinutes", + "type": "float", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "videoCount": { + "name": "videoCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "microCreditsCharged": { + "name": "microCreditsCharged", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { + "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", + "tableFrom": "developer_daily_storage_snapshots", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_daily_storage_snapshots_id": { + "name": "developer_daily_storage_snapshots_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_date_unique": { + "name": "app_date_unique", + "columns": ["appId", "snapshotDate"] + } + }, + "checkConstraint": {} + }, + "developer_videos": { + "name": "developer_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalUserId": { + "name": "externalUserId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3Key": { + "name": "s3Key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_created_idx": { + "name": "app_created_idx", + "columns": ["appId", "createdAt"], + "isUnique": false + }, + "app_user_idx": { + "name": "app_user_idx", + "columns": ["appId", "externalUserId"], + "isUnique": false + }, + "app_deleted_idx": { + "name": "app_deleted_idx", + "columns": ["appId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_videos_appId_developer_apps_id_fk": { + "name": "developer_videos_appId_developer_apps_id_fk", + "tableFrom": "developer_videos", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_videos_id": { + "name": "developer_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "folders": { + "name": "folders", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "parent_id_idx": { + "name": "parent_id_idx", + "columns": ["parentId"], + "isUnique": false + }, + "space_id_idx": { + "name": "space_id_idx", + "columns": ["spaceId"], + "isUnique": false + }, + "public_parent_id_idx": { + "name": "public_parent_id_idx", + "columns": ["public", "parentId"], + "isUnique": false + }, + "public_space_parent_id_idx": { + "name": "public_space_parent_id_idx", + "columns": ["public", "spaceId", "parentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "folders_id": { + "name": "folders_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "imported_videos": { + "name": "imported_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "imported_videos_orgId_source_source_id_pk": { + "name": "imported_videos_orgId_source_source_id_pk", + "columns": ["orgId", "source", "source_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "integration_installations": { + "name": "integration_installations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalId": { + "name": "externalId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "installedByUserId": { + "name": "installedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedCredentials": { + "name": "encryptedCredentials", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "provider_external_id_idx": { + "name": "provider_external_id_idx", + "columns": ["provider", "externalId"], + "isUnique": true + }, + "organization_provider_display_name_idx": { + "name": "organization_provider_display_name_idx", + "columns": ["organizationId", "provider", "displayName"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "integration_installations_id": { + "name": "integration_installations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_conversations": { + "name": "messenger_conversations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverByUserId": { + "name": "takeoverByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverAt": { + "name": "takeoverAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastMessageAt": { + "name": "lastMessageAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_last_message_idx": { + "name": "user_last_message_idx", + "columns": ["userId", "lastMessageAt"], + "isUnique": false + }, + "anonymous_last_message_idx": { + "name": "anonymous_last_message_idx", + "columns": ["anonymousId", "lastMessageAt"], + "isUnique": false + }, + "mode_last_message_idx": { + "name": "mode_last_message_idx", + "columns": ["mode", "lastMessageAt"], + "isUnique": false + }, + "updated_at_idx": { + "name": "updated_at_idx", + "columns": ["updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "messenger_conversations_id": { + "name": "messenger_conversations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_messages": { + "name": "messenger_messages", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "conversation_created_at_idx": { + "name": "conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + }, + "role_created_at_idx": { + "name": "role_created_at_idx", + "columns": ["role", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "messenger_messages_conversationId_messenger_conversations_id_fk": { + "name": "messenger_messages_conversationId_messenger_conversations_id_fk", + "tableFrom": "messenger_messages", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_messages_id": { + "name": "messenger_messages_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_support_emails": { + "name": "messenger_support_emails", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "support_email_user_created_at_idx": { + "name": "support_email_user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "support_email_conversation_created_at_idx": { + "name": "support_email_conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "support_email_conversation_fk": { + "name": "support_email_conversation_fk", + "tableFrom": "messenger_support_emails", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_support_emails_id": { + "name": "messenger_support_emails_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipientId": { + "name": "recipientId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "org_id_idx": { + "name": "org_id_idx", + "columns": ["orgId"], + "isUnique": false + }, + "type_idx": { + "name": "type_idx", + "columns": ["type"], + "isUnique": false + }, + "read_at_idx": { + "name": "read_at_idx", + "columns": ["readAt"], + "isUnique": false + }, + "created_at_idx": { + "name": "created_at_idx", + "columns": ["createdAt"], + "isUnique": false + }, + "recipient_read_idx": { + "name": "recipient_read_idx", + "columns": ["recipientId", "readAt"], + "isUnique": false + }, + "recipient_created_idx": { + "name": "recipient_created_idx", + "columns": ["recipientId", "createdAt"], + "isUnique": false + }, + "dedup_key_idx": { + "name": "dedup_key_idx", + "columns": ["dedupKey"], + "isUnique": true + }, + "type_recipient_created_idx": { + "name": "type_recipient_created_idx", + "columns": ["type", "recipientId", "createdAt"], + "isUnique": false + }, + "type_recipient_video_created_idx": { + "name": "type_recipient_video_created_idx", + "columns": ["type", "recipientId", "videoId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "notifications_id": { + "name": "notifications_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_invites": { + "name": "organization_invites", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmail": { + "name": "invitedEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmailNormalized": { + "name": "invitedEmailNormalized", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invitedByUserId": { + "name": "invitedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "emailDeliveryState": { + "name": "emailDeliveryState", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'legacy'" + }, + "emailDeliveryAttemptCount": { + "name": "emailDeliveryAttemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "emailDeliveryNextAttemptAt": { + "name": "emailDeliveryNextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryErrorCode": { + "name": "emailDeliveryErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryLeaseOwnerId": { + "name": "emailDeliveryLeaseOwnerId", + "type": "varchar(36)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryLeaseExpiresAt": { + "name": "emailDeliveryLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailProviderMessageId": { + "name": "emailProviderMessageId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailSentAt": { + "name": "emailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "invited_email_idx": { + "name": "invited_email_idx", + "columns": ["invitedEmail"], + "isUnique": false + }, + "normalized_email_idx": { + "name": "normalized_email_idx", + "columns": ["organizationId", "invitedEmailNormalized"], + "isUnique": true + }, + "invited_by_user_id_idx": { + "name": "invited_by_user_id_idx", + "columns": ["invitedByUserId"], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": ["status"], + "isUnique": false + }, + "email_delivery_idx": { + "name": "email_delivery_idx", + "columns": ["emailDeliveryState", "emailDeliveryNextAttemptAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_invites_id": { + "name": "organization_invites_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hasProSeat": { + "name": "hasProSeat", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "user_id_organization_id_idx": { + "name": "user_id_organization_id_idx", + "columns": ["userId", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_members_id": { + "name": "organization_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tombstoneAt": { + "name": "tombstoneAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowedEmailDomain": { + "name": "allowedEmailDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customDomain": { + "name": "customDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domainVerified": { + "name": "domainVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "shareableLinkIconUrl": { + "name": "shareableLinkIconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "workosOrganizationId": { + "name": "workosOrganizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workosConnectionId": { + "name": "workosConnectionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "owner_id_tombstone_idx": { + "name": "owner_id_tombstone_idx", + "columns": ["ownerId", "tombstoneAt"], + "isUnique": false + }, + "custom_domain_idx": { + "name": "custom_domain_idx", + "columns": ["customDomain"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organizations_id": { + "name": "organizations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_leases": { + "name": "product_analytics_erasure_leases", + "columns": { + "name": { + "name": "name", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requestId": { + "name": "requestId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fencingToken": { + "name": "fencingToken", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "pausedPipes": { + "name": "pausedPipes", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_leases_name": { + "name": "product_analytics_erasure_leases_name", + "columns": ["name"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_requests": { + "name": "product_analytics_erasure_requests", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopeHash": { + "name": "scopeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "nextAttemptAt": { + "name": "nextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "leaseOwnerId": { + "name": "leaseOwnerId", + "type": "varchar(36)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "scope_hash_idx": { + "name": "scope_hash_idx", + "columns": ["scopeHash"], + "isUnique": true + }, + "queue_idx": { + "name": "queue_idx", + "columns": ["status", "nextAttemptAt", "createdAt"], + "isUnique": false + }, + "lease_idx": { + "name": "lease_idx", + "columns": ["leaseExpiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_requests_id": { + "name": "product_analytics_erasure_requests_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_event_receipts": { + "name": "product_analytics_event_receipts", + "columns": { + "eventIdHash": { + "name": "eventIdHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadHash": { + "name": "payloadHash", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anonymousIdentityHash": { + "name": "anonymousIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userIdentityHash": { + "name": "userIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationIdentityHash": { + "name": "organizationIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "conflictCount": { + "name": "conflictCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "firstSeenAt": { + "name": "firstSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastSeenAt": { + "name": "lastSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "retainUntil": { + "name": "retainUntil", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "anonymous_identity_idx": { + "name": "anonymous_identity_idx", + "columns": ["anonymousIdentityHash"], + "isUnique": false + }, + "user_identity_idx": { + "name": "user_identity_idx", + "columns": ["userIdentityHash"], + "isUnique": false + }, + "organization_identity_idx": { + "name": "organization_identity_idx", + "columns": ["organizationIdentityHash"], + "isUnique": false + }, + "retention_idx": { + "name": "retention_idx", + "columns": ["retainUntil"], + "isUnique": false + }, + "conflict_idx": { + "name": "conflict_idx", + "columns": ["conflictCount"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_event_receipts_eventIdHash": { + "name": "product_analytics_event_receipts_eventIdHash", + "columns": ["eventIdHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_identity_links": { + "name": "product_analytics_identity_links", + "columns": { + "anonymousIdentityHash": { + "name": "anonymousIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userIdentityHash": { + "name": "userIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationIdentityHash": { + "name": "organizationIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_identity_idx": { + "name": "user_identity_idx", + "columns": ["userIdentityHash"], + "isUnique": false + }, + "organization_identity_idx": { + "name": "organization_identity_idx", + "columns": ["organizationIdentityHash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "identity_link_pk": { + "name": "identity_link_pk", + "columns": ["anonymousIdentityHash", "userIdentityHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_identity_state": { + "name": "product_analytics_identity_state", + "columns": { + "identityHash": { + "name": "identityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "identityKind": { + "name": "identityKind", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "blockedAt": { + "name": "blockedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "blocked_idx": { + "name": "blocked_idx", + "columns": ["blockedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_identity_state_identityHash": { + "name": "product_analytics_identity_state_identityHash", + "columns": ["identityHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_ingestion_leases": { + "name": "product_analytics_ingestion_leases", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "expiry_idx": { + "name": "expiry_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_ingestion_leases_id": { + "name": "product_analytics_ingestion_leases_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_outbox": { + "name": "product_analytics_outbox", + "columns": { + "eventId": { + "name": "eventId", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deliveryKey": { + "name": "deliveryKey", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadHash": { + "name": "payloadHash", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "eventName": { + "name": "eventName", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadKind": { + "name": "payloadKind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'product_event_row_v1'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "nextAttemptAt": { + "name": "nextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "leaseOwnerId": { + "name": "leaseOwnerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workflowRunId": { + "name": "workflowRunId", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payloadConflict": { + "name": "payloadConflict", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deliveredAt": { + "name": "deliveredAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadLetteredAt": { + "name": "deadLetteredAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "delivery_key_idx": { + "name": "delivery_key_idx", + "columns": ["deliveryKey"], + "isUnique": true + }, + "delivery_idx": { + "name": "delivery_idx", + "columns": ["status", "nextAttemptAt", "createdAt"], + "isUnique": false + }, + "lease_idx": { + "name": "lease_idx", + "columns": ["leaseExpiresAt"], + "isUnique": false + }, + "retention_idx": { + "name": "retention_idx", + "columns": ["status", "deliveredAt"], + "isUnique": false + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "anonymous_id_idx": { + "name": "anonymous_id_idx", + "columns": ["anonymousId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_outbox_eventId": { + "name": "product_analytics_outbox_eventId", + "columns": ["eventId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "s3_buckets": { + "name": "s3_buckets", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucketName": { + "name": "bucketName", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessKeyId": { + "name": "accessKeyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretAccessKey": { + "name": "secretAccessKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('aws')" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_organization_idx": { + "name": "owner_organization_idx", + "columns": ["ownerId", "organizationId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "s3_buckets_id": { + "name": "s3_buckets_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "session_token_idx": { + "name": "session_token_idx", + "columns": ["sessionToken"], + "isUnique": true + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sessions_id": { + "name": "sessions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "shared_videos": { + "name": "shared_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedByUserId": { + "name": "sharedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedAt": { + "name": "sharedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "shared_by_user_id_idx": { + "name": "shared_by_user_id_idx", + "columns": ["sharedByUserId"], + "isUnique": false + }, + "video_id_organization_id_idx": { + "name": "video_id_organization_id_idx", + "columns": ["videoId", "organizationId"], + "isUnique": false + }, + "video_id_folder_id_idx": { + "name": "video_id_folder_id_idx", + "columns": ["videoId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "shared_videos_id": { + "name": "shared_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_members_id": { + "name": "space_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "space_id_user_id_unique": { + "name": "space_id_user_id_unique", + "columns": ["spaceId", "userId"] + } + }, + "checkConstraint": {} + }, + "space_videos": { + "name": "space_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedById": { + "name": "addedById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "added_by_id_idx": { + "name": "added_by_id_idx", + "columns": ["addedById"], + "isUnique": false + }, + "space_id_video_id_idx": { + "name": "space_id_video_id_idx", + "columns": ["spaceId", "videoId"], + "isUnique": false + }, + "space_id_folder_id_idx": { + "name": "space_id_folder_id_idx", + "columns": ["spaceId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_videos_id": { + "name": "space_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "primary": { + "name": "primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "privacy": { + "name": "privacy", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Private'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "public_organization_id_idx": { + "name": "public_organization_id_idx", + "columns": ["public", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "spaces_id": { + "name": "spaces_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_integrations": { + "name": "storage_integrations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "encryptedConfig": { + "name": "encryptedConfig", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "googleDriveAccessToken": { + "name": "googleDriveAccessToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveAccessTokenExpiresAt": { + "name": "googleDriveAccessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseId": { + "name": "googleDriveTokenRefreshLeaseId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseExpiresAt": { + "name": "googleDriveTokenRefreshLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveStorageQuotaCache": { + "name": "googleDriveStorageQuotaCache", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_provider_idx": { + "name": "owner_provider_idx", + "columns": ["ownerId", "provider"], + "isUnique": false + }, + "owner_active_idx": { + "name": "owner_active_idx", + "columns": ["ownerId", "active"], + "isUnique": false + }, + "organization_provider_idx": { + "name": "organization_provider_idx", + "columns": ["organizationId", "provider"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "storage_integrations_id": { + "name": "storage_integrations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_objects": { + "name": "storage_objects", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrationId": { + "name": "integrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "objectKey": { + "name": "objectKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "objectKeyHash": { + "name": "objectKeyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerObjectId": { + "name": "providerObjectId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploadSessionUrl": { + "name": "uploadSessionUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploadStatus": { + "name": "uploadStatus", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "contentType": { + "name": "contentType", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentLength": { + "name": "contentLength", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "integration_key_hash_idx": { + "name": "integration_key_hash_idx", + "columns": ["integrationId", "objectKeyHash"], + "isUnique": true + }, + "integration_status_idx": { + "name": "integration_status_idx", + "columns": ["integrationId", "uploadStatus"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + } + }, + "foreignKeys": { + "storage_objects_integrationId_storage_integrations_id_fk": { + "name": "storage_objects_integrationId_storage_integrations_id_fk", + "tableFrom": "storage_objects", + "tableTo": "storage_integrations", + "columnsFrom": ["integrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "storage_objects_id": { + "name": "storage_objects_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastName": { + "name": "lastName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionId": { + "name": "stripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thirdPartyStripeSubscriptionId": { + "name": "thirdPartyStripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionStatus": { + "name": "stripeSubscriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionPriceId": { + "name": "stripeSubscriptionPriceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferences": { + "name": "preferences", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "('null')" + }, + "activeOrganizationId": { + "name": "activeOrganizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "onboardingSteps": { + "name": "onboardingSteps", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customBucket": { + "name": "customBucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "inviteQuota": { + "name": "inviteQuota", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "defaultOrgId": { + "name": "defaultOrgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authSessionVersion": { + "name": "authSessionVersion", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "email_idx": { + "name": "email_idx", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "identifier": { + "name": "identifier", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier": { + "name": "verification_tokens_identifier", + "columns": ["identifier"] + } + }, + "uniqueConstraints": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": ["token"] + } + }, + "checkConstraint": {} + }, + "video_edits": { + "name": "video_edits", + "columns": { + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceKey": { + "name": "sourceKey", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "editSpec": { + "name": "editSpec", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "video_edits_videoId_videos_id_fk": { + "name": "video_edits_videoId_videos_id_fk", + "tableFrom": "video_edits", + "tableTo": "videos", + "columnsFrom": ["videoId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "video_edits_videoId": { + "name": "video_edits_videoId", + "columns": ["videoId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "video_uploads": { + "name": "video_uploads", + "columns": { + "video_id": { + "name": "video_id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded": { + "name": "uploaded", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "mode": { + "name": "mode", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'uploading'" + }, + "processing_progress": { + "name": "processing_progress", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processing_message": { + "name": "processing_message", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_file_key": { + "name": "raw_file_key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "phase_updated_at_video_id_idx": { + "name": "phase_updated_at_video_id_idx", + "columns": ["phase", "updated_at", "video_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "video_uploads_video_id": { + "name": "video_uploads_video_id", + "columns": ["video_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'My Video'" + }, + "bucket": { + "name": "bucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageIntegrationId": { + "name": "storageIntegrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('{\"type\":\"MediaConvert\"}')" + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "effectiveCreatedAt": { + "name": "effectiveCreatedAt", + "type": "datetime", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "generated": { + "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", + "type": "stored" + } + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "xStreamInfo": { + "name": "xStreamInfo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstExternalViewAt": { + "name": "firstExternalViewAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstViewEmailSentAt": { + "name": "firstViewEmailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isScreenshot": { + "name": "isScreenshot", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "awsRegion": { + "name": "awsRegion", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "awsBucket": { + "name": "awsBucket", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoStartTime": { + "name": "videoStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audioStartTime": { + "name": "audioStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobId": { + "name": "jobId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobStatus": { + "name": "jobStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipProcessing": { + "name": "skipProcessing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + }, + "is_public_idx": { + "name": "is_public_idx", + "columns": ["public"], + "isUnique": false + }, + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "storage_integration_id_idx": { + "name": "storage_integration_id_idx", + "columns": ["storageIntegrationId"], + "isUnique": false + }, + "first_external_view_at_idx": { + "name": "first_external_view_at_idx", + "columns": ["firstExternalViewAt"], + "isUnique": false + }, + "org_owner_folder_idx": { + "name": "org_owner_folder_idx", + "columns": ["orgId", "ownerId", "folderId"], + "isUnique": false + }, + "org_effective_created_idx": { + "name": "org_effective_created_idx", + "columns": ["orgId", "effectiveCreatedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "videos_storageIntegrationId_storage_integrations_id_fk": { + "name": "videos_storageIntegrationId_storage_integrations_id_fk", + "tableFrom": "videos", + "tableTo": "storage_integrations", + "columnsFrom": ["storageIntegrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "videos_id": { + "name": "videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} diff --git a/packages/database/migrations/meta/0041_snapshot.json b/packages/database/migrations/meta/0041_snapshot.json new file mode 100644 index 00000000000..d89647d1298 --- /dev/null +++ b/packages/database/migrations/meta/0041_snapshot.json @@ -0,0 +1,4834 @@ +{ + "version": "5", + "dialect": "mysql", + "id": "ecee2d0f-264e-410c-aa49-0f3b724d432b", + "prevId": "e7dd8548-04d7-4482-9b35-a23dba999c86", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_in": { + "name": "expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_in": { + "name": "refresh_token_expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "tempColumn": { + "name": "tempColumn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "provider_account_id_idx": { + "name": "provider_account_id_idx", + "columns": ["providerAccountId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accounts_id": { + "name": "accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_authorization_codes": { + "name": "agent_api_authorization_codes", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeHash": { + "name": "codeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeChallenge": { + "name": "codeChallenge", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirectUri": { + "name": "redirectUri", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consumedAt": { + "name": "consumedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "code_hash_idx": { + "name": "code_hash_idx", + "columns": ["codeHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_authorization_codes_id": { + "name": "agent_api_authorization_codes_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_idempotency": { + "name": "agent_api_idempotency", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestHash": { + "name": "requestHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "statusCode": { + "name": "statusCode", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response": { + "name": "response", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_operation_key_idx": { + "name": "user_operation_key_idx", + "columns": ["userId", "operation", "keyHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_idempotency_id": { + "name": "agent_api_idempotency_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_keys": { + "name": "agent_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tokenHash": { + "name": "tokenHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Cap CLI'" + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "token_hash_idx": { + "name": "token_hash_idx", + "columns": ["tokenHash"], + "isUnique": true + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_keys_id": { + "name": "agent_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_operations": { + "name": "agent_api_operations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resourceId": { + "name": "resourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resultResourceId": { + "name": "resultResourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "state_updated_at_idx": { + "name": "state_updated_at_idx", + "columns": ["state", "updatedAt"], + "isUnique": false + }, + "resource_id_idx": { + "name": "resource_id_idx", + "columns": ["resourceId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_operations_id": { + "name": "agent_api_operations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "auth_api_keys": { + "name": "auth_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_id_created_at_idx": { + "name": "user_id_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "auth_api_keys_id": { + "name": "auth_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "comments": { + "name": "comments", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authorId": { + "name": "authorId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "parentCommentId": { + "name": "parentCommentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "analytics_reconciliation_idx": { + "name": "analytics_reconciliation_idx", + "columns": ["createdAt", "id"], + "isUnique": false + }, + "video_type_created_idx": { + "name": "video_type_created_idx", + "columns": ["videoId", "type", "createdAt", "id"], + "isUnique": false + }, + "author_id_idx": { + "name": "author_id_idx", + "columns": ["authorId"], + "isUnique": false + }, + "parent_comment_id_idx": { + "name": "parent_comment_id_idx", + "columns": ["parentCommentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "comments_id": { + "name": "comments_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_api_keys": { + "name": "developer_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyType": { + "name": "keyType", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyPrefix": { + "name": "keyPrefix", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedKey": { + "name": "encryptedKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "key_hash_idx": { + "name": "key_hash_idx", + "columns": ["keyHash"], + "isUnique": true + }, + "app_key_type_idx": { + "name": "app_key_type_idx", + "columns": ["appId", "keyType"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_api_keys_appId_developer_apps_id_fk": { + "name": "developer_api_keys_appId_developer_apps_id_fk", + "tableFrom": "developer_api_keys", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_api_keys_id": { + "name": "developer_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_app_domains": { + "name": "developer_app_domains", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_app_domains_appId_developer_apps_id_fk": { + "name": "developer_app_domains_appId_developer_apps_id_fk", + "tableFrom": "developer_app_domains", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_app_domains_id": { + "name": "developer_app_domains_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_domain_unique": { + "name": "app_domain_unique", + "columns": ["appId", "domain"] + } + }, + "checkConstraint": {} + }, + "developer_apps": { + "name": "developer_apps", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logoUrl": { + "name": "logoUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_deleted_idx": { + "name": "owner_deleted_idx", + "columns": ["ownerId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "developer_apps_id": { + "name": "developer_apps_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_accounts": { + "name": "developer_credit_accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceMicroCredits": { + "name": "balanceMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripePaymentMethodId": { + "name": "stripePaymentMethodId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoTopUpEnabled": { + "name": "autoTopUpEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "autoTopUpThresholdMicroCredits": { + "name": "autoTopUpThresholdMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoTopUpAmountCents": { + "name": "autoTopUpAmountCents", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_id_unique": { + "name": "app_id_unique", + "columns": ["appId"], + "isUnique": true + } + }, + "foreignKeys": { + "developer_credit_accounts_appId_developer_apps_id_fk": { + "name": "developer_credit_accounts_appId_developer_apps_id_fk", + "tableFrom": "developer_credit_accounts", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_accounts_id": { + "name": "developer_credit_accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_transactions": { + "name": "developer_credit_transactions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accountId": { + "name": "accountId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amountMicroCredits": { + "name": "amountMicroCredits", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceAfterMicroCredits": { + "name": "balanceAfterMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referenceType": { + "name": "referenceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "account_type_created_idx": { + "name": "account_type_created_idx", + "columns": ["accountId", "type", "createdAt"], + "isUnique": false + }, + "account_ref_dedup_idx": { + "name": "account_ref_dedup_idx", + "columns": ["accountId", "referenceId", "referenceType"], + "isUnique": false + } + }, + "foreignKeys": { + "dev_credit_txn_account_fk": { + "name": "dev_credit_txn_account_fk", + "tableFrom": "developer_credit_transactions", + "tableTo": "developer_credit_accounts", + "columnsFrom": ["accountId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_transactions_id": { + "name": "developer_credit_transactions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_daily_storage_snapshots": { + "name": "developer_daily_storage_snapshots", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshotDate": { + "name": "snapshotDate", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalDurationMinutes": { + "name": "totalDurationMinutes", + "type": "float", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "videoCount": { + "name": "videoCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "microCreditsCharged": { + "name": "microCreditsCharged", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { + "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", + "tableFrom": "developer_daily_storage_snapshots", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_daily_storage_snapshots_id": { + "name": "developer_daily_storage_snapshots_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_date_unique": { + "name": "app_date_unique", + "columns": ["appId", "snapshotDate"] + } + }, + "checkConstraint": {} + }, + "developer_videos": { + "name": "developer_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalUserId": { + "name": "externalUserId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3Key": { + "name": "s3Key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_created_idx": { + "name": "app_created_idx", + "columns": ["appId", "createdAt"], + "isUnique": false + }, + "app_user_idx": { + "name": "app_user_idx", + "columns": ["appId", "externalUserId"], + "isUnique": false + }, + "app_deleted_idx": { + "name": "app_deleted_idx", + "columns": ["appId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_videos_appId_developer_apps_id_fk": { + "name": "developer_videos_appId_developer_apps_id_fk", + "tableFrom": "developer_videos", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_videos_id": { + "name": "developer_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "folders": { + "name": "folders", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "parent_id_idx": { + "name": "parent_id_idx", + "columns": ["parentId"], + "isUnique": false + }, + "space_id_idx": { + "name": "space_id_idx", + "columns": ["spaceId"], + "isUnique": false + }, + "public_parent_id_idx": { + "name": "public_parent_id_idx", + "columns": ["public", "parentId"], + "isUnique": false + }, + "public_space_parent_id_idx": { + "name": "public_space_parent_id_idx", + "columns": ["public", "spaceId", "parentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "folders_id": { + "name": "folders_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "imported_videos": { + "name": "imported_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "imported_videos_orgId_source_source_id_pk": { + "name": "imported_videos_orgId_source_source_id_pk", + "columns": ["orgId", "source", "source_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "integration_installations": { + "name": "integration_installations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalId": { + "name": "externalId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "installedByUserId": { + "name": "installedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedCredentials": { + "name": "encryptedCredentials", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "provider_external_id_idx": { + "name": "provider_external_id_idx", + "columns": ["provider", "externalId"], + "isUnique": true + }, + "organization_provider_display_name_idx": { + "name": "organization_provider_display_name_idx", + "columns": ["organizationId", "provider", "displayName"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "integration_installations_id": { + "name": "integration_installations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_conversations": { + "name": "messenger_conversations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverByUserId": { + "name": "takeoverByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverAt": { + "name": "takeoverAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastMessageAt": { + "name": "lastMessageAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_last_message_idx": { + "name": "user_last_message_idx", + "columns": ["userId", "lastMessageAt"], + "isUnique": false + }, + "anonymous_last_message_idx": { + "name": "anonymous_last_message_idx", + "columns": ["anonymousId", "lastMessageAt"], + "isUnique": false + }, + "mode_last_message_idx": { + "name": "mode_last_message_idx", + "columns": ["mode", "lastMessageAt"], + "isUnique": false + }, + "updated_at_idx": { + "name": "updated_at_idx", + "columns": ["updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "messenger_conversations_id": { + "name": "messenger_conversations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_messages": { + "name": "messenger_messages", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "conversation_created_at_idx": { + "name": "conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + }, + "role_created_at_idx": { + "name": "role_created_at_idx", + "columns": ["role", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "messenger_messages_conversationId_messenger_conversations_id_fk": { + "name": "messenger_messages_conversationId_messenger_conversations_id_fk", + "tableFrom": "messenger_messages", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_messages_id": { + "name": "messenger_messages_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_support_emails": { + "name": "messenger_support_emails", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "support_email_user_created_at_idx": { + "name": "support_email_user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "support_email_conversation_created_at_idx": { + "name": "support_email_conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "support_email_conversation_fk": { + "name": "support_email_conversation_fk", + "tableFrom": "messenger_support_emails", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_support_emails_id": { + "name": "messenger_support_emails_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipientId": { + "name": "recipientId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "org_id_idx": { + "name": "org_id_idx", + "columns": ["orgId"], + "isUnique": false + }, + "type_idx": { + "name": "type_idx", + "columns": ["type"], + "isUnique": false + }, + "read_at_idx": { + "name": "read_at_idx", + "columns": ["readAt"], + "isUnique": false + }, + "created_at_idx": { + "name": "created_at_idx", + "columns": ["createdAt"], + "isUnique": false + }, + "recipient_read_idx": { + "name": "recipient_read_idx", + "columns": ["recipientId", "readAt"], + "isUnique": false + }, + "recipient_created_idx": { + "name": "recipient_created_idx", + "columns": ["recipientId", "createdAt"], + "isUnique": false + }, + "dedup_key_idx": { + "name": "dedup_key_idx", + "columns": ["dedupKey"], + "isUnique": true + }, + "type_recipient_created_idx": { + "name": "type_recipient_created_idx", + "columns": ["type", "recipientId", "createdAt"], + "isUnique": false + }, + "type_recipient_video_created_idx": { + "name": "type_recipient_video_created_idx", + "columns": ["type", "recipientId", "videoId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "notifications_id": { + "name": "notifications_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_invites": { + "name": "organization_invites", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmail": { + "name": "invitedEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmailNormalized": { + "name": "invitedEmailNormalized", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invitedByUserId": { + "name": "invitedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "emailDeliveryState": { + "name": "emailDeliveryState", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'legacy'" + }, + "emailDeliveryAttemptCount": { + "name": "emailDeliveryAttemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "emailDeliveryNextAttemptAt": { + "name": "emailDeliveryNextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryErrorCode": { + "name": "emailDeliveryErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryLeaseOwnerId": { + "name": "emailDeliveryLeaseOwnerId", + "type": "varchar(36)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryLeaseExpiresAt": { + "name": "emailDeliveryLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailProviderMessageId": { + "name": "emailProviderMessageId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailSentAt": { + "name": "emailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "invited_email_idx": { + "name": "invited_email_idx", + "columns": ["invitedEmail"], + "isUnique": false + }, + "normalized_email_idx": { + "name": "normalized_email_idx", + "columns": ["organizationId", "invitedEmailNormalized"], + "isUnique": true + }, + "invited_by_user_id_idx": { + "name": "invited_by_user_id_idx", + "columns": ["invitedByUserId"], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": ["status"], + "isUnique": false + }, + "email_delivery_idx": { + "name": "email_delivery_idx", + "columns": ["emailDeliveryState", "emailDeliveryNextAttemptAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_invites_id": { + "name": "organization_invites_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hasProSeat": { + "name": "hasProSeat", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "user_id_organization_id_idx": { + "name": "user_id_organization_id_idx", + "columns": ["userId", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_members_id": { + "name": "organization_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tombstoneAt": { + "name": "tombstoneAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowedEmailDomain": { + "name": "allowedEmailDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customDomain": { + "name": "customDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domainVerified": { + "name": "domainVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "shareableLinkIconUrl": { + "name": "shareableLinkIconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "workosOrganizationId": { + "name": "workosOrganizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workosConnectionId": { + "name": "workosConnectionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "owner_id_tombstone_idx": { + "name": "owner_id_tombstone_idx", + "columns": ["ownerId", "tombstoneAt"], + "isUnique": false + }, + "custom_domain_idx": { + "name": "custom_domain_idx", + "columns": ["customDomain"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organizations_id": { + "name": "organizations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_leases": { + "name": "product_analytics_erasure_leases", + "columns": { + "name": { + "name": "name", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requestId": { + "name": "requestId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fencingToken": { + "name": "fencingToken", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "pausedPipes": { + "name": "pausedPipes", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_leases_name": { + "name": "product_analytics_erasure_leases_name", + "columns": ["name"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_requests": { + "name": "product_analytics_erasure_requests", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopeHash": { + "name": "scopeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "nextAttemptAt": { + "name": "nextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "leaseOwnerId": { + "name": "leaseOwnerId", + "type": "varchar(36)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "scope_hash_idx": { + "name": "scope_hash_idx", + "columns": ["scopeHash"], + "isUnique": true + }, + "queue_idx": { + "name": "queue_idx", + "columns": ["status", "nextAttemptAt", "createdAt"], + "isUnique": false + }, + "lease_idx": { + "name": "lease_idx", + "columns": ["leaseExpiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_requests_id": { + "name": "product_analytics_erasure_requests_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_event_receipts": { + "name": "product_analytics_event_receipts", + "columns": { + "eventIdHash": { + "name": "eventIdHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadHash": { + "name": "payloadHash", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anonymousIdentityHash": { + "name": "anonymousIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userIdentityHash": { + "name": "userIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationIdentityHash": { + "name": "organizationIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "conflictCount": { + "name": "conflictCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "firstSeenAt": { + "name": "firstSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastSeenAt": { + "name": "lastSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "retainUntil": { + "name": "retainUntil", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "anonymous_identity_idx": { + "name": "anonymous_identity_idx", + "columns": ["anonymousIdentityHash"], + "isUnique": false + }, + "user_identity_idx": { + "name": "user_identity_idx", + "columns": ["userIdentityHash"], + "isUnique": false + }, + "organization_identity_idx": { + "name": "organization_identity_idx", + "columns": ["organizationIdentityHash"], + "isUnique": false + }, + "retention_idx": { + "name": "retention_idx", + "columns": ["retainUntil"], + "isUnique": false + }, + "conflict_idx": { + "name": "conflict_idx", + "columns": ["conflictCount"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_event_receipts_eventIdHash": { + "name": "product_analytics_event_receipts_eventIdHash", + "columns": ["eventIdHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_identity_links": { + "name": "product_analytics_identity_links", + "columns": { + "anonymousIdentityHash": { + "name": "anonymousIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userIdentityHash": { + "name": "userIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationIdentityHash": { + "name": "organizationIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_identity_idx": { + "name": "user_identity_idx", + "columns": ["userIdentityHash"], + "isUnique": false + }, + "organization_identity_idx": { + "name": "organization_identity_idx", + "columns": ["organizationIdentityHash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "identity_link_pk": { + "name": "identity_link_pk", + "columns": ["anonymousIdentityHash", "userIdentityHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_identity_state": { + "name": "product_analytics_identity_state", + "columns": { + "identityHash": { + "name": "identityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "identityKind": { + "name": "identityKind", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "blockedAt": { + "name": "blockedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "blocked_idx": { + "name": "blocked_idx", + "columns": ["blockedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_identity_state_identityHash": { + "name": "product_analytics_identity_state_identityHash", + "columns": ["identityHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_ingestion_leases": { + "name": "product_analytics_ingestion_leases", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fencingToken": { + "name": "fencingToken", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "expiry_idx": { + "name": "expiry_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_ingestion_leases_id": { + "name": "product_analytics_ingestion_leases_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_outbox": { + "name": "product_analytics_outbox", + "columns": { + "eventId": { + "name": "eventId", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deliveryKey": { + "name": "deliveryKey", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadHash": { + "name": "payloadHash", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "eventName": { + "name": "eventName", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadKind": { + "name": "payloadKind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'product_event_row_v1'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "nextAttemptAt": { + "name": "nextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "leaseOwnerId": { + "name": "leaseOwnerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workflowRunId": { + "name": "workflowRunId", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payloadConflict": { + "name": "payloadConflict", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deliveredAt": { + "name": "deliveredAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadLetteredAt": { + "name": "deadLetteredAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "delivery_key_idx": { + "name": "delivery_key_idx", + "columns": ["deliveryKey"], + "isUnique": true + }, + "delivery_idx": { + "name": "delivery_idx", + "columns": ["status", "nextAttemptAt", "createdAt"], + "isUnique": false + }, + "lease_idx": { + "name": "lease_idx", + "columns": ["leaseExpiresAt"], + "isUnique": false + }, + "retention_idx": { + "name": "retention_idx", + "columns": ["status", "deliveredAt"], + "isUnique": false + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "anonymous_id_idx": { + "name": "anonymous_id_idx", + "columns": ["anonymousId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_outbox_eventId": { + "name": "product_analytics_outbox_eventId", + "columns": ["eventId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_reconciliation_failures": { + "name": "product_analytics_reconciliation_failures", + "columns": { + "sourceHash": { + "name": "sourceHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceType": { + "name": "sourceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "firstSeenAt": { + "name": "firstSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastSeenAt": { + "name": "lastSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "source_type_idx": { + "name": "source_type_idx", + "columns": ["sourceType", "lastSeenAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_reconciliation_failures_sourceHash": { + "name": "product_analytics_reconciliation_failures_sourceHash", + "columns": ["sourceHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "s3_buckets": { + "name": "s3_buckets", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucketName": { + "name": "bucketName", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessKeyId": { + "name": "accessKeyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretAccessKey": { + "name": "secretAccessKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('aws')" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_organization_idx": { + "name": "owner_organization_idx", + "columns": ["ownerId", "organizationId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "s3_buckets_id": { + "name": "s3_buckets_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "session_token_idx": { + "name": "session_token_idx", + "columns": ["sessionToken"], + "isUnique": true + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sessions_id": { + "name": "sessions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "shared_videos": { + "name": "shared_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedByUserId": { + "name": "sharedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedAt": { + "name": "sharedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "shared_by_user_id_idx": { + "name": "shared_by_user_id_idx", + "columns": ["sharedByUserId"], + "isUnique": false + }, + "video_id_organization_id_idx": { + "name": "video_id_organization_id_idx", + "columns": ["videoId", "organizationId"], + "isUnique": false + }, + "video_id_folder_id_idx": { + "name": "video_id_folder_id_idx", + "columns": ["videoId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "shared_videos_id": { + "name": "shared_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_members_id": { + "name": "space_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "space_id_user_id_unique": { + "name": "space_id_user_id_unique", + "columns": ["spaceId", "userId"] + } + }, + "checkConstraint": {} + }, + "space_videos": { + "name": "space_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedById": { + "name": "addedById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "added_by_id_idx": { + "name": "added_by_id_idx", + "columns": ["addedById"], + "isUnique": false + }, + "space_id_video_id_idx": { + "name": "space_id_video_id_idx", + "columns": ["spaceId", "videoId"], + "isUnique": false + }, + "space_id_folder_id_idx": { + "name": "space_id_folder_id_idx", + "columns": ["spaceId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_videos_id": { + "name": "space_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "primary": { + "name": "primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "privacy": { + "name": "privacy", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Private'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "public_organization_id_idx": { + "name": "public_organization_id_idx", + "columns": ["public", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "spaces_id": { + "name": "spaces_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_integrations": { + "name": "storage_integrations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "encryptedConfig": { + "name": "encryptedConfig", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "googleDriveAccessToken": { + "name": "googleDriveAccessToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveAccessTokenExpiresAt": { + "name": "googleDriveAccessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseId": { + "name": "googleDriveTokenRefreshLeaseId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseExpiresAt": { + "name": "googleDriveTokenRefreshLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveStorageQuotaCache": { + "name": "googleDriveStorageQuotaCache", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_provider_idx": { + "name": "owner_provider_idx", + "columns": ["ownerId", "provider"], + "isUnique": false + }, + "owner_active_idx": { + "name": "owner_active_idx", + "columns": ["ownerId", "active"], + "isUnique": false + }, + "organization_provider_idx": { + "name": "organization_provider_idx", + "columns": ["organizationId", "provider"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "storage_integrations_id": { + "name": "storage_integrations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_objects": { + "name": "storage_objects", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrationId": { + "name": "integrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "objectKey": { + "name": "objectKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "objectKeyHash": { + "name": "objectKeyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerObjectId": { + "name": "providerObjectId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploadSessionUrl": { + "name": "uploadSessionUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploadStatus": { + "name": "uploadStatus", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "contentType": { + "name": "contentType", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentLength": { + "name": "contentLength", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "integration_key_hash_idx": { + "name": "integration_key_hash_idx", + "columns": ["integrationId", "objectKeyHash"], + "isUnique": true + }, + "integration_status_idx": { + "name": "integration_status_idx", + "columns": ["integrationId", "uploadStatus"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + } + }, + "foreignKeys": { + "storage_objects_integrationId_storage_integrations_id_fk": { + "name": "storage_objects_integrationId_storage_integrations_id_fk", + "tableFrom": "storage_objects", + "tableTo": "storage_integrations", + "columnsFrom": ["integrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "storage_objects_id": { + "name": "storage_objects_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastName": { + "name": "lastName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionId": { + "name": "stripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thirdPartyStripeSubscriptionId": { + "name": "thirdPartyStripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionStatus": { + "name": "stripeSubscriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionPriceId": { + "name": "stripeSubscriptionPriceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferences": { + "name": "preferences", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "('null')" + }, + "activeOrganizationId": { + "name": "activeOrganizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "onboardingSteps": { + "name": "onboardingSteps", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customBucket": { + "name": "customBucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "inviteQuota": { + "name": "inviteQuota", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "defaultOrgId": { + "name": "defaultOrgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authSessionVersion": { + "name": "authSessionVersion", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "email_idx": { + "name": "email_idx", + "columns": ["email"], + "isUnique": true + }, + "analytics_reconciliation_idx": { + "name": "analytics_reconciliation_idx", + "columns": ["created_at", "id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "identifier": { + "name": "identifier", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier": { + "name": "verification_tokens_identifier", + "columns": ["identifier"] + } + }, + "uniqueConstraints": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": ["token"] + } + }, + "checkConstraint": {} + }, + "video_edits": { + "name": "video_edits", + "columns": { + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceKey": { + "name": "sourceKey", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "editSpec": { + "name": "editSpec", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "video_edits_videoId_videos_id_fk": { + "name": "video_edits_videoId_videos_id_fk", + "tableFrom": "video_edits", + "tableTo": "videos", + "columnsFrom": ["videoId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "video_edits_videoId": { + "name": "video_edits_videoId", + "columns": ["videoId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "video_uploads": { + "name": "video_uploads", + "columns": { + "video_id": { + "name": "video_id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded": { + "name": "uploaded", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "mode": { + "name": "mode", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'uploading'" + }, + "processing_progress": { + "name": "processing_progress", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processing_message": { + "name": "processing_message", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_file_key": { + "name": "raw_file_key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "phase_updated_at_video_id_idx": { + "name": "phase_updated_at_video_id_idx", + "columns": ["phase", "updated_at", "video_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "video_uploads_video_id": { + "name": "video_uploads_video_id", + "columns": ["video_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'My Video'" + }, + "bucket": { + "name": "bucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageIntegrationId": { + "name": "storageIntegrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('{\"type\":\"MediaConvert\"}')" + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "effectiveCreatedAt": { + "name": "effectiveCreatedAt", + "type": "datetime", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "generated": { + "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", + "type": "stored" + } + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "xStreamInfo": { + "name": "xStreamInfo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstExternalViewAt": { + "name": "firstExternalViewAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstViewEmailSentAt": { + "name": "firstViewEmailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isScreenshot": { + "name": "isScreenshot", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "awsRegion": { + "name": "awsRegion", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "awsBucket": { + "name": "awsBucket", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoStartTime": { + "name": "videoStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audioStartTime": { + "name": "audioStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobId": { + "name": "jobId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobStatus": { + "name": "jobStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipProcessing": { + "name": "skipProcessing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + }, + "is_public_idx": { + "name": "is_public_idx", + "columns": ["public"], + "isUnique": false + }, + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "storage_integration_id_idx": { + "name": "storage_integration_id_idx", + "columns": ["storageIntegrationId"], + "isUnique": false + }, + "first_external_view_at_idx": { + "name": "first_external_view_at_idx", + "columns": ["firstExternalViewAt"], + "isUnique": false + }, + "analytics_created_at_idx": { + "name": "analytics_created_at_idx", + "columns": ["createdAt", "id"], + "isUnique": false + }, + "analytics_first_view_at_idx": { + "name": "analytics_first_view_at_idx", + "columns": ["firstExternalViewAt", "id"], + "isUnique": false + }, + "org_owner_folder_idx": { + "name": "org_owner_folder_idx", + "columns": ["orgId", "ownerId", "folderId"], + "isUnique": false + }, + "org_effective_created_idx": { + "name": "org_effective_created_idx", + "columns": ["orgId", "effectiveCreatedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "videos_storageIntegrationId_storage_integrations_id_fk": { + "name": "videos_storageIntegrationId_storage_integrations_id_fk", + "tableFrom": "videos", + "tableTo": "storage_integrations", + "columnsFrom": ["storageIntegrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "videos_id": { + "name": "videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} diff --git a/packages/database/migrations/meta/0042_snapshot.json b/packages/database/migrations/meta/0042_snapshot.json new file mode 100644 index 00000000000..fc60f54753b --- /dev/null +++ b/packages/database/migrations/meta/0042_snapshot.json @@ -0,0 +1,4935 @@ +{ + "version": "5", + "dialect": "mysql", + "id": "8a28a044-0e09-4121-87d0-388c3ed0e8b9", + "prevId": "ecee2d0f-264e-410c-aa49-0f3b724d432b", + "tables": { + "accounts": { + "name": "accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerAccountId": { + "name": "providerAccountId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_in": { + "name": "expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_in": { + "name": "refresh_token_expires_in", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "token_type": { + "name": "token_type", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "tempColumn": { + "name": "tempColumn", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "provider_account_id_idx": { + "name": "provider_account_id_idx", + "columns": ["providerAccountId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "accounts_id": { + "name": "accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_authorization_codes": { + "name": "agent_api_authorization_codes", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeHash": { + "name": "codeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "codeChallenge": { + "name": "codeChallenge", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirectUri": { + "name": "redirectUri", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "consumedAt": { + "name": "consumedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "code_hash_idx": { + "name": "code_hash_idx", + "columns": ["codeHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_authorization_codes_id": { + "name": "agent_api_authorization_codes_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_idempotency": { + "name": "agent_api_idempotency", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "operation": { + "name": "operation", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "requestHash": { + "name": "requestHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "statusCode": { + "name": "statusCode", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "response": { + "name": "response", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_operation_key_idx": { + "name": "user_operation_key_idx", + "columns": ["userId", "operation", "keyHash"], + "isUnique": true + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_idempotency_id": { + "name": "agent_api_idempotency_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_keys": { + "name": "agent_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tokenHash": { + "name": "tokenHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(100)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Cap CLI'" + }, + "scopes": { + "name": "scopes", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "token_hash_idx": { + "name": "token_hash_idx", + "columns": ["tokenHash"], + "isUnique": true + }, + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "expires_at_idx": { + "name": "expires_at_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_keys_id": { + "name": "agent_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "agent_api_operations": { + "name": "agent_api_operations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resourceId": { + "name": "resourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "resultResourceId": { + "name": "resultResourceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "result": { + "name": "result", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "errorMessage": { + "name": "errorMessage", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "completedAt": { + "name": "completedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "user_created_at_idx": { + "name": "user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "state_updated_at_idx": { + "name": "state_updated_at_idx", + "columns": ["state", "updatedAt"], + "isUnique": false + }, + "resource_id_idx": { + "name": "resource_id_idx", + "columns": ["resourceId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "agent_api_operations_id": { + "name": "agent_api_operations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "auth_api_keys": { + "name": "auth_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unknown'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_id_created_at_idx": { + "name": "user_id_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "auth_api_keys_id": { + "name": "auth_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "comments": { + "name": "comments", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(6)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "timestamp": { + "name": "timestamp", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authorId": { + "name": "authorId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "parentCommentId": { + "name": "parentCommentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "analytics_reconciliation_idx": { + "name": "analytics_reconciliation_idx", + "columns": ["createdAt", "id"], + "isUnique": false + }, + "video_type_created_idx": { + "name": "video_type_created_idx", + "columns": ["videoId", "type", "createdAt", "id"], + "isUnique": false + }, + "author_id_idx": { + "name": "author_id_idx", + "columns": ["authorId"], + "isUnique": false + }, + "parent_comment_id_idx": { + "name": "parent_comment_id_idx", + "columns": ["parentCommentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "comments_id": { + "name": "comments_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_api_keys": { + "name": "developer_api_keys", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyType": { + "name": "keyType", + "type": "varchar(8)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyPrefix": { + "name": "keyPrefix", + "type": "varchar(12)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "keyHash": { + "name": "keyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedKey": { + "name": "encryptedKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "lastUsedAt": { + "name": "lastUsedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "revokedAt": { + "name": "revokedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "key_hash_idx": { + "name": "key_hash_idx", + "columns": ["keyHash"], + "isUnique": true + }, + "app_key_type_idx": { + "name": "app_key_type_idx", + "columns": ["appId", "keyType"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_api_keys_appId_developer_apps_id_fk": { + "name": "developer_api_keys_appId_developer_apps_id_fk", + "tableFrom": "developer_api_keys", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_api_keys_id": { + "name": "developer_api_keys_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_app_domains": { + "name": "developer_app_domains", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "domain": { + "name": "domain", + "type": "varchar(253)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_app_domains_appId_developer_apps_id_fk": { + "name": "developer_app_domains_appId_developer_apps_id_fk", + "tableFrom": "developer_app_domains", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_app_domains_id": { + "name": "developer_app_domains_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_domain_unique": { + "name": "app_domain_unique", + "columns": ["appId", "domain"] + } + }, + "checkConstraint": {} + }, + "developer_apps": { + "name": "developer_apps", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "environment": { + "name": "environment", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logoUrl": { + "name": "logoUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_deleted_idx": { + "name": "owner_deleted_idx", + "columns": ["ownerId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "developer_apps_id": { + "name": "developer_apps_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_accounts": { + "name": "developer_credit_accounts", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceMicroCredits": { + "name": "balanceMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripePaymentMethodId": { + "name": "stripePaymentMethodId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "autoTopUpEnabled": { + "name": "autoTopUpEnabled", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "autoTopUpThresholdMicroCredits": { + "name": "autoTopUpThresholdMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "autoTopUpAmountCents": { + "name": "autoTopUpAmountCents", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_id_unique": { + "name": "app_id_unique", + "columns": ["appId"], + "isUnique": true + } + }, + "foreignKeys": { + "developer_credit_accounts_appId_developer_apps_id_fk": { + "name": "developer_credit_accounts_appId_developer_apps_id_fk", + "tableFrom": "developer_credit_accounts", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_accounts_id": { + "name": "developer_credit_accounts_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_credit_transactions": { + "name": "developer_credit_transactions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accountId": { + "name": "accountId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amountMicroCredits": { + "name": "amountMicroCredits", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "balanceAfterMicroCredits": { + "name": "balanceAfterMicroCredits", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "referenceId": { + "name": "referenceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "referenceType": { + "name": "referenceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "account_type_created_idx": { + "name": "account_type_created_idx", + "columns": ["accountId", "type", "createdAt"], + "isUnique": false + }, + "account_ref_dedup_idx": { + "name": "account_ref_dedup_idx", + "columns": ["accountId", "referenceId", "referenceType"], + "isUnique": false + } + }, + "foreignKeys": { + "dev_credit_txn_account_fk": { + "name": "dev_credit_txn_account_fk", + "tableFrom": "developer_credit_transactions", + "tableTo": "developer_credit_accounts", + "columnsFrom": ["accountId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_credit_transactions_id": { + "name": "developer_credit_transactions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "developer_daily_storage_snapshots": { + "name": "developer_daily_storage_snapshots", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "snapshotDate": { + "name": "snapshotDate", + "type": "varchar(10)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "totalDurationMinutes": { + "name": "totalDurationMinutes", + "type": "float", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "videoCount": { + "name": "videoCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "microCreditsCharged": { + "name": "microCreditsCharged", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processedAt": { + "name": "processedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "developer_daily_storage_snapshots_appId_developer_apps_id_fk": { + "name": "developer_daily_storage_snapshots_appId_developer_apps_id_fk", + "tableFrom": "developer_daily_storage_snapshots", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_daily_storage_snapshots_id": { + "name": "developer_daily_storage_snapshots_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "app_date_unique": { + "name": "app_date_unique", + "columns": ["appId", "snapshotDate"] + } + }, + "checkConstraint": {} + }, + "developer_videos": { + "name": "developer_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "appId": { + "name": "appId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalUserId": { + "name": "externalUserId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Untitled'" + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "s3Key": { + "name": "s3Key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(16)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deletedAt": { + "name": "deletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "app_created_idx": { + "name": "app_created_idx", + "columns": ["appId", "createdAt"], + "isUnique": false + }, + "app_user_idx": { + "name": "app_user_idx", + "columns": ["appId", "externalUserId"], + "isUnique": false + }, + "app_deleted_idx": { + "name": "app_deleted_idx", + "columns": ["appId", "deletedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "developer_videos_appId_developer_apps_id_fk": { + "name": "developer_videos_appId_developer_apps_id_fk", + "tableFrom": "developer_videos", + "tableTo": "developer_apps", + "columnsFrom": ["appId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "developer_videos_id": { + "name": "developer_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "folders": { + "name": "folders", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "color": { + "name": "color", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'normal'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parentId": { + "name": "parentId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "parent_id_idx": { + "name": "parent_id_idx", + "columns": ["parentId"], + "isUnique": false + }, + "space_id_idx": { + "name": "space_id_idx", + "columns": ["spaceId"], + "isUnique": false + }, + "public_parent_id_idx": { + "name": "public_parent_id_idx", + "columns": ["public", "parentId"], + "isUnique": false + }, + "public_space_parent_id_idx": { + "name": "public_space_parent_id_idx", + "columns": ["public", "spaceId", "parentId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "folders_id": { + "name": "folders_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "imported_videos": { + "name": "imported_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "source_id": { + "name": "source_id", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "imported_videos_orgId_source_source_id_pk": { + "name": "imported_videos_orgId_source_source_id_pk", + "columns": ["orgId", "source", "source_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "integration_installations": { + "name": "integration_installations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "externalId": { + "name": "externalId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "installedByUserId": { + "name": "installedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "encryptedCredentials": { + "name": "encryptedCredentials", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "provider_external_id_idx": { + "name": "provider_external_id_idx", + "columns": ["provider", "externalId"], + "isUnique": true + }, + "organization_provider_display_name_idx": { + "name": "organization_provider_display_name_idx", + "columns": ["organizationId", "provider", "displayName"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "integration_installations_id": { + "name": "integration_installations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_conversations": { + "name": "messenger_conversations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent": { + "name": "agent", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mode": { + "name": "mode", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'agent'" + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverByUserId": { + "name": "takeoverByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "takeoverAt": { + "name": "takeoverAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "lastMessageAt": { + "name": "lastMessageAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "user_last_message_idx": { + "name": "user_last_message_idx", + "columns": ["userId", "lastMessageAt"], + "isUnique": false + }, + "anonymous_last_message_idx": { + "name": "anonymous_last_message_idx", + "columns": ["anonymousId", "lastMessageAt"], + "isUnique": false + }, + "mode_last_message_idx": { + "name": "mode_last_message_idx", + "columns": ["mode", "lastMessageAt"], + "isUnique": false + }, + "updated_at_idx": { + "name": "updated_at_idx", + "columns": ["updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "messenger_conversations_id": { + "name": "messenger_conversations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_messages": { + "name": "messenger_messages", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "conversation_created_at_idx": { + "name": "conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + }, + "role_created_at_idx": { + "name": "role_created_at_idx", + "columns": ["role", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "messenger_messages_conversationId_messenger_conversations_id_fk": { + "name": "messenger_messages_conversationId_messenger_conversations_id_fk", + "tableFrom": "messenger_messages", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_messages_id": { + "name": "messenger_messages_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "messenger_support_emails": { + "name": "messenger_support_emails", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "conversationId": { + "name": "conversationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userEmail": { + "name": "userEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "subject": { + "name": "subject", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "support_email_user_created_at_idx": { + "name": "support_email_user_created_at_idx", + "columns": ["userId", "createdAt"], + "isUnique": false + }, + "support_email_conversation_created_at_idx": { + "name": "support_email_conversation_created_at_idx", + "columns": ["conversationId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": { + "support_email_conversation_fk": { + "name": "support_email_conversation_fk", + "tableFrom": "messenger_support_emails", + "tableTo": "messenger_conversations", + "columnsFrom": ["conversationId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "messenger_support_emails_id": { + "name": "messenger_support_emails_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "notifications": { + "name": "notifications", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "recipientId": { + "name": "recipientId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "data": { + "name": "data", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(50)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "dedupKey": { + "name": "dedupKey", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "readAt": { + "name": "readAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "org_id_idx": { + "name": "org_id_idx", + "columns": ["orgId"], + "isUnique": false + }, + "type_idx": { + "name": "type_idx", + "columns": ["type"], + "isUnique": false + }, + "read_at_idx": { + "name": "read_at_idx", + "columns": ["readAt"], + "isUnique": false + }, + "created_at_idx": { + "name": "created_at_idx", + "columns": ["createdAt"], + "isUnique": false + }, + "recipient_read_idx": { + "name": "recipient_read_idx", + "columns": ["recipientId", "readAt"], + "isUnique": false + }, + "recipient_created_idx": { + "name": "recipient_created_idx", + "columns": ["recipientId", "createdAt"], + "isUnique": false + }, + "dedup_key_idx": { + "name": "dedup_key_idx", + "columns": ["dedupKey"], + "isUnique": true + }, + "type_recipient_created_idx": { + "name": "type_recipient_created_idx", + "columns": ["type", "recipientId", "createdAt"], + "isUnique": false + }, + "type_recipient_video_created_idx": { + "name": "type_recipient_video_created_idx", + "columns": ["type", "recipientId", "videoId", "createdAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "notifications_id": { + "name": "notifications_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_invites": { + "name": "organization_invites", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmail": { + "name": "invitedEmail", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "invitedEmailNormalized": { + "name": "invitedEmailNormalized", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "invitedByUserId": { + "name": "invitedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "emailDeliveryState": { + "name": "emailDeliveryState", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'legacy'" + }, + "emailDeliveryAttemptCount": { + "name": "emailDeliveryAttemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "emailDeliveryNextAttemptAt": { + "name": "emailDeliveryNextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryErrorCode": { + "name": "emailDeliveryErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryLeaseOwnerId": { + "name": "emailDeliveryLeaseOwnerId", + "type": "varchar(36)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailDeliveryLeaseExpiresAt": { + "name": "emailDeliveryLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailProviderMessageId": { + "name": "emailProviderMessageId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "emailSentAt": { + "name": "emailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "invited_email_idx": { + "name": "invited_email_idx", + "columns": ["invitedEmail"], + "isUnique": false + }, + "normalized_email_idx": { + "name": "normalized_email_idx", + "columns": ["organizationId", "invitedEmailNormalized"], + "isUnique": true + }, + "invited_by_user_id_idx": { + "name": "invited_by_user_id_idx", + "columns": ["invitedByUserId"], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": ["status"], + "isUnique": false + }, + "email_delivery_idx": { + "name": "email_delivery_idx", + "columns": ["emailDeliveryState", "emailDeliveryNextAttemptAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_invites_id": { + "name": "organization_invites_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organization_members": { + "name": "organization_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hasProSeat": { + "name": "hasProSeat", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "user_id_organization_id_idx": { + "name": "user_id_organization_id_idx", + "columns": ["userId", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organization_members_id": { + "name": "organization_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "organizations": { + "name": "organizations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tombstoneAt": { + "name": "tombstoneAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "allowedEmailDomain": { + "name": "allowedEmailDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customDomain": { + "name": "customDomain", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "domainVerified": { + "name": "domainVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "shareableLinkIconUrl": { + "name": "shareableLinkIconUrl", + "type": "varchar(1024)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "workosOrganizationId": { + "name": "workosOrganizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workosConnectionId": { + "name": "workosConnectionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "owner_id_tombstone_idx": { + "name": "owner_id_tombstone_idx", + "columns": ["ownerId", "tombstoneAt"], + "isUnique": false + }, + "custom_domain_idx": { + "name": "custom_domain_idx", + "columns": ["customDomain"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "organizations_id": { + "name": "organizations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_leases": { + "name": "product_analytics_erasure_leases", + "columns": { + "name": { + "name": "name", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "requestId": { + "name": "requestId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fencingToken": { + "name": "fencingToken", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "pausedPipes": { + "name": "pausedPipes", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_leases_name": { + "name": "product_analytics_erasure_leases_name", + "columns": ["name"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_erasure_requests": { + "name": "product_analytics_erasure_requests", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scopeHash": { + "name": "scopeHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "nextAttemptAt": { + "name": "nextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "leaseOwnerId": { + "name": "leaseOwnerId", + "type": "varchar(36)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "scope_hash_idx": { + "name": "scope_hash_idx", + "columns": ["scopeHash"], + "isUnique": true + }, + "queue_idx": { + "name": "queue_idx", + "columns": ["status", "nextAttemptAt", "createdAt"], + "isUnique": false + }, + "lease_idx": { + "name": "lease_idx", + "columns": ["leaseExpiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_erasure_requests_id": { + "name": "product_analytics_erasure_requests_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_event_receipts": { + "name": "product_analytics_event_receipts", + "columns": { + "eventIdHash": { + "name": "eventIdHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadHash": { + "name": "payloadHash", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anonymousIdentityHash": { + "name": "anonymousIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userIdentityHash": { + "name": "userIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationIdentityHash": { + "name": "organizationIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "conflictCount": { + "name": "conflictCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "firstSeenAt": { + "name": "firstSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastSeenAt": { + "name": "lastSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "retainUntil": { + "name": "retainUntil", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "anonymous_identity_idx": { + "name": "anonymous_identity_idx", + "columns": ["anonymousIdentityHash"], + "isUnique": false + }, + "user_identity_idx": { + "name": "user_identity_idx", + "columns": ["userIdentityHash"], + "isUnique": false + }, + "organization_identity_idx": { + "name": "organization_identity_idx", + "columns": ["organizationIdentityHash"], + "isUnique": false + }, + "retention_idx": { + "name": "retention_idx", + "columns": ["retainUntil"], + "isUnique": false + }, + "conflict_idx": { + "name": "conflict_idx", + "columns": ["conflictCount"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_event_receipts_eventIdHash": { + "name": "product_analytics_event_receipts_eventIdHash", + "columns": ["eventIdHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_identity_links": { + "name": "product_analytics_identity_links", + "columns": { + "anonymousIdentityHash": { + "name": "anonymousIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userIdentityHash": { + "name": "userIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationIdentityHash": { + "name": "organizationIdentityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_identity_idx": { + "name": "user_identity_idx", + "columns": ["userIdentityHash"], + "isUnique": false + }, + "organization_identity_idx": { + "name": "organization_identity_idx", + "columns": ["organizationIdentityHash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "identity_link_pk": { + "name": "identity_link_pk", + "columns": ["anonymousIdentityHash", "userIdentityHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_identity_state": { + "name": "product_analytics_identity_state", + "columns": { + "identityHash": { + "name": "identityHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "identityKind": { + "name": "identityKind", + "type": "varchar(16)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "blockedAt": { + "name": "blockedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "blocked_idx": { + "name": "blocked_idx", + "columns": ["blockedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_identity_state_identityHash": { + "name": "product_analytics_identity_state_identityHash", + "columns": ["identityHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_ingestion_leases": { + "name": "product_analytics_ingestion_leases", + "columns": { + "id": { + "name": "id", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "fencingToken": { + "name": "fencingToken", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expiresAt": { + "name": "expiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "expiry_idx": { + "name": "expiry_idx", + "columns": ["expiresAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_ingestion_leases_id": { + "name": "product_analytics_ingestion_leases_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_outbox": { + "name": "product_analytics_outbox", + "columns": { + "eventId": { + "name": "eventId", + "type": "varchar(128)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deliveryKey": { + "name": "deliveryKey", + "type": "varchar(36)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadHash": { + "name": "payloadHash", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "eventName": { + "name": "eventName", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payloadKind": { + "name": "payloadKind", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'product_event_row_v1'" + }, + "payload": { + "name": "payload", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "anonymousId": { + "name": "anonymousId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "nextAttemptAt": { + "name": "nextAttemptAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "leaseOwnerId": { + "name": "leaseOwnerId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workflowRunId": { + "name": "workflowRunId", + "type": "varchar(128)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payloadConflict": { + "name": "payloadConflict", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deliveredAt": { + "name": "deliveredAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deadLetteredAt": { + "name": "deadLetteredAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "delivery_key_idx": { + "name": "delivery_key_idx", + "columns": ["deliveryKey"], + "isUnique": true + }, + "delivery_idx": { + "name": "delivery_idx", + "columns": ["status", "nextAttemptAt", "createdAt"], + "isUnique": false + }, + "lease_idx": { + "name": "lease_idx", + "columns": ["leaseExpiresAt"], + "isUnique": false + }, + "retention_idx": { + "name": "retention_idx", + "columns": ["status", "deliveredAt"], + "isUnique": false + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "anonymous_id_idx": { + "name": "anonymous_id_idx", + "columns": ["anonymousId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_outbox_eventId": { + "name": "product_analytics_outbox_eventId", + "columns": ["eventId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_reconciliation_failures": { + "name": "product_analytics_reconciliation_failures", + "columns": { + "sourceHash": { + "name": "sourceHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceType": { + "name": "sourceType", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "errorCode": { + "name": "errorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "attemptCount": { + "name": "attemptCount", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "firstSeenAt": { + "name": "firstSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "lastSeenAt": { + "name": "lastSeenAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "source_type_idx": { + "name": "source_type_idx", + "columns": ["sourceType", "lastSeenAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_reconciliation_failures_sourceHash": { + "name": "product_analytics_reconciliation_failures_sourceHash", + "columns": ["sourceHash"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "product_analytics_refresh_leases": { + "name": "product_analytics_refresh_leases", + "columns": { + "name": { + "name": "name", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(36)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "generation": { + "name": "generation", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "sourceCutoff": { + "name": "sourceCutoff", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "leaseExpiresAt": { + "name": "leaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'idle'" + }, + "lastCompletedAt": { + "name": "lastCompletedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastErrorCode": { + "name": "lastErrorCode", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "expiry_idx": { + "name": "expiry_idx", + "columns": ["leaseExpiresAt"], + "isUnique": false + }, + "status_idx": { + "name": "status_idx", + "columns": ["status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "product_analytics_refresh_leases_name": { + "name": "product_analytics_refresh_leases_name", + "columns": ["name"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "s3_buckets": { + "name": "s3_buckets", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucketName": { + "name": "bucketName", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "accessKeyId": { + "name": "accessKeyId", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secretAccessKey": { + "name": "secretAccessKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('aws')" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_organization_idx": { + "name": "owner_organization_idx", + "columns": ["ownerId", "organizationId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "updatedAt"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "s3_buckets_id": { + "name": "s3_buckets_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "sessions": { + "name": "sessions", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sessionToken": { + "name": "sessionToken", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "session_token_idx": { + "name": "session_token_idx", + "columns": ["sessionToken"], + "isUnique": true + }, + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "sessions_id": { + "name": "sessions_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "shared_videos": { + "name": "shared_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedByUserId": { + "name": "sharedByUserId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sharedAt": { + "name": "sharedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "shared_by_user_id_idx": { + "name": "shared_by_user_id_idx", + "columns": ["sharedByUserId"], + "isUnique": false + }, + "video_id_organization_id_idx": { + "name": "video_id_organization_id_idx", + "columns": ["videoId", "organizationId"], + "isUnique": false + }, + "video_id_folder_id_idx": { + "name": "video_id_folder_id_idx", + "columns": ["videoId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "shared_videos_id": { + "name": "shared_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "space_members": { + "name": "space_members", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "userId": { + "name": "userId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "user_id_idx": { + "name": "user_id_idx", + "columns": ["userId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_members_id": { + "name": "space_members_id", + "columns": ["id"] + } + }, + "uniqueConstraints": { + "space_id_user_id_unique": { + "name": "space_id_user_id_unique", + "columns": ["spaceId", "userId"] + } + }, + "checkConstraint": {} + }, + "space_videos": { + "name": "space_videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "spaceId": { + "name": "spaceId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedById": { + "name": "addedById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "addedAt": { + "name": "addedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + } + }, + "indexes": { + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "added_by_id_idx": { + "name": "added_by_id_idx", + "columns": ["addedById"], + "isUnique": false + }, + "space_id_video_id_idx": { + "name": "space_id_video_id_idx", + "columns": ["spaceId", "videoId"], + "isUnique": false + }, + "space_id_folder_id_idx": { + "name": "space_id_folder_id_idx", + "columns": ["spaceId", "folderId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "space_videos_id": { + "name": "space_videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "spaces": { + "name": "spaces", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "primary": { + "name": "primary", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdById": { + "name": "createdById", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "iconUrl": { + "name": "iconUrl", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "varchar(1000)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "privacy": { + "name": "privacy", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'Private'" + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_id_idx": { + "name": "organization_id_idx", + "columns": ["organizationId"], + "isUnique": false + }, + "created_by_id_idx": { + "name": "created_by_id_idx", + "columns": ["createdById"], + "isUnique": false + }, + "public_organization_id_idx": { + "name": "public_organization_id_idx", + "columns": ["public", "organizationId"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "spaces_id": { + "name": "spaces_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_integrations": { + "name": "storage_integrations", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "organizationId": { + "name": "organizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "provider": { + "name": "provider", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "displayName": { + "name": "displayName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "active": { + "name": "active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "encryptedConfig": { + "name": "encryptedConfig", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "googleDriveAccessToken": { + "name": "googleDriveAccessToken", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveAccessTokenExpiresAt": { + "name": "googleDriveAccessTokenExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseId": { + "name": "googleDriveTokenRefreshLeaseId", + "type": "varchar(64)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveTokenRefreshLeaseExpiresAt": { + "name": "googleDriveTokenRefreshLeaseExpiresAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "googleDriveStorageQuotaCache": { + "name": "googleDriveStorageQuotaCache", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "owner_provider_idx": { + "name": "owner_provider_idx", + "columns": ["ownerId", "provider"], + "isUnique": false + }, + "owner_active_idx": { + "name": "owner_active_idx", + "columns": ["ownerId", "active"], + "isUnique": false + }, + "organization_provider_idx": { + "name": "organization_provider_idx", + "columns": ["organizationId", "provider"], + "isUnique": false + }, + "organization_active_idx": { + "name": "organization_active_idx", + "columns": ["organizationId", "active", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "storage_integrations_id": { + "name": "storage_integrations_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "storage_objects": { + "name": "storage_objects", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "integrationId": { + "name": "integrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "objectKey": { + "name": "objectKey", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "objectKeyHash": { + "name": "objectKeyHash", + "type": "varchar(64)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "providerObjectId": { + "name": "providerObjectId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploadSessionUrl": { + "name": "uploadSessionUrl", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "uploadStatus": { + "name": "uploadStatus", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "contentType": { + "name": "contentType", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "contentLength": { + "name": "contentLength", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": { + "integration_key_hash_idx": { + "name": "integration_key_hash_idx", + "columns": ["integrationId", "objectKeyHash"], + "isUnique": true + }, + "integration_status_idx": { + "name": "integration_status_idx", + "columns": ["integrationId", "uploadStatus"], + "isUnique": false + }, + "video_id_idx": { + "name": "video_id_idx", + "columns": ["videoId"], + "isUnique": false + }, + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + } + }, + "foreignKeys": { + "storage_objects_integrationId_storage_integrations_id_fk": { + "name": "storage_objects_integrationId_storage_integrations_id_fk", + "tableFrom": "storage_objects", + "tableTo": "storage_integrations", + "columnsFrom": ["integrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "storage_objects_id": { + "name": "storage_objects_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "users": { + "name": "users", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lastName": { + "name": "lastName", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "emailVerified": { + "name": "emailVerified", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "image": { + "name": "image", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeCustomerId": { + "name": "stripeCustomerId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionId": { + "name": "stripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "thirdPartyStripeSubscriptionId": { + "name": "thirdPartyStripeSubscriptionId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionStatus": { + "name": "stripeSubscriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stripeSubscriptionPriceId": { + "name": "stripeSubscriptionPriceId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "preferences": { + "name": "preferences", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": "('null')" + }, + "activeOrganizationId": { + "name": "activeOrganizationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "onboardingSteps": { + "name": "onboardingSteps", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "onboarding_completed_at": { + "name": "onboarding_completed_at", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "customBucket": { + "name": "customBucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "inviteQuota": { + "name": "inviteQuota", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "defaultOrgId": { + "name": "defaultOrgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "authSessionVersion": { + "name": "authSessionVersion", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + } + }, + "indexes": { + "email_idx": { + "name": "email_idx", + "columns": ["email"], + "isUnique": true + }, + "analytics_reconciliation_idx": { + "name": "analytics_reconciliation_idx", + "columns": ["created_at", "id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "users_id": { + "name": "users_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "verification_tokens": { + "name": "verification_tokens", + "columns": { + "identifier": { + "name": "identifier", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires": { + "name": "expires", + "type": "datetime", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "verification_tokens_identifier": { + "name": "verification_tokens_identifier", + "columns": ["identifier"] + } + }, + "uniqueConstraints": { + "verification_tokens_token_unique": { + "name": "verification_tokens_token_unique", + "columns": ["token"] + } + }, + "checkConstraint": {} + }, + "video_edits": { + "name": "video_edits", + "columns": { + "videoId": { + "name": "videoId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sourceKey": { + "name": "sourceKey", + "type": "varchar(512)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "editSpec": { + "name": "editSpec", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + } + }, + "indexes": {}, + "foreignKeys": { + "video_edits_videoId_videos_id_fk": { + "name": "video_edits_videoId_videos_id_fk", + "tableFrom": "video_edits", + "tableTo": "videos", + "columnsFrom": ["videoId"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "video_edits_videoId": { + "name": "video_edits_videoId", + "columns": ["videoId"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "video_uploads": { + "name": "video_uploads", + "columns": { + "video_id": { + "name": "video_id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "uploaded": { + "name": "uploaded", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total": { + "name": "total", + "type": "bigint unsigned", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "mode": { + "name": "mode", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "phase": { + "name": "phase", + "type": "varchar(32)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'uploading'" + }, + "processing_progress": { + "name": "processing_progress", + "type": "int", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "processing_message": { + "name": "processing_message", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "processing_error": { + "name": "processing_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw_file_key": { + "name": "raw_file_key", + "type": "varchar(512)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "phase_updated_at_video_id_idx": { + "name": "phase_updated_at_video_id_idx", + "columns": ["phase", "updated_at", "video_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "video_uploads_video_id": { + "name": "video_uploads_video_id", + "columns": ["video_id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + }, + "videos": { + "name": "videos", + "columns": { + "id": { + "name": "id", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ownerId": { + "name": "ownerId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "orgId": { + "name": "orgId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "varchar(255)", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'My Video'" + }, + "bucket": { + "name": "bucket", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "storageIntegrationId": { + "name": "storageIntegrationId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration": { + "name": "duration", + "type": "float", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "width": { + "name": "width", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "height": { + "name": "height", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fps": { + "name": "fps", + "type": "int", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "settings": { + "name": "settings", + "type": "json", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "transcriptionStatus": { + "name": "transcriptionStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "json", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "('{\"type\":\"MediaConvert\"}')" + }, + "folderId": { + "name": "folderId", + "type": "varchar(15)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "createdAt": { + "name": "createdAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(now())" + }, + "effectiveCreatedAt": { + "name": "effectiveCreatedAt", + "type": "datetime", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "generated": { + "as": "COALESCE(\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%s.%fZ'),\n STR_TO_DATE(JSON_UNQUOTE(JSON_EXTRACT(`metadata`, '$.customCreatedAt')), '%Y-%m-%dT%H:%i:%sZ'),\n `createdAt`\n )", + "type": "stored" + } + }, + "updatedAt": { + "name": "updatedAt", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "onUpdate": true, + "default": "(now())" + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "xStreamInfo": { + "name": "xStreamInfo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstExternalViewAt": { + "name": "firstExternalViewAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "firstViewEmailSentAt": { + "name": "firstViewEmailSentAt", + "type": "timestamp", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "isScreenshot": { + "name": "isScreenshot", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "awsRegion": { + "name": "awsRegion", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "awsBucket": { + "name": "awsBucket", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "videoStartTime": { + "name": "videoStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "audioStartTime": { + "name": "audioStartTime", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobId": { + "name": "jobId", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "jobStatus": { + "name": "jobStatus", + "type": "varchar(255)", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "skipProcessing": { + "name": "skipProcessing", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "owner_id_idx": { + "name": "owner_id_idx", + "columns": ["ownerId"], + "isUnique": false + }, + "is_public_idx": { + "name": "is_public_idx", + "columns": ["public"], + "isUnique": false + }, + "folder_id_idx": { + "name": "folder_id_idx", + "columns": ["folderId"], + "isUnique": false + }, + "storage_integration_id_idx": { + "name": "storage_integration_id_idx", + "columns": ["storageIntegrationId"], + "isUnique": false + }, + "first_external_view_at_idx": { + "name": "first_external_view_at_idx", + "columns": ["firstExternalViewAt"], + "isUnique": false + }, + "analytics_created_at_idx": { + "name": "analytics_created_at_idx", + "columns": ["createdAt", "id"], + "isUnique": false + }, + "analytics_first_view_at_idx": { + "name": "analytics_first_view_at_idx", + "columns": ["firstExternalViewAt", "id"], + "isUnique": false + }, + "org_owner_folder_idx": { + "name": "org_owner_folder_idx", + "columns": ["orgId", "ownerId", "folderId"], + "isUnique": false + }, + "org_effective_created_idx": { + "name": "org_effective_created_idx", + "columns": ["orgId", "effectiveCreatedAt"], + "isUnique": false + } + }, + "foreignKeys": { + "videos_storageIntegrationId_storage_integrations_id_fk": { + "name": "videos_storageIntegrationId_storage_integrations_id_fk", + "tableFrom": "videos", + "tableTo": "storage_integrations", + "columnsFrom": ["storageIntegrationId"], + "columnsTo": ["id"], + "onDelete": "restrict", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "videos_id": { + "name": "videos_id", + "columns": ["id"] + } + }, + "uniqueConstraints": {}, + "checkConstraint": {} + } + }, + "views": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "tables": {}, + "indexes": {} + } +} diff --git a/packages/database/migrations/meta/_journal.json b/packages/database/migrations/meta/_journal.json index 60232fd0eb6..f5eba8111c6 100644 --- a/packages/database/migrations/meta/_journal.json +++ b/packages/database/migrations/meta/_journal.json @@ -1,286 +1,307 @@ { - "version": "5", - "dialect": "mysql", - "entries": [ - { - "idx": 0, - "version": "5", - "when": 1743020179593, - "tag": "0000_brown_sunfire", - "breakpoints": true - }, - { - "idx": 1, - "version": "5", - "when": 1749268354138, - "tag": "0001_white_young_avengers", - "breakpoints": true - }, - { - "idx": 2, - "version": "5", - "when": 1750935538683, - "tag": "0002_dusty_maginty", - "breakpoints": true - }, - { - "idx": 3, - "version": "5", - "when": 1761710697286, - "tag": "0003_thin_gressill", - "breakpoints": true - }, - { - "idx": 4, - "version": "5", - "when": 1761711378574, - "tag": "0004_video-org-id", - "breakpoints": true - }, - { - "idx": 5, - "version": "5", - "when": 1761711605408, - "tag": "0005_video-org-id-required", - "breakpoints": true - }, - { - "idx": 6, - "version": "5", - "when": 1762410865419, - "tag": "0006_hesitant_stone_men", - "breakpoints": true - }, - { - "idx": 7, - "version": "5", - "when": 1762428551824, - "tag": "0007_public_toxin", - "breakpoints": true - }, - { - "idx": 8, - "version": "5", - "when": 1762428905323, - "tag": "0008_fat_ender_wiggin", - "breakpoints": true - }, - { - "idx": 9, - "version": "5", - "when": 1768139432782, - "tag": "0009_easy_ulik", - "breakpoints": true - }, - { - "idx": 10, - "version": "5", - "when": 1738505600000, - "tag": "0010_video_uploads_bigint", - "breakpoints": true - }, - { - "idx": 11, - "version": "5", - "when": 1771325011010, - "tag": "0011_public_inhumans", - "breakpoints": true - }, - { - "idx": 12, - "version": "5", - "when": 1772534950328, - "tag": "0012_lethal_lilith", - "breakpoints": true - }, - { - "idx": 13, - "version": "5", - "when": 1772573402457, - "tag": "0013_faithful_marvex", - "breakpoints": true - }, - { - "idx": 14, - "version": "5", - "when": 1772576220593, - "tag": "0014_modern_triton", - "breakpoints": true - }, - { - "idx": 15, - "version": "5", - "when": 1772582468257, - "tag": "0015_closed_tigra", - "breakpoints": true - }, - { - "idx": 16, - "version": "5", - "when": 1772641549840, - "tag": "0016_bouncy_hellcat", - "breakpoints": true - }, - { - "idx": 17, - "version": "5", - "when": 1778099532336, - "tag": "0017_productive_betty_brant", - "breakpoints": true - }, - { - "idx": 18, - "version": "5", - "when": 1778153157657, - "tag": "0018_loud_mongu", - "breakpoints": true - }, - { - "idx": 19, - "version": "5", - "when": 1778154209547, - "tag": "0019_solid_paibok", - "breakpoints": true - }, - { - "idx": 20, - "version": "5", - "when": 1778157016497, - "tag": "0020_orange_talkback", - "breakpoints": true - }, - { - "idx": 21, - "version": "5", - "when": 1778249922872, - "tag": "0021_glorious_khan", - "breakpoints": true - }, - { - "idx": 22, - "version": "5", - "when": 1778252207674, - "tag": "0022_dazzling_namor", - "breakpoints": true - }, - { - "idx": 23, - "version": "5", - "when": 1778434170430, - "tag": "0023_misty_luckman", - "breakpoints": true - }, - { - "idx": 24, - "version": "5", - "when": 1778776694053, - "tag": "0024_many_speed", - "breakpoints": true - }, - { - "idx": 25, - "version": "5", - "when": 1778858762935, - "tag": "0025_demonic_mother_askani", - "breakpoints": true - }, - { - "idx": 26, - "version": "5", - "when": 1779283712737, - "tag": "0026_pretty_the_professor", - "breakpoints": true - }, - { - "idx": 27, - "version": "5", - "when": 1780932760351, - "tag": "0027_giant_shinko_yamashiro", - "breakpoints": true - }, - { - "idx": 28, - "version": "5", - "when": 1780937790068, - "tag": "0028_nebulous_madame_masque", - "breakpoints": true - }, - { - "idx": 29, - "version": "5", - "when": 1780937848351, - "tag": "0029_blushing_pretty_boy", - "breakpoints": true - }, - { - "idx": 30, - "version": "5", - "when": 1780940100362, - "tag": "0030_add_folder_settings", - "breakpoints": true - }, - { - "idx": 31, - "version": "5", - "when": 1781271269353, - "tag": "0031_add_auth_api_keys_user_created_index", - "breakpoints": true - }, - { - "idx": 32, - "version": "5", - "when": 1782850635482, - "tag": "0032_past_lester", - "breakpoints": true - }, - { - "idx": 33, - "version": "5", - "when": 1782853232527, - "tag": "0033_fluffy_gamora", - "breakpoints": true - }, - { - "idx": 34, - "version": "5", - "when": 1782854064505, - "tag": "0034_fluffy_falcon", - "breakpoints": true - }, - { - "idx": 35, - "version": "5", - "when": 1784393810560, - "tag": "0035_agent_api", - "breakpoints": true - }, - { - "idx": 36, - "version": "5", - "when": 1784402487715, - "tag": "0036_premium_master_chief", - "breakpoints": true - }, - { - "idx": 37, - "version": "5", - "when": 1785452585612, - "tag": "0037_integration_installations", - "breakpoints": true - }, - { - "idx": 38, - "version": "5", - "when": 1785568617812, - "tag": "0038_messy_stephen_strange", - "breakpoints": true - }, - { - "idx": 39, - "version": "5", - "when": 1785569502530, - "tag": "0039_bumpy_phil_sheldon", - "breakpoints": true - } - ] -} \ No newline at end of file + "version": "5", + "dialect": "mysql", + "entries": [ + { + "idx": 0, + "version": "5", + "when": 1743020179593, + "tag": "0000_brown_sunfire", + "breakpoints": true + }, + { + "idx": 1, + "version": "5", + "when": 1749268354138, + "tag": "0001_white_young_avengers", + "breakpoints": true + }, + { + "idx": 2, + "version": "5", + "when": 1750935538683, + "tag": "0002_dusty_maginty", + "breakpoints": true + }, + { + "idx": 3, + "version": "5", + "when": 1761710697286, + "tag": "0003_thin_gressill", + "breakpoints": true + }, + { + "idx": 4, + "version": "5", + "when": 1761711378574, + "tag": "0004_video-org-id", + "breakpoints": true + }, + { + "idx": 5, + "version": "5", + "when": 1761711605408, + "tag": "0005_video-org-id-required", + "breakpoints": true + }, + { + "idx": 6, + "version": "5", + "when": 1762410865419, + "tag": "0006_hesitant_stone_men", + "breakpoints": true + }, + { + "idx": 7, + "version": "5", + "when": 1762428551824, + "tag": "0007_public_toxin", + "breakpoints": true + }, + { + "idx": 8, + "version": "5", + "when": 1762428905323, + "tag": "0008_fat_ender_wiggin", + "breakpoints": true + }, + { + "idx": 9, + "version": "5", + "when": 1768139432782, + "tag": "0009_easy_ulik", + "breakpoints": true + }, + { + "idx": 10, + "version": "5", + "when": 1738505600000, + "tag": "0010_video_uploads_bigint", + "breakpoints": true + }, + { + "idx": 11, + "version": "5", + "when": 1771325011010, + "tag": "0011_public_inhumans", + "breakpoints": true + }, + { + "idx": 12, + "version": "5", + "when": 1772534950328, + "tag": "0012_lethal_lilith", + "breakpoints": true + }, + { + "idx": 13, + "version": "5", + "when": 1772573402457, + "tag": "0013_faithful_marvex", + "breakpoints": true + }, + { + "idx": 14, + "version": "5", + "when": 1772576220593, + "tag": "0014_modern_triton", + "breakpoints": true + }, + { + "idx": 15, + "version": "5", + "when": 1772582468257, + "tag": "0015_closed_tigra", + "breakpoints": true + }, + { + "idx": 16, + "version": "5", + "when": 1772641549840, + "tag": "0016_bouncy_hellcat", + "breakpoints": true + }, + { + "idx": 17, + "version": "5", + "when": 1778099532336, + "tag": "0017_productive_betty_brant", + "breakpoints": true + }, + { + "idx": 18, + "version": "5", + "when": 1778153157657, + "tag": "0018_loud_mongu", + "breakpoints": true + }, + { + "idx": 19, + "version": "5", + "when": 1778154209547, + "tag": "0019_solid_paibok", + "breakpoints": true + }, + { + "idx": 20, + "version": "5", + "when": 1778157016497, + "tag": "0020_orange_talkback", + "breakpoints": true + }, + { + "idx": 21, + "version": "5", + "when": 1778249922872, + "tag": "0021_glorious_khan", + "breakpoints": true + }, + { + "idx": 22, + "version": "5", + "when": 1778252207674, + "tag": "0022_dazzling_namor", + "breakpoints": true + }, + { + "idx": 23, + "version": "5", + "when": 1778434170430, + "tag": "0023_misty_luckman", + "breakpoints": true + }, + { + "idx": 24, + "version": "5", + "when": 1778776694053, + "tag": "0024_many_speed", + "breakpoints": true + }, + { + "idx": 25, + "version": "5", + "when": 1778858762935, + "tag": "0025_demonic_mother_askani", + "breakpoints": true + }, + { + "idx": 26, + "version": "5", + "when": 1779283712737, + "tag": "0026_pretty_the_professor", + "breakpoints": true + }, + { + "idx": 27, + "version": "5", + "when": 1780932760351, + "tag": "0027_giant_shinko_yamashiro", + "breakpoints": true + }, + { + "idx": 28, + "version": "5", + "when": 1780937790068, + "tag": "0028_nebulous_madame_masque", + "breakpoints": true + }, + { + "idx": 29, + "version": "5", + "when": 1780937848351, + "tag": "0029_blushing_pretty_boy", + "breakpoints": true + }, + { + "idx": 30, + "version": "5", + "when": 1780940100362, + "tag": "0030_add_folder_settings", + "breakpoints": true + }, + { + "idx": 31, + "version": "5", + "when": 1781271269353, + "tag": "0031_add_auth_api_keys_user_created_index", + "breakpoints": true + }, + { + "idx": 32, + "version": "5", + "when": 1782850635482, + "tag": "0032_past_lester", + "breakpoints": true + }, + { + "idx": 33, + "version": "5", + "when": 1782853232527, + "tag": "0033_fluffy_gamora", + "breakpoints": true + }, + { + "idx": 34, + "version": "5", + "when": 1782854064505, + "tag": "0034_fluffy_falcon", + "breakpoints": true + }, + { + "idx": 35, + "version": "5", + "when": 1784393810560, + "tag": "0035_agent_api", + "breakpoints": true + }, + { + "idx": 36, + "version": "5", + "when": 1784402487715, + "tag": "0036_premium_master_chief", + "breakpoints": true + }, + { + "idx": 37, + "version": "5", + "when": 1785452585612, + "tag": "0037_integration_installations", + "breakpoints": true + }, + { + "idx": 38, + "version": "5", + "when": 1785568617812, + "tag": "0038_messy_stephen_strange", + "breakpoints": true + }, + { + "idx": 39, + "version": "5", + "when": 1785569502530, + "tag": "0039_bumpy_phil_sheldon", + "breakpoints": true + }, + { + "idx": 40, + "version": "5", + "when": 1785580245023, + "tag": "0040_chief_skreet", + "breakpoints": true + }, + { + "idx": 41, + "version": "5", + "when": 1785582713497, + "tag": "0041_complex_outlaw_kid", + "breakpoints": true + }, + { + "idx": 42, + "version": "5", + "when": 1785585973508, + "tag": "0042_lying_sharon_ventura", + "breakpoints": true + } + ] +} diff --git a/packages/database/schema.ts b/packages/database/schema.ts index 6996889e80f..028931fa340 100644 --- a/packages/database/schema.ts +++ b/packages/database/schema.ts @@ -131,6 +131,10 @@ export const users = mysqlTable( }, (table) => ({ emailIndex: uniqueIndex("email_idx").on(table.email), + analyticsReconciliationIndex: index("analytics_reconciliation_idx").on( + table.created_at, + table.id, + ), }), ); @@ -209,6 +213,206 @@ export const productAnalyticsErasureLeases = mysqlTable( }, ); +export const productAnalyticsOutbox = mysqlTable( + "product_analytics_outbox", + { + eventId: varchar("eventId", { length: 128 }).notNull().primaryKey(), + deliveryKey: varchar("deliveryKey", { length: 36 }).notNull(), + payloadHash: varchar("payloadHash", { length: 32 }).notNull(), + eventName: varchar("eventName", { length: 64 }).notNull(), + payloadKind: varchar("payloadKind", { length: 32 }) + .notNull() + .default("product_event_row_v1"), + payload: json("payload").$type().notNull(), + anonymousId: varchar("anonymousId", { length: 255 }), + userId: varchar("userId", { length: 255 }), + organizationId: varchar("organizationId", { length: 255 }), + status: varchar("status", { length: 32 }).notNull().default("pending"), + attemptCount: int("attemptCount").notNull().default(0), + nextAttemptAt: timestamp("nextAttemptAt").notNull().defaultNow(), + leaseOwnerId: varchar("leaseOwnerId", { length: 64 }), + leaseExpiresAt: timestamp("leaseExpiresAt"), + workflowRunId: varchar("workflowRunId", { length: 128 }), + payloadConflict: boolean("payloadConflict").notNull().default(false), + lastErrorCode: varchar("lastErrorCode", { length: 64 }), + deliveredAt: timestamp("deliveredAt"), + deadLetteredAt: timestamp("deadLetteredAt"), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow().onUpdateNow(), + }, + (table) => ({ + deliveryKeyIndex: uniqueIndex("delivery_key_idx").on(table.deliveryKey), + deliveryIndex: index("delivery_idx").on( + table.status, + table.nextAttemptAt, + table.createdAt, + ), + leaseIndex: index("lease_idx").on(table.leaseExpiresAt), + retentionIndex: index("retention_idx").on(table.status, table.deliveredAt), + userIdIndex: index("user_id_idx").on(table.userId), + organizationIdIndex: index("organization_id_idx").on(table.organizationId), + anonymousIdIndex: index("anonymous_id_idx").on(table.anonymousId), + }), +); + +export const productAnalyticsIdentityState = mysqlTable( + "product_analytics_identity_state", + { + identityHash: varchar("identityHash", { length: 64 }) + .notNull() + .primaryKey(), + identityKind: varchar("identityKind", { length: 16 }) + .notNull() + .$type<"anonymous" | "organization" | "user">(), + blockedAt: timestamp("blockedAt"), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow().onUpdateNow(), + }, + (table) => ({ + blockedIndex: index("blocked_idx").on(table.blockedAt), + }), +); + +export const productAnalyticsIdentityLinks = mysqlTable( + "product_analytics_identity_links", + { + anonymousIdentityHash: varchar("anonymousIdentityHash", { + length: 64, + }).notNull(), + userIdentityHash: varchar("userIdentityHash", { length: 64 }).notNull(), + organizationIdentityHash: varchar("organizationIdentityHash", { + length: 64, + }), + anonymousId: varchar("anonymousId", { length: 255 }).notNull(), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow().onUpdateNow(), + }, + (table) => ({ + primary: primaryKey({ + name: "identity_link_pk", + columns: [table.anonymousIdentityHash, table.userIdentityHash], + }), + userIndex: index("user_identity_idx").on(table.userIdentityHash), + organizationIndex: index("organization_identity_idx").on( + table.organizationIdentityHash, + ), + }), +); + +export const productAnalyticsEventReceipts = mysqlTable( + "product_analytics_event_receipts", + { + eventIdHash: varchar("eventIdHash", { length: 64 }).notNull().primaryKey(), + payloadHash: varchar("payloadHash", { length: 32 }).notNull(), + anonymousIdentityHash: varchar("anonymousIdentityHash", { length: 64 }), + userIdentityHash: varchar("userIdentityHash", { length: 64 }), + organizationIdentityHash: varchar("organizationIdentityHash", { + length: 64, + }), + conflictCount: int("conflictCount").notNull().default(0), + firstSeenAt: timestamp("firstSeenAt").notNull().defaultNow(), + lastSeenAt: timestamp("lastSeenAt").notNull().defaultNow().onUpdateNow(), + retainUntil: timestamp("retainUntil").notNull(), + }, + (table) => ({ + anonymousIdentityIndex: index("anonymous_identity_idx").on( + table.anonymousIdentityHash, + ), + userIdentityIndex: index("user_identity_idx").on(table.userIdentityHash), + organizationIdentityIndex: index("organization_identity_idx").on( + table.organizationIdentityHash, + ), + retentionIndex: index("retention_idx").on(table.retainUntil), + conflictIndex: index("conflict_idx").on(table.conflictCount), + }), +); + +export const productAnalyticsIngestionLeases = mysqlTable( + "product_analytics_ingestion_leases", + { + id: varchar("id", { length: 36 }).notNull().primaryKey(), + fencingToken: bigint("fencingToken", { + mode: "number", + unsigned: true, + }).notNull(), + expiresAt: timestamp("expiresAt").notNull(), + createdAt: timestamp("createdAt").notNull().defaultNow(), + }, + (table) => ({ + expiryIndex: index("expiry_idx").on(table.expiresAt), + }), +); + +export const productAnalyticsRefreshLeases = mysqlTable( + "product_analytics_refresh_leases", + { + name: varchar("name", { length: 64 }).notNull().primaryKey(), + ownerId: varchar("ownerId", { length: 36 }), + generation: bigint("generation", { mode: "number", unsigned: true }) + .notNull() + .default(0), + sourceCutoff: timestamp("sourceCutoff"), + leaseExpiresAt: timestamp("leaseExpiresAt"), + status: varchar("status", { length: 32 }).notNull().default("idle"), + lastCompletedAt: timestamp("lastCompletedAt"), + lastErrorCode: varchar("lastErrorCode", { length: 64 }), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow().onUpdateNow(), + }, + (table) => ({ + expiryIndex: index("expiry_idx").on(table.leaseExpiresAt), + statusIndex: index("status_idx").on(table.status), + }), +); + +export const productAnalyticsReconciliationFailures = mysqlTable( + "product_analytics_reconciliation_failures", + { + sourceHash: varchar("sourceHash", { length: 64 }).notNull().primaryKey(), + sourceType: varchar("sourceType", { length: 32 }).notNull(), + errorCode: varchar("errorCode", { length: 64 }).notNull(), + attemptCount: int("attemptCount").notNull().default(1), + firstSeenAt: timestamp("firstSeenAt").notNull().defaultNow(), + lastSeenAt: timestamp("lastSeenAt").notNull().defaultNow().onUpdateNow(), + }, + (table) => ({ + sourceTypeIndex: index("source_type_idx").on( + table.sourceType, + table.lastSeenAt, + ), + }), +); + +export const productAnalyticsErasureRequests = mysqlTable( + "product_analytics_erasure_requests", + { + id: varchar("id", { length: 36 }).notNull().primaryKey(), + scopeHash: varchar("scopeHash", { length: 64 }).notNull(), + userId: varchar("userId", { length: 255 }), + organizationId: varchar("organizationId", { length: 255 }), + status: varchar("status", { length: 32 }) + .notNull() + .default("pending") + .$type<"dead_letter" | "pending" | "processing">(), + attemptCount: int("attemptCount").notNull().default(0), + nextAttemptAt: timestamp("nextAttemptAt").notNull().defaultNow(), + leaseOwnerId: varchar("leaseOwnerId", { length: 36 }), + leaseExpiresAt: timestamp("leaseExpiresAt"), + lastErrorCode: varchar("lastErrorCode", { length: 64 }), + createdAt: timestamp("createdAt").notNull().defaultNow(), + updatedAt: timestamp("updatedAt").notNull().defaultNow().onUpdateNow(), + }, + (table) => ({ + scopeIndex: uniqueIndex("scope_hash_idx").on(table.scopeHash), + queueIndex: index("queue_idx").on( + table.status, + table.nextAttemptAt, + table.createdAt, + ), + leaseIndex: index("lease_idx").on(table.leaseExpiresAt), + }), +); + export const organizations = mysqlTable( "organizations", { @@ -317,11 +521,27 @@ export const organizationInvites = mysqlTable( .notNull() .$type(), invitedEmail: varchar("invitedEmail", { length: 255 }).notNull(), + invitedEmailNormalized: varchar("invitedEmailNormalized", { length: 255 }), invitedByUserId: nanoId("invitedByUserId").notNull().$type(), role: varchar("role", { length: 255 }) .notNull() .$type(), status: varchar("status", { length: 255 }).notNull().default("pending"), + emailDeliveryState: varchar("emailDeliveryState", { length: 32 }) + .notNull() + .default("legacy") + .$type<"dead_letter" | "legacy" | "pending" | "sent">(), + emailDeliveryAttemptCount: int("emailDeliveryAttemptCount") + .notNull() + .default(0), + emailDeliveryNextAttemptAt: timestamp("emailDeliveryNextAttemptAt"), + emailDeliveryErrorCode: varchar("emailDeliveryErrorCode", { length: 64 }), + emailDeliveryLeaseOwnerId: varchar("emailDeliveryLeaseOwnerId", { + length: 36, + }), + emailDeliveryLeaseExpiresAt: timestamp("emailDeliveryLeaseExpiresAt"), + emailProviderMessageId: varchar("emailProviderMessageId", { length: 255 }), + emailSentAt: timestamp("emailSentAt"), createdAt: timestamp("createdAt").notNull().defaultNow(), updatedAt: timestamp("updatedAt").notNull().defaultNow().onUpdateNow(), expiresAt: timestamp("expiresAt"), @@ -329,10 +549,18 @@ export const organizationInvites = mysqlTable( (table) => ({ organizationIdIndex: index("organization_id_idx").on(table.organizationId), invitedEmailIndex: index("invited_email_idx").on(table.invitedEmail), + normalizedEmailIndex: uniqueIndex("normalized_email_idx").on( + table.organizationId, + table.invitedEmailNormalized, + ), invitedByUserIdIndex: index("invited_by_user_id_idx").on( table.invitedByUserId, ), statusIndex: index("status_idx").on(table.status), + emailDeliveryIndex: index("email_delivery_idx").on( + table.emailDeliveryState, + table.emailDeliveryNextAttemptAt, + ), }), ); @@ -449,6 +677,11 @@ export const videos = mysqlTable( index("folder_id_idx").on(table.folderId), index("storage_integration_id_idx").on(table.storageIntegrationId), index("first_external_view_at_idx").on(table.firstExternalViewAt), + index("analytics_created_at_idx").on(table.createdAt, table.id), + index("analytics_first_view_at_idx").on( + table.firstExternalViewAt, + table.id, + ), index("org_owner_folder_idx").on( table.orgId, table.ownerId, @@ -517,6 +750,10 @@ export const comments = mysqlTable( nanoIdNullable("parentCommentId").$type(), }, (table) => ({ + analyticsReconciliationIndex: index("analytics_reconciliation_idx").on( + table.createdAt, + table.id, + ), videoTypeCreatedIndex: index("video_type_created_idx").on( table.videoId, table.type, diff --git a/packages/database/types/metadata.ts b/packages/database/types/metadata.ts index 86a5e501066..f70c3636a7f 100644 --- a/packages/database/types/metadata.ts +++ b/packages/database/types/metadata.ts @@ -38,7 +38,10 @@ export interface VideoMetadata { agentUpload?: { state: "pending" | "accepted" | "rejected"; rawFileKey?: string; + acceptedAt?: string; + rejectedAt?: string; }; + initiatingPlatform?: "cli" | "desktop" | "mobile" | "server" | "web"; } export type VideoEditRange = { diff --git a/packages/web-backend/src/Organisations/index.ts b/packages/web-backend/src/Organisations/index.ts index d7cb6b236c3..013484767a3 100644 --- a/packages/web-backend/src/Organisations/index.ts +++ b/packages/web-backend/src/Organisations/index.ts @@ -94,16 +94,26 @@ export class Organisations extends Effect.Service()( ); if (!organisation.tombstoneAt) { - yield* db.use((db) => - db - .update(Db.organizations) - .set({ tombstoneAt: new Date() }) - .where( - Dz.and( - Dz.eq(Db.organizations.id, id), - Dz.isNull(Db.organizations.tombstoneAt), - ), - ), + yield* db.use((database) => + database.transaction(async (tx) => { + await tx + .update(Db.organizations) + .set({ tombstoneAt: new Date() }) + .where( + Dz.and( + Dz.eq(Db.organizations.id, id), + Dz.isNull(Db.organizations.tombstoneAt), + ), + ); + await tx + .delete(Db.organizationInvites) + .where( + Dz.and( + Dz.eq(Db.organizationInvites.organizationId, id), + Dz.eq(Db.organizationInvites.status, "pending"), + ), + ); + }), ); } yield* Effect.sleep(PRODUCT_ANALYTICS_DELIVERY_DRAIN_MS); diff --git a/packages/web-backend/src/ProductAnalytics/index.ts b/packages/web-backend/src/ProductAnalytics/index.ts index bc41a466219..9395b7c4986 100644 --- a/packages/web-backend/src/ProductAnalytics/index.ts +++ b/packages/web-backend/src/ProductAnalytics/index.ts @@ -1,11 +1,15 @@ +import { randomUUID } from "node:crypto"; import { PRODUCT_ANALYTICS_ACCOUNT_DELETION_PENDING_SUBJECT, ProductAnalyticsError, type ProductEventRow, + productAnalyticsEventIdHash, + productAnalyticsIdentityHash, sendProductAnalyticsRows, } from "@cap/analytics"; import * as Db from "@cap/database/schema"; import { serverEnv } from "@cap/env"; +import type { DatabaseError } from "@cap/web-domain"; import { HttpServerRequest } from "@effect/platform"; import * as Dz from "drizzle-orm"; import { Effect, Option, Schema } from "effect"; @@ -22,8 +26,53 @@ export interface ProductAnalyticsActor { organizationId: string; } +export type { ProductAnalyticsIdentityKind } from "@cap/analytics"; +export { + productAnalyticsEventIdHash, + productAnalyticsIdentityHash, +} from "@cap/analytics"; + +function productAnalyticsRowIdentities(rows: readonly ProductEventRow[]) { + return [ + ...new Map( + rows + .flatMap((row) => [ + row.anonymous_id + ? { + identityHash: productAnalyticsIdentityHash( + "anonymous", + row.anonymous_id, + ), + identityKind: "anonymous" as const, + } + : undefined, + row.user_id + ? { + identityHash: productAnalyticsIdentityHash("user", row.user_id), + identityKind: "user" as const, + } + : undefined, + row.organization_id + ? { + identityHash: productAnalyticsIdentityHash( + "organization", + row.organization_id, + ), + identityKind: "organization" as const, + } + : undefined, + ]) + .filter((identity) => identity !== undefined) + .map((identity) => [identity.identityHash, identity]), + ).values(), + ].sort((left, right) => left.identityHash.localeCompare(right.identityHash)); +} + type VercelEnvironment = "production" | "preview" | "development"; +const PRODUCT_ANALYTICS_INGESTION_LEASE_MS = 5 * 60 * 1_000; +const PRODUCT_ANALYTICS_RECEIPT_RETENTION_MS = 800 * 24 * 60 * 60 * 1_000; + interface ProductAnalyticsServiceOptions { host?: string; token?: string; @@ -170,9 +219,10 @@ export const resolveProductAnalyticsActor = Effect.gen(function* () { export class ProductAnalytics extends Effect.Service()( "ProductAnalytics", { - effect: Effect.sync(() => { + effect: Effect.gen(function* () { + const database = yield* Database; const env = serverEnv(); - return createProductAnalyticsService({ + const service = createProductAnalyticsService({ host: env.PRODUCT_ANALYTICS_TINYBIRD_HOST, token: env.PRODUCT_ANALYTICS_TINYBIRD_TOKEN, required: isOfficialProductAnalyticsDeployment({ @@ -180,6 +230,241 @@ export class ProductAnalytics extends Effect.Service()( vercelEnvironment: env.VERCEL_ENV, }), }); + const appendWithIdentityFence = ( + rows: readonly ProductEventRow[], + ): Effect.Effect => { + const identities = productAnalyticsRowIdentities(rows); + return database + .use(async (db) => { + await db + .insert(Db.productAnalyticsErasureLeases) + .values({ name: "global" }) + .onDuplicateKeyUpdate({ set: { name: "global" } }); + const [fence] = await db + .select({ + fencingToken: Db.productAnalyticsErasureLeases.fencingToken, + phase: Db.productAnalyticsErasureLeases.phase, + }) + .from(Db.productAnalyticsErasureLeases) + .where(Dz.eq(Db.productAnalyticsErasureLeases.name, "global")) + .limit(1); + if (!fence || fence.phase !== "idle") { + throw new Error("Product analytics erasure is in progress"); + } + const leaseId = randomUUID(); + await db.insert(Db.productAnalyticsIngestionLeases).values({ + id: leaseId, + fencingToken: fence.fencingToken, + expiresAt: new Date( + Date.now() + PRODUCT_ANALYTICS_INGESTION_LEASE_MS, + ), + }); + const [confirmed] = await db + .select({ + fencingToken: Db.productAnalyticsErasureLeases.fencingToken, + phase: Db.productAnalyticsErasureLeases.phase, + }) + .from(Db.productAnalyticsErasureLeases) + .where(Dz.eq(Db.productAnalyticsErasureLeases.name, "global")) + .limit(1); + if ( + !confirmed || + confirmed.phase !== "idle" || + confirmed.fencingToken !== fence.fencingToken + ) { + await db + .delete(Db.productAnalyticsIngestionLeases) + .where(Dz.eq(Db.productAnalyticsIngestionLeases.id, leaseId)); + throw new Error("Product analytics erasure fence changed"); + } + return leaseId; + }) + .pipe( + Effect.flatMap((leaseId) => + database + .use((db) => + db.transaction(async (tx) => { + await tx + .insert(Db.productAnalyticsIdentityState) + .values(identities) + .onDuplicateKeyUpdate({ + set: { + identityHash: Dz.sql`${Db.productAnalyticsIdentityState.identityHash}`, + }, + }); + const states = await tx + .select({ + blockedAt: Db.productAnalyticsIdentityState.blockedAt, + identityKind: + Db.productAnalyticsIdentityState.identityKind, + }) + .from(Db.productAnalyticsIdentityState) + .where( + Dz.inArray( + Db.productAnalyticsIdentityState.identityHash, + identities.map((identity) => identity.identityHash), + ), + ) + .for("update"); + const blockedAt = states.find( + (state) => state.blockedAt !== null, + )?.blockedAt; + if (blockedAt) { + const blockedPrincipal = states.some( + (state) => + state.blockedAt !== null && + state.identityKind !== "anonymous", + ); + const anonymousHashes = identities + .filter( + (identity) => identity.identityKind === "anonymous", + ) + .map((identity) => identity.identityHash); + if (blockedPrincipal && anonymousHashes.length > 0) { + await tx + .update(Db.productAnalyticsIdentityState) + .set({ blockedAt }) + .where( + Dz.inArray( + Db.productAnalyticsIdentityState.identityHash, + anonymousHashes, + ), + ); + } + return undefined; + } + const anonymousIdentity = identities.find( + (identity) => identity.identityKind === "anonymous", + ); + const userIdentity = identities.find( + (identity) => identity.identityKind === "user", + ); + const organizationIdentity = identities.find( + (identity) => identity.identityKind === "organization", + ); + const anonymousId = rows.find( + (row) => row.anonymous_id, + )?.anonymous_id; + if (anonymousIdentity && userIdentity && anonymousId) { + await tx + .insert(Db.productAnalyticsIdentityLinks) + .values({ + anonymousIdentityHash: anonymousIdentity.identityHash, + userIdentityHash: userIdentity.identityHash, + organizationIdentityHash: + organizationIdentity?.identityHash ?? null, + anonymousId, + }) + .onDuplicateKeyUpdate({ + set: { + organizationIdentityHash: + organizationIdentity?.identityHash ?? null, + updatedAt: new Date(), + }, + }); + } + const now = new Date(); + const retainUntil = new Date( + now.getTime() + PRODUCT_ANALYTICS_RECEIPT_RETENTION_MS, + ); + const admittedRows: ProductEventRow[] = []; + let conflicts = 0; + for (const row of rows) { + const eventIdHash = productAnalyticsEventIdHash( + row.event_id, + ); + const anonymousIdentityHash = row.anonymous_id + ? productAnalyticsIdentityHash( + "anonymous", + row.anonymous_id, + ) + : null; + const userIdentityHash = row.user_id + ? productAnalyticsIdentityHash("user", row.user_id) + : null; + const organizationIdentityHash = row.organization_id + ? productAnalyticsIdentityHash( + "organization", + row.organization_id, + ) + : null; + await tx + .insert(Db.productAnalyticsEventReceipts) + .values({ + eventIdHash, + payloadHash: row.payload_hash, + anonymousIdentityHash, + userIdentityHash, + organizationIdentityHash, + retainUntil, + }) + .onDuplicateKeyUpdate({ + set: { + conflictCount: Dz.sql`IF(${Db.productAnalyticsEventReceipts.payloadHash} <> ${row.payload_hash}, ${Db.productAnalyticsEventReceipts.conflictCount} + 1, ${Db.productAnalyticsEventReceipts.conflictCount})`, + lastSeenAt: now, + retainUntil: Dz.sql`GREATEST(${Db.productAnalyticsEventReceipts.retainUntil}, ${retainUntil})`, + }, + }); + const [receipt] = await tx + .select({ + payloadHash: + Db.productAnalyticsEventReceipts.payloadHash, + }) + .from(Db.productAnalyticsEventReceipts) + .where( + Dz.eq( + Db.productAnalyticsEventReceipts.eventIdHash, + eventIdHash, + ), + ) + .limit(1) + .for("update"); + if (receipt?.payloadHash === row.payload_hash) { + admittedRows.push(row); + } else { + conflicts += 1; + } + } + return { conflicts, rows: admittedRows }; + }), + ) + .pipe( + Effect.flatMap((admission) => + !admission + ? Effect.void + : admission.conflicts > 0 + ? Effect.fail( + new ProductAnalyticsError({ + cause: + "Product analytics event ID was reused with a different payload", + retryable: false, + status: 409, + }), + ) + : admission.rows.length === 0 + ? Effect.void + : service.append(admission.rows), + ), + Effect.ensuring( + database + .use((db) => + db + .delete(Db.productAnalyticsIngestionLeases) + .where( + Dz.eq( + Db.productAnalyticsIngestionLeases.id, + leaseId, + ), + ), + ) + .pipe(Effect.catchAll(() => Effect.void)), + ), + ), + ), + ); + }; + + return { ...service, appendWithIdentityFence } as const; }), }, ) {} diff --git a/packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts b/packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts index a4618f29fc2..ff969949684 100644 --- a/packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts +++ b/packages/web-backend/src/Tinybird/ProductAnalyticsErasureLeaseRepo.ts @@ -1,9 +1,10 @@ -import { randomUUID } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import * as Db from "@cap/database/schema"; import * as Dz from "drizzle-orm"; import { Effect } from "effect"; import { Database } from "../Database.ts"; +import { productAnalyticsIdentityHash } from "../ProductAnalytics/index.ts"; const LEASE_NAME = "global"; const LEASE_DURATION_MS = 5 * 60 * 1_000; @@ -30,9 +31,32 @@ export type ProductAnalyticsErasureLease = { scope: ProductAnalyticsErasureScope; }; +export type ProductAnalyticsErasureRequest = { + id: string; + ownerId: string; + scope: ProductAnalyticsErasureScope; +}; + export interface ProductAnalyticsErasureLeaseStore { + enqueueErasureRequest: ( + scope: ProductAnalyticsErasureScope, + ) => Effect.Effect; + claimErasureRequest: ( + requestId?: string, + ) => Effect.Effect; + completeErasureRequest: (requestId: string) => Effect.Effect; + deferErasureRequest: ( + request: ProductAnalyticsErasureRequest, + errorCode: "erase_failed" | "lease_unavailable", + ) => Effect.Effect; + waitForIngestionQuiescence: () => Effect.Effect; + discardPendingEvents: ( + scope: ProductAnalyticsErasureScope, + anonymousIds?: readonly string[], + ) => Effect.Effect; claimNew: ( scope: ProductAnalyticsErasureScope, + requestId?: string, ) => Effect.Effect; claimRecovery: () => Effect.Effect< ProductAnalyticsErasureLease | null, @@ -67,11 +91,427 @@ const affectedRows = (result: unknown) => { const leaseExpiry = () => new Date(Date.now() + LEASE_DURATION_MS); +const erasureScopeHash = (scope: ProductAnalyticsErasureScope) => + createHash("sha256") + .update(`erasure\0${scope.userId ?? ""}\0${scope.organizationId ?? ""}`) + .digest("hex"); + export class ProductAnalyticsErasureLeaseRepo extends Effect.Service()( "ProductAnalyticsErasureLeaseRepo", { effect: Effect.gen(function* () { const database = yield* Database; + const enqueueErasureRequest: ProductAnalyticsErasureLeaseStore["enqueueErasureRequest"] = + (scope) => + database.use(async (db) => { + if (!scope.userId && !scope.organizationId) { + throw new Error("Product analytics erasure requires an identity"); + } + const id = randomUUID(); + const scopeHash = erasureScopeHash(scope); + await db + .insert(Db.productAnalyticsErasureRequests) + .values({ + id, + scopeHash, + userId: scope.userId ?? null, + organizationId: scope.organizationId ?? null, + }) + .onDuplicateKeyUpdate({ + set: { + status: Dz.sql`IF(${Db.productAnalyticsErasureRequests.status} = 'dead_letter', 'pending', ${Db.productAnalyticsErasureRequests.status})`, + nextAttemptAt: Dz.sql`IF(${Db.productAnalyticsErasureRequests.status} = 'dead_letter', CURRENT_TIMESTAMP, ${Db.productAnalyticsErasureRequests.nextAttemptAt})`, + lastErrorCode: Dz.sql`IF(${Db.productAnalyticsErasureRequests.status} = 'dead_letter', NULL, ${Db.productAnalyticsErasureRequests.lastErrorCode})`, + }, + }); + const [stored] = await db + .select({ id: Db.productAnalyticsErasureRequests.id }) + .from(Db.productAnalyticsErasureRequests) + .where( + Dz.eq(Db.productAnalyticsErasureRequests.scopeHash, scopeHash), + ) + .limit(1); + if (!stored) throw new Error("Erasure request was not persisted"); + return stored.id; + }); + + const claimErasureRequest: ProductAnalyticsErasureLeaseStore["claimErasureRequest"] = + (requestId) => + Effect.gen(function* () { + const ownerId = randomUUID(); + return yield* database.use((db) => + db.transaction(async (tx) => { + const now = new Date(); + const [request] = await tx + .select({ + id: Db.productAnalyticsErasureRequests.id, + userId: Db.productAnalyticsErasureRequests.userId, + organizationId: + Db.productAnalyticsErasureRequests.organizationId, + }) + .from(Db.productAnalyticsErasureRequests) + .where( + Dz.and( + requestId + ? Dz.eq( + Db.productAnalyticsErasureRequests.id, + requestId, + ) + : undefined, + Dz.or( + Dz.and( + Dz.eq( + Db.productAnalyticsErasureRequests.status, + "pending", + ), + Dz.lte( + Db.productAnalyticsErasureRequests.nextAttemptAt, + now, + ), + ), + Dz.and( + Dz.eq( + Db.productAnalyticsErasureRequests.status, + "processing", + ), + Dz.or( + Dz.isNull( + Db.productAnalyticsErasureRequests.leaseExpiresAt, + ), + Dz.lte( + Db.productAnalyticsErasureRequests.leaseExpiresAt, + now, + ), + ), + ), + ), + ), + ) + .orderBy(Dz.asc(Db.productAnalyticsErasureRequests.createdAt)) + .limit(1) + .for("update", { skipLocked: true }); + if (!request) return null; + await tx + .update(Db.productAnalyticsErasureRequests) + .set({ + status: "processing", + attemptCount: Dz.sql`${Db.productAnalyticsErasureRequests.attemptCount} + 1`, + leaseOwnerId: ownerId, + leaseExpiresAt: leaseExpiry(), + lastErrorCode: null, + }) + .where( + Dz.eq(Db.productAnalyticsErasureRequests.id, request.id), + ); + return { + id: request.id, + ownerId, + scope: { + userId: request.userId ?? undefined, + organizationId: request.organizationId ?? undefined, + }, + }; + }), + ); + }); + + const completeErasureRequest: ProductAnalyticsErasureLeaseStore["completeErasureRequest"] = + (requestId) => + database + .use((db) => + db + .delete(Db.productAnalyticsErasureRequests) + .where(Dz.eq(Db.productAnalyticsErasureRequests.id, requestId)), + ) + .pipe(Effect.asVoid); + + const deferErasureRequest: ProductAnalyticsErasureLeaseStore["deferErasureRequest"] = + (request, errorCode) => + database + .use((db) => + db + .update(Db.productAnalyticsErasureRequests) + .set({ + status: "pending", + nextAttemptAt: new Date(Date.now() + 30_000), + leaseOwnerId: null, + leaseExpiresAt: null, + lastErrorCode: errorCode, + }) + .where( + Dz.and( + Dz.eq(Db.productAnalyticsErasureRequests.id, request.id), + Dz.eq( + Db.productAnalyticsErasureRequests.leaseOwnerId, + request.ownerId, + ), + ), + ), + ) + .pipe(Effect.asVoid); + const waitForIngestionQuiescence: ProductAnalyticsErasureLeaseStore["waitForIngestionQuiescence"] = + () => + Effect.gen(function* () { + for (let attempt = 0; attempt < 300; attempt += 1) { + const active = yield* database.use(async (db) => { + const now = new Date(); + await db + .delete(Db.productAnalyticsIngestionLeases) + .where( + Dz.lte(Db.productAnalyticsIngestionLeases.expiresAt, now), + ); + const [row] = await db + .select({ count: Dz.count() }) + .from(Db.productAnalyticsIngestionLeases) + .where( + Dz.gt(Db.productAnalyticsIngestionLeases.expiresAt, now), + ); + return Number(row?.count ?? 0); + }); + if (active === 0) return; + yield* Effect.sleep(1_000); + } + return yield* Effect.fail( + new Error("Product analytics ingestion did not quiesce"), + ); + }); + const discardPendingEvents: ProductAnalyticsErasureLeaseStore["discardPendingEvents"] = + (scope, anonymousIds = []) => + database.use((db) => + db.transaction(async (tx) => { + const userIdentityHash = scope.userId + ? productAnalyticsIdentityHash("user", scope.userId) + : undefined; + const organizationIdentityHash = scope.organizationId + ? productAnalyticsIdentityHash( + "organization", + scope.organizationId, + ) + : undefined; + const linkedAliases = userIdentityHash + ? await tx + .select({ + anonymousId: Db.productAnalyticsIdentityLinks.anonymousId, + }) + .from(Db.productAnalyticsIdentityLinks) + .where( + Dz.eq( + Db.productAnalyticsIdentityLinks.userIdentityHash, + userIdentityHash, + ), + ) + .for("update") + : []; + const localAliases = scope.userId + ? await tx + .select({ + anonymousId: Db.productAnalyticsOutbox.anonymousId, + }) + .from(Db.productAnalyticsOutbox) + .where( + Dz.eq(Db.productAnalyticsOutbox.userId, scope.userId), + ) + : []; + const candidateAliases = [ + ...new Set([ + ...anonymousIds, + ...linkedAliases.map(({ anonymousId }) => anonymousId), + ...localAliases.flatMap(({ anonymousId }) => + anonymousId ? [anonymousId] : [], + ), + ]), + ]; + const candidateAliasHashes = candidateAliases.map((anonymousId) => + productAnalyticsIdentityHash("anonymous", anonymousId), + ); + const sharedLinkedAliases = + userIdentityHash && candidateAliasHashes.length > 0 + ? await tx + .select({ + anonymousIdentityHash: + Db.productAnalyticsIdentityLinks + .anonymousIdentityHash, + }) + .from(Db.productAnalyticsIdentityLinks) + .where( + Dz.and( + Dz.inArray( + Db.productAnalyticsIdentityLinks + .anonymousIdentityHash, + candidateAliasHashes, + ), + Dz.ne( + Db.productAnalyticsIdentityLinks.userIdentityHash, + userIdentityHash, + ), + ), + ) + .for("update") + : []; + const sharedAliases = + scope.userId && candidateAliases.length > 0 + ? await tx + .select({ + anonymousId: Db.productAnalyticsOutbox.anonymousId, + }) + .from(Db.productAnalyticsOutbox) + .where( + Dz.and( + Dz.inArray( + Db.productAnalyticsOutbox.anonymousId, + candidateAliases, + ), + Dz.isNotNull(Db.productAnalyticsOutbox.userId), + Dz.ne(Db.productAnalyticsOutbox.userId, scope.userId), + ), + ) + .for("update") + : []; + const sharedAliasSet = new Set( + sharedAliases.flatMap(({ anonymousId }) => + anonymousId ? [anonymousId] : [], + ), + ); + const sharedLinkedAliasHashes = new Set( + sharedLinkedAliases.map( + ({ anonymousIdentityHash }) => anonymousIdentityHash, + ), + ); + const aliases = candidateAliases.filter( + (anonymousId) => + !sharedAliasSet.has(anonymousId) && + !sharedLinkedAliasHashes.has( + productAnalyticsIdentityHash("anonymous", anonymousId), + ), + ); + const anonymousIdentityHashes = aliases.map((anonymousId) => + productAnalyticsIdentityHash("anonymous", anonymousId), + ); + const conditions = [ + scope.userId + ? Dz.eq(Db.productAnalyticsOutbox.userId, scope.userId) + : undefined, + scope.organizationId + ? Dz.eq( + Db.productAnalyticsOutbox.organizationId, + scope.organizationId, + ) + : undefined, + aliases.length > 0 && scope.userId + ? Dz.and( + Dz.inArray( + Db.productAnalyticsOutbox.anonymousId, + aliases, + ), + Dz.or( + Dz.isNull(Db.productAnalyticsOutbox.userId), + Dz.eq(Db.productAnalyticsOutbox.userId, scope.userId), + ), + ) + : undefined, + ].filter((condition) => condition !== undefined); + if (conditions.length === 0) { + throw new Error( + "Product analytics erasure requires an identity", + ); + } + const states = [ + scope.userId + ? { + identityHash: productAnalyticsIdentityHash( + "user", + scope.userId, + ), + identityKind: "user" as const, + } + : undefined, + scope.organizationId + ? { + identityHash: productAnalyticsIdentityHash( + "organization", + scope.organizationId, + ), + identityKind: "organization" as const, + } + : undefined, + ...aliases.map((anonymousId) => ({ + identityHash: productAnalyticsIdentityHash( + "anonymous", + anonymousId, + ), + identityKind: "anonymous" as const, + })), + ] + .filter((state) => state !== undefined) + .sort((left, right) => + left.identityHash.localeCompare(right.identityHash), + ); + const blockedAt = new Date(); + await tx + .insert(Db.productAnalyticsIdentityState) + .values(states.map((state) => ({ ...state, blockedAt }))) + .onDuplicateKeyUpdate({ set: { blockedAt } }); + await tx + .delete(Db.productAnalyticsOutbox) + .where(Dz.or(...conditions)); + const receiptConditions = [ + userIdentityHash + ? Dz.eq( + Db.productAnalyticsEventReceipts.userIdentityHash, + userIdentityHash, + ) + : undefined, + organizationIdentityHash + ? Dz.eq( + Db.productAnalyticsEventReceipts.organizationIdentityHash, + organizationIdentityHash, + ) + : undefined, + anonymousIdentityHashes.length > 0 && userIdentityHash + ? Dz.and( + Dz.inArray( + Db.productAnalyticsEventReceipts.anonymousIdentityHash, + anonymousIdentityHashes, + ), + Dz.or( + Dz.isNull( + Db.productAnalyticsEventReceipts.userIdentityHash, + ), + Dz.eq( + Db.productAnalyticsEventReceipts.userIdentityHash, + userIdentityHash, + ), + ), + ) + : undefined, + ].filter((condition) => condition !== undefined); + if (receiptConditions.length > 0) { + await tx + .delete(Db.productAnalyticsEventReceipts) + .where(Dz.or(...receiptConditions)); + } + if (userIdentityHash) { + await tx + .delete(Db.productAnalyticsIdentityLinks) + .where( + Dz.eq( + Db.productAnalyticsIdentityLinks.userIdentityHash, + userIdentityHash, + ), + ); + } + const [remaining] = await tx + .select({ count: Dz.count() }) + .from(Db.productAnalyticsOutbox) + .where(Dz.or(...conditions)); + if (Number(remaining?.count ?? 0) !== 0) { + throw new Error( + "Product analytics outbox erasure was incomplete", + ); + } + return aliases; + }), + ); const readOwned = (ownerId: string) => database.use(async (db) => { @@ -110,42 +550,45 @@ export class ProductAnalyticsErasureLeaseRepo extends Effect.Service + const claimNew: ProductAnalyticsErasureLeaseStore["claimNew"] = ( + scope, + queuedRequestId, + ) => Effect.gen(function* () { const ownerId = randomUUID(); - const requestId = randomUUID(); - const available = Dz.sql`${Db.productAnalyticsErasureLeases.ownerId} IS NULL AND ${Db.productAnalyticsErasureLeases.phase} = 'idle'`; - yield* database.use((db) => - db - .insert(Db.productAnalyticsErasureLeases) - .values({ - name: LEASE_NAME, - ownerId, - requestId, - fencingToken: 1, - leaseExpiresAt: leaseExpiry(), - phase: "claimed", - pausedPipes: [], - userId: scope.userId ?? null, - organizationId: scope.organizationId ?? null, - attemptCount: 1, - }) - .onDuplicateKeyUpdate({ - set: { - ownerId: Dz.sql`IF(${available}, ${ownerId}, ${Db.productAnalyticsErasureLeases.ownerId})`, - requestId: Dz.sql`IF(${available}, ${requestId}, ${Db.productAnalyticsErasureLeases.requestId})`, - fencingToken: Dz.sql`IF(${available}, ${Db.productAnalyticsErasureLeases.fencingToken} + 1, ${Db.productAnalyticsErasureLeases.fencingToken})`, - leaseExpiresAt: Dz.sql`IF(${available}, ${leaseExpiry()}, ${Db.productAnalyticsErasureLeases.leaseExpiresAt})`, - phase: Dz.sql`IF(${available}, 'claimed', ${Db.productAnalyticsErasureLeases.phase})`, - pausedPipes: Dz.sql`IF(${available}, JSON_ARRAY(), ${Db.productAnalyticsErasureLeases.pausedPipes})`, - userId: Dz.sql`IF(${available}, ${scope.userId ?? null}, ${Db.productAnalyticsErasureLeases.userId})`, - organizationId: Dz.sql`IF(${available}, ${scope.organizationId ?? null}, ${Db.productAnalyticsErasureLeases.organizationId})`, - attemptCount: Dz.sql`IF(${available}, ${Db.productAnalyticsErasureLeases.attemptCount} + 1, ${Db.productAnalyticsErasureLeases.attemptCount})`, - lastErrorCode: Dz.sql`IF(${available}, NULL, ${Db.productAnalyticsErasureLeases.lastErrorCode})`, - updatedAt: Dz.sql`IF(${available}, CURRENT_TIMESTAMP, ${Db.productAnalyticsErasureLeases.updatedAt})`, - }, - }), + const requestId = queuedRequestId ?? randomUUID(); + const claimed = yield* database.use((db) => + db.transaction(async (tx) => { + await tx + .insert(Db.productAnalyticsErasureLeases) + .values({ name: LEASE_NAME }) + .onDuplicateKeyUpdate({ set: { name: LEASE_NAME } }); + const result = await tx + .update(Db.productAnalyticsErasureLeases) + .set({ + ownerId, + requestId, + fencingToken: Dz.sql`${Db.productAnalyticsErasureLeases.fencingToken} + 1`, + leaseExpiresAt: leaseExpiry(), + phase: "claimed", + pausedPipes: [], + userId: scope.userId ?? null, + organizationId: scope.organizationId ?? null, + attemptCount: Dz.sql`${Db.productAnalyticsErasureLeases.attemptCount} + 1`, + lastErrorCode: null, + updatedAt: new Date(), + }) + .where( + Dz.and( + Dz.eq(Db.productAnalyticsErasureLeases.name, LEASE_NAME), + Dz.isNull(Db.productAnalyticsErasureLeases.ownerId), + Dz.eq(Db.productAnalyticsErasureLeases.phase, "idle"), + ), + ); + return affectedRows(result) > 0; + }), ); + if (!claimed) return null; return yield* readOwned(ownerId); }); @@ -266,6 +709,12 @@ export class ProductAnalyticsErasureLeaseRepo extends Effect.Service { return typeof id === "string" && id ? id : undefined; }; +const tinybirdJobStatus = (response: TinybirdJobResponse) => + String( + response.status ?? + response.state ?? + response.job?.status ?? + response.job?.state ?? + "", + ).toLowerCase(); + const PRODUCT_ANALYTICS_COPY_MARKERS = { snapshot_product_events_daily_exact: "decision_markers", snapshot_product_traffic_daily_exact: "traffic_markers", @@ -54,6 +67,8 @@ const PRODUCT_ANALYTICS_COPY_MARKERS = { snapshot_product_activation_daily_exact: "activation_markers", snapshot_product_creator_retention_exact: "retention_markers", snapshot_product_identity_funnel_exact: "identity_markers", + snapshot_product_attribution_daily_exact: "attribution_markers", + snapshot_product_experiment_outcomes_exact: "experiment_markers", snapshot_product_events_health_hourly: "health_markers", } as const; @@ -416,19 +431,38 @@ export class Tinybird extends Effect.Service()("Tinybird", { const runProductAnalyticsCopyPipe = (name: string, copyRunId?: string) => { const search = new URLSearchParams({ _mode: "replace" }); if (copyRunId) search.set("copy_run_id", copyRunId); - return productAnalyticsRequest( - `/v0/pipes/${encodeURIComponent(name)}/copy?${search.toString()}`, - { token: productAnalyticsCopyToken, purpose: "Copy execution" }, - { method: "POST" }, - ).pipe( - Effect.flatMap((copy) => - tinybirdJobId(copy) - ? Effect.void - : Effect.fail( - new Error("Product analytics copy did not return a job ID"), - ), - ), - ); + return Effect.gen(function* () { + const copy = yield* productAnalyticsRequest( + `/v0/pipes/${encodeURIComponent(name)}/copy?${search.toString()}`, + { token: productAnalyticsCopyToken, purpose: "Copy execution" }, + { method: "POST" }, + ); + const id = tinybirdJobId(copy); + if (!id) { + return yield* Effect.fail( + new Error("Product analytics copy did not return a job ID"), + ); + } + for (let attempt = 0; attempt < 450; attempt += 1) { + const job = yield* productAnalyticsRequest( + `/v0/jobs/${encodeURIComponent(id)}`, + { token: productAnalyticsCopyToken, purpose: "Copy execution" }, + ); + const status = tinybirdJobStatus(job); + if (["done", "success", "finished", "completed"].includes(status)) { + return; + } + if (["failed", "error", "cancelled", "canceled"].includes(status)) { + return yield* Effect.fail( + new Error(`Product analytics copy ${name} ended in ${status}`), + ); + } + yield* Effect.sleep(2_000); + } + return yield* Effect.fail( + new Error(`Product analytics copy ${name} timed out`), + ); + }); }; const queryProductAnalyticsSql = (sql: string) => @@ -460,6 +494,7 @@ export class Tinybird extends Effect.Service()("Tinybird", { auth, ).pipe( Effect.flatMap((pipe) => { + if (!pipe.schedule) return Effect.void; const status = pipe.schedule?.status?.toLowerCase() ?? ""; const matches = paused ? status === "paused" @@ -583,6 +618,37 @@ export class Tinybird extends Effect.Service()("Tinybird", { ); const operation = Effect.gen(function* () { + yield* erasureLeases.waitForIngestionQuiescence(); + let anonymousIds = yield* erasureLeases.discardPendingEvents( + lease.scope, + ); + if (lease.scope.userId) { + const escapedUserId = escapeTinybirdString(lease.scope.userId); + const anonymousRows = yield* queryProductAnalyticsSql<{ + anonymous_id: string; + }>( + `SELECT anonymous_id FROM product_events_v1 WHERE anonymous_id != '' GROUP BY anonymous_id HAVING countIf(user_id = '${escapedUserId}') > 0 AND countIf(user_id != '' AND user_id != '${escapedUserId}') = 0 LIMIT 1001`, + ); + if (anonymousRows.length > 1000) { + return yield* Effect.fail( + new Error( + "Product analytics identity fanout exceeded the erasure bound", + ), + ); + } + anonymousIds = [ + ...new Set([ + ...anonymousIds, + ...anonymousRows + .map(({ anonymous_id: anonymousId }) => anonymousId) + .filter(Boolean), + ]), + ]; + } + anonymousIds = yield* erasureLeases.discardPendingEvents( + lease.scope, + anonymousIds, + ); if (pausedPipes.length > 0) { yield* advance("resuming"); yield* resumeProductAnalyticsCopySchedules(pausedPipes); @@ -602,22 +668,7 @@ export class Tinybird extends Effect.Service()("Tinybird", { } if (lease.scope.userId) { const escapedUserId = escapeTinybirdString(lease.scope.userId); - const anonymousRows = yield* queryProductAnalyticsSql<{ - anonymous_id: string; - }>( - `SELECT anonymous_id FROM product_events_v1 WHERE anonymous_id != '' GROUP BY anonymous_id HAVING countIf(user_id = '${escapedUserId}') > 0 AND countIf(user_id != '' AND user_id != '${escapedUserId}') = 0 LIMIT 1001`, - ); - if (anonymousRows.length > 1000) { - return yield* Effect.fail( - new Error( - "Product analytics identity fanout exceeded the erasure bound", - ), - ); - } conditions.push(`user_id = '${escapedUserId}'`); - const anonymousIds = anonymousRows - .map(({ anonymous_id: anonymousId }) => anonymousId) - .filter(Boolean); if (anonymousIds.length > 0) { conditions.push( `(anonymous_id IN (${anonymousIds @@ -652,6 +703,14 @@ export class Tinybird extends Effect.Service()("Tinybird", { ); } yield* advance("rebuilding"); + yield* runProductAnalyticsCopyPipe( + "snapshot_product_event_id_states_v2", + ); + yield* advance("rebuilding"); + yield* runProductAnalyticsCopyPipe( + "snapshot_product_event_day_states_v2", + ); + yield* advance("rebuilding"); yield* runProductAnalyticsCopyPipe( "snapshot_product_events_canonical_v1", ); @@ -682,6 +741,7 @@ export class Tinybird extends Effect.Service()("Tinybird", { ([rawCount, canonicalCount]) => rawCount === 0 && canonicalCount === 0, ); + yield* erasureLeases.discardPendingEvents(lease.scope, anonymousIds); yield* advance("resuming"); yield* resumeProductAnalyticsCopySchedules(pausedPipes); pausedPipes = []; @@ -714,9 +774,24 @@ export class Tinybird extends Effect.Service()("Tinybird", { const recoverProductAnalyticsErasure = Effect.gen(function* () { const lease = yield* erasureLeases.claimRecovery(); - if (!lease) return { recovered: false as const }; - yield* runProductAnalyticsErasure(lease); - return { recovered: true as const, requestId: lease.requestId }; + if (lease) { + yield* runProductAnalyticsErasure(lease); + yield* erasureLeases.completeErasureRequest(lease.requestId); + return { recovered: true as const, requestId: lease.requestId }; + } + const request = yield* erasureLeases.claimErasureRequest(); + if (!request) return { recovered: false as const }; + const nextLease = yield* erasureLeases.claimNew( + request.scope, + request.id, + ); + if (!nextLease) { + yield* erasureLeases.deferErasureRequest(request, "lease_unavailable"); + return { recovered: false as const }; + } + yield* runProductAnalyticsErasure(nextLease); + yield* erasureLeases.completeErasureRequest(request.id); + return { recovered: true as const, requestId: request.id }; }); const eraseProductAnalytics = ({ @@ -732,17 +807,21 @@ export class Tinybird extends Effect.Service()("Tinybird", { new Error("Product analytics erasure requires an identity"), ); } - let lease = yield* erasureLeases.claimNew({ userId, organizationId }); - if (!lease) { - yield* recoverProductAnalyticsErasure; - lease = yield* erasureLeases.claimNew({ userId, organizationId }); - } + const scope = { userId, organizationId }; + const requestId = yield* erasureLeases.enqueueErasureRequest(scope); + const request = yield* erasureLeases.claimErasureRequest(requestId); + if (!request) return { queued: true as const, requestId }; + const lease = yield* erasureLeases.claimNew(scope, requestId); if (!lease) { - return yield* Effect.fail( - new Error("Product analytics erasure is already in progress"), + yield* erasureLeases.deferErasureRequest( + request, + "lease_unavailable", ); + return { queued: true as const, requestId }; } yield* runProductAnalyticsErasure(lease); + yield* erasureLeases.completeErasureRequest(requestId); + return { queued: false as const, requestId }; }); return { diff --git a/packages/web-backend/src/index.ts b/packages/web-backend/src/index.ts index 7067cb60259..8b5f155708b 100644 --- a/packages/web-backend/src/index.ts +++ b/packages/web-backend/src/index.ts @@ -15,6 +15,9 @@ export { hasAnalyticsSessionCookie, ProductAnalytics, ProductAnalyticsError, + type ProductAnalyticsIdentityKind, + productAnalyticsEventIdHash, + productAnalyticsIdentityHash, resolveProductAnalyticsActor, sendProductAnalyticsRows, } from "./ProductAnalytics/index.ts"; diff --git a/packages/web-domain/src/Agent.ts b/packages/web-domain/src/Agent.ts index 991f058a839..59634c85506 100644 --- a/packages/web-domain/src/Agent.ts +++ b/packages/web-domain/src/Agent.ts @@ -1095,6 +1095,7 @@ export const AgentMutationResponse = Schema.Struct({ }); export const AgentUploadCreateInput = Schema.Struct({ + initiatingPlatform: Schema.optional(Schema.Literal("cli", "server")), organizationId: Schema.optional(OrganisationId), folderId: Schema.optional(FolderId), fileName: Schema.String, From d7d80a9652b546e0f9d2f43de6cef353c81a0dc7 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:40:08 +0100 Subject: [PATCH 083/110] feat: harden analytics decision aggregates --- .../unit/admin-product-analytics.test.ts | 139 +++++++- apps/web/app/admin/analytics/page.tsx | 209 ++++++++---- apps/web/app/admin/analytics/tinybird.ts | 205 ++++++++++- scripts/analytics/tests/datafiles.test.js | 317 +++++++++++++++++- scripts/analytics/tests/tooling.test.js | 19 +- ...roduct_analytics_generations_v2.datasource | 18 + ...product_attribution_daily_exact.datasource | 27 ++ .../product_event_day_states_v2.datasource | 41 +++ .../product_event_id_states_v2.datasource | 40 +++ .../product_events_daily_cold_v2.datasource | 63 ++++ .../product_events_daily_exact.datasource | 3 +- .../product_events_daily_hot_v2.datasource | 63 ++++ ...oduct_experiment_outcomes_exact.datasource | 25 ++ .../product_traffic_daily_cold_v2.datasource | 36 ++ .../product_traffic_daily_exact.datasource | 4 +- .../product_traffic_daily_hot_v2.datasource | 36 ++ ...uct_traffic_pages_daily_cold_v2.datasource | 35 ++ ...oduct_traffic_pages_daily_exact.datasource | 4 +- ...duct_traffic_pages_daily_hot_v2.datasource | 35 ++ .../fixtures/product_events_v1.ndjson | 12 +- ...terialize_product_event_day_states_v2.pipe | 40 +++ ...aterialize_product_event_id_states_v2.pipe | 39 +++ ...erialize_product_events_health_hourly.pipe | 23 ++ .../product_analytics_copy_assertions.pipe | 2 + .../pipes/product_analytics_freshness.pipe | 4 +- .../tinybird/pipes/product_attribution.pipe | 40 +++ .../product_events_canonical_current.pipe | 39 +++ .../product_events_canonical_window.pipe | 60 ++++ .../tinybird/pipes/product_events_daily.pipe | 9 +- .../product_events_daily_bounded_v2.pipe | 112 +++++++ .../product_events_daily_current_v2.pipe | 36 ++ .../pipes/product_experiment_outcomes.pipe | 42 +++ .../pipes/product_traffic_countries.pipe | 2 + .../product_traffic_daily_bounded_v2.pipe | 98 ++++++ .../product_traffic_daily_current_v2.pipe | 36 ++ .../pipes/product_traffic_overview.pipe | 2 + .../tinybird/pipes/product_traffic_pages.pipe | 2 + ...roduct_traffic_pages_daily_bounded_v2.pipe | 105 ++++++ ...roduct_traffic_pages_daily_current_v2.pipe | 36 ++ .../pipes/product_traffic_sources.pipe | 2 + .../pipes/product_traffic_technology.pipe | 2 + .../pipes/product_traffic_totals.pipe | 37 ++ ...blish_product_analytics_generation_v2.pipe | 43 +++ ...apshot_product_activation_daily_exact.pipe | 6 +- ...pshot_product_attribution_daily_exact.pipe | 105 ++++++ ...pshot_product_creator_retention_exact.pipe | 4 +- .../snapshot_product_event_day_states_v2.pipe | 51 +++ .../snapshot_product_event_id_states_v2.pipe | 50 +++ .../snapshot_product_events_canonical_v1.pipe | 67 +--- ...snapshot_product_events_daily_cold_v2.pipe | 18 + .../snapshot_product_events_daily_exact.pipe | 18 +- .../snapshot_product_events_daily_hot_v2.pipe | 14 + ...snapshot_product_events_health_hourly.pipe | 2 +- ...hot_product_experiment_outcomes_exact.pipe | 145 ++++++++ ...napshot_product_identity_funnel_exact.pipe | 40 +-- ...napshot_product_traffic_daily_cold_v2.pipe | 18 + .../snapshot_product_traffic_daily_exact.pipe | 34 +- ...snapshot_product_traffic_daily_hot_v2.pipe | 14 + ...t_product_traffic_pages_daily_cold_v2.pipe | 18 + ...hot_product_traffic_pages_daily_exact.pipe | 20 +- ...ot_product_traffic_pages_daily_hot_v2.pipe | 14 + scripts/analytics/tooling.js | 229 +++++++++++-- scripts/analytics/verify-local.js | 2 + 63 files changed, 2766 insertions(+), 245 deletions(-) create mode 100644 scripts/analytics/tinybird/datasources/product_analytics_generations_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_attribution_daily_exact.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_event_day_states_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_event_id_states_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_events_daily_cold_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_events_daily_hot_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_experiment_outcomes_exact.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_traffic_daily_cold_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_traffic_daily_hot_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_traffic_pages_daily_cold_v2.datasource create mode 100644 scripts/analytics/tinybird/datasources/product_traffic_pages_daily_hot_v2.datasource create mode 100644 scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/materialize_product_events_health_hourly.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_attribution.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_events_canonical_window.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/product_traffic_totals.pipe create mode 100644 scripts/analytics/tinybird/pipes/publish_product_analytics_generation_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_attribution_daily_exact.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_events_daily_cold_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_events_daily_hot_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_cold_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_hot_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_cold_v2.pipe create mode 100644 scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_hot_v2.pipe diff --git a/apps/web/__tests__/unit/admin-product-analytics.test.ts b/apps/web/__tests__/unit/admin-product-analytics.test.ts index 985999405e0..aedbed8b541 100644 --- a/apps/web/__tests__/unit/admin-product-analytics.test.ts +++ b/apps/web/__tests__/unit/admin-product-analytics.test.ts @@ -9,8 +9,11 @@ import { buildAdminAnalyticsEndpointUrl, calculateHealthWindowStart, decodeAnalyticsFreshnessResponse, + decodeExperimentOutcomesResponse, decodeProductEventsResponse, + decodeTrafficAttributionResponse, decodeTrafficOverviewResponse, + decodeTrafficTotalsResponse, fetchOptionalRollbackEndpoint, } from "@/app/admin/analytics/tinybird"; @@ -122,7 +125,41 @@ describe("admin analytics Tinybird client", () => { }, ], }), - ).toMatchObject([{ changeKind: "", revenueMinor: 2_500 }]); + ).toMatchObject([ + { changeKind: "", recordingStatus: "", revenueMinor: 2_500 }, + ]); + expect( + decodeProductEventsResponse({ + data: [ + { + date: "2026-07-31", + event_name: "recording_completed", + source: "client", + platform: "desktop", + app_version: "1.0.0", + hostname: "", + country: "GB", + device: "desktop", + browser: "", + os: "macOS", + channel: "direct", + plan_id: "", + recording_status: "failed", + payment_status: "", + subscription_status: "", + currency: "", + billing_interval: "", + events: 2, + actors: 2, + users: 2, + organizations: 1, + revenue_minor: 0, + }, + ], + }), + ).toMatchObject([ + { eventName: "recording_completed", recordingStatus: "failed" }, + ]); expect( decodeAnalyticsFreshnessResponse({ data: [ @@ -174,4 +211,104 @@ describe("admin analytics Tinybird client", () => { AdminAnalyticsRequestError, ); }); + + it("decodes exact traffic totals and explicit attribution models", () => { + expect( + decodeTrafficTotalsResponse({ + data: [ + { + visitors: "9", + visits: 12, + pageviews: "24", + views_per_visit: 2, + bounce_rate: "25", + visit_duration_ms: 30_000, + engaged_ms: "60000", + }, + ], + }), + ).toEqual([ + { + visitors: 9, + visits: 12, + pageviews: 24, + viewsPerVisit: 2, + bounceRate: 25, + visitDurationMs: 30_000, + engagedMs: 60_000, + }, + ]); + expect( + decodeTrafficAttributionResponse({ + data: [ + { + attribution_model: "last", + source: "newsletter", + medium: "email", + campaign: "launch", + visitors: "7", + visits: "8", + pageviews: "11", + }, + ], + }), + ).toEqual([ + { + attributionModel: "last", + source: "newsletter", + medium: "email", + campaign: "launch", + visitors: 7, + visits: 8, + pageviews: 11, + }, + ]); + expect(() => + decodeTrafficAttributionResponse({ + data: [ + { + attribution_model: "inferred", + source: "", + medium: "", + campaign: "", + visitors: 1, + visits: 1, + pageviews: 1, + }, + ], + }), + ).toThrow(AdminAnalyticsRequestError); + }); + + it("decodes aggregate experiment outcomes without identifiers", () => { + expect( + decodeExperimentOutcomesResponse({ + data: [ + { + experiment_id: "pricing-copy", + assignment_version: "v2", + variant: "concise", + platform: "web", + app_version: "web", + outcome_name: "paid_purchase", + exposed_actors: "40", + converted_actors: "6", + conversion_rate: "15", + }, + ], + }), + ).toEqual([ + { + experimentId: "pricing-copy", + assignmentVersion: "v2", + variant: "concise", + platform: "web", + appVersion: "web", + outcomeName: "paid_purchase", + exposedActors: 40, + convertedActors: 6, + conversionRate: 15, + }, + ]); + }); }); diff --git a/apps/web/app/admin/analytics/page.tsx b/apps/web/app/admin/analytics/page.tsx index 11c59d6eed3..848819c4886 100644 --- a/apps/web/app/admin/analytics/page.tsx +++ b/apps/web/app/admin/analytics/page.tsx @@ -158,9 +158,11 @@ function parseFilters(params: SearchParams): AdminAnalyticsFilters { } function trafficTotals(data: AdminAnalyticsDashboard) { - const visitorDays = sumBy(data.trafficOverview, (row) => row.visitors); - const visits = sumBy(data.trafficOverview, (row) => row.visits); - const pageviews = sumBy(data.trafficOverview, (row) => row.pageviews); + const exact = data.trafficTotals[0]; + const visits = + exact?.visits ?? sumBy(data.trafficOverview, (row) => row.visits); + const pageviews = + exact?.pageviews ?? sumBy(data.trafficOverview, (row) => row.pageviews); const weightedBounces = sumBy( data.trafficOverview, (row) => row.bounceRate * row.visits, @@ -171,12 +173,15 @@ function trafficTotals(data: AdminAnalyticsDashboard) { ); const latestDay = data.trafficOverview.at(-1); return { - visitorDays, + visitors: data.trafficTotalsAvailable ? (exact?.visitors ?? 0) : undefined, visits, pageviews, - viewsPerVisit: visits === 0 ? 0 : pageviews / visits, - bounceRate: visits === 0 ? 0 : weightedBounces / visits, - visitDurationMs: visits === 0 ? 0 : weightedDuration / visits, + viewsPerVisit: + exact?.viewsPerVisit ?? (visits === 0 ? 0 : pageviews / visits), + bounceRate: + exact?.bounceRate ?? (visits === 0 ? 0 : weightedBounces / visits), + visitDurationMs: + exact?.visitDurationMs ?? (visits === 0 ? 0 : weightedDuration / visits), latestDay, }; } @@ -207,6 +212,20 @@ function eventCount(data: AdminAnalyticsDashboard, eventName: string): number { ); } +function recordingCount( + data: AdminAnalyticsDashboard, + status: "success" | "degraded" | "failed" | "unknown", +): number { + return sumBy( + data.productEvents.filter( + (row) => + row.eventName === "recording_completed" && + (row.recordingStatus || "unknown") === status, + ), + (row) => row.events, + ); +} + function eventValue( data: AdminAnalyticsDashboard, eventName: string, @@ -448,13 +467,22 @@ function TrafficSection({ data }: { data: AdminAnalyticsDashboard }) { const totals = trafficTotals(data); return (
@@ -543,9 +571,60 @@ function PagesSection({ data }: { data: AdminAnalyticsDashboard }) { function AcquisitionSection({ data }: { data: AdminAnalyticsDashboard }) { return (
+ {data.trafficAttributionAvailable ? ( +
+

+ Campaign attribution models +

+ + + + + + + + + + + + + {data.trafficAttribution.slice(0, 45).map((row, index) => ( + + + + + + + + + ))} + +
ModelSourceCampaignVisitorsVisitsPageviews
+ {row.attributionModel} + + {row.source || "Direct"} + {row.medium ? ` · ${row.medium}` : ""} + + {row.campaign || "Unattributed"} + + {formatInteger(row.visitors)} + + {formatInteger(row.visits)} + + {formatInteger(row.pageviews)} +
+
+ ) : ( + + Campaign attribution models are unavailable on the active rollback + target. + + )}

@@ -736,9 +815,23 @@ function ProductSection({

+ + + +
+
(); - for (const row of data.productEvents) { - if (row.eventName !== "experiment_exposed" || !row.experimentId) continue; - const key = [ - row.experimentId, - row.experimentVariant, - row.assignmentVersion, - ].join("\u0000"); - const current = exposures.get(key) ?? { - experimentId: row.experimentId, - variant: row.experimentVariant, - assignmentVersion: row.assignmentVersion, - exposures: 0, - actorDays: 0, - userDays: 0, - }; - current.exposures += row.events; - current.actorDays += row.actors; - current.userDays += row.users; - exposures.set(key, current); - } - const rows = [...exposures.values()].sort( - (left, right) => - left.experimentId.localeCompare(right.experimentId) || - left.assignmentVersion.localeCompare(right.assignmentVersion) || - left.variant.localeCompare(right.variant), - ); - return (
- {rows.length === 0 ? ( + {!data.experimentOutcomesAvailable ? ( + + Experiment outcomes are unavailable on the active rollback target. + + ) : data.experimentOutcomes.length === 0 ? ( No experiment exposures in this period. ) : (
@@ -963,15 +1022,16 @@ function ExperimentationSection({ data }: { data: AdminAnalyticsDashboard }) { Experiment Version Variant - Exposures - Actor-days - User-days + Outcome + Exposed + Converted + Rate - {rows.map((row) => ( + {data.experimentOutcomes.map((row) => ( {row.experimentId} @@ -980,14 +1040,17 @@ function ExperimentationSection({ data }: { data: AdminAnalyticsDashboard }) { {row.assignmentVersion} {row.variant} + + {row.outcomeName.replaceAll("_", " ")} + - {formatInteger(row.exposures)} + {formatInteger(row.exposedActors)} - {formatInteger(row.actorDays)} + {formatInteger(row.convertedActors)} - {formatInteger(row.userDays)} + {formatPercent(row.conversionRate)} ))} @@ -1160,6 +1223,16 @@ function QualitySection({ data }: { data: AdminAnalyticsDashboard }) { label="Identity funnel calculated" value={formatTimestamp(freshness?.identityCalculatedAt ?? "")} /> + +
); @@ -1170,7 +1243,7 @@ function Definitions() { ["Metric timezone", "All cohorts, sessions, and reporting dates use UTC."], [ "Visitor", - "An anonymous, privacy-safe visitor key. The overview exposes daily uniques, so the selected-range card is visitor-days, not period-unique people.", + "An anonymous, privacy-safe visitor key. The Visitors card merges exact aggregate states across the complete selected range and counts a visitor once.", ], [ "Visit", @@ -1204,9 +1277,17 @@ function Definitions() { "Deduplication", "Decision-facing endpoints are built from stable event IDs. Duplicate deliveries remain visible in health but count once in metrics.", ], + [ + "Campaign attribution", + "First touch is the visitor's earliest stored campaign, session touch is fixed when a 30-minute session begins, and last touch is the latest campaign-bearing navigation. Each model is reported independently.", + ], [ "Experiment exposure", - "A typed, stable assignment rendered to an actor. Variants are read only from exposure events and are never inferred from conversion behavior.", + "A typed, stable assignment rendered to an actor. The outcome cohorts include only recorded exposures, exclude conflicting variants, and measure the first authoritative signup, share creation, or paid purchase within 30 days.", + ], + [ + "Recording terminal status", + "Successful, degraded, failed, and unknown terminal recordings are counted separately. A failed terminal event never contributes to the successful recording metric.", ], [ "Privacy", @@ -1214,7 +1295,7 @@ function Definitions() { ], [ "Filter coverage", - "Date applies throughout. Platform applies to product, creator, retention, and health aggregates. App version, source, and plan apply to product aggregates; app version also applies to health. Country applies to product and supported traffic aggregates. Organization cohort selects a first-value UTC date for creator and organization retention.", + "Date applies throughout. Platform and app version apply to traffic, attribution, product, experiment, creator, retention, and health aggregates where those dimensions exist. Source and plan apply to product aggregates; source also filters exact traffic totals and attribution. Country applies to product and supported traffic aggregates. Organization cohort selects a first-value UTC date for creator and organization retention.", ], ]; return ( diff --git a/apps/web/app/admin/analytics/tinybird.ts b/apps/web/app/admin/analytics/tinybird.ts index e25f9b2a9b1..793581e644f 100644 --- a/apps/web/app/admin/analytics/tinybird.ts +++ b/apps/web/app/admin/analytics/tinybird.ts @@ -2,8 +2,10 @@ import "server-only"; const ADMIN_ANALYTICS_ENDPOINTS = [ "product_traffic_overview", + "product_traffic_totals", "product_traffic_pages", "product_traffic_sources", + "product_attribution", "product_traffic_countries", "product_traffic_technology", "product_activation", @@ -12,6 +14,7 @@ const ADMIN_ANALYTICS_ENDPOINTS = [ "product_identity_funnel", "product_events_daily", "product_feature_adoption", + "product_experiment_outcomes", "product_events_health", "product_analytics_freshness", ] as const; @@ -43,6 +46,20 @@ export type TrafficOverviewRow = { engagedMs: number; }; +export type TrafficTotalsRow = Omit; + +export type AttributionModel = "first" | "session" | "last"; + +export type TrafficAttributionRow = { + attributionModel: AttributionModel; + source: string; + medium: string; + campaign: string; + visitors: number; + visits: number; + pageviews: number; +}; + export type TrafficPageRow = { pathname: string; visitors: number; @@ -124,6 +141,7 @@ export type ProductEventRow = { os: string; channel: string; planId: string; + recordingStatus: string; paymentStatus: string; subscriptionStatus: string; currency: string; @@ -182,6 +200,18 @@ export type FeatureAdoptionRow = { organizationDays: number; }; +export type ExperimentOutcomeRow = { + experimentId: string; + assignmentVersion: string; + variant: string; + platform: string; + appVersion: string; + outcomeName: string; + exposedActors: number; + convertedActors: number; + conversionRate: number; +}; + export type ProductEventsHealthRow = { receivedRows: number; uniqueEvents: number; @@ -201,12 +231,18 @@ export type AnalyticsFreshnessRow = { trafficCalculatedAt: string; retentionCalculatedAt: string; identityCalculatedAt: string; + attributionCalculatedAt: string; + experimentCalculatedAt: string; }; export type AdminAnalyticsDashboard = { trafficOverview: TrafficOverviewRow[]; + trafficTotals: TrafficTotalsRow[]; + trafficTotalsAvailable: boolean; trafficPages: TrafficPageRow[]; trafficSources: TrafficSourceRow[]; + trafficAttribution: TrafficAttributionRow[]; + trafficAttributionAvailable: boolean; trafficCountries: TrafficCountryRow[]; trafficTechnology: TrafficTechnologyRow[]; activation: ActivationRow[]; @@ -216,6 +252,8 @@ export type AdminAnalyticsDashboard = { identityFunnelAvailable: boolean; productEvents: ProductEventRow[]; featureAdoption: FeatureAdoptionRow[]; + experimentOutcomes: ExperimentOutcomeRow[]; + experimentOutcomesAvailable: boolean; health: ProductEventsHealthRow[]; freshness: AnalyticsFreshnessRow[]; healthWindowStart: string; @@ -354,6 +392,57 @@ export function decodeTrafficOverviewResponse( return decodeTinybirdRows(value, decodeTrafficOverviewRow); } +function decodeTrafficTotalsRow(row: UnknownRecord): TrafficTotalsRow { + return { + visitors: readNumber(row, "visitors"), + visits: readNumber(row, "visits"), + pageviews: readNumber(row, "pageviews"), + viewsPerVisit: readNumber(row, "views_per_visit"), + bounceRate: readNumber(row, "bounce_rate"), + visitDurationMs: readNumber(row, "visit_duration_ms"), + engagedMs: readNumber(row, "engaged_ms"), + }; +} + +export function decodeTrafficTotalsResponse( + value: unknown, +): TrafficTotalsRow[] { + return decodeTinybirdRows(value, decodeTrafficTotalsRow); +} + +function decodeAttributionModel( + row: UnknownRecord, + key: string, +): AttributionModel { + const value = readString(row, key); + if (value !== "first" && value !== "session" && value !== "last") { + throw new AdminAnalyticsRequestError( + `Tinybird returned an invalid ${key} value`, + ); + } + return value; +} + +function decodeTrafficAttributionRow( + row: UnknownRecord, +): TrafficAttributionRow { + return { + attributionModel: decodeAttributionModel(row, "attribution_model"), + source: readString(row, "source"), + medium: readString(row, "medium"), + campaign: readString(row, "campaign"), + visitors: readNumber(row, "visitors"), + visits: readNumber(row, "visits"), + pageviews: readNumber(row, "pageviews"), + }; +} + +export function decodeTrafficAttributionResponse( + value: unknown, +): TrafficAttributionRow[] { + return decodeTinybirdRows(value, decodeTrafficAttributionRow); +} + function decodeTrafficPageRow(row: UnknownRecord): TrafficPageRow { return { pathname: readString(row, "pathname"), @@ -470,6 +559,7 @@ function decodeProductEventRow(row: UnknownRecord): ProductEventRow { os: readString(row, "os"), channel: readString(row, "channel"), planId: readString(row, "plan_id"), + recordingStatus: readOptionalString(row, "recording_status"), paymentStatus: readString(row, "payment_status"), subscriptionStatus: readString(row, "subscription_status"), currency: readString(row, "currency"), @@ -513,6 +603,26 @@ function decodeFeatureAdoptionRow(row: UnknownRecord): FeatureAdoptionRow { }; } +function decodeExperimentOutcomeRow(row: UnknownRecord): ExperimentOutcomeRow { + return { + experimentId: readString(row, "experiment_id"), + assignmentVersion: readString(row, "assignment_version"), + variant: readString(row, "variant"), + platform: readString(row, "platform"), + appVersion: readString(row, "app_version"), + outcomeName: readString(row, "outcome_name"), + exposedActors: readNumber(row, "exposed_actors"), + convertedActors: readNumber(row, "converted_actors"), + conversionRate: readNumber(row, "conversion_rate"), + }; +} + +export function decodeExperimentOutcomesResponse( + value: unknown, +): ExperimentOutcomeRow[] { + return decodeTinybirdRows(value, decodeExperimentOutcomeRow); +} + function decodeProductEventsHealthRow( row: UnknownRecord, ): ProductEventsHealthRow { @@ -539,6 +649,11 @@ function decodeAnalyticsFreshnessRow( trafficCalculatedAt: readString(row, "traffic_calculated_at"), retentionCalculatedAt: readString(row, "retention_calculated_at"), identityCalculatedAt: readOptionalString(row, "identity_calculated_at"), + attributionCalculatedAt: readOptionalString( + row, + "attribution_calculated_at", + ), + experimentCalculatedAt: readOptionalString(row, "experiment_calculated_at"), }; } @@ -672,8 +787,10 @@ export async function fetchAdminAnalyticsDashboard( const [ trafficOverview, + trafficTotalsResult, trafficPages, trafficSources, + trafficAttributionResult, trafficCountries, trafficTechnology, activation, @@ -682,32 +799,83 @@ export async function fetchAdminAnalyticsDashboard( identityFunnelResult, productEvents, featureAdoption, + experimentOutcomesResult, health, freshness, ] = await Promise.all([ fetchEndpoint( "product_traffic_overview", - { ...dateParams, country: filters.country }, + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + country: filters.country, + }, decodeTrafficOverviewRow, ), + fetchOptionalRollbackEndpoint( + fetchEndpoint, + "product_traffic_totals", + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + source: filters.source, + country: filters.country, + }, + decodeTrafficTotalsRow, + ), fetchEndpoint( "product_traffic_pages", - { ...dateParams, country: filters.country, limit: 100 }, + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + country: filters.country, + limit: 100, + }, decodeTrafficPageRow, ), fetchEndpoint( "product_traffic_sources", - { ...dateParams, limit: 100 }, + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + limit: 100, + }, decodeTrafficSourceRow, ), + fetchOptionalRollbackEndpoint( + fetchEndpoint, + "product_attribution", + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + source: filters.source, + country: filters.country, + limit: 300, + }, + decodeTrafficAttributionRow, + ), fetchEndpoint( "product_traffic_countries", - dateParams, + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + }, decodeTrafficCountryRow, ), fetchEndpoint( "product_traffic_technology", - { ...dateParams, country: filters.country }, + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + country: filters.country, + }, decodeTrafficTechnologyRow, ), fetchEndpoint("product_activation", dateParams, decodeActivationRow), @@ -760,6 +928,17 @@ export async function fetchAdminAnalyticsDashboard( }, decodeFeatureAdoptionRow, ), + fetchOptionalRollbackEndpoint( + fetchEndpoint, + "product_experiment_outcomes", + { + ...dateParams, + platform: filters.platform, + app_version: filters.appVersion, + limit: 1000, + }, + decodeExperimentOutcomeRow, + ), fetchEndpoint( "product_events_health", { @@ -782,6 +961,16 @@ export async function fetchAdminAnalyticsDashboard( "Product analytics reached the aggregate endpoint row limit. Narrow the date range or filters before using these metrics for decisions.", ); } + if (trafficAttributionResult.rows.length >= 300) { + throw new AdminAnalyticsRequestError( + "Campaign attribution reached the aggregate endpoint row limit. Narrow the date range or filters before using these metrics for decisions.", + ); + } + if (experimentOutcomesResult.rows.length >= 1000) { + throw new AdminAnalyticsRequestError( + "Experiment outcomes reached the aggregate endpoint row limit. Narrow the date range or filters before using these metrics for decisions.", + ); + } if (creatorRetention.length >= 5000) { throw new AdminAnalyticsRequestError( "Creator retention reached the aggregate endpoint row limit. Narrow the date range or platform before using these metrics for decisions.", @@ -790,8 +979,12 @@ export async function fetchAdminAnalyticsDashboard( return { trafficOverview, + trafficTotals: trafficTotalsResult.rows, + trafficTotalsAvailable: trafficTotalsResult.available, trafficPages, trafficSources, + trafficAttribution: trafficAttributionResult.rows, + trafficAttributionAvailable: trafficAttributionResult.available, trafficCountries, trafficTechnology, activation, @@ -801,6 +994,8 @@ export async function fetchAdminAnalyticsDashboard( identityFunnelAvailable: identityFunnelResult.available, productEvents, featureAdoption, + experimentOutcomes: experimentOutcomesResult.rows, + experimentOutcomesAvailable: experimentOutcomesResult.available, health, freshness, healthWindowStart, diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index 4c9ddbc3250..8276d1dabc8 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -84,6 +84,8 @@ test("product datasource matches the runtime event contract", () => { "product_activation_daily_exact", "product_creator_retention_exact", "product_identity_funnel_exact", + "product_attribution_daily_exact", + "product_experiment_outcomes_exact", ]; for (const name of retainedDatasources) { const retained = project.datasources.find( @@ -100,6 +102,8 @@ test("product datasource matches the runtime event contract", () => { "product_activation_daily_exact", "product_creator_retention_exact", "product_identity_funnel_exact", + "product_attribution_daily_exact", + "product_experiment_outcomes_exact", "product_events_health_hourly_exact", ]; for (const name of copyTargets) { @@ -162,6 +166,10 @@ test("daily product queries read the exact snapshot instead of raw deliveries", /subscription_status = \{\{String\(subscription_status\)\}\}/, ); assert.match(contents, /app_version = \{\{String\(app_version\)\}\}/); + assert.match( + contents, + /recording_status = \{\{String\(recording_status\)\}\}/, + ); assert.match(contents, /currency = upper\(\{\{String\(currency\)\}\}\)/); assert.match(contents, /revenue_minor/); assert.doesNotMatch(contents, /FROM product_events_v1/); @@ -181,6 +189,154 @@ test("feature adoption merges daily identities after applying filters", () => { assert.doesNotMatch(contents, /FROM product_events_v1/); }); +test("traffic totals merge visitor states across the selected range", () => { + const contents = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", "product_traffic_totals.pipe"), + "utf8", + ); + assert.match(contents, /uniqExactMerge\(visitors\) AS visitors/); + assert.match(contents, /FROM product_traffic_daily_exact/); + assert.match(contents, /platform = \{\{String\(platform\)\}\}/); + assert.match(contents, /app_version = \{\{String\(app_version\)\}\}/); + assert.doesNotMatch(contents, /GROUP BY date/); + assert.doesNotMatch(contents, /FROM product_events_v1/); +}); + +test("traffic endpoints apply platform and app-version filters consistently", () => { + const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); + for (const name of [ + "product_traffic_daily_exact", + "product_traffic_pages_daily_exact", + ]) { + const datasource = project.datasources.find( + (candidate) => candidate.name === name, + ); + assert.ok(datasource); + const columnNames = new Set(datasource.columns.map(({ name }) => name)); + assert.ok(columnNames.has("platform")); + assert.ok(columnNames.has("app_version")); + } + const trafficSnapshot = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "snapshot_product_traffic_daily_exact.pipe", + ), + "utf8", + ); + const pagesSnapshot = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "snapshot_product_traffic_pages_daily_exact.pipe", + ), + "utf8", + ); + assert.match( + trafficSnapshot, + /argMin\(platform, tuple\(occurred_at, event_id\)\) AS platform/, + ); + assert.match( + trafficSnapshot, + /argMin\(app_version, tuple\(occurred_at, event_id\)\) AS app_version/, + ); + assert.match( + pagesSnapshot, + /argMin\(event_id, tuple\(occurred_at, event_id\)\) AS landing_event_id/, + ); + assert.match( + pagesSnapshot, + /argMax\(event_id, tuple\(occurred_at, event_id\)\) AS exit_event_id/, + ); + assert.match(pagesSnapshot, /\n\s+platform,\n\s+app_version,/); + for (const name of [ + "product_traffic_overview", + "product_traffic_totals", + "product_traffic_pages", + "product_traffic_sources", + "product_traffic_countries", + "product_traffic_technology", + ]) { + const contents = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), + "utf8", + ); + assert.match(contents, /platform = \{\{String\(platform\)\}\}/); + assert.match(contents, /app_version = \{\{String\(app_version\)\}\}/); + } +}); + +test("campaign attribution materializes explicit first, session, and last models", () => { + const snapshot = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "snapshot_product_attribution_daily_exact.pipe", + ), + "utf8", + ); + const endpoint = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", "product_attribution.pipe"), + "utf8", + ); + assert.match(snapshot, /ARRAY JOIN \['first', 'session', 'last'\]/); + assert.match( + snapshot, + /attribution_model = 'last'.*argMax\(attribution_source, tuple\(occurred_at, event_id\)\).*argMin\(attribution_source, tuple\(occurred_at, event_id\)\)/, + ); + for (const prefix of ["first_touch", "session_touch", "last_touch"]) { + assert.match(snapshot, new RegExp(`${prefix}_source`)); + assert.match(snapshot, new RegExp(`${prefix}_medium`)); + assert.match(snapshot, new RegExp(`${prefix}_campaign`)); + } + assert.match(snapshot, /uniqExactState\(visitor_id\) AS visitors/); + assert.match(endpoint, /uniqExactMerge\(visitors\) AS visitors/); + assert.match(endpoint, /FROM product_attribution_daily_exact/); + assert.doesNotMatch( + endpoint, + /anonymous_id|session_id|user_id|organization_id/, + ); +}); + +test("experiment outcomes are aggregate-only and anchored to explicit exposures", () => { + const snapshot = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "snapshot_product_experiment_outcomes_exact.pipe", + ), + "utf8", + ); + const endpoint = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "product_experiment_outcomes.pipe", + ), + "utf8", + ); + assert.match(snapshot, /event_name = 'experiment_exposed'/); + assert.match(snapshot, /HAVING uniqExact\(user_id\) = 1/); + assert.match(snapshot, /HAVING uniqExact\(variant\) = 1/); + assert.match(snapshot, /outcome_candidates AS/); + assert.match( + snapshot, + /minIf\(outcomes\.occurred_at, outcomes\.event_name = 'share_link_created' AND outcomes\.occurred_at >= exposures\.exposed_at/, + ); + assert.match(snapshot, /outcome_at >= exposed_at/); + assert.match(snapshot, /outcome_at < exposed_at \+ INTERVAL 30 DAY/); + assert.match( + snapshot, + /JSONExtractString\(properties, 'payment_status'\) = 'paid'/, + ); + assert.match(endpoint, /FROM product_experiment_outcomes_exact/); + assert.match(endpoint, /sum\(converted_actors\)/); + assert.doesNotMatch( + endpoint, + /anonymous_id|session_id|user_id|organization_id/, + ); +}); + test("retention merges identities across activity platforms", () => { const contents = fs.readFileSync( path.join(TINYBIRD_PROJECT_DIR, "pipes", "product_creator_retention.pipe"), @@ -222,6 +378,12 @@ test("identity cohorts stitch acquisition and guest purchases without exposing m test("daily snapshot quarantines payload conflicts and rebuilds exact metrics", () => { const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); + const materializedState = project.pipes.find( + (pipe) => pipe.name === "materialize_product_event_id_states_v2", + ); + const canonicalCurrent = project.pipes.find( + (pipe) => pipe.name === "product_events_canonical_current", + ); const canonical = project.pipes.find( (pipe) => pipe.name === "snapshot_product_events_canonical_v1", ); @@ -230,6 +392,13 @@ test("daily snapshot quarantines payload conflicts and rebuilds exact metrics", ); assert.ok(snapshot); assert.ok(canonical); + assert.ok(materializedState); + assert.ok(canonicalCurrent); + assert.equal(materializedState.type, "materialized"); + assert.equal( + materializedState.targetDatasource, + "product_event_id_states_v2", + ); assert.equal(snapshot.type, "copy"); assert.equal(snapshot.targetDatasource, "product_events_daily_exact"); @@ -241,9 +410,30 @@ test("daily snapshot quarantines payload conflicts and rebuilds exact metrics", ), "utf8", ); - assert.match(canonicalContents, /GROUP BY event_id/); - assert.match(canonicalContents, /HAVING uniqExact\(raw_payload_hash\) = 1/); + assert.match(canonicalContents, /FROM product_events_canonical_current/); + assert.doesNotMatch(canonicalContents, /FROM product_events_v1/); assert.match(canonicalContents, /^COPY_MODE replace$/m); + const stateContents = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "materialize_product_event_id_states_v2.pipe", + ), + "utf8", + ); + assert.match(stateContents, /uniqExactState\(payload_hash\)/); + assert.match(stateContents, /GROUP BY event_id/); + assert.match(stateContents, /^TYPE MATERIALIZED$/m); + const currentContents = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "product_events_canonical_current.pipe", + ), + "utf8", + ); + assert.match(currentContents, /uniqExactMerge\(payload_hashes\) = 1/); + assert.match(currentContents, /GROUP BY event_id/); const contents = fs.readFileSync( path.join( @@ -255,9 +445,13 @@ test("daily snapshot quarantines payload conflicts and rebuilds exact metrics", ); assert.match(contents, /^TYPE COPY$/m); assert.match(contents, /^COPY_MODE replace$/m); - assert.match(contents, /FROM product_events_canonical_v1/); + assert.match(contents, /FROM product_events_canonical_current/); assert.match(contents, /toUInt64\(count\(\)\) AS events/); assert.match(contents, /uniqExactState\(/); + assert.match(contents, /AS recording_status/); + assert.match(contents, /'success', 'success'/); + assert.match(contents, /'degraded', 'degraded'/); + assert.match(contents, /'failed', 'failed'/); assert.match(contents, /JSONExtractString\(properties, 'payment_status'\)/); assert.match( contents, @@ -267,28 +461,110 @@ test("daily snapshot quarantines payload conflicts and rebuilds exact metrics", assert.doesNotMatch(contents, /uniqState\(/); }); -test("copy schedules serialize canonical and derived rebuilds", () => { - const schedules = new Map([ - ["snapshot_product_events_canonical_v1", "0-59/8 * * * *"], - ["snapshot_product_events_health_hourly", "1-59/8 * * * *"], - ["snapshot_product_events_daily_exact", "2-59/8 * * * *"], - ["snapshot_product_traffic_daily_exact", "3-59/8 * * * *"], - ["snapshot_product_traffic_pages_daily_exact", "4-59/8 * * * *"], - ["snapshot_product_activation_daily_exact", "5-59/8 * * * *"], - ["snapshot_product_creator_retention_exact", "6-59/8 * * * *"], - ["snapshot_product_identity_funnel_exact", "7-59/8 * * * *"], - ]); - for (const [name, schedule] of schedules) { +test("bounded v2 aggregates publish one common generation without historical scans", () => { + const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); + const dayState = project.datasources.find( + (datasource) => datasource.name === "product_event_day_states_v2", + ); + assert.ok(dayState); + assert.equal(dayState.engine, "AggregatingMergeTree"); + assert.equal(dayState.partitionKey, "toYYYYMM(occurred_date)"); + assert.equal(dayState.sortingKey, "(occurred_date, event_id)"); + assert.equal(dayState.ttl, "occurred_date + INTERVAL 807 DAY"); + const dayMaterialization = project.pipes.find( + (pipe) => pipe.name === "materialize_product_event_day_states_v2", + ); + assert.ok(dayMaterialization); + assert.equal(dayMaterialization.type, "materialized"); + assert.equal( + dayMaterialization.targetDatasource, + "product_event_day_states_v2", + ); + + for (const prefix of [ + "product_events_daily", + "product_traffic_daily", + "product_traffic_pages_daily", + ]) { + const bounded = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", `${prefix}_bounded_v2.pipe`), + "utf8", + ); + assert.match(bounded, /source_cutoff is required/); + assert.match(bounded, /generation_id is required/); + assert.match(bounded, /FROM product_events_canonical_window/); + assert.match(bounded, /INTERVAL 8 DAY/); + assert.doesNotMatch(bounded, /INTERVAL 800 DAY|FROM product_events_v1/); + + for (const [temperature, mode] of [ + ["hot", "replace"], + ["cold", "append"], + ]) { + const copy = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + `snapshot_${prefix}_${temperature}_v2.pipe`, + ), + "utf8", + ); + assert.match(copy, new RegExp(`^COPY_MODE ${mode}$`, "m")); + assert.match(copy, /^COPY_SCHEDULE @on-demand$/m); + assert.match(copy, /^TOKEN product_events_copy_runner READ$/m); + } + + const current = fs.readFileSync( + path.join(TINYBIRD_PROJECT_DIR, "pipes", `${prefix}_current_v2.pipe`), + "utf8", + ); + assert.match(current, /FROM product_analytics_generations_v2/); + assert.match(current, /generation_kind = 'hot'/); + assert.match(current, /generation_kind = 'cold'/); + assert.match(current, /is_marker = 0/); + assert.match(current, /INTERVAL 8 DAY/); + } + + const publication = fs.readFileSync( + path.join( + TINYBIRD_PROJECT_DIR, + "pipes", + "publish_product_analytics_generation_v2.pipe", + ), + "utf8", + ); + assert.match(publication, /event hot generation is incomplete/); + assert.match(publication, /traffic hot generation is incomplete/); + assert.match(publication, /page hot generation is incomplete/); + assert.match(publication, /event cold generation is incomplete/); + assert.match(publication, /traffic cold generation is incomplete/); + assert.match(publication, /page cold generation is incomplete/); + assert.match(publication, /^COPY_MODE append$/m); + assert.match(publication, /^COPY_SCHEDULE @on-demand$/m); +}); + +test("copy rebuilds are on-demand and require the sequential controller", () => { + const copies = [ + "snapshot_product_events_canonical_v1", + "snapshot_product_events_health_hourly", + "snapshot_product_events_daily_exact", + "snapshot_product_traffic_daily_exact", + "snapshot_product_traffic_pages_daily_exact", + "snapshot_product_activation_daily_exact", + "snapshot_product_creator_retention_exact", + "snapshot_product_identity_funnel_exact", + "snapshot_product_attribution_daily_exact", + "snapshot_product_experiment_outcomes_exact", + ]; + for (const name of copies) { const contents = fs.readFileSync( path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), "utf8", ); - assert.ok(contents.split("\n").includes(`COPY_SCHEDULE ${schedule}`)); + assert.ok(contents.split("\n").includes("COPY_SCHEDULE @on-demand")); assert.match(contents, /^TOKEN product_events_copy_runner READ$/m); assert.doesNotMatch(contents, /^TOKEN product_events_agent_read READ$/m); assert.match(contents, /\{\{max_threads\(Int32\(copy_max_threads\)\)\}\}/); } - assert.equal(new Set(schedules.values()).size, schedules.size); }); test("decision endpoints cannot be executed with the Copy runner token", () => { @@ -316,6 +592,9 @@ test("staging copy markers and synthetic rows are excluded from decision endpoin "product_events_daily", "product_feature_adoption", "product_identity_funnel", + "product_traffic_totals", + "product_attribution", + "product_experiment_outcomes", ]; for (const name of syntheticEndpoints) { const contents = fs.readFileSync( @@ -342,6 +621,8 @@ test("staging copy markers and synthetic rows are excluded from decision endpoin "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", "snapshot_product_identity_funnel_exact", + "snapshot_product_attribution_daily_exact", + "snapshot_product_experiment_outcomes_exact", ]) { const contents = fs.readFileSync( path.join(TINYBIRD_PROJECT_DIR, "pipes", `${name}.pipe`), @@ -364,6 +645,8 @@ test("staging copy markers and synthetic rows are excluded from decision endpoin assert.doesNotMatch(assertions, /requested_copy_run_id/); assert.match(assertions, /retention_markers/); assert.match(assertions, /identity_markers/); + assert.match(assertions, /attribution_markers/); + assert.match(assertions, /experiment_markers/); }); test("health queries use stable hourly aggregates and a bounded window", () => { diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index e73617d7054..ffc6b4165f3 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -78,19 +78,32 @@ test("local setup builds, verifies copied endpoints and writes its deterministic .map((step, index) => ({ index, command: step.args?.join(" ") ?? "" })) .filter(({ command }) => command.includes("--local copy pause")) .map(({ index }) => index); - assert.equal(pauseIndexes.length, 8); + assert.equal(pauseIndexes.length, 19); assert.ok(pauseIndexes.every((index) => index < appendIndex)); const copyCommands = commands.filter((command) => command.includes("--local copy run"), ); - assert.equal(copyCommands.length, 8); + assert.equal(copyCommands.length, 16); assert.ok( copyCommands.every((command) => command.includes( - "--param copy_max_threads=1 --param copy_run_id=run_local_copy_assertions --wait", + "--param copy_max_threads=1 --param copy_run_id=run_local_copy_assertions", ), ), ); + assert.equal( + copyCommands.filter((command) => command.includes("source_cutoff=")).length, + 4, + ); + assert.equal( + copyCommands.filter((command) => command.includes("generation_id=")).length, + 4, + ); + assert.equal( + copyCommands.filter((command) => command.includes("generation_kind=hot")) + .length, + 1, + ); assert.ok(steps.some((step) => step.type === "verify-local")); assert.ok(steps.some((step) => step.type === "write-local-env")); assert.ok( diff --git a/scripts/analytics/tinybird/datasources/product_analytics_generations_v2.datasource b/scripts/analytics/tinybird/datasources/product_analytics_generations_v2.datasource new file mode 100644 index 00000000000..75ae450b29c --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_analytics_generations_v2.datasource @@ -0,0 +1,18 @@ +DESCRIPTION > + Publication barrier for complete cross-model bounded analytics generations. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + generation_id String, + source_cutoff DateTime64(3), + generation_kind LowCardinality(String), + settled_date Date, + published_at DateTime64(3), + copy_run_id String + +ENGINE MergeTree +ENGINE_PARTITION_KEY toYYYYMM(source_cutoff) +ENGINE_SORTING_KEY (generation_kind, settled_date, source_cutoff, generation_id) +ENGINE_TTL toDateTime(source_cutoff) + INTERVAL 807 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_attribution_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_attribution_daily_exact.datasource new file mode 100644 index 00000000000..5df95e5b93f --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_attribution_daily_exact.datasource @@ -0,0 +1,27 @@ +DESCRIPTION > + Eight-hundred-day privacy-safe campaign attribution with exact visitor states for first, session, and last touch. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + calculated_at DateTime64(3), + copy_run_id String, + synthetic_run_id String, + date Date, + attribution_model LowCardinality(String), + platform LowCardinality(String), + app_version LowCardinality(String), + hostname LowCardinality(String), + country LowCardinality(String), + attribution_source LowCardinality(String), + attribution_medium LowCardinality(String), + campaign LowCardinality(String), + visitors AggregateFunction(uniqExact, String), + visits UInt64, + pageviews UInt64 + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, date, attribution_model, platform, app_version, hostname, country, attribution_source, attribution_medium, campaign) +ENGINE_TTL date + INTERVAL 800 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_event_day_states_v2.datasource b/scripts/analytics/tinybird/datasources/product_event_day_states_v2.datasource new file mode 100644 index 00000000000..7525d6ce8d9 --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_event_day_states_v2.datasource @@ -0,0 +1,41 @@ +DESCRIPTION > + Time-partitioned per-event aggregate state for bounded exact decision refreshes. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + occurred_date Date, + event_id String, + payload_hashes AggregateFunction(uniqExact, FixedString(32)), + payload_hash AggregateFunction(argMin, FixedString(32), DateTime64(3)), + occurred_at AggregateFunction(argMin, DateTime64(3), DateTime64(3)), + first_received_at SimpleAggregateFunction(min, DateTime64(3)), + received_at SimpleAggregateFunction(max, DateTime64(3)), + event_name AggregateFunction(argMin, String, DateTime64(3)), + schema_version AggregateFunction(argMin, UInt16, DateTime64(3)), + source AggregateFunction(argMin, String, DateTime64(3)), + platform AggregateFunction(argMin, String, DateTime64(3)), + anonymous_id AggregateFunction(argMin, String, DateTime64(3)), + session_id AggregateFunction(argMin, String, DateTime64(3)), + user_id AggregateFunction(argMin, String, DateTime64(3)), + organization_id AggregateFunction(argMin, String, DateTime64(3)), + app_version AggregateFunction(argMin, String, DateTime64(3)), + pathname AggregateFunction(argMin, String, DateTime64(3)), + referrer AggregateFunction(argMin, String, DateTime64(3)), + country AggregateFunction(argMin, String, DateTime64(3)), + region AggregateFunction(argMin, String, DateTime64(3)), + city AggregateFunction(argMin, String, DateTime64(3)), + hostname AggregateFunction(argMin, String, DateTime64(3)), + browser AggregateFunction(argMin, String, DateTime64(3)), + device AggregateFunction(argMin, String, DateTime64(3)), + os AggregateFunction(argMin, String, DateTime64(3)), + channel AggregateFunction(argMin, String, DateTime64(3)), + traffic_class AggregateFunction(argMin, String, DateTime64(3)), + synthetic_run_id AggregateFunction(argMin, String, DateTime64(3)), + properties AggregateFunction(argMin, String, DateTime64(3)) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(occurred_date) +ENGINE_SORTING_KEY (occurred_date, event_id) +ENGINE_TTL occurred_date + INTERVAL 807 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_event_id_states_v2.datasource b/scripts/analytics/tinybird/datasources/product_event_id_states_v2.datasource new file mode 100644 index 00000000000..e878ba8ed4c --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_event_id_states_v2.datasource @@ -0,0 +1,40 @@ +DESCRIPTION > + Streaming per-event aggregate state used to deduplicate delivery retries before decision aggregation. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + event_id String, + payload_hashes AggregateFunction(uniqExact, FixedString(32)), + payload_hash AggregateFunction(argMin, FixedString(32), DateTime64(3)), + occurred_at AggregateFunction(argMin, DateTime64(3), DateTime64(3)), + first_received_at SimpleAggregateFunction(min, DateTime64(3)), + received_at SimpleAggregateFunction(max, DateTime64(3)), + event_name AggregateFunction(argMin, String, DateTime64(3)), + schema_version AggregateFunction(argMin, UInt16, DateTime64(3)), + source AggregateFunction(argMin, String, DateTime64(3)), + platform AggregateFunction(argMin, String, DateTime64(3)), + anonymous_id AggregateFunction(argMin, String, DateTime64(3)), + session_id AggregateFunction(argMin, String, DateTime64(3)), + user_id AggregateFunction(argMin, String, DateTime64(3)), + organization_id AggregateFunction(argMin, String, DateTime64(3)), + app_version AggregateFunction(argMin, String, DateTime64(3)), + pathname AggregateFunction(argMin, String, DateTime64(3)), + referrer AggregateFunction(argMin, String, DateTime64(3)), + country AggregateFunction(argMin, String, DateTime64(3)), + region AggregateFunction(argMin, String, DateTime64(3)), + city AggregateFunction(argMin, String, DateTime64(3)), + hostname AggregateFunction(argMin, String, DateTime64(3)), + browser AggregateFunction(argMin, String, DateTime64(3)), + device AggregateFunction(argMin, String, DateTime64(3)), + os AggregateFunction(argMin, String, DateTime64(3)), + channel AggregateFunction(argMin, String, DateTime64(3)), + traffic_class AggregateFunction(argMin, String, DateTime64(3)), + synthetic_run_id AggregateFunction(argMin, String, DateTime64(3)), + properties AggregateFunction(argMin, String, DateTime64(3)) + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY cityHash64(event_id) % 64 +ENGINE_SORTING_KEY event_id +ENGINE_TTL toDateTime(received_at) + INTERVAL 800 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_events_daily_cold_v2.datasource b/scripts/analytics/tinybird/datasources/product_events_daily_cold_v2.datasource new file mode 100644 index 00000000000..c64f9ec6893 --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_events_daily_cold_v2.datasource @@ -0,0 +1,63 @@ +DESCRIPTION > + Settled long-lived event-decision partitions with one active generation per UTC date. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + source_cutoff DateTime64(3), + generation_id String, + is_marker UInt8, + calculated_at DateTime64(3), + copy_run_id String, + date Date, + event_name LowCardinality(String), + schema_version UInt16, + source LowCardinality(String), + platform LowCardinality(String), + app_version LowCardinality(String), + hostname LowCardinality(String), + country LowCardinality(String), + device LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + channel LowCardinality(String), + traffic_class LowCardinality(String), + synthetic_run_id String, + plan_id LowCardinality(String), + recording_status LowCardinality(String), + payment_status LowCardinality(String), + subscription_status LowCardinality(String), + currency LowCardinality(String), + billing_interval LowCardinality(String), + change_kind LowCardinality(String), + previous_status LowCardinality(String), + new_status LowCardinality(String), + previous_plan_id LowCardinality(String), + quantity Int64, + previous_quantity Int64, + new_quantity Int64, + seat_delta Int64, + first_purchase LowCardinality(String), + guest_checkout LowCardinality(String), + onboarding LowCardinality(String), + cancel_at_period_end LowCardinality(String), + fully_refunded LowCardinality(String), + ended_at Int64, + trial_end_at Int64, + amount_due_minor Int64, + attempt_count Int64, + experiment_id LowCardinality(String), + experiment_variant LowCardinality(String), + assignment_version LowCardinality(String), + delivery_loss_count UInt64, + events UInt64, + actors AggregateFunction(uniqExact, String), + users AggregateFunction(uniqExact, String), + organizations AggregateFunction(uniqExact, String), + revenue_minor Int64 + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY (generation_id, is_marker, date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, recording_status, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version) +ENGINE_TTL date + INTERVAL 800 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource index 9d450bf6833..d8e2b6fbd01 100644 --- a/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_events_daily_exact.datasource @@ -21,6 +21,7 @@ SCHEMA > traffic_class LowCardinality(String), synthetic_run_id String, plan_id LowCardinality(String), + recording_status LowCardinality(String), payment_status LowCardinality(String), subscription_status LowCardinality(String), currency LowCardinality(String), @@ -54,6 +55,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (copy_run_id, date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version) +ENGINE_SORTING_KEY (copy_run_id, date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, recording_status, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version) ENGINE_TTL date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_events_daily_hot_v2.datasource b/scripts/analytics/tinybird/datasources/product_events_daily_hot_v2.datasource new file mode 100644 index 00000000000..5c23120dd4b --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_events_daily_hot_v2.datasource @@ -0,0 +1,63 @@ +DESCRIPTION > + Bounded mutable event-decision window rebuilt from exact canonical event states. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + source_cutoff DateTime64(3), + generation_id String, + is_marker UInt8, + calculated_at DateTime64(3), + copy_run_id String, + date Date, + event_name LowCardinality(String), + schema_version UInt16, + source LowCardinality(String), + platform LowCardinality(String), + app_version LowCardinality(String), + hostname LowCardinality(String), + country LowCardinality(String), + device LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + channel LowCardinality(String), + traffic_class LowCardinality(String), + synthetic_run_id String, + plan_id LowCardinality(String), + recording_status LowCardinality(String), + payment_status LowCardinality(String), + subscription_status LowCardinality(String), + currency LowCardinality(String), + billing_interval LowCardinality(String), + change_kind LowCardinality(String), + previous_status LowCardinality(String), + new_status LowCardinality(String), + previous_plan_id LowCardinality(String), + quantity Int64, + previous_quantity Int64, + new_quantity Int64, + seat_delta Int64, + first_purchase LowCardinality(String), + guest_checkout LowCardinality(String), + onboarding LowCardinality(String), + cancel_at_period_end LowCardinality(String), + fully_refunded LowCardinality(String), + ended_at Int64, + trial_end_at Int64, + amount_due_minor Int64, + attempt_count Int64, + experiment_id LowCardinality(String), + experiment_variant LowCardinality(String), + assignment_version LowCardinality(String), + delivery_loss_count UInt64, + events UInt64, + actors AggregateFunction(uniqExact, String), + users AggregateFunction(uniqExact, String), + organizations AggregateFunction(uniqExact, String), + revenue_minor Int64 + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY (generation_id, is_marker, date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, recording_status, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version) +ENGINE_TTL date + INTERVAL 16 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_experiment_outcomes_exact.datasource b/scripts/analytics/tinybird/datasources/product_experiment_outcomes_exact.datasource new file mode 100644 index 00000000000..50ddbff6f27 --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_experiment_outcomes_exact.datasource @@ -0,0 +1,25 @@ +DESCRIPTION > + Eight-hundred-day aggregate experiment cohorts anchored only to explicit exposures and first authoritative outcomes. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + calculated_at DateTime64(3), + copy_run_id String, + synthetic_run_id String, + cohort_date Date, + experiment_id LowCardinality(String), + assignment_version LowCardinality(String), + variant LowCardinality(String), + platform LowCardinality(String), + app_version LowCardinality(String), + outcome_name LowCardinality(String), + exposed_actors UInt64, + converted_actors UInt64, + copy_marker UInt8 + +ENGINE SummingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(cohort_date) +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, cohort_date, experiment_id, assignment_version, variant, platform, app_version, outcome_name) +ENGINE_TTL cohort_date + INTERVAL 800 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_daily_cold_v2.datasource b/scripts/analytics/tinybird/datasources/product_traffic_daily_cold_v2.datasource new file mode 100644 index 00000000000..94ed1574d5c --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_traffic_daily_cold_v2.datasource @@ -0,0 +1,36 @@ +DESCRIPTION > + Settled long-lived traffic-session partitions with one active generation per UTC date. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + source_cutoff DateTime64(3), + generation_id String, + is_marker UInt8, + calculated_at DateTime64(3), + copy_run_id String, + synthetic_run_id String, + date Date, + platform LowCardinality(String), + app_version LowCardinality(String), + hostname LowCardinality(String), + country LowCardinality(String), + device LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + channel LowCardinality(String), + attribution_source LowCardinality(String), + attribution_medium LowCardinality(String), + campaign LowCardinality(String), + visitors AggregateFunction(uniqExact, String), + visits UInt64, + pageviews UInt64, + bounces UInt64, + visit_duration_ms UInt64, + engaged_ms UInt64 + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY (generation_id, is_marker, synthetic_run_id, date, platform, app_version, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) +ENGINE_TTL date + INTERVAL 800 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource index 8ca28cdb67b..c431d05b520 100644 --- a/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_traffic_daily_exact.datasource @@ -8,6 +8,8 @@ SCHEMA > copy_run_id String, synthetic_run_id String, date Date, + platform LowCardinality(String), + app_version LowCardinality(String), hostname LowCardinality(String), country LowCardinality(String), device LowCardinality(String), @@ -26,6 +28,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, date, platform, app_version, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) ENGINE_TTL date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_daily_hot_v2.datasource b/scripts/analytics/tinybird/datasources/product_traffic_daily_hot_v2.datasource new file mode 100644 index 00000000000..3acf1419b38 --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_traffic_daily_hot_v2.datasource @@ -0,0 +1,36 @@ +DESCRIPTION > + Bounded mutable 30-minute traffic-session window rebuilt from canonical event states. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + source_cutoff DateTime64(3), + generation_id String, + is_marker UInt8, + calculated_at DateTime64(3), + copy_run_id String, + synthetic_run_id String, + date Date, + platform LowCardinality(String), + app_version LowCardinality(String), + hostname LowCardinality(String), + country LowCardinality(String), + device LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + channel LowCardinality(String), + attribution_source LowCardinality(String), + attribution_medium LowCardinality(String), + campaign LowCardinality(String), + visitors AggregateFunction(uniqExact, String), + visits UInt64, + pageviews UInt64, + bounces UInt64, + visit_duration_ms UInt64, + engaged_ms UInt64 + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY (generation_id, is_marker, synthetic_run_id, date, platform, app_version, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign) +ENGINE_TTL date + INTERVAL 16 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_cold_v2.datasource b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_cold_v2.datasource new file mode 100644 index 00000000000..f6ee4d96e78 --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_cold_v2.datasource @@ -0,0 +1,35 @@ +DESCRIPTION > + Settled long-lived page and engagement partitions with one active generation per UTC date. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + source_cutoff DateTime64(3), + generation_id String, + is_marker UInt8, + calculated_at DateTime64(3), + copy_run_id String, + synthetic_run_id String, + date Date, + platform LowCardinality(String), + app_version LowCardinality(String), + hostname LowCardinality(String), + pathname String, + country LowCardinality(String), + device LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + channel LowCardinality(String), + visitors AggregateFunction(uniqExact, String), + visits AggregateFunction(uniqExact, String), + pageviews UInt64, + landings UInt64, + exits UInt64, + engaged_ms UInt64, + scroll_depth_sum Float64 + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY (generation_id, is_marker, synthetic_run_id, date, platform, app_version, hostname, pathname, country, device, browser, os, channel) +ENGINE_TTL date + INTERVAL 800 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource index 267fe688ee6..458c65bd0c8 100644 --- a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource +++ b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_exact.datasource @@ -8,6 +8,8 @@ SCHEMA > copy_run_id String, synthetic_run_id String, date Date, + platform LowCardinality(String), + app_version LowCardinality(String), hostname LowCardinality(String), pathname String, country LowCardinality(String), @@ -25,6 +27,6 @@ SCHEMA > ENGINE AggregatingMergeTree ENGINE_PARTITION_KEY toYYYYMM(date) -ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, date, hostname, pathname, country, device, browser, os, channel) +ENGINE_SORTING_KEY (copy_run_id, synthetic_run_id, date, platform, app_version, hostname, pathname, country, device, browser, os, channel) ENGINE_TTL date + INTERVAL 800 DAY ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_hot_v2.datasource b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_hot_v2.datasource new file mode 100644 index 00000000000..ac76022e505 --- /dev/null +++ b/scripts/analytics/tinybird/datasources/product_traffic_pages_daily_hot_v2.datasource @@ -0,0 +1,35 @@ +DESCRIPTION > + Bounded mutable page, landing, exit, and foreground-engagement window. + +TOKEN product_events_copy_runner APPEND + +SCHEMA > + source_cutoff DateTime64(3), + generation_id String, + is_marker UInt8, + calculated_at DateTime64(3), + copy_run_id String, + synthetic_run_id String, + date Date, + platform LowCardinality(String), + app_version LowCardinality(String), + hostname LowCardinality(String), + pathname String, + country LowCardinality(String), + device LowCardinality(String), + browser LowCardinality(String), + os LowCardinality(String), + channel LowCardinality(String), + visitors AggregateFunction(uniqExact, String), + visits AggregateFunction(uniqExact, String), + pageviews UInt64, + landings UInt64, + exits UInt64, + engaged_ms UInt64, + scroll_depth_sum Float64 + +ENGINE AggregatingMergeTree +ENGINE_PARTITION_KEY toYYYYMM(date) +ENGINE_SORTING_KEY (generation_id, is_marker, synthetic_run_id, date, platform, app_version, hostname, pathname, country, device, browser, os, channel) +ENGINE_TTL date + INTERVAL 16 DAY +ENGINE_SETTINGS index_granularity = 256 diff --git a/scripts/analytics/tinybird/fixtures/product_events_v1.ndjson b/scripts/analytics/tinybird/fixtures/product_events_v1.ndjson index 3d5cca66106..15a237de966 100644 --- a/scripts/analytics/tinybird/fixtures/product_events_v1.ndjson +++ b/scripts/analytics/tinybird/fixtures/product_events_v1.ndjson @@ -3,12 +3,12 @@ {"event_id":"evt_fixture_recording_2","payload_hash":"33333333333333333333333333333333","occurred_at":"2099-01-10 09:10:00.000","received_at":"2099-01-10 09:10:00.100","event_name":"recording_started","schema_version":1,"source":"client","platform":"desktop","anonymous_id":"anon_fixture_2","session_id":"session_fixture_2","user_id":"","organization_id":"","app_version":"1.0.0","pathname":"","referrer":"","country":"GB","region":"ENG","city":"London","hostname":"","browser":"","device":"desktop","os":"Windows","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"mode\":\"camera\"}"} {"event_id":"evt_fixture_purchase_1","payload_hash":"44444444444444444444444444444444","occurred_at":"2099-01-11 10:00:00.000","received_at":"2099-01-11 10:00:00.100","event_name":"purchase_completed","schema_version":1,"source":"server","platform":"web","anonymous_id":"anon_fixture_1","session_id":"session_fixture_1","user_id":"user_fixture_1","organization_id":"org_fixture_1","app_version":"web","pathname":"/dashboard/settings/billing","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"cap.so","browser":"Chrome","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"payment_status\":\"paid\",\"subscription_status\":\"active\",\"amount_total_minor\":2500,\"currency\":\"usd\",\"billing_interval\":\"month\"}"} {"event_id":"evt_fixture_trial_1","payload_hash":"55555555555555555555555555555555","occurred_at":"2099-01-11 11:00:00.000","received_at":"2099-01-11 11:00:00.100","event_name":"trial_started","schema_version":1,"source":"server","platform":"web","anonymous_id":"anon_fixture_trial","session_id":"session_fixture_trial","user_id":"user_fixture_trial","organization_id":"org_fixture_trial","app_version":"web","pathname":"/dashboard/settings/billing","referrer":"","country":"US","region":"NY","city":"New York","hostname":"cap.so","browser":"Chrome","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"subscription_status\":\"trialing\",\"currency\":\"usd\",\"billing_interval\":\"month\"}"} -{"event_id":"evt_fixture_page_1","payload_hash":"66666666666666666666666666666666","occurred_at":"2099-01-10 10:00:00.000","received_at":"2099-01-10 10:00:00.100","event_name":"page_view","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_1","session_id":"visit_fixture_1","user_id":"","organization_id":"","app_version":"web","pathname":"/","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"cap.so","browser":"Chrome","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"hostname\":\"cap.so\",\"is_session_entry\":true}"} -{"event_id":"evt_fixture_engagement_1","payload_hash":"77777777777777777777777777777777","occurred_at":"2099-01-10 10:00:05.000","received_at":"2099-01-10 10:00:05.100","event_name":"page_engagement","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_1","session_id":"visit_fixture_1","user_id":"","organization_id":"","app_version":"web","pathname":"/","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"cap.so","browser":"Chrome","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"page_view_id\":\"evt_fixture_page_1\",\"engaged_ms\":5000,\"max_scroll_depth\":30}"} -{"event_id":"evt_fixture_page_2","payload_hash":"88888888888888888888888888888888","occurred_at":"2099-01-10 10:10:00.000","received_at":"2099-01-10 10:10:00.100","event_name":"page_view","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/pricing","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"hostname\":\"cap.so\",\"is_session_entry\":true,\"session_touch_source\":\"google\",\"session_touch_medium\":\"organic\",\"session_touch_campaign\":\"launch\"}"} -{"event_id":"evt_fixture_engagement_2","payload_hash":"99999999999999999999999999999999","occurred_at":"2099-01-10 10:10:15.000","received_at":"2099-01-10 10:10:15.100","event_name":"page_engagement","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/pricing","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"page_view_id\":\"evt_fixture_page_2\",\"engaged_ms\":15000,\"max_scroll_depth\":80}"} -{"event_id":"evt_fixture_page_3","payload_hash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","occurred_at":"2099-01-10 10:12:00.000","received_at":"2099-01-10 10:12:00.100","event_name":"page_view","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/download","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"hostname\":\"cap.so\",\"is_session_entry\":false,\"session_touch_source\":\"google\",\"session_touch_medium\":\"organic\",\"session_touch_campaign\":\"launch\"}"} -{"event_id":"evt_fixture_engagement_3","payload_hash":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","occurred_at":"2099-01-10 10:12:03.000","received_at":"2099-01-10 10:12:03.100","event_name":"page_engagement","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/download","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"page_view_id\":\"evt_fixture_page_3\",\"engaged_ms\":3000,\"max_scroll_depth\":20}"} +{"event_id":"evt_fixture_page_1","payload_hash":"66666666666666666666666666666666","occurred_at":"2099-01-10 10:00:00.000","received_at":"2099-01-10 10:00:00.100","event_name":"page_view","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_1","session_id":"visit_fixture_1","user_id":"","organization_id":"","app_version":"web","pathname":"/","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"cap.so","browser":"Chrome","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"hostname\":\"cap.so\",\"is_session_entry\":true,\"session_started_at\":\"2026-07-29T10:00:00.000Z\"}"} +{"event_id":"evt_fixture_engagement_1","payload_hash":"77777777777777777777777777777777","occurred_at":"2099-01-10 10:00:05.000","received_at":"2099-01-10 10:00:05.100","event_name":"page_engagement","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_1","session_id":"visit_fixture_1","user_id":"","organization_id":"","app_version":"web","pathname":"/","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"cap.so","browser":"Chrome","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"page_view_id\":\"evt_fixture_page_1\",\"engaged_ms\":5000,\"max_scroll_depth\":30,\"session_started_at\":\"2026-07-29T10:00:00.000Z\"}"} +{"event_id":"evt_fixture_page_2","payload_hash":"88888888888888888888888888888888","occurred_at":"2099-01-10 10:10:00.000","received_at":"2099-01-10 10:10:00.100","event_name":"page_view","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/pricing","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"hostname\":\"cap.so\",\"is_session_entry\":true,\"session_started_at\":\"2026-07-29T10:10:00.000Z\",\"session_touch_source\":\"google\",\"session_touch_medium\":\"organic\",\"session_touch_campaign\":\"launch\"}"} +{"event_id":"evt_fixture_engagement_2","payload_hash":"99999999999999999999999999999999","occurred_at":"2099-01-10 10:10:15.000","received_at":"2099-01-10 10:10:15.100","event_name":"page_engagement","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/pricing","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"page_view_id\":\"evt_fixture_page_2\",\"engaged_ms\":15000,\"max_scroll_depth\":80,\"session_started_at\":\"2026-07-29T10:10:00.000Z\"}"} +{"event_id":"evt_fixture_page_3","payload_hash":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","occurred_at":"2099-01-10 10:12:00.000","received_at":"2099-01-10 10:12:00.100","event_name":"page_view","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/download","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"hostname\":\"cap.so\",\"is_session_entry\":false,\"session_started_at\":\"2026-07-29T10:10:00.000Z\",\"session_touch_source\":\"google\",\"session_touch_medium\":\"organic\",\"session_touch_campaign\":\"launch\"}"} +{"event_id":"evt_fixture_engagement_3","payload_hash":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","occurred_at":"2099-01-10 10:12:03.000","received_at":"2099-01-10 10:12:03.100","event_name":"page_engagement","schema_version":1,"source":"client","platform":"web","anonymous_id":"anon_traffic_2","session_id":"visit_fixture_2","user_id":"","organization_id":"","app_version":"web","pathname":"/download","referrer":"google.com","country":"GB","region":"ENG","city":"London","hostname":"cap.so","browser":"Safari","device":"mobile","os":"iOS","channel":"organic_search","traffic_class":"external","synthetic_run_id":"","properties":"{\"page_view_id\":\"evt_fixture_page_3\",\"engaged_ms\":3000,\"max_scroll_depth\":20,\"session_started_at\":\"2026-07-29T10:10:00.000Z\"}"} {"event_id":"evt_fixture_recording_d1_desktop","payload_hash":"cccccccccccccccccccccccccccccccc","occurred_at":"2099-01-11 09:00:00.000","received_at":"2099-01-11 09:00:00.100","event_name":"recording_started","schema_version":1,"source":"client","platform":"desktop","anonymous_id":"anon_fixture_1","session_id":"session_fixture_1","user_id":"user_fixture_1","organization_id":"org_fixture_1","app_version":"1.0.0","pathname":"","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"","browser":"","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"mode\":\"screen\",\"target_kind\":\"display\",\"has_camera\":false,\"has_mic\":true,\"has_system_audio\":true,\"target_fps\":30,\"target_width\":1920,\"target_height\":1080,\"fragmented\":true,\"custom_cursor_capture\":true}"} {"event_id":"evt_fixture_recording_d1_mobile","payload_hash":"dddddddddddddddddddddddddddddddd","occurred_at":"2099-01-11 09:05:00.000","received_at":"2099-01-11 09:05:00.100","event_name":"recording_started","schema_version":1,"source":"client","platform":"mobile","anonymous_id":"anon_fixture_1","session_id":"session_fixture_mobile","user_id":"user_fixture_1","organization_id":"org_fixture_1","app_version":"1.0.0","pathname":"","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"","browser":"","device":"mobile","os":"iOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"mode\":\"screen\",\"target_kind\":\"display\",\"has_camera\":false,\"has_mic\":true,\"has_system_audio\":false,\"target_fps\":30,\"target_width\":1179,\"target_height\":2556,\"fragmented\":true,\"custom_cursor_capture\":false}"} {"event_id":"evt_fixture_checkout_web","payload_hash":"eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","occurred_at":"2099-01-12 09:00:00.000","received_at":"2099-01-12 09:00:00.100","event_name":"checkout_started","schema_version":1,"source":"server","platform":"web","anonymous_id":"anon_fixture_1","session_id":"session_fixture_web_checkout","user_id":"user_fixture_1","organization_id":"org_fixture_1","app_version":"web","pathname":"/pricing","referrer":"","country":"US","region":"CA","city":"San Francisco","hostname":"cap.so","browser":"Chrome","device":"desktop","os":"macOS","channel":"direct","traffic_class":"external","synthetic_run_id":"","properties":"{\"price_id\":\"price_fixture\",\"quantity\":1,\"is_onboarding\":false}"} diff --git a/scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe b/scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe new file mode 100644 index 00000000000..b1136d2459f --- /dev/null +++ b/scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe @@ -0,0 +1,40 @@ +DESCRIPTION > + Continuously merge retry attempts into exact event states partitioned by UTC occurrence date. + +NODE materialize_product_event_day_states_v2_node +SQL > + SELECT + toDate(occurred_at, 'UTC') AS occurred_date, + event_id, + uniqExactState(payload_hash) AS payload_hashes, + argMinState(payload_hash, received_at) AS payload_hash, + argMinState(occurred_at, received_at) AS occurred_at, + min(received_at) AS first_received_at, + max(received_at) AS received_at, + argMinState(toString(event_name), received_at) AS event_name, + argMinState(schema_version, received_at) AS schema_version, + argMinState(toString(source), received_at) AS source, + argMinState(toString(platform), received_at) AS platform, + argMinState(anonymous_id, received_at) AS anonymous_id, + argMinState(session_id, received_at) AS session_id, + argMinState(user_id, received_at) AS user_id, + argMinState(organization_id, received_at) AS organization_id, + argMinState(toString(app_version), received_at) AS app_version, + argMinState(pathname, received_at) AS pathname, + argMinState(referrer, received_at) AS referrer, + argMinState(toString(country), received_at) AS country, + argMinState(toString(region), received_at) AS region, + argMinState(toString(city), received_at) AS city, + argMinState(toString(hostname), received_at) AS hostname, + argMinState(toString(browser), received_at) AS browser, + argMinState(toString(device), received_at) AS device, + argMinState(toString(os), received_at) AS os, + argMinState(toString(channel), received_at) AS channel, + argMinState(toString(traffic_class), received_at) AS traffic_class, + argMinState(synthetic_run_id, received_at) AS synthetic_run_id, + argMinState(properties, received_at) AS properties + FROM product_events_v1 + GROUP BY occurred_date, event_id + +TYPE MATERIALIZED +DATASOURCE product_event_day_states_v2 diff --git a/scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe b/scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe new file mode 100644 index 00000000000..aa26d0d922d --- /dev/null +++ b/scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe @@ -0,0 +1,39 @@ +DESCRIPTION > + Continuously merge retry attempts into one exact aggregate state per immutable event ID. + +NODE materialize_product_event_id_states_v2_node +SQL > + SELECT + event_id, + uniqExactState(payload_hash) AS payload_hashes, + argMinState(payload_hash, received_at) AS payload_hash, + argMinState(occurred_at, received_at) AS occurred_at, + min(received_at) AS first_received_at, + max(received_at) AS received_at, + argMinState(toString(event_name), received_at) AS event_name, + argMinState(schema_version, received_at) AS schema_version, + argMinState(toString(source), received_at) AS source, + argMinState(toString(platform), received_at) AS platform, + argMinState(anonymous_id, received_at) AS anonymous_id, + argMinState(session_id, received_at) AS session_id, + argMinState(user_id, received_at) AS user_id, + argMinState(organization_id, received_at) AS organization_id, + argMinState(toString(app_version), received_at) AS app_version, + argMinState(pathname, received_at) AS pathname, + argMinState(referrer, received_at) AS referrer, + argMinState(toString(country), received_at) AS country, + argMinState(toString(region), received_at) AS region, + argMinState(toString(city), received_at) AS city, + argMinState(toString(hostname), received_at) AS hostname, + argMinState(toString(browser), received_at) AS browser, + argMinState(toString(device), received_at) AS device, + argMinState(toString(os), received_at) AS os, + argMinState(toString(channel), received_at) AS channel, + argMinState(toString(traffic_class), received_at) AS traffic_class, + argMinState(synthetic_run_id, received_at) AS synthetic_run_id, + argMinState(properties, received_at) AS properties + FROM product_events_v1 + GROUP BY event_id + +TYPE MATERIALIZED +DATASOURCE product_event_id_states_v2 diff --git a/scripts/analytics/tinybird/pipes/materialize_product_events_health_hourly.pipe b/scripts/analytics/tinybird/pipes/materialize_product_events_health_hourly.pipe new file mode 100644 index 00000000000..7cecec0808d --- /dev/null +++ b/scripts/analytics/tinybird/pipes/materialize_product_events_health_hourly.pipe @@ -0,0 +1,23 @@ +DESCRIPTION > + Continuously aggregate delivery attempts, duplicates, clock skew, identity gaps, and ingestion lag. + +NODE materialize_product_events_health_hourly_node +SQL > + SELECT + toStartOfHour(received_at) AS hour, + now64(3) AS calculated_at, + '' AS copy_run_id, + platform, + app_version, + count() AS received_rows, + uniqExactState(event_id) AS unique_events, + uniqExactState(tuple(event_id, payload_hash)) AS unique_payloads, + countIf(occurred_at > received_at + INTERVAL 5 MINUTE) AS future_events, + countIf(occurred_at < received_at - INTERVAL 1 DAY) AS late_events, + countIf(user_id = '' AND anonymous_id = '' AND session_id = '') AS missing_identity_events, + quantilesTDigestState(0.5, 0.95, 0.99)(toFloat64(dateDiff('millisecond', occurred_at, received_at))) AS ingestion_lag_ms + FROM product_events_v1 + GROUP BY hour, platform, app_version + +TYPE MATERIALIZED +DATASOURCE product_events_health_hourly_exact diff --git a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe index 06682f6d7b2..da8b484ef40 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe @@ -16,6 +16,8 @@ SQL > (SELECT count() FROM product_activation_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS activation_markers, (SELECT count() FROM product_creator_retention_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS retention_markers, (SELECT count() FROM product_identity_funnel_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS identity_markers, + (SELECT count() FROM product_attribution_daily_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS attribution_markers, + (SELECT count() FROM product_experiment_outcomes_exact WHERE copy_run_id = {{String(copy_run_id)}} AND copy_marker = 1) AS experiment_markers, (SELECT count() FROM product_events_health_hourly_exact WHERE copy_run_id = {{String(copy_run_id)}}) AS health_markers WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 diff --git a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe index 66d3295e55f..74892360b67 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe @@ -11,7 +11,9 @@ SQL > (SELECT max(calculated_at) FROM product_events_daily_exact WHERE copy_run_id = '') AS product_calculated_at, (SELECT max(calculated_at) FROM product_traffic_daily_exact) AS traffic_calculated_at, (SELECT max(calculated_at) FROM product_creator_retention_exact) AS retention_calculated_at, - (SELECT max(calculated_at) FROM product_identity_funnel_exact) AS identity_calculated_at + (SELECT max(calculated_at) FROM product_identity_funnel_exact) AS identity_calculated_at, + (SELECT max(calculated_at) FROM product_attribution_daily_exact) AS attribution_calculated_at, + (SELECT max(calculated_at) FROM product_experiment_outcomes_exact) AS experiment_calculated_at FROM product_events_health_hourly_exact WHERE copy_run_id = '' diff --git a/scripts/analytics/tinybird/pipes/product_attribution.pipe b/scripts/analytics/tinybird/pipes/product_attribution.pipe new file mode 100644 index 00000000000..387b3516e02 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_attribution.pipe @@ -0,0 +1,40 @@ +DESCRIPTION > + Query aggregate first, session, and last campaign attribution without returning visitor or session identifiers. + +TOKEN product_events_agent_read READ + +NODE product_attribution_node +SQL > + % + SELECT + attribution_model, + attribution_source AS source, + attribution_medium AS medium, + campaign, + uniqExactMerge(visitors) AS visitors, + toUInt64(sum(visits)) AS visits, + toUInt64(sum(pageviews)) AS pageviews + FROM product_attribution_daily_exact + WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} + AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(attribution_model) %} + AND attribution_model = {{String(attribution_model)}} + AND throwIf(NOT has(['first', 'session', 'last'], {{String(attribution_model)}}), 'attribution_model must be first, session, or last') = 0 + {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} + {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} + {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} + {% if defined(source) %} AND attribution_source = {{String(source)}} {% end %} + GROUP BY attribution_model, attribution_source, attribution_medium, campaign + ORDER BY attribution_model ASC, visits DESC, source ASC, medium ASC, campaign ASC + LIMIT greatest(1, least({{Int32(limit, 300)}}, 1000)) + +TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe b/scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe new file mode 100644 index 00000000000..9fb3ec0ec74 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe @@ -0,0 +1,39 @@ +DESCRIPTION > + Finalize streaming event states, count exact retries once, and quarantine payload conflicts. + +TOKEN product_events_copy_runner READ + +NODE product_events_canonical_current +SQL > + % + SELECT + event_id, + argMinMerge(payload_hash) AS payload_hash, + argMinMerge(occurred_at) AS occurred_at, + max(received_at) AS received_at, + argMinMerge(event_name) AS event_name, + argMinMerge(schema_version) AS schema_version, + argMinMerge(source) AS source, + argMinMerge(platform) AS platform, + argMinMerge(anonymous_id) AS anonymous_id, + argMinMerge(session_id) AS session_id, + argMinMerge(user_id) AS user_id, + argMinMerge(organization_id) AS organization_id, + argMinMerge(app_version) AS app_version, + argMinMerge(pathname) AS pathname, + argMinMerge(referrer) AS referrer, + argMinMerge(country) AS country, + argMinMerge(region) AS region, + argMinMerge(city) AS city, + argMinMerge(hostname) AS hostname, + argMinMerge(browser) AS browser, + argMinMerge(device) AS device, + argMinMerge(os) AS os, + argMinMerge(channel) AS channel, + argMinMerge(traffic_class) AS traffic_class, + argMinMerge(synthetic_run_id) AS synthetic_run_id, + argMinMerge(properties) AS properties + FROM product_event_id_states_v2 + WHERE first_received_at <= {% if defined(source_cutoff) %} toDateTime64({{DateTime64(source_cutoff)}}, 3) {% else %} now64(3) {% end %} + GROUP BY event_id + HAVING uniqExactMerge(payload_hashes) = 1 diff --git a/scripts/analytics/tinybird/pipes/product_events_canonical_window.pipe b/scripts/analytics/tinybird/pipes/product_events_canonical_window.pipe new file mode 100644 index 00000000000..51bad1cab51 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_events_canonical_window.pipe @@ -0,0 +1,60 @@ +DESCRIPTION > + Finalize one bounded UTC event window while retaining global payload-conflict quarantine. + +TOKEN product_events_copy_runner READ + +NODE product_events_canonical_window_node +SQL > + % + {% if not defined(source_cutoff) %} + {{ error('source_cutoff is required') }} + {% end %} + WITH candidate_ids AS ( + SELECT event_id + FROM product_event_day_states_v2 + WHERE occurred_date >= {% if defined(settled_date) %} toDate({{Date(settled_date)}}) - INTERVAL 1 DAY {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 9 DAY {% end %} + AND occurred_date < {% if defined(settled_date) %} toDate({{Date(settled_date)}}) + INTERVAL 2 DAY {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) + INTERVAL 1 DAY {% end %} + AND first_received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + GROUP BY event_id + ), valid_ids AS ( + SELECT event_id + FROM product_event_id_states_v2 + WHERE event_id IN (SELECT event_id FROM candidate_ids) + AND first_received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + GROUP BY event_id + HAVING uniqExactMerge(payload_hashes) = 1 + ) + SELECT + event_id, + argMinMerge(payload_hash) AS payload_hash, + argMinMerge(occurred_at) AS occurred_at, + max(received_at) AS received_at, + argMinMerge(event_name) AS event_name, + argMinMerge(schema_version) AS schema_version, + argMinMerge(source) AS source, + argMinMerge(platform) AS platform, + argMinMerge(anonymous_id) AS anonymous_id, + argMinMerge(session_id) AS session_id, + argMinMerge(user_id) AS user_id, + argMinMerge(organization_id) AS organization_id, + argMinMerge(app_version) AS app_version, + argMinMerge(pathname) AS pathname, + argMinMerge(referrer) AS referrer, + argMinMerge(country) AS country, + argMinMerge(region) AS region, + argMinMerge(city) AS city, + argMinMerge(hostname) AS hostname, + argMinMerge(browser) AS browser, + argMinMerge(device) AS device, + argMinMerge(os) AS os, + argMinMerge(channel) AS channel, + argMinMerge(traffic_class) AS traffic_class, + argMinMerge(synthetic_run_id) AS synthetic_run_id, + argMinMerge(properties) AS properties + FROM product_event_day_states_v2 + INNER JOIN valid_ids USING event_id + WHERE occurred_date >= {% if defined(settled_date) %} toDate({{Date(settled_date)}}) - INTERVAL 1 DAY {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 9 DAY {% end %} + AND occurred_date < {% if defined(settled_date) %} toDate({{Date(settled_date)}}) + INTERVAL 2 DAY {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) + INTERVAL 1 DAY {% end %} + AND first_received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + GROUP BY event_id + HAVING uniqExactMerge(payload_hashes) = 1 diff --git a/scripts/analytics/tinybird/pipes/product_events_daily.pipe b/scripts/analytics/tinybird/pipes/product_events_daily.pipe index 714bd2eb476..e28c61f70ea 100644 --- a/scripts/analytics/tinybird/pipes/product_events_daily.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_daily.pipe @@ -20,6 +20,7 @@ SQL > os, channel, plan_id, + recording_status, payment_status, subscription_status, currency, @@ -73,6 +74,10 @@ SQL > {% if defined(os) %} AND os = {{String(os)}} {% end %} {% if defined(channel) %} AND channel = {{String(channel)}} {% end %} {% if defined(plan_id) %} AND plan_id = {{String(plan_id)}} {% end %} + {% if defined(recording_status) %} + AND recording_status = {{String(recording_status)}} + AND throwIf({{String(recording_status)}} NOT IN ('success', 'degraded', 'failed', 'unknown'), 'recording_status is invalid') = 0 + {% end %} {% if defined(payment_status) %} AND payment_status = {{String(payment_status)}} {% end %} {% if defined(subscription_status) %} AND subscription_status = {{String(subscription_status)}} {% end %} {% if defined(currency) %} AND currency = upper({{String(currency)}}) {% end %} @@ -86,8 +91,8 @@ SQL > {% if defined(experiment_id) %} AND experiment_id = {{String(experiment_id)}} {% end %} {% if defined(experiment_variant) %} AND experiment_variant = {{String(experiment_variant)}} {% end %} {% if defined(assignment_version) %} AND assignment_version = {{String(assignment_version)}} {% end %} - GROUP BY date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, plan_id, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version, delivery_loss_count, events, revenue_minor, calculated_at - ORDER BY date DESC, event_name ASC, schema_version DESC, source ASC, platform ASC, app_version ASC, hostname ASC, country ASC, device ASC, browser ASC, os ASC, channel ASC, plan_id ASC, payment_status ASC, subscription_status ASC, currency ASC, billing_interval ASC, change_kind ASC, previous_status ASC, new_status ASC, previous_plan_id ASC, experiment_id ASC, experiment_variant ASC, assignment_version ASC + GROUP BY date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, plan_id, recording_status, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version, delivery_loss_count, events, revenue_minor, calculated_at + ORDER BY date DESC, event_name ASC, schema_version DESC, source ASC, platform ASC, app_version ASC, hostname ASC, country ASC, device ASC, browser ASC, os ASC, channel ASC, plan_id ASC, recording_status ASC, payment_status ASC, subscription_status ASC, currency ASC, billing_interval ASC, change_kind ASC, previous_status ASC, new_status ASC, previous_plan_id ASC, experiment_id ASC, experiment_variant ASC, assignment_version ASC LIMIT greatest(1, least({{Int32(limit, 1000)}}, 1000)) TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe b/scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe new file mode 100644 index 00000000000..63053a09f90 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe @@ -0,0 +1,112 @@ +DESCRIPTION > + Build one generation of bounded hot or settled daily event decisions from canonical event states. + +TOKEN product_events_copy_runner READ + +NODE product_events_daily_bounded_v2_node +SQL > + % + {% if not defined(source_cutoff) %} + {{ error('source_cutoff is required') }} + {% end %} + {% if not defined(generation_id) %} + {{ error('generation_id is required') }} + {% end %} + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + SELECT + toDateTime64({{DateTime64(source_cutoff)}}, 3) AS source_cutoff, + {{String(generation_id)}} AS generation_id, + toUInt8(0) AS is_marker, + now64(3) AS calculated_at, + '' AS copy_run_id, + toDate(occurred_at, 'UTC') AS date, + event_name, + schema_version, + source, + platform, + app_version, + hostname, + country, + device, + browser, + os, + channel, + traffic_class, + synthetic_run_id, + if(event_name = 'subscription_changed', JSONExtractString(properties, 'new_price_id'), JSONExtractString(properties, 'price_id')) AS plan_id, + if( + event_name = 'recording_completed', + multiIf( + lower(JSONExtractString(properties, 'status')) = 'success', 'success', + lower(JSONExtractString(properties, 'status')) = 'degraded', 'degraded', + lower(JSONExtractString(properties, 'status')) = 'failed', 'failed', + 'unknown' + ), + '' + ) AS recording_status, + if(event_name = 'purchase_completed', JSONExtractString(properties, 'payment_status'), '') AS payment_status, + multiIf( + event_name IN ('purchase_completed', 'trial_started'), JSONExtractString(properties, 'subscription_status'), + event_name = 'trial_converted', JSONExtractString(properties, 'new_status'), + event_name = 'subscription_changed', JSONExtractString(properties, 'new_status'), + event_name = 'subscription_cancelled', JSONExtractString(properties, 'status'), + '' + ) AS subscription_status, + if(event_name IN ('purchase_completed', 'trial_started', 'subscription_renewed', 'subscription_refunded', 'subscription_payment_failed'), upper(JSONExtractString(properties, 'currency')), '') AS currency, + if(event_name IN ('purchase_completed', 'trial_started'), JSONExtractString(properties, 'billing_interval'), '') AS billing_interval, + if(event_name = 'subscription_changed', JSONExtractString(properties, 'change_kind'), '') AS change_kind, + if(event_name IN ('trial_converted', 'subscription_changed'), JSONExtractString(properties, 'previous_status'), '') AS previous_status, + if(event_name IN ('trial_converted', 'subscription_changed'), JSONExtractString(properties, 'new_status'), '') AS new_status, + if(event_name = 'subscription_changed', JSONExtractString(properties, 'previous_price_id'), '') AS previous_plan_id, + if(event_name IN ('checkout_started', 'guest_checkout_started', 'purchase_completed', 'trial_started'), JSONExtractInt(properties, 'quantity'), 0) AS quantity, + if(event_name = 'subscription_changed', JSONExtractInt(properties, 'previous_quantity'), 0) AS previous_quantity, + if(event_name = 'subscription_changed', JSONExtractInt(properties, 'new_quantity'), 0) AS new_quantity, + if(event_name = 'subscription_changed' AND JSONExtractString(properties, 'change_kind') = 'seats', JSONExtractInt(properties, 'new_quantity') - JSONExtractInt(properties, 'previous_quantity'), 0) AS seat_delta, + if(event_name = 'purchase_completed', if(JSONExtractBool(properties, 'is_first_purchase'), 'true', 'false'), '') AS first_purchase, + if(event_name IN ('purchase_completed', 'trial_started'), if(JSONExtractBool(properties, 'is_guest_checkout'), 'true', 'false'), '') AS guest_checkout, + if(event_name IN ('checkout_started', 'purchase_completed', 'trial_started'), if(JSONExtractBool(properties, 'is_onboarding'), 'true', 'false'), '') AS onboarding, + if(event_name = 'subscription_cancelled', if(JSONExtractBool(properties, 'cancel_at_period_end'), 'true', 'false'), '') AS cancel_at_period_end, + if(event_name = 'subscription_refunded', if(JSONExtractBool(properties, 'fully_refunded'), 'true', 'false'), '') AS fully_refunded, + if(event_name = 'subscription_cancelled', JSONExtractInt(properties, 'ended_at'), 0) AS ended_at, + if(event_name = 'trial_started', JSONExtractInt(properties, 'trial_end_at'), 0) AS trial_end_at, + if(event_name = 'subscription_payment_failed', JSONExtractInt(properties, 'amount_due_minor'), 0) AS amount_due_minor, + if(event_name = 'subscription_payment_failed', JSONExtractInt(properties, 'attempt_count'), 0) AS attempt_count, + if(event_name = 'experiment_exposed', JSONExtractString(properties, 'experiment_id'), '') AS experiment_id, + if(event_name = 'experiment_exposed', JSONExtractString(properties, 'variant'), '') AS experiment_variant, + if(event_name = 'experiment_exposed', JSONExtractString(properties, 'assignment_version'), '') AS assignment_version, + toUInt64(sum(if(event_name = 'analytics_delivery_loss', greatest(JSONExtractInt(properties, 'count'), 0), 0))) AS delivery_loss_count, + toUInt64(count()) AS events, + uniqExactState(if(user_id != '', concat('user:', user_id), if(anonymous_id != '', concat('anonymous:', anonymous_id), concat('session:', session_id)))) AS actors, + uniqExactStateIf(user_id, user_id != '') AS users, + uniqExactStateIf(organization_id, organization_id != '') AS organizations, + toInt64(sum(multiIf( + event_name = 'purchase_completed' AND payment_status = 'paid', JSONExtractInt(properties, 'amount_total_minor'), + event_name = 'subscription_renewed', JSONExtractInt(properties, 'amount_paid_minor'), + event_name = 'subscription_refunded', -JSONExtractInt(properties, 'amount_refunded_minor'), + 0 + ))) AS revenue_minor + FROM product_events_canonical_window + WHERE toDate(occurred_at, 'UTC') >= {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY {% end %} + AND toDate(occurred_at, 'UTC') < {% if defined(settled_date) %} toDate({{Date(settled_date)}}) + INTERVAL 1 DAY {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) + INTERVAL 1 DAY {% end %} + GROUP BY date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, recording_status, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version + UNION ALL + SELECT + toDateTime64({{DateTime64(source_cutoff)}}, 3), + {{String(generation_id)}}, + toUInt8(1), + now64(3), + {% if defined(copy_run_id) %} {{String(copy_run_id)}} {% else %} '' {% end %}, + {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) {% end %}, + '', toUInt16(0), '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + toInt64(0), toInt64(0), toInt64(0), toInt64(0), + '', '', '', '', '', + toInt64(0), toInt64(0), toInt64(0), toInt64(0), + '', '', '', + toUInt64(0), toUInt64(0), + uniqExactStateIf('', 0), uniqExactStateIf('', 0), uniqExactStateIf('', 0), + toInt64(0) + WHERE throwIf(length({{String(generation_id)}}) < 8 OR length({{String(generation_id)}}) > 128, 'generation_id has an invalid length') = 0 + {% if defined(settled_date) %} AND throwIf(toDate({{Date(settled_date)}}) > toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY, 'settled_date is inside the mutable window') = 0 {% end %} + diff --git a/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe new file mode 100644 index 00000000000..c29e7abdee8 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe @@ -0,0 +1,36 @@ +DESCRIPTION > + Expose the active bounded event-decision generation and latest settled UTC partitions. + +NODE product_events_daily_current_v2_node +SQL > + WITH hot_generation AS ( + SELECT + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'hot' + ), cold_generations AS ( + SELECT + settled_date AS date, + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'cold' + GROUP BY settled_date + ) + SELECT hot.* + FROM product_events_daily_hot_v2 AS hot + INNER JOIN hot_generation AS generation + ON hot.generation_id = generation.generation_id + AND hot.source_cutoff = generation.source_cutoff + WHERE hot.is_marker = 0 + UNION ALL + SELECT cold.* + FROM product_events_daily_cold_v2 AS cold + INNER JOIN cold_generations AS generation + ON cold.date = generation.date + AND cold.generation_id = generation.generation_id + AND cold.source_cutoff = generation.source_cutoff + CROSS JOIN hot_generation AS hot_generation + WHERE cold.is_marker = 0 + AND cold.date < toDate(hot_generation.source_cutoff, 'UTC') - INTERVAL 8 DAY diff --git a/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe b/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe new file mode 100644 index 00000000000..2a3469d9dd0 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe @@ -0,0 +1,42 @@ +DESCRIPTION > + Query aggregate 30-day first authoritative outcomes for actors with an explicit stable experiment exposure. + +TOKEN product_events_agent_read READ + +NODE product_experiment_outcomes_node +SQL > + % + SELECT + experiment_id, + assignment_version, + variant, + platform, + app_version, + outcome_name, + toUInt64(sum(exposed_actors)) AS exposed_actors, + toUInt64(sum(converted_actors)) AS converted_actors, + round(if(sum(exposed_actors) = 0, 0, 100 * sum(converted_actors) / sum(exposed_actors)), 2) AS conversion_rate + FROM product_experiment_outcomes_exact + WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} + AND cohort_date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + AND cohort_date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(experiment_id) %} AND experiment_id = {{String(experiment_id)}} {% end %} + {% if defined(assignment_version) %} AND assignment_version = {{String(assignment_version)}} {% end %} + {% if defined(variant) %} AND variant = {{String(variant)}} {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} + {% if defined(outcome_name) %} + AND outcome_name = {{String(outcome_name)}} + AND throwIf(NOT has(['signup', 'share_created', 'paid_purchase'], {{String(outcome_name)}}), 'outcome_name is invalid') = 0 + {% end %} + GROUP BY experiment_id, assignment_version, variant, platform, app_version, outcome_name + ORDER BY experiment_id ASC, assignment_version ASC, variant ASC, outcome_name ASC, platform ASC, app_version ASC + LIMIT greatest(1, least({{Int32(limit, 1000)}}, 1000)) + +TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe index c51cc33e973..905097520a2 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe @@ -21,6 +21,8 @@ SQL > {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} GROUP BY country ORDER BY visits DESC, country ASC diff --git a/scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe new file mode 100644 index 00000000000..474c925f36d --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe @@ -0,0 +1,98 @@ +DESCRIPTION > + Build one generation of bounded hot or settled 30-minute traffic-session decisions. + +TOKEN product_events_copy_runner READ + +NODE product_traffic_daily_bounded_v2_node +SQL > + % + {% if not defined(source_cutoff) %} + {{ error('source_cutoff is required') }} + {% end %} + {% if not defined(generation_id) %} + {{ error('generation_id is required') }} + {% end %} + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + WITH session_engagement AS ( + SELECT + synthetic_run_id, + session_id, + toUInt64(sum(greatest(0, JSONExtractInt(properties, 'engaged_ms')))) AS session_engaged_ms, + max(greatest(0, least(100, JSONExtractInt(properties, 'max_scroll_depth')))) AS session_max_scroll_depth + FROM product_events_canonical_window + WHERE event_name = 'page_engagement' + AND (traffic_class = 'external' OR synthetic_run_id != '') + AND session_id != '' + GROUP BY synthetic_run_id, session_id + ), sessions AS ( + SELECT + synthetic_run_id, + session_id, + concat('anonymous:', argMin(anonymous_id, tuple(occurred_at, event_id))) AS visitor_id, + min(occurred_at) AS started_at, + argMin(platform, tuple(occurred_at, event_id)) AS platform, + argMin(app_version, tuple(occurred_at, event_id)) AS app_version, + argMin(hostname, tuple(occurred_at, event_id)) AS hostname, + argMin(country, tuple(occurred_at, event_id)) AS country, + argMin(device, tuple(occurred_at, event_id)) AS device, + argMin(browser, tuple(occurred_at, event_id)) AS browser, + argMin(os, tuple(occurred_at, event_id)) AS os, + argMin(channel, tuple(occurred_at, event_id)) AS channel, + argMin(JSONExtractString(properties, 'session_touch_source'), tuple(occurred_at, event_id)) AS attribution_source, + argMin(JSONExtractString(properties, 'session_touch_medium'), tuple(occurred_at, event_id)) AS attribution_medium, + argMin(JSONExtractString(properties, 'session_touch_campaign'), tuple(occurred_at, event_id)) AS campaign, + toUInt64(count()) AS session_pageviews + FROM product_events_canonical_window + WHERE event_name = 'page_view' + AND (traffic_class = 'external' OR synthetic_run_id != '') + AND session_id != '' + AND anonymous_id != '' + GROUP BY synthetic_run_id, session_id + ) + SELECT + toDateTime64({{DateTime64(source_cutoff)}}, 3) AS source_cutoff, + {{String(generation_id)}} AS generation_id, + toUInt8(0) AS is_marker, + now64(3) AS calculated_at, + '' AS copy_run_id, + synthetic_run_id, + toDate(started_at, 'UTC') AS date, + platform, + app_version, + hostname, + country, + device, + browser, + os, + channel, + attribution_source, + attribution_medium, + campaign, + uniqExactState(visitor_id) AS visitors, + toUInt64(count()) AS visits, + toUInt64(sum(session_pageviews)) AS pageviews, + toUInt64(sum(if(session_pageviews = 1 AND coalesce(session_engaged_ms, 0) < 10000 AND coalesce(session_max_scroll_depth, 0) < 50, 1, 0))) AS bounces, + toUInt64(sum(coalesce(session_engaged_ms, 0))) AS visit_duration_ms, + toUInt64(sum(coalesce(session_engaged_ms, 0))) AS engaged_ms + FROM sessions + LEFT JOIN session_engagement USING (synthetic_run_id, session_id) + GROUP BY synthetic_run_id, date, platform, app_version, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign + HAVING date >= {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY {% end %} + AND date < {% if defined(settled_date) %} toDate({{Date(settled_date)}}) + INTERVAL 1 DAY {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) + INTERVAL 1 DAY {% end %} + UNION ALL + SELECT + toDateTime64({{DateTime64(source_cutoff)}}, 3), + {{String(generation_id)}}, + toUInt8(1), + now64(3), + {% if defined(copy_run_id) %} {{String(copy_run_id)}} {% else %} '' {% end %}, + '', + {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) {% end %}, + '', '', '', '', '', '', '', '', '', '', '', + uniqExactStateIf('', 0), + toUInt64(0), toUInt64(0), toUInt64(0), toUInt64(0), toUInt64(0) + WHERE throwIf(length({{String(generation_id)}}) < 8 OR length({{String(generation_id)}}) > 128, 'generation_id has an invalid length') = 0 + {% if defined(settled_date) %} AND throwIf(toDate({{Date(settled_date)}}) > toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY, 'settled_date is inside the mutable window') = 0 {% end %} + diff --git a/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe new file mode 100644 index 00000000000..538ebf50935 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe @@ -0,0 +1,36 @@ +DESCRIPTION > + Expose the active bounded traffic generation and latest settled UTC partitions. + +NODE product_traffic_daily_current_v2_node +SQL > + WITH hot_generation AS ( + SELECT + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'hot' + ), cold_generations AS ( + SELECT + settled_date AS date, + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'cold' + GROUP BY settled_date + ) + SELECT hot.* + FROM product_traffic_daily_hot_v2 AS hot + INNER JOIN hot_generation AS generation + ON hot.generation_id = generation.generation_id + AND hot.source_cutoff = generation.source_cutoff + WHERE hot.is_marker = 0 + UNION ALL + SELECT cold.* + FROM product_traffic_daily_cold_v2 AS cold + INNER JOIN cold_generations AS generation + ON cold.date = generation.date + AND cold.generation_id = generation.generation_id + AND cold.source_cutoff = generation.source_cutoff + CROSS JOIN hot_generation AS hot_generation + WHERE cold.is_marker = 0 + AND cold.date < toDate(hot_generation.source_cutoff, 'UTC') - INTERVAL 8 DAY diff --git a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe index b9710e0ba12..0d204b857c5 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe @@ -25,6 +25,8 @@ SQL > {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} {% if defined(device) %} AND device = {{String(device)}} {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe index fc8ca3a429b..a8aa92b0740 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe @@ -26,6 +26,8 @@ SQL > {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} {% if defined(device) %} AND device = {{String(device)}} {% end %} diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe new file mode 100644 index 00000000000..aaaf2d1b7a5 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe @@ -0,0 +1,105 @@ +DESCRIPTION > + Build one generation of bounded hot or settled page and engagement decisions. + +TOKEN product_events_copy_runner READ + +NODE product_traffic_pages_daily_bounded_v2_node +SQL > + % + {% if not defined(source_cutoff) %} + {{ error('source_cutoff is required') }} + {% end %} + {% if not defined(generation_id) %} + {{ error('generation_id is required') }} + {% end %} + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + WITH sessions AS ( + SELECT + synthetic_run_id, + session_id, + argMin(event_id, tuple(occurred_at, event_id)) AS landing_event_id, + argMax(event_id, tuple(occurred_at, event_id)) AS exit_event_id + FROM product_events_canonical_window + WHERE event_name = 'page_view' + AND (traffic_class = 'external' OR synthetic_run_id != '') + AND session_id != '' + GROUP BY synthetic_run_id, session_id + ), engagements AS ( + SELECT + synthetic_run_id, + JSONExtractString(properties, 'page_view_id') AS page_view_id, + toUInt64(sum(greatest(0, JSONExtractInt(properties, 'engaged_ms')))) AS engaged_ms, + toFloat64(max(greatest(0, least(100, JSONExtractInt(properties, 'max_scroll_depth'))))) AS scroll_depth + FROM product_events_canonical_window + WHERE event_name = 'page_engagement' + AND (traffic_class = 'external' OR synthetic_run_id != '') + GROUP BY synthetic_run_id, page_view_id + ), page_views AS ( + SELECT + synthetic_run_id, + event_id, + occurred_at, + session_id, + concat('anonymous:', anonymous_id) AS visitor_id, + platform, + app_version, + hostname, + pathname, + country, + device, + browser, + os, + channel + FROM product_events_canonical_window + WHERE event_name = 'page_view' + AND (traffic_class = 'external' OR synthetic_run_id != '') + AND session_id != '' + AND anonymous_id != '' + ) + SELECT + toDateTime64({{DateTime64(source_cutoff)}}, 3) AS source_cutoff, + {{String(generation_id)}} AS generation_id, + toUInt8(0) AS is_marker, + now64(3) AS calculated_at, + '' AS copy_run_id, + page_views.synthetic_run_id AS synthetic_run_id, + toDate(occurred_at, 'UTC') AS date, + platform, + app_version, + hostname, + pathname, + country, + device, + browser, + os, + channel, + uniqExactState(visitor_id) AS visitors, + uniqExactState(page_views.session_id) AS visits, + toUInt64(count()) AS pageviews, + toUInt64(countIf(event_id = landing_event_id)) AS landings, + toUInt64(countIf(event_id = exit_event_id)) AS exits, + toUInt64(sum(coalesce(engaged_ms, 0))) AS engaged_ms, + toFloat64(sum(coalesce(scroll_depth, 0))) AS scroll_depth_sum + FROM page_views + INNER JOIN sessions ON page_views.synthetic_run_id = sessions.synthetic_run_id AND page_views.session_id = sessions.session_id + LEFT JOIN engagements ON page_views.synthetic_run_id = engagements.synthetic_run_id AND page_views.event_id = engagements.page_view_id + GROUP BY page_views.synthetic_run_id, date, platform, app_version, hostname, pathname, country, device, browser, os, channel + HAVING date >= {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY {% end %} + AND date < {% if defined(settled_date) %} toDate({{Date(settled_date)}}) + INTERVAL 1 DAY {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) + INTERVAL 1 DAY {% end %} + UNION ALL + SELECT + toDateTime64({{DateTime64(source_cutoff)}}, 3), + {{String(generation_id)}}, + toUInt8(1), + now64(3), + {% if defined(copy_run_id) %} {{String(copy_run_id)}} {% else %} '' {% end %}, + '', + {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) {% end %}, + '', '', '', '', '', '', '', '', '', + uniqExactStateIf('', 0), uniqExactStateIf('', 0), + toUInt64(0), toUInt64(0), toUInt64(0), toUInt64(0), toFloat64(0) + WHERE throwIf(length({{String(generation_id)}}) < 8 OR length({{String(generation_id)}}) > 128, 'generation_id has an invalid length') = 0 + {% if defined(settled_date) %} AND throwIf(toDate({{Date(settled_date)}}) > toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY, 'settled_date is inside the mutable window') = 0 {% end %} + diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe new file mode 100644 index 00000000000..3cd7559f723 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe @@ -0,0 +1,36 @@ +DESCRIPTION > + Expose the active bounded page generation and latest settled UTC partitions. + +NODE product_traffic_pages_daily_current_v2_node +SQL > + WITH hot_generation AS ( + SELECT + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'hot' + ), cold_generations AS ( + SELECT + settled_date AS date, + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'cold' + GROUP BY settled_date + ) + SELECT hot.* + FROM product_traffic_pages_daily_hot_v2 AS hot + INNER JOIN hot_generation AS generation + ON hot.generation_id = generation.generation_id + AND hot.source_cutoff = generation.source_cutoff + WHERE hot.is_marker = 0 + UNION ALL + SELECT cold.* + FROM product_traffic_pages_daily_cold_v2 AS cold + INNER JOIN cold_generations AS generation + ON cold.date = generation.date + AND cold.generation_id = generation.generation_id + AND cold.source_cutoff = generation.source_cutoff + CROSS JOIN hot_generation AS hot_generation + WHERE cold.is_marker = 0 + AND cold.date < toDate(hot_generation.source_cutoff, 'UTC') - INTERVAL 8 DAY diff --git a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe index ed75f6476f8..003efb59144 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe @@ -26,6 +26,8 @@ SQL > {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} GROUP BY channel, attribution_source, attribution_medium, campaign ) diff --git a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe index c2438c5f7af..11cd3966595 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe @@ -23,6 +23,8 @@ SQL > {% end %} AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} GROUP BY device, browser, os diff --git a/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe b/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe new file mode 100644 index 00000000000..c08588fd565 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe @@ -0,0 +1,37 @@ +DESCRIPTION > + Query selected-range unique visitors and exact traffic totals by merging privacy-safe aggregate states across UTC dates. + +TOKEN product_events_agent_read READ + +NODE product_traffic_totals_node +SQL > + % + SELECT + uniqExactMerge(visitors) AS visitors, + toUInt64(sum(visits)) AS visits, + toUInt64(sum(pageviews)) AS pageviews, + round(if(sum(visits) = 0, 0, sum(pageviews) / sum(visits)), 2) AS views_per_visit, + round(if(sum(visits) = 0, 0, 100 * sum(bounces) / sum(visits)), 2) AS bounce_rate, + toUInt64(if(sum(visits) = 0, 0, sum(visit_duration_ms) / sum(visits))) AS visit_duration_ms, + toUInt64(sum(engaged_ms)) AS engaged_ms + FROM product_traffic_daily_exact + WHERE copy_run_id = '' + {% if defined(synthetic_run_id) %} + AND synthetic_run_id = {{String(synthetic_run_id)}} + AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 + {% else %} + AND synthetic_run_id = '' + {% end %} + AND date >= {% if defined(start_date) %} {{Date(start_date)}} {% else %} today() - INTERVAL 30 DAY {% end %} + AND date <= {% if defined(end_date) %} {{Date(end_date)}} {% else %} today() {% end %} + {% if defined(platform) %} AND platform = {{String(platform)}} {% end %} + {% if defined(app_version) %} AND app_version = {{String(app_version)}} {% end %} + {% if defined(hostname) %} AND hostname = lower({{String(hostname)}}) {% end %} + {% if defined(country) %} AND country = upper({{String(country)}}) {% end %} + {% if defined(device) %} AND device = {{String(device)}} {% end %} + {% if defined(browser) %} AND browser = {{String(browser)}} {% end %} + {% if defined(os) %} AND os = {{String(os)}} {% end %} + {% if defined(channel) %} AND channel = {{String(channel)}} {% end %} + {% if defined(source) %} AND attribution_source = {{String(source)}} {% end %} + +TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/publish_product_analytics_generation_v2.pipe b/scripts/analytics/tinybird/pipes/publish_product_analytics_generation_v2.pipe new file mode 100644 index 00000000000..0a46ea1639a --- /dev/null +++ b/scripts/analytics/tinybird/pipes/publish_product_analytics_generation_v2.pipe @@ -0,0 +1,43 @@ +DESCRIPTION > + Publish one generation only after every bounded decision model has completed. + +TOKEN product_events_copy_runner READ + +NODE publish_product_analytics_generation_v2_node +SQL > + % + {% if not defined(source_cutoff) %} + {{ error('source_cutoff is required') }} + {% end %} + {% if not defined(generation_id) %} + {{ error('generation_id is required') }} + {% end %} + {% if not defined(generation_kind) %} + {{ error('generation_kind is required') }} + {% end %} + SELECT + {{String(generation_id)}} AS generation_id, + toDateTime64({{DateTime64(source_cutoff)}}, 3) AS source_cutoff, + {{String(generation_kind)}} AS generation_kind, + {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate('1970-01-01') {% end %} AS settled_date, + now64(3) AS published_at, + {% if defined(copy_run_id) %} {{String(copy_run_id)}} {% else %} '' {% end %} AS copy_run_id + WHERE throwIf(length({{String(generation_id)}}) < 8 OR length({{String(generation_id)}}) > 128, 'generation_id has an invalid length') = 0 + AND throwIf({{String(generation_kind)}} NOT IN ('hot', 'cold'), 'generation_kind is invalid') = 0 + {% if defined(settled_date) %} + AND throwIf({{String(generation_kind)}} != 'cold', 'settled_date requires a cold generation') = 0 + AND throwIf(toDate({{Date(settled_date)}}) > toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY, 'settled_date is inside the mutable window') = 0 + AND throwIf((SELECT count() FROM product_events_daily_cold_v2 WHERE generation_id = {{String(generation_id)}} AND source_cutoff = toDateTime64({{DateTime64(source_cutoff)}}, 3) AND date = toDate({{Date(settled_date)}}) AND is_marker = 1) = 0, 'event cold generation is incomplete') = 0 + AND throwIf((SELECT count() FROM product_traffic_daily_cold_v2 WHERE generation_id = {{String(generation_id)}} AND source_cutoff = toDateTime64({{DateTime64(source_cutoff)}}, 3) AND date = toDate({{Date(settled_date)}}) AND is_marker = 1) = 0, 'traffic cold generation is incomplete') = 0 + AND throwIf((SELECT count() FROM product_traffic_pages_daily_cold_v2 WHERE generation_id = {{String(generation_id)}} AND source_cutoff = toDateTime64({{DateTime64(source_cutoff)}}, 3) AND date = toDate({{Date(settled_date)}}) AND is_marker = 1) = 0, 'page cold generation is incomplete') = 0 + {% else %} + AND throwIf({{String(generation_kind)}} != 'hot', 'cold generation requires settled_date') = 0 + AND throwIf((SELECT count() FROM product_events_daily_hot_v2 WHERE generation_id = {{String(generation_id)}} AND source_cutoff = toDateTime64({{DateTime64(source_cutoff)}}, 3) AND is_marker = 1) = 0, 'event hot generation is incomplete') = 0 + AND throwIf((SELECT count() FROM product_traffic_daily_hot_v2 WHERE generation_id = {{String(generation_id)}} AND source_cutoff = toDateTime64({{DateTime64(source_cutoff)}}, 3) AND is_marker = 1) = 0, 'traffic hot generation is incomplete') = 0 + AND throwIf((SELECT count() FROM product_traffic_pages_daily_hot_v2 WHERE generation_id = {{String(generation_id)}} AND source_cutoff = toDateTime64({{DateTime64(source_cutoff)}}, 3) AND is_marker = 1) = 0, 'page hot generation is incomplete') = 0 + {% end %} + +TYPE COPY +TARGET_DATASOURCE product_analytics_generations_v2 +COPY_MODE append +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe index a222ee9d0bb..bb0f5042439 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_activation_daily_exact.pipe @@ -14,7 +14,7 @@ SQL > synthetic_run_id, user_id, min(occurred_at) AS signed_up_at - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name = 'user_signed_up' AND user_id != '' GROUP BY synthetic_run_id, user_id @@ -23,7 +23,7 @@ SQL > synthetic_run_id, user_id, min(occurred_at) AS activated_at - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name = 'share_link_created' AND user_id != '' GROUP BY synthetic_run_id, user_id @@ -55,4 +55,4 @@ SQL > TYPE COPY TARGET_DATASOURCE product_activation_daily_exact COPY_MODE replace -COPY_SCHEDULE 5-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_attribution_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_attribution_daily_exact.pipe new file mode 100644 index 00000000000..a09efd0ed3c --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_attribution_daily_exact.pipe @@ -0,0 +1,105 @@ +DESCRIPTION > + Rebuild exact campaign attribution for first, session, and last touch from canonical page views. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_attribution_daily_exact_node +SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + WITH expanded_pageviews AS ( + SELECT + synthetic_run_id, + session_id, + anonymous_id, + event_id, + occurred_at, + platform, + app_version, + hostname, + country, + attribution_model, + multiIf( + attribution_model = 'first', JSONExtractString(properties, 'first_touch_source'), + attribution_model = 'session', JSONExtractString(properties, 'session_touch_source'), + JSONExtractString(properties, 'last_touch_source') + ) AS attribution_source, + multiIf( + attribution_model = 'first', JSONExtractString(properties, 'first_touch_medium'), + attribution_model = 'session', JSONExtractString(properties, 'session_touch_medium'), + JSONExtractString(properties, 'last_touch_medium') + ) AS attribution_medium, + multiIf( + attribution_model = 'first', JSONExtractString(properties, 'first_touch_campaign'), + attribution_model = 'session', JSONExtractString(properties, 'session_touch_campaign'), + JSONExtractString(properties, 'last_touch_campaign') + ) AS campaign + FROM product_events_canonical_current + ARRAY JOIN ['first', 'session', 'last'] AS attribution_model + WHERE event_name = 'page_view' + AND (traffic_class = 'external' OR synthetic_run_id != '') + AND session_id != '' + AND anonymous_id != '' + ), sessions AS ( + SELECT + synthetic_run_id, + session_id, + attribution_model, + concat('anonymous:', argMin(anonymous_id, tuple(occurred_at, event_id))) AS visitor_id, + min(occurred_at) AS started_at, + argMin(platform, tuple(occurred_at, event_id)) AS platform, + argMin(app_version, tuple(occurred_at, event_id)) AS app_version, + argMin(hostname, tuple(occurred_at, event_id)) AS hostname, + argMin(country, tuple(occurred_at, event_id)) AS country, + if(attribution_model = 'last', argMax(attribution_source, tuple(occurred_at, event_id)), argMin(attribution_source, tuple(occurred_at, event_id))) AS attribution_source, + if(attribution_model = 'last', argMax(attribution_medium, tuple(occurred_at, event_id)), argMin(attribution_medium, tuple(occurred_at, event_id))) AS attribution_medium, + if(attribution_model = 'last', argMax(campaign, tuple(occurred_at, event_id)), argMin(campaign, tuple(occurred_at, event_id))) AS campaign, + toUInt64(count()) AS session_pageviews + FROM expanded_pageviews + GROUP BY synthetic_run_id, session_id, attribution_model + ) + SELECT + now64(3) AS calculated_at, + '' AS copy_run_id, + synthetic_run_id, + toDate(started_at, 'UTC') AS date, + attribution_model, + platform, + app_version, + hostname, + country, + attribution_source, + attribution_medium, + campaign, + uniqExactState(visitor_id) AS visitors, + toUInt64(count()) AS visits, + toUInt64(sum(session_pageviews)) AS pageviews + FROM sessions + GROUP BY synthetic_run_id, date, attribution_model, platform, app_version, hostname, country, attribution_source, attribution_medium, campaign + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3) AS calculated_at, + {{String(copy_run_id)}} AS copy_run_id, + '' AS synthetic_run_id, + today() AS date, + 'session' AS attribution_model, + '' AS platform, + '' AS app_version, + '' AS hostname, + '' AS country, + '' AS attribution_source, + '' AS attribution_medium, + '' AS campaign, + uniqExactStateIf('', 0) AS visitors, + toUInt64(0) AS visits, + toUInt64(0) AS pageviews + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} + +TYPE COPY +TARGET_DATASOURCE product_attribution_daily_exact +COPY_MODE replace +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe index 93ee5eda729..71e476a539f 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_creator_retention_exact.pipe @@ -16,7 +16,7 @@ SQL > organization_id, toDate(occurred_at, 'UTC') AS activity_date, platform - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name IN ('recording_started', 'recording_completed', 'multipart_upload_complete', 'share_link_created', 'loom_import_completed') AND user_id != '' AND (event_name != 'recording_completed' OR JSONExtractString(properties, 'status') IN ('success', 'degraded')) @@ -84,4 +84,4 @@ SQL > TYPE COPY TARGET_DATASOURCE product_creator_retention_exact COPY_MODE replace -COPY_SCHEDULE 6-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe new file mode 100644 index 00000000000..57cc6025f07 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe @@ -0,0 +1,51 @@ +DESCRIPTION > + Exceptionally rebuild time-partitioned event state after erasure or during a controlled backfill. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_event_day_states_v2_node +SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + SELECT + toDate(occurred_at, 'UTC') AS occurred_date, + event_id, + uniqExactState(payload_hash) AS payload_hashes, + argMinState(payload_hash, received_at) AS payload_hash, + argMinState(occurred_at, received_at) AS occurred_at, + min(received_at) AS first_received_at, + max(received_at) AS received_at, + argMinState(toString(event_name), received_at) AS event_name, + argMinState(schema_version, received_at) AS schema_version, + argMinState(toString(source), received_at) AS source, + argMinState(toString(platform), received_at) AS platform, + argMinState(anonymous_id, received_at) AS anonymous_id, + argMinState(session_id, received_at) AS session_id, + argMinState(user_id, received_at) AS user_id, + argMinState(organization_id, received_at) AS organization_id, + argMinState(toString(app_version), received_at) AS app_version, + argMinState(pathname, received_at) AS pathname, + argMinState(referrer, received_at) AS referrer, + argMinState(toString(country), received_at) AS country, + argMinState(toString(region), received_at) AS region, + argMinState(toString(city), received_at) AS city, + argMinState(toString(hostname), received_at) AS hostname, + argMinState(toString(browser), received_at) AS browser, + argMinState(toString(device), received_at) AS device, + argMinState(toString(os), received_at) AS os, + argMinState(toString(channel), received_at) AS channel, + argMinState(toString(traffic_class), received_at) AS traffic_class, + argMinState(synthetic_run_id, received_at) AS synthetic_run_id, + argMinState(properties, received_at) AS properties + FROM product_events_v1 + {% if defined(source_cutoff) %} + WHERE received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + {% end %} + GROUP BY occurred_date, event_id + +TYPE COPY +TARGET_DATASOURCE product_event_day_states_v2 +COPY_MODE replace +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe new file mode 100644 index 00000000000..7af8f1c1ac3 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe @@ -0,0 +1,50 @@ +DESCRIPTION > + Rebuild per-event aggregate state after erasure or during a controlled backfill. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_event_id_states_v2_node +SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + SELECT + event_id, + uniqExactState(payload_hash) AS payload_hashes, + argMinState(payload_hash, received_at) AS payload_hash, + argMinState(occurred_at, received_at) AS occurred_at, + min(received_at) AS first_received_at, + max(received_at) AS received_at, + argMinState(toString(event_name), received_at) AS event_name, + argMinState(schema_version, received_at) AS schema_version, + argMinState(toString(source), received_at) AS source, + argMinState(toString(platform), received_at) AS platform, + argMinState(anonymous_id, received_at) AS anonymous_id, + argMinState(session_id, received_at) AS session_id, + argMinState(user_id, received_at) AS user_id, + argMinState(organization_id, received_at) AS organization_id, + argMinState(toString(app_version), received_at) AS app_version, + argMinState(pathname, received_at) AS pathname, + argMinState(referrer, received_at) AS referrer, + argMinState(toString(country), received_at) AS country, + argMinState(toString(region), received_at) AS region, + argMinState(toString(city), received_at) AS city, + argMinState(toString(hostname), received_at) AS hostname, + argMinState(toString(browser), received_at) AS browser, + argMinState(toString(device), received_at) AS device, + argMinState(toString(os), received_at) AS os, + argMinState(toString(channel), received_at) AS channel, + argMinState(toString(traffic_class), received_at) AS traffic_class, + argMinState(synthetic_run_id, received_at) AS synthetic_run_id, + argMinState(properties, received_at) AS properties + FROM product_events_v1 + {% if defined(source_cutoff) %} + WHERE received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + {% end %} + GROUP BY event_id + +TYPE COPY +TARGET_DATASOURCE product_event_id_states_v2 +COPY_MODE replace +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe index ac23b9a92e9..cc6d5807bfb 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_canonical_v1.pipe @@ -9,68 +9,13 @@ SQL > {% if defined(copy_max_threads) %} {{max_threads(Int32(copy_max_threads))}} {% end %} - SELECT - event_id, - any(raw_payload_hash) AS payload_hash, - argMax(raw_occurred_at, raw_received_at) AS occurred_at, - max(raw_received_at) AS received_at, - argMax(raw_event_name, raw_received_at) AS event_name, - argMax(raw_schema_version, raw_received_at) AS schema_version, - argMax(raw_source, raw_received_at) AS source, - argMax(raw_platform, raw_received_at) AS platform, - argMax(raw_anonymous_id, raw_received_at) AS anonymous_id, - argMax(raw_session_id, raw_received_at) AS session_id, - argMax(raw_user_id, raw_received_at) AS user_id, - argMax(raw_organization_id, raw_received_at) AS organization_id, - argMax(raw_app_version, raw_received_at) AS app_version, - argMax(raw_pathname, raw_received_at) AS pathname, - argMax(raw_referrer, raw_received_at) AS referrer, - argMax(raw_country, raw_received_at) AS country, - argMax(raw_region, raw_received_at) AS region, - argMax(raw_city, raw_received_at) AS city, - argMax(raw_hostname, raw_received_at) AS hostname, - argMax(raw_browser, raw_received_at) AS browser, - argMax(raw_device, raw_received_at) AS device, - argMax(raw_os, raw_received_at) AS os, - argMax(raw_channel, raw_received_at) AS channel, - argMax(raw_traffic_class, raw_received_at) AS traffic_class, - argMax(raw_synthetic_run_id, raw_received_at) AS synthetic_run_id, - argMax(raw_properties, raw_received_at) AS properties - FROM - ( - SELECT - event_id, - payload_hash AS raw_payload_hash, - occurred_at AS raw_occurred_at, - received_at AS raw_received_at, - event_name AS raw_event_name, - schema_version AS raw_schema_version, - source AS raw_source, - platform AS raw_platform, - anonymous_id AS raw_anonymous_id, - session_id AS raw_session_id, - user_id AS raw_user_id, - organization_id AS raw_organization_id, - app_version AS raw_app_version, - pathname AS raw_pathname, - referrer AS raw_referrer, - country AS raw_country, - region AS raw_region, - city AS raw_city, - hostname AS raw_hostname, - browser AS raw_browser, - device AS raw_device, - os AS raw_os, - channel AS raw_channel, - traffic_class AS raw_traffic_class, - synthetic_run_id AS raw_synthetic_run_id, - properties AS raw_properties - FROM product_events_v1 - ) - GROUP BY event_id - HAVING uniqExact(raw_payload_hash) = 1 + SELECT * + FROM product_events_canonical_current + {% if defined(source_cutoff) %} + WHERE received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + {% end %} TYPE COPY TARGET_DATASOURCE product_events_canonical_v1 COPY_MODE replace -COPY_SCHEDULE 0-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_cold_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_cold_v2.pipe new file mode 100644 index 00000000000..5cc05162815 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_cold_v2.pipe @@ -0,0 +1,18 @@ +DESCRIPTION > + Append one settled UTC event-decision partition for a controller generation. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_events_daily_cold_v2_node +SQL > + % + {% if not defined(settled_date) %} + {{ error('settled_date is required') }} + {% end %} + SELECT * + FROM product_events_daily_bounded_v2 + +TYPE COPY +TARGET_DATASOURCE product_events_daily_cold_v2 +COPY_MODE append +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe index 44434f70f6e..da6b0e19c44 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_exact.pipe @@ -27,6 +27,16 @@ SQL > traffic_class, synthetic_run_id, if(event_name = 'subscription_changed', JSONExtractString(properties, 'new_price_id'), JSONExtractString(properties, 'price_id')) AS plan_id, + if( + event_name = 'recording_completed', + multiIf( + lower(JSONExtractString(properties, 'status')) = 'success', 'success', + lower(JSONExtractString(properties, 'status')) = 'degraded', 'degraded', + lower(JSONExtractString(properties, 'status')) = 'failed', 'failed', + 'unknown' + ), + '' + ) AS recording_status, if(event_name = 'purchase_completed', JSONExtractString(properties, 'payment_status'), '') AS payment_status, multiIf( event_name IN ('purchase_completed', 'trial_started'), JSONExtractString(properties, 'subscription_status'), @@ -68,8 +78,8 @@ SQL > event_name = 'subscription_refunded', -JSONExtractInt(properties, 'amount_refunded_minor'), 0 ))) AS revenue_minor - FROM product_events_canonical_v1 - GROUP BY date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version + FROM product_events_canonical_current + GROUP BY date, event_name, schema_version, source, platform, app_version, hostname, country, device, browser, os, channel, traffic_class, synthetic_run_id, plan_id, recording_status, payment_status, subscription_status, currency, billing_interval, change_kind, previous_status, new_status, previous_plan_id, quantity, previous_quantity, new_quantity, seat_delta, first_purchase, guest_checkout, onboarding, cancel_at_period_end, fully_refunded, ended_at, trial_end_at, amount_due_minor, attempt_count, experiment_id, experiment_variant, assignment_version {% if defined(copy_run_id) %} UNION ALL SELECT @@ -78,7 +88,7 @@ SQL > today(), '', toUInt16(0), - '', '', '', '', '', '', '', '', '', '', + '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', toInt64(0), toInt64(0), toInt64(0), toInt64(0), '', '', '', '', '', @@ -96,4 +106,4 @@ SQL > TYPE COPY TARGET_DATASOURCE product_events_daily_exact COPY_MODE replace -COPY_SCHEDULE 2-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_hot_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_hot_v2.pipe new file mode 100644 index 00000000000..7b2dffda3a8 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_daily_hot_v2.pipe @@ -0,0 +1,14 @@ +DESCRIPTION > + Replace only the bounded mutable event-decision window for one controller generation. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_events_daily_hot_v2_node +SQL > + SELECT * + FROM product_events_daily_bounded_v2 + +TYPE COPY +TARGET_DATASOURCE product_events_daily_hot_v2 +COPY_MODE replace +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe index 5e3e9c2274e..93a5f7039b6 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_events_health_hourly.pipe @@ -45,4 +45,4 @@ SQL > TYPE COPY TARGET_DATASOURCE product_events_health_hourly_exact COPY_MODE replace -COPY_SCHEDULE 1-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe new file mode 100644 index 00000000000..a953a9bdaec --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe @@ -0,0 +1,145 @@ +DESCRIPTION > + Rebuild aggregate 30-day first-outcome cohorts anchored only to explicit, stable experiment exposures. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_experiment_outcomes_exact_node +SQL > + % + {% if defined(copy_max_threads) %} + {{max_threads(Int32(copy_max_threads))}} + {% end %} + WITH identity_candidates AS ( + SELECT synthetic_run_id, anonymous_id, user_id, event_id, occurred_at + FROM product_events_canonical_current + WHERE event_name = 'identity_linked' + AND anonymous_id != '' + AND user_id != '' + UNION ALL + SELECT synthetic_run_id, anonymous_id, user_id, event_id, occurred_at + FROM product_events_canonical_current + WHERE event_name = 'purchase_completed' + AND JSONExtractString(properties, 'payment_status') = 'paid' + AND JSONExtractBool(properties, 'is_guest_checkout') + AND anonymous_id != '' + AND user_id != '' + ), identity_links AS ( + SELECT + synthetic_run_id, + anonymous_id, + argMin(user_id, tuple(occurred_at, event_id)) AS user_id + FROM identity_candidates + GROUP BY synthetic_run_id, anonymous_id + HAVING uniqExact(user_id) = 1 + ), resolved_exposures AS ( + SELECT + exposures.synthetic_run_id AS synthetic_run_id, + if(exposures.user_id != '', exposures.user_id, coalesce(identity_links.user_id, '')) AS resolved_user_id, + if( + if(exposures.user_id != '', exposures.user_id, coalesce(identity_links.user_id, '')) != '', + concat('user:', if(exposures.user_id != '', exposures.user_id, coalesce(identity_links.user_id, ''))), + concat('anonymous:', exposures.anonymous_id) + ) AS actor_id, + exposures.occurred_at AS occurred_at, + exposures.event_id AS event_id, + JSONExtractString(exposures.properties, 'experiment_id') AS experiment_id, + JSONExtractString(exposures.properties, 'assignment_version') AS assignment_version, + JSONExtractString(exposures.properties, 'variant') AS variant, + exposures.platform AS platform, + exposures.app_version AS app_version + FROM product_events_canonical_current AS exposures + LEFT JOIN identity_links ON exposures.synthetic_run_id = identity_links.synthetic_run_id AND exposures.anonymous_id = identity_links.anonymous_id + WHERE exposures.event_name = 'experiment_exposed' + AND (exposures.traffic_class = 'external' OR exposures.synthetic_run_id != '') + AND (exposures.user_id != '' OR exposures.anonymous_id != '') + ), exposures AS ( + SELECT + synthetic_run_id, + actor_id, + argMin(resolved_user_id, tuple(occurred_at, event_id)) AS user_id, + min(occurred_at) AS exposed_at, + experiment_id, + assignment_version, + argMin(variant, tuple(occurred_at, event_id)) AS variant, + argMin(platform, tuple(occurred_at, event_id)) AS platform, + argMin(app_version, tuple(occurred_at, event_id)) AS app_version + FROM resolved_exposures + WHERE experiment_id != '' + AND assignment_version != '' + AND variant != '' + GROUP BY synthetic_run_id, actor_id, experiment_id, assignment_version + HAVING uniqExact(variant) = 1 + ), outcome_candidates AS ( + SELECT + synthetic_run_id, + user_id, + event_name, + occurred_at + FROM product_events_canonical_current + WHERE user_id != '' + AND (traffic_class = 'external' OR synthetic_run_id != '') + AND event_name IN ('user_signed_up', 'share_link_created', 'purchase_completed') + AND (event_name != 'purchase_completed' OR JSONExtractString(properties, 'payment_status') = 'paid') + ), exposure_outcomes AS ( + SELECT + exposures.synthetic_run_id AS synthetic_run_id, + exposures.actor_id AS actor_id, + exposures.user_id AS user_id, + exposures.exposed_at AS exposed_at, + exposures.experiment_id AS experiment_id, + exposures.assignment_version AS assignment_version, + exposures.variant AS variant, + exposures.platform AS platform, + exposures.app_version AS app_version, + minIf(outcomes.occurred_at, outcomes.event_name = 'user_signed_up' AND outcomes.occurred_at >= exposures.exposed_at AND outcomes.occurred_at < exposures.exposed_at + INTERVAL 30 DAY) AS signed_up_at, + minIf(outcomes.occurred_at, outcomes.event_name = 'share_link_created' AND outcomes.occurred_at >= exposures.exposed_at AND outcomes.occurred_at < exposures.exposed_at + INTERVAL 30 DAY) AS shared_at, + minIf(outcomes.occurred_at, outcomes.event_name = 'purchase_completed' AND outcomes.occurred_at >= exposures.exposed_at AND outcomes.occurred_at < exposures.exposed_at + INTERVAL 30 DAY) AS purchased_at + FROM exposures + LEFT JOIN outcome_candidates AS outcomes ON exposures.synthetic_run_id = outcomes.synthetic_run_id AND exposures.user_id = outcomes.user_id + GROUP BY exposures.synthetic_run_id, exposures.actor_id, exposures.user_id, exposures.exposed_at, exposures.experiment_id, exposures.assignment_version, exposures.variant, exposures.platform, exposures.app_version + ), outcome_rows AS ( + SELECT *, 'signup' AS outcome_name, signed_up_at AS outcome_at FROM exposure_outcomes + UNION ALL + SELECT *, 'share_created' AS outcome_name, shared_at AS outcome_at FROM exposure_outcomes + UNION ALL + SELECT *, 'paid_purchase' AS outcome_name, purchased_at AS outcome_at FROM exposure_outcomes + ) + SELECT + now64(3) AS calculated_at, + '' AS copy_run_id, + synthetic_run_id, + toDate(exposed_at, 'UTC') AS cohort_date, + experiment_id, + assignment_version, + variant, + platform, + app_version, + outcome_name, + toUInt64(count()) AS exposed_actors, + toUInt64(countIf(outcome_at >= exposed_at AND outcome_at < exposed_at + INTERVAL 30 DAY)) AS converted_actors, + toUInt8(0) AS copy_marker + FROM outcome_rows + GROUP BY synthetic_run_id, cohort_date, experiment_id, assignment_version, variant, platform, app_version, outcome_name + {% if defined(copy_run_id) %} + UNION ALL + SELECT + now64(3) AS calculated_at, + {{String(copy_run_id)}} AS copy_run_id, + '' AS synthetic_run_id, + today() AS cohort_date, + '' AS experiment_id, + '' AS assignment_version, + '' AS variant, + '' AS platform, + '' AS app_version, + '' AS outcome_name, + toUInt64(0) AS exposed_actors, + toUInt64(0) AS converted_actors, + toUInt8(1) AS copy_marker + WHERE throwIf(length({{String(copy_run_id)}}) < 8 OR length({{String(copy_run_id)}}) > 128, 'copy_run_id has an invalid length') = 0 + {% end %} + +TYPE COPY +TARGET_DATASOURCE product_experiment_outcomes_exact +COPY_MODE replace +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe index ee3d7fadb3e..dce54b6623b 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe @@ -10,14 +10,14 @@ SQL > {{max_threads(Int32(copy_max_threads))}} {% end %} WITH identity_links AS ( - SELECT synthetic_run_id, user_id, anonymous_id, organization_id, occurred_at - FROM product_events_canonical_v1 + SELECT synthetic_run_id, user_id, anonymous_id, organization_id, event_id, occurred_at + FROM product_events_canonical_current WHERE event_name = 'identity_linked' AND user_id != '' AND anonymous_id != '' ), paid_guest_links AS ( - SELECT synthetic_run_id, user_id, anonymous_id, organization_id, occurred_at - FROM product_events_canonical_v1 + SELECT synthetic_run_id, user_id, anonymous_id, organization_id, event_id, occurred_at + FROM product_events_canonical_current WHERE event_name = 'purchase_completed' AND JSONExtractString(properties, 'payment_status') = 'paid' AND JSONExtractBool(properties, 'is_guest_checkout') @@ -31,8 +31,8 @@ SQL > SELECT synthetic_run_id, user_id, - argMin(anonymous_id, occurred_at) AS anonymous_id, - argMinIf(organization_id, occurred_at, organization_id != '') AS organization_id, + argMin(anonymous_id, tuple(occurred_at, event_id)) AS anonymous_id, + argMinIf(organization_id, tuple(occurred_at, event_id), organization_id != '') AS organization_id, min(occurred_at) AS linked_at FROM link_candidates GROUP BY synthetic_run_id, user_id @@ -40,7 +40,7 @@ SQL > SELECT synthetic_run_id, organization_id, - argMin(user_id, linked_at) AS first_user_id + argMin(user_id, tuple(linked_at, user_id)) AS first_user_id FROM first_links WHERE organization_id != '' GROUP BY synthetic_run_id, organization_id @@ -49,12 +49,12 @@ SQL > synthetic_run_id, anonymous_id, min(occurred_at) AS acquired_at, - argMin(channel, occurred_at) AS channel, - argMin(JSONExtractString(properties, 'session_touch_source'), occurred_at) AS attribution_source, - argMin(JSONExtractString(properties, 'session_touch_medium'), occurred_at) AS attribution_medium, - argMin(JSONExtractString(properties, 'session_touch_campaign'), occurred_at) AS campaign, - argMin(country, occurred_at) AS country - FROM product_events_canonical_v1 + argMin(channel, tuple(occurred_at, event_id)) AS channel, + argMin(JSONExtractString(properties, 'session_touch_source'), tuple(occurred_at, event_id)) AS attribution_source, + argMin(JSONExtractString(properties, 'session_touch_medium'), tuple(occurred_at, event_id)) AS attribution_medium, + argMin(JSONExtractString(properties, 'session_touch_campaign'), tuple(occurred_at, event_id)) AS campaign, + argMin(country, tuple(occurred_at, event_id)) AS country + FROM product_events_canonical_current WHERE event_name = 'page_view' AND anonymous_id != '' AND (traffic_class = 'external' OR synthetic_run_id != '') @@ -72,7 +72,7 @@ SQL > toUInt64(countIf(event_name = 'trial_started') > 0) AS trial_started, toUInt64(countIf(event_name = 'purchase_completed' AND JSONExtractString(properties, 'payment_status') = 'paid') > 0) AS purchased, toUInt64(countIf(event_name = 'purchase_completed' AND JSONExtractString(properties, 'payment_status') = 'paid' AND JSONExtractBool(properties, 'is_guest_checkout')) > 0) AS guest_purchased - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE user_id != '' GROUP BY synthetic_run_id, user_id ), guest_checkouts AS ( @@ -81,7 +81,7 @@ SQL > anonymous_id, min(occurred_at) AS checkout_at, toUInt64(count() > 0) AS guest_checkout - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name = 'guest_checkout_started' AND anonymous_id != '' GROUP BY synthetic_run_id, anonymous_id @@ -109,9 +109,9 @@ SQL > SELECT synthetic_run_id, user_id, - argMin(anonymous_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at)) AS anonymous_id, - argMin(organization_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at)) AS organization_id, - argMin(path_at, tuple(if(paid_guest_purchase > 0, 0, 1), path_at)) AS linked_at, + argMin(anonymous_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at, anonymous_id)) AS anonymous_id, + argMin(organization_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at, anonymous_id)) AS organization_id, + argMin(path_at, tuple(if(paid_guest_purchase > 0, 0, 1), path_at, anonymous_id)) AS linked_at, toUInt64(max(paid_guest_purchase) > 0) AS guest_purchased FROM guest_path_candidates GROUP BY synthetic_run_id, user_id @@ -212,7 +212,7 @@ SQL > '' AS attribution_medium, '' AS campaign, '' AS country, - toUInt64(0) AS linked_visitors, + toUInt64(1) AS linked_visitors, toUInt64(0) AS linked_users, toUInt64(0) AS signup_users, toUInt64(0) AS organizations, @@ -231,4 +231,4 @@ SQL > TYPE COPY TARGET_DATASOURCE product_identity_funnel_exact COPY_MODE replace -COPY_SCHEDULE 7-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_cold_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_cold_v2.pipe new file mode 100644 index 00000000000..79c11736847 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_cold_v2.pipe @@ -0,0 +1,18 @@ +DESCRIPTION > + Append one settled UTC traffic-session partition for a controller generation. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_traffic_daily_cold_v2_node +SQL > + % + {% if not defined(settled_date) %} + {{ error('settled_date is required') }} + {% end %} + SELECT * + FROM product_traffic_daily_bounded_v2 + +TYPE COPY +TARGET_DATASOURCE product_traffic_daily_cold_v2 +COPY_MODE append +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe index 2d5eb2bc4ff..260fccc3028 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_exact.pipe @@ -15,7 +15,7 @@ SQL > session_id, toUInt64(sum(greatest(0, JSONExtractInt(properties, 'engaged_ms')))) AS session_engaged_ms, max(greatest(0, least(100, JSONExtractInt(properties, 'max_scroll_depth')))) AS session_max_scroll_depth - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name = 'page_engagement' AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' @@ -24,19 +24,21 @@ SQL > SELECT synthetic_run_id, session_id, - concat('anonymous:', argMin(anonymous_id, occurred_at)) AS visitor_id, + concat('anonymous:', argMin(anonymous_id, tuple(occurred_at, event_id))) AS visitor_id, min(occurred_at) AS started_at, - argMin(hostname, occurred_at) AS hostname, - argMin(country, occurred_at) AS country, - argMin(device, occurred_at) AS device, - argMin(browser, occurred_at) AS browser, - argMin(os, occurred_at) AS os, - argMin(channel, occurred_at) AS channel, - argMin(JSONExtractString(properties, 'session_touch_source'), occurred_at) AS attribution_source, - argMin(JSONExtractString(properties, 'session_touch_medium'), occurred_at) AS attribution_medium, - argMin(JSONExtractString(properties, 'session_touch_campaign'), occurred_at) AS campaign, + argMin(platform, tuple(occurred_at, event_id)) AS platform, + argMin(app_version, tuple(occurred_at, event_id)) AS app_version, + argMin(hostname, tuple(occurred_at, event_id)) AS hostname, + argMin(country, tuple(occurred_at, event_id)) AS country, + argMin(device, tuple(occurred_at, event_id)) AS device, + argMin(browser, tuple(occurred_at, event_id)) AS browser, + argMin(os, tuple(occurred_at, event_id)) AS os, + argMin(channel, tuple(occurred_at, event_id)) AS channel, + argMin(JSONExtractString(properties, 'session_touch_source'), tuple(occurred_at, event_id)) AS attribution_source, + argMin(JSONExtractString(properties, 'session_touch_medium'), tuple(occurred_at, event_id)) AS attribution_medium, + argMin(JSONExtractString(properties, 'session_touch_campaign'), tuple(occurred_at, event_id)) AS campaign, toUInt64(count()) AS session_pageviews - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name = 'page_view' AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' @@ -48,6 +50,8 @@ SQL > '' AS copy_run_id, synthetic_run_id, toDate(started_at, 'UTC') AS date, + platform, + app_version, hostname, country, device, @@ -65,7 +69,7 @@ SQL > toUInt64(sum(coalesce(session_engaged_ms, 0))) AS engaged_ms FROM sessions LEFT JOIN session_engagement USING (synthetic_run_id, session_id) - GROUP BY synthetic_run_id, date, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign + GROUP BY synthetic_run_id, date, platform, app_version, hostname, country, device, browser, os, channel, attribution_source, attribution_medium, campaign {% if defined(copy_run_id) %} UNION ALL SELECT @@ -73,6 +77,8 @@ SQL > {{String(copy_run_id)}} AS copy_run_id, '' AS synthetic_run_id, today() AS date, + '' AS platform, + '' AS app_version, '' AS hostname, '' AS country, '' AS device, @@ -94,4 +100,4 @@ SQL > TYPE COPY TARGET_DATASOURCE product_traffic_daily_exact COPY_MODE replace -COPY_SCHEDULE 3-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_hot_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_hot_v2.pipe new file mode 100644 index 00000000000..5e3d436007e --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_daily_hot_v2.pipe @@ -0,0 +1,14 @@ +DESCRIPTION > + Replace only the bounded mutable traffic-session window for one controller generation. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_traffic_daily_hot_v2_node +SQL > + SELECT * + FROM product_traffic_daily_bounded_v2 + +TYPE COPY +TARGET_DATASOURCE product_traffic_daily_hot_v2 +COPY_MODE replace +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_cold_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_cold_v2.pipe new file mode 100644 index 00000000000..311b6049de8 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_cold_v2.pipe @@ -0,0 +1,18 @@ +DESCRIPTION > + Append one settled UTC page-decision partition for a controller generation. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_traffic_pages_daily_cold_v2_node +SQL > + % + {% if not defined(settled_date) %} + {{ error('settled_date is required') }} + {% end %} + SELECT * + FROM product_traffic_pages_daily_bounded_v2 + +TYPE COPY +TARGET_DATASOURCE product_traffic_pages_daily_cold_v2 +COPY_MODE append +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe index e173d6fd16f..8b6e1d65c6a 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_exact.pipe @@ -13,9 +13,9 @@ SQL > SELECT synthetic_run_id, session_id, - argMin(event_id, occurred_at) AS landing_event_id, - argMax(event_id, occurred_at) AS exit_event_id - FROM product_events_canonical_v1 + argMin(event_id, tuple(occurred_at, event_id)) AS landing_event_id, + argMax(event_id, tuple(occurred_at, event_id)) AS exit_event_id + FROM product_events_canonical_current WHERE event_name = 'page_view' AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' @@ -26,7 +26,7 @@ SQL > JSONExtractString(properties, 'page_view_id') AS page_view_id, toUInt64(sum(greatest(0, JSONExtractInt(properties, 'engaged_ms')))) AS engaged_ms, toFloat64(max(greatest(0, least(100, JSONExtractInt(properties, 'max_scroll_depth'))))) AS scroll_depth - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name = 'page_engagement' AND (traffic_class = 'external' OR synthetic_run_id != '') GROUP BY synthetic_run_id, page_view_id @@ -37,6 +37,8 @@ SQL > occurred_at, session_id, concat('anonymous:', anonymous_id) AS visitor_id, + platform, + app_version, hostname, pathname, country, @@ -44,7 +46,7 @@ SQL > browser, os, channel - FROM product_events_canonical_v1 + FROM product_events_canonical_current WHERE event_name = 'page_view' AND (traffic_class = 'external' OR synthetic_run_id != '') AND session_id != '' @@ -55,6 +57,8 @@ SQL > '' AS copy_run_id, page_views.synthetic_run_id AS synthetic_run_id, toDate(occurred_at, 'UTC') AS date, + platform, + app_version, hostname, pathname, country, @@ -72,7 +76,7 @@ SQL > FROM page_views INNER JOIN sessions ON page_views.synthetic_run_id = sessions.synthetic_run_id AND page_views.session_id = sessions.session_id LEFT JOIN engagements ON page_views.synthetic_run_id = engagements.synthetic_run_id AND page_views.event_id = engagements.page_view_id - GROUP BY page_views.synthetic_run_id, date, hostname, pathname, country, device, browser, os, channel + GROUP BY page_views.synthetic_run_id, date, platform, app_version, hostname, pathname, country, device, browser, os, channel {% if defined(copy_run_id) %} UNION ALL SELECT @@ -80,6 +84,8 @@ SQL > {{String(copy_run_id)}} AS copy_run_id, '' AS synthetic_run_id, today() AS date, + '' AS platform, + '' AS app_version, '' AS hostname, '' AS pathname, '' AS country, @@ -100,4 +106,4 @@ SQL > TYPE COPY TARGET_DATASOURCE product_traffic_pages_daily_exact COPY_MODE replace -COPY_SCHEDULE 4-59/8 * * * * +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_hot_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_hot_v2.pipe new file mode 100644 index 00000000000..ccfa2db9060 --- /dev/null +++ b/scripts/analytics/tinybird/pipes/snapshot_product_traffic_pages_daily_hot_v2.pipe @@ -0,0 +1,14 @@ +DESCRIPTION > + Replace only the bounded mutable page-decision window for one controller generation. + +TOKEN product_events_copy_runner READ + +NODE snapshot_product_traffic_pages_daily_hot_v2_node +SQL > + SELECT * + FROM product_traffic_pages_daily_bounded_v2 + +TYPE COPY +TARGET_DATASOURCE product_traffic_pages_daily_hot_v2 +COPY_MODE replace +COPY_SCHEDULE @on-demand diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index 40498d10647..a802f921a30 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -36,7 +36,9 @@ const TEST_FILES = fs const CLOUD_URL_DEFAULT = "https://api.tinybird.co"; const STAGING_WORKSPACE_ID = "37b8fef9-817f-4c3c-b21f-218c36a6077d"; const LOCAL_COPY_RUN_ID = "run_local_copy_assertions"; -const PRODUCT_COPY_PIPES = [ +const EXCEPTIONAL_PRODUCT_COPY_PIPES = [ + "snapshot_product_event_id_states_v2", + "snapshot_product_event_day_states_v2", "snapshot_product_events_canonical_v1", "snapshot_product_events_daily_exact", "snapshot_product_traffic_daily_exact", @@ -44,9 +46,33 @@ const PRODUCT_COPY_PIPES = [ "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", "snapshot_product_identity_funnel_exact", + "snapshot_product_attribution_daily_exact", + "snapshot_product_experiment_outcomes_exact", "snapshot_product_events_health_hourly", ]; +const BOUNDED_PRODUCT_COPY_PIPES = [ + "snapshot_product_events_daily_hot_v2", + "snapshot_product_events_daily_cold_v2", + "snapshot_product_traffic_daily_hot_v2", + "snapshot_product_traffic_daily_cold_v2", + "snapshot_product_traffic_pages_daily_hot_v2", + "snapshot_product_traffic_pages_daily_cold_v2", +]; +const PRODUCT_COPY_PIPES = [ + ...EXCEPTIONAL_PRODUCT_COPY_PIPES, + ...BOUNDED_PRODUCT_COPY_PIPES, + "publish_product_analytics_generation_v2", +]; +const LOCAL_PRODUCT_COPY_PIPES = [ + ...EXCEPTIONAL_PRODUCT_COPY_PIPES, + "snapshot_product_events_daily_hot_v2", + "snapshot_product_traffic_daily_hot_v2", + "snapshot_product_traffic_pages_daily_hot_v2", + "publish_product_analytics_generation_v2", +]; const PRODUCT_COPY_TARGETS = [ + "product_event_id_states_v2", + "product_event_day_states_v2", "product_events_canonical_v1", "product_events_daily_exact", "product_traffic_daily_exact", @@ -54,7 +80,23 @@ const PRODUCT_COPY_TARGETS = [ "product_activation_daily_exact", "product_creator_retention_exact", "product_identity_funnel_exact", + "product_attribution_daily_exact", + "product_experiment_outcomes_exact", "product_events_health_hourly_exact", + "product_events_daily_hot_v2", + "product_events_daily_cold_v2", + "product_traffic_daily_hot_v2", + "product_traffic_daily_cold_v2", + "product_traffic_pages_daily_hot_v2", + "product_traffic_pages_daily_cold_v2", + "product_analytics_generations_v2", +]; +const PRODUCT_COPY_SOURCES = [ + "product_events_canonical_current", + "product_events_canonical_window", + "product_events_daily_bounded_v2", + "product_traffic_daily_bounded_v2", + "product_traffic_pages_daily_bounded_v2", ]; const WORKSPACE_ID_SOURCE = "[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; @@ -147,6 +189,35 @@ const cloudCliStep = (...args) => ({ }); const operationPlan = (operation) => { + const localSourceCutoff = new Date().toISOString(); + const localGenerationId = `local_${localSourceCutoff.replaceAll(/[^0-9]/g, "")}`; + const localCopyStep = (name) => { + const parameters = [ + "--param", + "copy_max_threads=1", + "--param", + `copy_run_id=${LOCAL_COPY_RUN_ID}`, + ]; + if ( + BOUNDED_PRODUCT_COPY_PIPES.includes(name) || + name === "publish_product_analytics_generation_v2" + ) { + parameters.push( + "--param", + `source_cutoff=${localSourceCutoff}`, + "--param", + `generation_id=${localGenerationId}`, + ); + } + if (name === "publish_product_analytics_generation_v2") { + parameters.push("--param", "generation_kind=hot"); + } + return { + ...localCliStep("--local", "copy", "run", name, ...parameters, "--wait"), + attempts: 8, + retryPattern: /CANNOT_SCHEDULE_TASK|no free thread/i, + }; + }; const plans = { validate: [{ type: "validate" }], test: [{ type: "validate" }, { type: "node-test" }], @@ -182,21 +253,7 @@ const operationPlan = (operation) => { "--file", "fixtures/product_events_v1.local.ndjson", ), - ...PRODUCT_COPY_PIPES.map((name) => ({ - ...localCliStep( - "--local", - "copy", - "run", - name, - "--param", - "copy_max_threads=1", - "--param", - `copy_run_id=${LOCAL_COPY_RUN_ID}`, - "--wait", - ), - attempts: 8, - retryPattern: /CANNOT_SCHEDULE_TASK|no free thread/i, - })), + ...LOCAL_PRODUCT_COPY_PIPES.map(localCopyStep), { type: "verify-local" }, { type: "write-local-env" }, ], @@ -227,21 +284,7 @@ const operationPlan = (operation) => { "--file", "fixtures/product_events_v1.local.ndjson", ), - ...PRODUCT_COPY_PIPES.map((name) => ({ - ...localCliStep( - "--local", - "copy", - "run", - name, - "--param", - "copy_max_threads=1", - "--param", - `copy_run_id=${LOCAL_COPY_RUN_ID}`, - "--wait", - ), - attempts: 8, - retryPattern: /CANNOT_SCHEDULE_TASK|no free thread/i, - })), + ...LOCAL_PRODUCT_COPY_PIPES.map(localCopyStep), { type: "verify-local" }, ], "local-tokens": [{ type: "write-local-env" }], @@ -399,6 +442,7 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { new Set([ ...PRODUCT_COPY_TARGETS.map((name) => `datasource:${name}:APPEND`), ...PRODUCT_COPY_PIPES.map((name) => `pipe:${name}:READ`), + ...PRODUCT_COPY_SOURCES.map((name) => `pipe:${name}:READ`), ]), issues, ); @@ -498,12 +542,132 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { issues.push("Canonical product events are missing erasure lookup access"); } } + const eventStates = project.datasources.find( + (datasource) => datasource.name === "product_event_id_states_v2", + ); + if (!eventStates || eventStates.engine !== "AggregatingMergeTree") { + issues.push("Missing streaming product event ID aggregate state"); + } else { + if (eventStates.sortingKey !== "event_id") { + issues.push("Product event ID state must be sorted by event ID"); + } + if (eventStates.ttl !== "toDateTime(received_at) + INTERVAL 800 DAY") { + issues.push("Product event ID state must retain the erasure horizon"); + } + } + const eventStateMaterialization = project.pipes.find( + (pipe) => pipe.name === "materialize_product_event_id_states_v2", + ); + if ( + !eventStateMaterialization || + eventStateMaterialization.type !== "materialized" || + eventStateMaterialization.targetDatasource !== "product_event_id_states_v2" + ) { + issues.push("Missing streaming product event ID materialization"); + } + const eventDayStates = project.datasources.find( + (datasource) => datasource.name === "product_event_day_states_v2", + ); + if (!eventDayStates || eventDayStates.engine !== "AggregatingMergeTree") { + issues.push("Missing bounded product event day aggregate state"); + } else { + if (eventDayStates.sortingKey !== "(occurred_date, event_id)") { + issues.push( + "Product event day state must be sorted by date and event ID", + ); + } + if (eventDayStates.partitionKey !== "toYYYYMM(occurred_date)") { + issues.push( + "Product event day state must use monthly occurrence partitions", + ); + } + if (eventDayStates.ttl !== "occurred_date + INTERVAL 807 DAY") { + issues.push("Product event day state must retain the erasure horizon"); + } + } + const eventDayStateMaterialization = project.pipes.find( + (pipe) => pipe.name === "materialize_product_event_day_states_v2", + ); + if ( + !eventDayStateMaterialization || + eventDayStateMaterialization.type !== "materialized" || + eventDayStateMaterialization.targetDatasource !== + "product_event_day_states_v2" + ) { + issues.push("Missing bounded product event day materialization"); + } + for (const [prefix, hotTtl, coldTtl] of [ + [ + "product_events_daily", + "date + INTERVAL 16 DAY", + "date + INTERVAL 800 DAY", + ], + [ + "product_traffic_daily", + "date + INTERVAL 16 DAY", + "date + INTERVAL 800 DAY", + ], + [ + "product_traffic_pages_daily", + "date + INTERVAL 16 DAY", + "date + INTERVAL 800 DAY", + ], + ]) { + for (const [suffix, ttl] of [ + ["hot_v2", hotTtl], + ["cold_v2", coldTtl], + ]) { + const name = `${prefix}_${suffix}`; + const datasource = project.datasources.find( + (candidate) => candidate.name === name, + ); + if (!datasource || datasource.engine !== "AggregatingMergeTree") { + issues.push(`Missing bounded aggregate datasource ${name}`); + } else if (datasource.ttl !== ttl) { + issues.push(`${name} has an invalid retention window`); + } + } + if ( + !project.pipes.some( + (pipe) => + pipe.name === `${prefix}_current_v2` && pipe.type === "generic", + ) + ) { + issues.push( + `Missing generation-selected aggregate source ${prefix}_current_v2`, + ); + } + } + for (const name of [ + "product_events_daily_bounded_v2", + "product_traffic_daily_bounded_v2", + "product_traffic_pages_daily_bounded_v2", + ]) { + const pipe = project.pipes.find((candidate) => candidate.name === name); + if (!pipe || !hasToken(pipe, "product_events_copy_runner", "READ")) { + issues.push(`Missing bounded Copy source ${name}`); + } + } + const generations = project.datasources.find( + (datasource) => datasource.name === "product_analytics_generations_v2", + ); + if (!generations || generations.engine !== "MergeTree") { + issues.push("Missing bounded analytics publication barrier"); + } else if ( + generations.ttl !== "toDateTime(source_cutoff) + INTERVAL 807 DAY" + ) { + issues.push( + "Analytics generation metadata must retain the erasure horizon", + ); + } for (const [name, engine] of [ ["product_traffic_daily_exact", "AggregatingMergeTree"], ["product_traffic_pages_daily_exact", "AggregatingMergeTree"], ["product_activation_daily_exact", "AggregatingMergeTree"], ["product_creator_retention_exact", "AggregatingMergeTree"], ["product_identity_funnel_exact", "SummingMergeTree"], + ["product_attribution_daily_exact", "AggregatingMergeTree"], + ["product_experiment_outcomes_exact", "SummingMergeTree"], ]) { const datasource = project.datasources.find( (candidate) => candidate.name === name, @@ -573,8 +737,10 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { "product_events_daily", "product_events_health", "product_traffic_overview", + "product_traffic_totals", "product_traffic_pages", "product_traffic_sources", + "product_attribution", "product_traffic_countries", "product_traffic_technology", "product_activation", @@ -582,6 +748,7 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { "product_creator_activity", "product_feature_adoption", "product_identity_funnel", + "product_experiment_outcomes", "product_analytics_freshness", "product_analytics_copy_assertions", ]) { diff --git a/scripts/analytics/verify-local.js b/scripts/analytics/verify-local.js index 23b4c8713cd..3757d5b66f0 100644 --- a/scripts/analytics/verify-local.js +++ b/scripts/analytics/verify-local.js @@ -133,11 +133,13 @@ const copyAssertions = await query("product_analytics_copy_assertions", { }); assert.deepEqual(copyAssertions, [ { + decision_markers: 1, traffic_markers: 1, traffic_page_markers: 1, activation_markers: 1, retention_markers: 1, identity_markers: 1, + health_markers: 1, }, ]); From c6101c810c3126779ff1b730bb93c4c514ff9039 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:40:18 +0100 Subject: [PATCH 084/110] ci: prove analytics staging at the exact SHA --- .github/workflows/analytics.yml | 217 ++- ...st-party-analytics-production-readiness.md | 39 +- .../e2e/analytics-staging.spec.ts | 211 ++- .../app/api/analytics/staging-test/route.ts | 656 +++++++- scripts/analytics/README.md | 12 +- scripts/analytics/staging-ci-lib.js | 555 ++++++- scripts/analytics/staging-ci.js | 1477 +++++++++++++++-- scripts/analytics/tests/staging-ci.test.js | 596 ++++++- 8 files changed, 3496 insertions(+), 267 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 8b7a512dd3e..df9681e4e57 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -12,6 +12,12 @@ on: - "packages/database/**" - "packages/env/server.ts" - "packages/web-backend/src/ProductAnalytics/**" + - "packages/web-backend/src/Tinybird/**" + - "packages/web-backend/src/Organisations/**" + - "packages/web-domain/**" + - "apps/cli/**" + - "apps/desktop/**" + - "apps/web/**" - "apps/cli/src/analytics.rs" - "apps/mobile/**" - "apps/web/actions/analytics/**" @@ -21,17 +27,26 @@ on: - "apps/web/app/Layout/*Analytics*.tsx" - "apps/web/app/admin/analytics/**" - "apps/web/app/api/analytics/**" + - "apps/web/app/api/cron/drain-product-analytics-outbox/**" - "apps/web/app/api/cron/reconcile-product-analytics/**" + - "apps/web/app/api/cron/recover-organization-invite-delivery/**" + - "apps/web/app/api/cron/recover-product-analytics-erasure/**" + - "apps/web/app/api/cron/refresh-product-analytics/**" - "apps/web/app/api/events/**" - "apps/web/app/api/invite/accept/route.ts" - "apps/web/app/api/settings/billing/**" - "apps/web/app/api/webhooks/stripe/route.ts" - "apps/web/app/mobile/checkout/**" + - "apps/web/app/api/mobile/[...route]/route.ts" - "apps/web/app/utils/analytics.ts" - "apps/web/app/utils/product-analytics.ts" - "apps/web/lib/analytics/**" + - "apps/web/lib/organization-invite-delivery.ts" + - "apps/web/proxy.ts" + - "apps/web/vercel.json" - "apps/web/lib/rate-limit.ts" - "apps/web/workflows/*product-analytics*" + - "apps/web/workflows/recover-organization-invite-delivery.ts" - "apps/desktop/src-tauri/src/lib.rs" - "apps/desktop/src-tauri/src/product_analytics.rs" - "apps/desktop/src/utils/analytics.ts" @@ -41,8 +56,10 @@ on: workflow_dispatch: permissions: + actions: read contents: read deployments: read + pull-requests: read statuses: read concurrency: @@ -64,6 +81,18 @@ jobs: github.ref == 'refs/heads/codex/first-party-analytics') runs-on: ubuntu-latest timeout-minutes: 30 + services: + mysql: + image: mysql:8.4 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: "yes" + ports: + - 3306:3306 + options: >- + --health-cmd="mysqladmin ping --silent" + --health-interval=5s + --health-timeout=5s + --health-retries=20 env: EXPECTED_SHA: ${{ github.event.pull_request.head.sha || github.sha }} EVENT_NUMBER: ${{ github.event.pull_request.number || '' }} @@ -79,6 +108,12 @@ jobs: run: node scripts/analytics/staging-ci.js verify-scope --actual-sha "$(git rev-parse HEAD)" - name: Run analytics contract and regression suites run: pnpm analytics:test + - name: Prove analytics leases, receipts, and shared-cookie erasure in MySQL + env: + CAP_PRODUCT_ANALYTICS_MYSQL_E2E: "1" + run: >- + pnpm --filter @cap/web exec vitest run + __tests__/e2e/product-analytics-mysql-e2e.test.ts - name: Validate Docker Compose run: node scripts/analytics/analytics-cli.js compose-check - name: Build and test Tinybird datafiles with fixtures @@ -107,9 +142,17 @@ jobs: uses: dtolnay/rust-toolchain@1.88.0 - uses: ./.github/actions/setup-rust-cache with: - target: x86_64-apple-darwin + target: aarch64-apple-darwin - name: Prepare pinned native dependencies run: node scripts/setup.js + - name: Create compile-only Tauri sidecars + run: | + target="$(rustc -vV | sed -n 's|host: ||p')" + binaries="apps/desktop/src-tauri/binaries" + install -d "$binaries" + for binary in cap-muxer cap-exporter cap-cli; do + install -m 755 /dev/null "$binaries/$binary-$target" + done - name: Compile the desktop analytics path run: cargo check -p cap-desktop --locked - name: Prove durable desktop analytics behavior @@ -120,7 +163,7 @@ jobs: needs: [validate, validate-desktop] if: needs.validate.result == 'success' && needs.validate-desktop.result == 'success' runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 180 environment: staging env: EXPECTED_SHA: ${{ github.event.pull_request.head.sha || github.sha }} @@ -152,6 +195,35 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: node scripts/analytics/staging-ci.js wait-vercel + - name: Refuse a preview bound outside Tinybird staging + env: + ANALYTICS_PREVIEW_URL: ${{ steps.vercel.outputs.url }} + CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} + TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} + TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN }} + TINYBIRD_STAGING_INGEST_TOKEN: ${{ secrets.TINYBIRD_STAGING_INGEST_TOKEN }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + run: node scripts/analytics/staging-ci.js attest-preview + - name: Persist the pre-create Tinybird recovery boundary + env: + VERCEL_DEPLOYMENT_ID: ${{ steps.vercel.outputs.deployment_id }} + VERCEL_PREVIEW_URL: ${{ steps.vercel.outputs.url }} + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js prepare-deployment-boundary + --output "$RUNNER_TEMP/analytics-deployment-boundary.json" + - name: Upload the immutable pre-create recovery boundary + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: analytics-recovery-${{ github.run_id }}-${{ github.run_attempt }}-00-precreate + path: ${{ runner.temp }}/analytics-deployment-boundary.json + if-no-files-found: error + retention-days: 30 - name: Record Tinybird deployment boundary run: echo "TINYBIRD_DEPLOYMENT_STARTED_AT=$(date -u +%Y-%m-%dT%H:%M:%S.000Z)" >> "$GITHUB_ENV" - name: Check Tinybird cloud deployment plan @@ -173,8 +245,11 @@ jobs: TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | set -o pipefail + set +e docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud deployment create --allow-destructive-operations --wait | tee "$RUNNER_TEMP/tinybird-create-output.txt" - node -e 'const fs = require("node:fs"); fs.writeFileSync(process.argv[2], JSON.stringify({ output: fs.readFileSync(process.argv[1], "utf8") }));' "$RUNNER_TEMP/tinybird-create-output.txt" "$RUNNER_TEMP/tinybird-create-output.json" + create_exit="${PIPESTATUS[0]}" + set -e + node -e 'const fs = require("node:fs"); fs.writeFileSync(process.argv[2], JSON.stringify({ exitCode: Number(process.argv[3]), output: fs.readFileSync(process.argv[1], "utf8") }));' "$RUNNER_TEMP/tinybird-create-output.txt" "$RUNNER_TEMP/tinybird-create-output.json" "$create_exit" - name: Resolve only the deployment created by this run id: tinybird env: @@ -184,7 +259,28 @@ jobs: TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: | docker compose --file packages/local-docker/docker-compose.yml --profile analytics run --rm tinybird-cloud-cli --cloud --output json deployment ls > "$RUNNER_TEMP/tinybird-deployments.json" - node scripts/analytics/staging-ci.js select-deployment --input "$RUNNER_TEMP/tinybird-deployments.json" --create-output "$RUNNER_TEMP/tinybird-create-output.json" --minimum-created-at "$TINYBIRD_DEPLOYMENT_STARTED_AT" + node scripts/analytics/staging-ci.js select-deployment --input "$RUNNER_TEMP/tinybird-deployments.json" --create-output "$RUNNER_TEMP/tinybird-create-output.json" --boundary "$RUNNER_TEMP/analytics-deployment-boundary.json" --minimum-created-at "$TINYBIRD_DEPLOYMENT_STARTED_AT" + - name: Prepare immutable synthetic cleanup state before ingestion + env: + VERCEL_DEPLOYMENT_ID: ${{ steps.vercel.outputs.deployment_id }} + VERCEL_PREVIEW_URL: ${{ steps.vercel.outputs.url }} + run: >- + node scripts/analytics/staging-ci.js prepare-seed + --run-id "$ANALYTICS_TEST_RUN_ID" + --deployment-id "${{ steps.tinybird.outputs.id }}" + --needs-promotion "${{ steps.tinybird.outputs.needs_promotion }}" + --boundary "$RUNNER_TEMP/analytics-deployment-boundary.json" + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Upload the immutable pre-ingestion recovery checkpoint + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: analytics-recovery-${{ github.run_id }}-${{ github.run_attempt }}-20-preseed + path: | + ${{ runner.temp }}/analytics-staging-state.json + ${{ runner.temp }}/analytics-staging-report.json + if-no-files-found: error + retention-days: 30 - name: Refuse source mutations after exact-SHA checkout run: | git diff --exit-code @@ -237,18 +333,40 @@ jobs: --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live' }}" --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Persist the exact Tinybird promotion plan + id: promotion-plan + if: steps.tinybird.outputs.needs_promotion == 'true' + env: + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js prepare-promotion + --deployment-id "${{ steps.tinybird.outputs.id }}" + --state "$RUNNER_TEMP/analytics-staging-state.json" + - name: Upload the immutable pre-promotion recovery checkpoint + if: steps.promotion-plan.outcome == 'success' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: analytics-recovery-${{ github.run_id }}-${{ github.run_attempt }}-30-prepromote + path: | + ${{ runner.temp }}/analytics-staging-state.json + ${{ runner.temp }}/analytics-staging-report.json + if-no-files-found: error + retention-days: 30 - name: Promote the verified staging deployment id: promote if: steps.tinybird.outputs.needs_promotion == 'true' continue-on-error: true env: + GITHUB_TOKEN: ${{ github.token }} TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: | test "$(git rev-parse HEAD)" = "$EXPECTED_SHA" git diff --exit-code test -z "$(git status --porcelain --untracked-files=no)" - node scripts/analytics/staging-ci.js promote-deployment --deployment-id "${{ steps.tinybird.outputs.id }}" + node scripts/analytics/staging-ci.js verify-pr-head + node scripts/analytics/staging-ci.js promote-deployment --deployment-id "${{ steps.tinybird.outputs.id }}" --state "$RUNNER_TEMP/analytics-staging-state.json" - name: Resolve authoritative Tinybird state after the promotion attempt id: deployment-state if: always() && steps.tinybird.outcome == 'success' @@ -411,9 +529,11 @@ jobs: id: cleanup if: always() && steps.seed.outcome != 'skipped' env: + CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} run: | if [[ ! -f "$RUNNER_TEMP/analytics-staging-state.json" ]]; then echo "required=false" >> "$GITHUB_OUTPUT" @@ -464,9 +584,26 @@ jobs: --action resume --state "$RUNNER_TEMP/analytics-staging-state.json" --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Persist finalization eligibility + id: recovery-ready + if: steps.verify-cleanup.outcome == 'success' && steps.rollback-drill.outcome == 'success' && steps.resume-copies.outcome == 'success' + run: >- + node scripts/analytics/staging-ci.js mark-recovery-ready + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Upload the immutable pre-finalization recovery checkpoint + if: steps.recovery-ready.outcome == 'success' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: analytics-recovery-${{ github.run_id }}-${{ github.run_attempt }}-40-ready-to-finalize + path: | + ${{ runner.temp }}/analytics-staging-state.json + ${{ runner.temp }}/analytics-staging-report.json + if-no-files-found: error + retention-days: 30 - name: Finalize the fully verified staging promotion id: finalize - if: steps.verify-cleanup.outcome == 'success' && steps.rollback-drill.outcome == 'success' && steps.resume-copies.outcome == 'success' + if: steps.recovery-ready.outcome == 'success' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} @@ -502,9 +639,75 @@ jobs: node scripts/analytics/staging-ci.js discard-deployment "${discard_args[@]}" - name: Upload redacted staging evidence if: always() && steps.cleanup.outputs.required == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: analytics-staging-${{ github.event.pull_request.head.sha || github.sha }}-${{ github.run_attempt }} path: ${{ runner.temp }}/analytics-staging-report.json if-no-files-found: error retention-days: 30 + + recover-staging: + name: Recover interrupted analytics staging + needs: deploy-staging + if: always() && needs.deploy-staging.result != 'success' + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: staging + permissions: + actions: read + contents: read + deployments: read + pull-requests: read + env: + EXPECTED_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + EVENT_NUMBER: ${{ github.event.pull_request.number || '' }} + HEAD_REF: ${{ github.event.pull_request.head.ref || '' }} + ANALYTICS_TEST_RUN_ID: run_${{ github.run_id }}_${{ github.run_attempt }}_${{ github.event.pull_request.head.sha || github.sha }} + steps: + - name: Check out the interrupted exact SHA + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + ref: ${{ github.event.pull_request.head.sha || github.sha }} + fetch-depth: 1 + - name: Locate immutable recovery checkpoints + id: checkpoints + env: + GH_TOKEN: ${{ github.token }} + CHECKPOINT_PREFIX: analytics-recovery-${{ github.run_id }}-${{ github.run_attempt }}- + run: | + count="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?per_page=100" --jq '[.artifacts[] | select(.expired == false and (.name | startswith(env.CHECKPOINT_PREFIX)))] | length')" + if [[ "$count" -eq 0 ]]; then + echo "found=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "found=true" >> "$GITHUB_OUTPUT" + - name: Download immutable recovery checkpoints + if: steps.checkpoints.outputs.found == 'true' + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + pattern: analytics-recovery-${{ github.run_id }}-${{ github.run_attempt }}-* + path: ${{ runner.temp }}/analytics-recovery-checkpoints + - name: Reconcile and recover only the owned staging run + id: recover + if: steps.checkpoints.outputs.found == 'true' + env: + CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} + VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + run: >- + node scripts/analytics/staging-ci.js recover + --checkpoint-directory "$RUNNER_TEMP/analytics-recovery-checkpoints" + --artifact "$RUNNER_TEMP/analytics-recovery-report.json" + - name: Upload redacted recovery evidence + if: always() && steps.checkpoints.outputs.found == 'true' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: analytics-recovery-result-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/analytics-recovery-report.json + if-no-files-found: error + retention-days: 30 diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index 08b2460d28f..e33e72b2bfb 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -6,7 +6,7 @@ Cap analytics is at-least-once at the delivery boundary and exactly once for dec The central registry is `packages/analytics/src/event-registry.ts`. It defines the version, authority, delivery class, permitted platforms, semantic, and property schema for every event. Contract CI rejects unknown literal names, templates, invalid property keys, missing emitters, and drift between the Rust desktop catalogue and the TypeScript registry. Invalid properties reject the whole event rather than being silently removed. -Critical server facts start a durable Vercel Workflow before the request succeeds. Each delivery step retries transient network, `429`, and `5xx` failures with the original ID. Permanent contract failures become visible failed workflow runs. Signup, share, and collaboration facts are also reconciled from authoritative database rows every day with deterministic event IDs, closing the database-commit to workflow-start gap. Stripe remains authoritative for purchases, trials, renewals, plan and seat changes, cancellation, churn, refunds, and payment failures; Stripe webhook retries close its enqueue gap. +Critical server facts are committed to a durable MySQL outbox before the capture call succeeds. Business paths that already own a database transaction write the business mutation and analytics fact atomically; other paths persist the analytics fact in their own transaction immediately after the authoritative mutation. A leased dispatcher starts Vercel Workflow outside the database transaction, retries transient workflow-start and provider failures with the original ID, reclaims unconfirmed runs, and exposes pending age, dead letters, and payload conflicts. Signup, share, and collaboration facts are also reconciled from authoritative database rows every day with deterministic event IDs, closing the remaining mutation-to-outbox gap. Stripe remains authoritative for purchases, trials, renewals, plan and seat changes, cancellation, churn, refunds, and payment failures; Stripe webhook retries and daily paginated reconciliation close its enqueue gap. Desktop critical events enter a bounded encrypted local outbox before network delivery. The encryption key is stored in the operating-system keyring, events survive restart and offline periods, retryable failures back off, permanent failures enter a bounded dead-letter queue, and counters expose accepted, retried, dropped, overflowed, and dead-lettered events. Mobile critical events enter a bounded atomic AsyncStorage outbox before capture returns; interrupted rotations recover the last valid backup, account-scoped queues are isolated, offline and restart delivery resumes with the original event ID, and terminal failures enter a bounded dead-letter ledger. Browser events are intentionally best effort: attempted, accepted, retried, dropped, overflowed, and oversize counts accompany later batches without recursively creating analytics events. @@ -28,19 +28,21 @@ Cap chose a persistent first-party pseudonymous visitor identifier, not Plausibl The collector rejects unregistered events, unknown properties, raw error fields, invalid identifiers, excessive cardinality, and oversized requests. Raw customer content, filenames, transcripts, email addresses, raw errors, raw IP addresses, and raw user-agent strings are not analytics properties. IP and user-agent data are used only in the request process to derive internal/bot classification and coarse country, region, city, browser, device, and operating-system dimensions; raw values are not stored. -Known bots, crawlers, previews, internal IP hashes, and synthetic runs are visible to health monitoring but excluded from decision snapshots. Anonymous write tokens are rate-limited by both source IP and anonymous ID. Replay and automation can be made expensive and observable, but public traffic analytics cannot cryptographically guarantee a human browser. Signup, organization, billing, and other business outcomes therefore remain server-authoritative. +Known bots, crawlers, previews, internal IP hashes, and synthetic runs are visible to health monitoring but excluded from decision snapshots. Short-lived signed browser claims are rate-limited by both source IP and anonymous ID; browsers never receive a Tinybird write token. Replay and automation can be made expensive and observable, but public traffic analytics cannot cryptographically guarantee a human browser. Signup, organization, billing, and other business outcomes therefore remain server-authoritative. Vercel request-IP buckets use only the platform-owned forwarding header and retain only its SHA-256 hash in process memory. A self-hosted deployment does not trust forwarding headers by default, so product-event collection and legacy public view tracking return `503` until `CAP_ANALYTICS_TRUST_PROXY=1` is configured. That setting uses the first `x-forwarded-for` address and is safe only behind a reverse proxy that overwrites, rather than appends or passes through, that header. Do not set it on Vercel. Client-controlled anonymous IDs and authorization strings never become network rate-limit keys. The desktop critical-event outbox uses AES-256-GCM. Its stable random key is stored in the OS keyring and in a separate app-local recovery-key file so queued events remain recoverable during temporary keyring outages; Unix recovery-key permissions are `0600`, while Windows relies on the per-user app-data ACL. This protects queue contents from accidental store disclosure but does not claim protection from a process or local account with access to both app-data files. An upgrade may read the earlier application-store fallback-key shape only to decrypt and re-encrypt pending queues, then removes that legacy value after both primary and recovery queues are safely consolidated. -Account and organization deletion writes its durable pending marker or tombstone before erasure begins. Delivery and reconciliation reject marked identities, then deletion waits longer than the bounded in-flight delivery attempt before removing matching raw rows with a dedicated erasure token and rebuilding every replace-mode canonical, traffic, activation, retention, and health snapshot. This closes the race where a workflow that passed its identity check could otherwise append after a completed erasure. Deletion fails closed if the erasure credential or any rebuild is unavailable. The staging suite deletes a synthetic user and organization, proves their raw-health and decision state is gone, and proves an out-of-scope row sharing the anonymous ID remains until final test cleanup. Tinybird's row-deletion API currently requires the general `DATASOURCES:CREATE` scope and rejects resource-scoped creation of that operator; the token therefore has no read or deployment scope, lives only in the protected staging environment, and is used by code that accepts bounded validated deletion predicates. Its broader same-workspace mutation capability is an explicit provider limitation rather than a least-privilege claim. +Account and organization deletion first fences the user and organization identities in MySQL, records linked anonymous identities in a durable identity ledger, and queues a recoverable erasure request. New delivery and reconciliation reject fenced identities. Erasure then acquires the global fencing lease, waits for bounded ingestion leases, removes matching raw rows with a dedicated erasure token, rebuilds every replace-mode canonical, traffic, activation, retention, attribution, experiment, and health snapshot, and deletes only anonymous aliases that are exclusive to the erased principals. Event receipts and queued rows for the erased scope are removed as part of the same recovery contract. This closes both append-after-delete and shared-browser over-deletion races. Deletion fails closed if the erasure credential or any rebuild is unavailable. The staging suite deletes a synthetic user and organization, proves their raw-health and decision state is gone, proves replay is suppressed, and proves an out-of-scope row sharing the anonymous ID remains until final test cleanup. Tinybird's row-deletion API currently requires the general `DATASOURCES:CREATE` scope and rejects resource-scoped creation of that operator; the token therefore has no read or deployment scope, lives only in the protected staging environment, and is used by code that accepts bounded validated deletion predicates. Its broader same-workspace mutation capability is an explicit provider limitation rather than a least-privilege claim. Raw, canonical, and decision data share an 800-day TTL. This supports two complete 365-day comparison windows plus a rebuild buffer for year-over-year decisions. The identity-bearing source is intentionally retained for the same horizon as its exact aggregates so a later account or organization deletion can still retract every historical contribution. Keeping only irreversible long-lived counts would use less storage but would break the derived-erasure guarantee. Health-detail aggregates remain limited to 90 days. This retention choice requires privacy-owner approval before production rollout and must be disclosed with the persistent first-party identifier model. ## Data quality and performance gates -The staging workflow is restricted to PR 2003 and `codex/first-party-analytics`, hard-codes the staging Tinybird workspace ID, checks the exact Git SHA, waits for the exact-SHA Vercel preview, creates an isolated Tinybird staging deployment, runs fixture and synthetic tests, and promotes only a verified deployment inside that staging workspace. Candidate reads use the exact numeric deployment ID and prove raw delivery, isolation, endpoint execution, and absolute latency before promotion. Tinybird's Copy API does not reliably route an on-demand Copy mutation into a candidate, so Copy mutations are prohibited until the exact candidate is live in staging. The prior staging deployment remains available until promoted Copy results, public business values, retained-deployment and representative-volume performance, erasure, cleanup, and a live rollback-and-restoration drill pass. The first rollout of a new endpoint compares only shared endpoints to the retained deployment, records the new endpoint as having no historical baseline, and still applies candidate and representative absolute budgets. The rollback drill executes every shared typed endpoint while the prior deployment is live; the exact-SHA admin client treats only an absent identity-funnel endpoint as optional, then the drill restores the candidate and re-proves the full endpoint suite, health, and synthetic business responses. Ambiguous live-switch responses are reconciled against the exact deployment pair before recovery continues. Any earlier failure restores a data-plane-proven prior deployment and removes the rejected candidate. The workflow has no push trigger, production environment, production token, or production deployment command. +The staging workflow is restricted to PR 2003 and `codex/first-party-analytics`, hard-codes the staging Tinybird workspace ID, checks the exact Git SHA, waits for the exact-SHA Vercel preview, and requires that preview to attest the reviewed staging database credential fingerprint and migration `0042_lying_sharon_ventura` before any synthetic mutation. It creates an isolated Tinybird staging deployment, runs fixture and synthetic tests, and promotes only a verified deployment inside that staging workspace. Candidate reads use the exact numeric deployment ID and prove raw delivery, isolation, endpoint execution, and absolute latency before promotion. Tinybird's Copy API does not reliably route an on-demand Copy mutation into a candidate, so Copy mutations are prohibited until the exact candidate is live in staging. The prior staging deployment remains available until promoted Copy results, public business values, retained-deployment and representative-volume performance, erasure, cleanup, durable MySQL outbox health, and a live rollback-and-restoration drill pass. The first rollout of a new endpoint compares only shared endpoints to the retained deployment, records the new endpoint as having no historical baseline, and still applies candidate and representative absolute budgets. A no-op deployment uses absolute budgets rather than presenting the same deployment as an independent regression baseline. The rollback drill probes candidate-only endpoints against the retained deployment and excludes only endpoints that return `404`; every available typed endpoint must remain readable. The exact-SHA admin client applies the same narrow rollback compatibility rule, then the drill restores the candidate and re-proves the full endpoint suite, health, and synthetic business responses. Ambiguous live-switch responses are reconciled against the exact deployment pair before recovery continues. Any earlier failure restores a data-plane-proven prior deployment and removes the rejected candidate. + +The workflow persists immutable checkpoints before deployment creation, seeding, promotion, and finalization. Its same-run recovery job resolves uncertain creation from the exact deployment-ID set difference, restores the prior read plane, resumes paused copies, removes owned synthetic state, and discards only the owned candidate. GitHub force-cancellation or a GitHub control-plane outage can still prevent that same-workflow recovery job from starting; the recorded checkpoint is the manual recovery input in that case. The workflow has no push trigger, production environment, production token, or production deployment command. The redacted evidence artifact records the Git SHA, GitHub run, Vercel and Tinybird deployment IDs, hashed synthetic-run identity, delivery attempts, ingestion throughput, endpoint and full-dashboard p50/p95/p99, measured-baseline regressions, visibility time, health totals, decision-dedup assertions, conflict quarantine, least-privilege token checks, scoped identity erasure, and cleanup. Synthetic rows are excluded from normal metrics and cleanup is verified after every promoted run. @@ -58,26 +60,29 @@ Required live gates are: - a populated-table performance pass measures every typed decision endpoint and full dashboard fanout after materialization; - wrong workspace, stale SHA, missing credentials, partial execution, failed promotion, and failed cleanup all fail closed. -Copy-backed decision tables rebuild on serialized eight-minute schedules to avoid competing for the same Tinybird worker pool. Exact-SHA staging runs invoke the eight copies in dependency order after an exact candidate is promoted, after identity erasure, and after final cleanup. A no-op deployment invokes them immediately after seeding. Candidate ingestion visibility is measured directly against the exact candidate and promoted Copy visibility is measured separately, so neither proof waits for the periodic schedule. Dashboard freshness exposes product, traffic, retention, and identity-funnel aggregate timestamps; scheduled production freshness is therefore bounded by the documented copy cadence plus provider execution time. +Copy-backed decision tables rebuild through a durable single-owner Vercel Workflow every ten minutes to avoid competing for the same Tinybird worker pool. The routine refresh runs the eight decision models from the streaming exact event-ID state. Exact-SHA staging and erasure runs additionally rebuild global and day-partitioned event state, canonical state, and health, for twelve ordered copies in total. Both event-state copies must reach terminal success before canonical rebuilding begins. Candidate ingestion visibility is measured directly against the exact candidate and promoted Copy visibility is measured separately, so neither proof waits for the periodic refresh. Dashboard freshness exposes product, traffic, retention, identity-funnel, attribution, and experiment-outcome aggregate timestamps; scheduled production freshness is therefore bounded by the documented workflow cadence plus provider execution time. + +The checked-in bounded v2 hot/cold models are deployment-validated side-by-side resources, not the current public read plane. Routine dashboard and agent endpoints continue to read the exact pre-aggregated decision tables and never scan raw events. A future cutover must backfill settled UTC dates, publish complete cross-model generations, prove parity against the current tables, add physical cold-state erasure rebuilding, and then switch endpoints. Until those gates exist, the v2 resources remain unscheduled and cannot affect decision metrics. -The staging run records absolute ingestion budgets, compares shared typed endpoints and dashboard fanout against the retained prior deployment, and separately measures bounded 1,000-row and 10,000-row mixed high-cardinality corpora with nonzero traffic, activation, retention, identity, product, and revenue assertions. It measures exact-SHA browser main-thread cost and append-batch p50, p95, p99, error rate, and throughput. A newly introduced endpoint without an honest prior baseline is labeled as such and must pass measured and representative absolute budgets. It reports p50, p95, and p99 for baseline, measured, and representative samples, and applies retained-baseline regression budgets to representative samples wherever a baseline exists. A green static/unit run alone is not rollout evidence. +The staging run records absolute ingestion budgets, compares shared typed endpoints and dashboard fanout against the retained prior deployment, and separately measures 1,000-row and 100,000-row mixed high-cardinality corpora spread across 30 and 80 UTC days with nonzero traffic, activation, retention, identity, attribution, experimentation, product, and revenue assertions. It separately measures direct provider ingestion and the exact-SHA `/api/events` collector with 20 full batches at controlled concurrency, zero-error and throughput gates. Browser capture uses 30 alternating matched-control samples on the deployed page; desktop capture measures 100 real encrypted and fsynced journal appends. Endpoint workloads use 30 interleaved samples and report p50, p95, p99, mean, standard deviation, and coefficient of variation. A newly introduced endpoint without an honest prior baseline is labeled as such and must pass measured and representative absolute budgets. A green static/unit run alone is not rollout evidence. ## Production rollout checklist -Production remains prohibited until the final staging artifact, relevant CI, security/data/performance reviews, and Greptile are green for the same branch SHA. +Production remains prohibited until the final staging artifact, relevant CI, security/data/performance reviews, and Greptile are green for the same branch SHA. It is additionally blocked until Cap provisions an analytics-only production Tinybird workspace, or Tinybird provides an equivalently protected resource-scoped row-deletion capability: the current `DATASOURCES:CREATE` requirement would otherwise let an erasure credential mutate unrelated datasources in a shared production workspace. The preview-only `/api/analytics/staging-test` route must not receive a production secret. Its `CAP_ANALYTICS_STAGING_TEST_SECRET` exists only in the Vercel Preview environment and the protected GitHub staging environment; the route returns `404` outside `VERCEL_ENV=preview` and requires the exact Vercel Git SHA. -1. Create production Tinybird resources from the reviewed datafiles with a deploy credential scoped only to the production workspace. Run `deployment create --check`, review destructive/schema changes, create an isolated deployment, run fixture tests, rebuild every copy, query all aggregate endpoints, then promote. Record the deployment ID. Do not reuse the staging workspace or tokens. -2. Create least-privilege Tinybird tokens: +1. Generate and review the database migrations from `packages/database/schema.ts`; confirm `0040_chief_skreet.sql`, `0041_complex_outlaw_kid.sql`, and `0042_lying_sharon_ventura.sql` exactly match the reviewed schema. Create a PlanetScale deploy request targeting production `main`, require zero destructive operations and a green schema lint, and obtain separate production approval before deployment. After deployment, attest the analytics outbox, receipt, identity, ingestion-lease, erasure-request, reconciliation-failure, refresh-lease, invite-delivery, and reconciliation index shapes before deploying application code. +2. Create an analytics-only production Tinybird workspace, then create its resources from the reviewed datafiles with a deploy credential scoped only to that workspace. Run `deployment create --check`, review destructive/schema changes, create an isolated deployment, run fixture tests, rebuild every copy, query all aggregate endpoints, then promote. Record the workspace and deployment IDs. Do not reuse the staging workspace or tokens. +3. Create least-privilege Tinybird tokens: - append-only token for `product_events_v1`; - aggregate endpoint read token with no raw or canonical datasource access; - - resource-scoped copy token for the eight reviewed copy pipes, with no raw identity datasource access; + - resource-scoped copy token for the reviewed copy pipes enumerated by the staging runner, with no raw identity datasource access; - erasure-lookup token limited to read access on `product_events_v1` and `product_events_canonical_v1`, protected from all agent and admin surfaces; - - schedule-controller token limited to cancelling, pausing, and resuming the eight reviewed Copy Pipes; + - schedule-controller token limited to cancelling, pausing, and resuming the reviewed Copy Pipes enumerated by the staging runner; - dedicated erasure token with Tinybird's required `DATASOURCES:CREATE` scope, no read/deploy scopes, protected as a high-impact operational secret until Tinybird offers resource-scoped row deletion; - deployment token used only by the controlled production release path. -3. Set these Vercel production variables without copying values into logs or artifacts: +4. Set these Vercel production variables without copying values into logs or artifacts: - `PRODUCT_ANALYTICS_TINYBIRD_HOST` - `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` - `PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN` @@ -89,10 +94,10 @@ The preview-only `/api/analytics/staging-test` route must not receive a producti - `CRON_SECRET` - `NEXTAUTH_SECRET` (existing application secret used to sign the short-lived anonymous browser token; do not create an analytics-specific duplicate) For a self-hosted release only, set `CAP_ANALYTICS_TRUST_PROXY=1` after verifying that the fronting reverse proxy overwrites `x-forwarded-for`. Omit it on Vercel. -4. Configure the Vercel firewall rule referenced by the collector so `/api/events` has the verified per-IP limit. The collector also applies the same rule by anonymous-ID key. Verify normal navigation is usable, the documented burst test receives `429`, forged forwarding headers do not bypass classification, and preview hostnames are excluded from decision metrics. -5. Deploy the web application through the normal production release process. Confirm the Vercel deployment Git SHA exactly matches the reviewed commit and confirm the configured Tinybird deployment ID before sending any test event. -6. Run a uniquely tagged production smoke test only if separately approved. Do not use customer identifiers. Confirm ingestion, health, decision deduplication, admin freshness, and strictly scoped cleanup. -7. Enable dashboards only after freshness, duplicates, conflicts, missing identity, clock skew, late events, queue drops, dead letters, and ingestion lag are healthy. Reconcile signup and billing totals against database and Stripe source-of-truth counts before using conversion or revenue decisions. +5. Configure the Vercel firewall rule referenced by the collector so `/api/events` has the verified per-IP limit. The collector also applies the same rule by anonymous-ID key. Verify normal navigation is usable, the documented burst test receives `429`, forged forwarding headers do not bypass classification, and preview hostnames are excluded from decision metrics. +6. Deploy the web application through the normal production release process. Confirm the Vercel deployment Git SHA exactly matches the reviewed commit, the database schema attestation matches, and the configured Tinybird workspace and deployment IDs match the reviewed release before sending any test event. +7. Run a uniquely tagged production smoke test only if separately approved. Do not use customer identifiers. Confirm ingestion, durable outbox drain, health, decision deduplication, admin freshness, and strictly scoped cleanup. +8. Enable dashboards only after freshness, duplicates, conflicts, missing identity, clock skew, late events, queue drops, dead letters, and ingestion lag are healthy. Reconcile signup and billing totals against database and Stripe source-of-truth counts before using conversion or revenue decisions. Rollback is fail closed: disable collection by removing the app append token or route feature switch, roll Vercel back to the recorded prior deployment, and promote the recorded prior Tinybird deployment if schema/query behavior is implicated. Do not delete raw data during rollback. Keep reconciliation paused until the prior collector contract is confirmed compatible, retain health visibility, and record the rollback SHA and deployment IDs. Token rotation follows containment, and erasure credentials must remain available for pending deletion requests. @@ -104,4 +109,6 @@ Rollback is fail closed: disable collection by removing the app append token or - Database reconciliation proves stored signup, share, and collaboration facts; Stripe remains the source of truth for money and subscription state. - Aggregated actor counts summed across days are actor-days, not period-unique people. The admin UI labels this explicitly. - Tinybird requires workspace-wide `DATASOURCES:CREATE` for row deletion; Cap narrows accepted predicates and isolates the credential, but the provider cannot technically restrict it to `product_events_v1` today. +- The bounded v2 hot/cold resources are not a production read-path claim until backfill, publication, parity, physical erasure, and cutover gates are implemented and proven. +- Same-workflow recovery cannot run after a GitHub force-cancel or GitHub control-plane outage; recovery then uses the last immutable checkpoint manually. - No production rollout is authorized by this document or by a green staging workflow. diff --git a/apps/chrome-extension/e2e/analytics-staging.spec.ts b/apps/chrome-extension/e2e/analytics-staging.spec.ts index 11eb7a6d5a4..7718ad3ff1b 100644 --- a/apps/chrome-extension/e2e/analytics-staging.spec.ts +++ b/apps/chrome-extension/e2e/analytics-staging.spec.ts @@ -1,3 +1,4 @@ +import { createHash } from "node:crypto"; import fs from "node:fs"; import { expect, test } from "@playwright/test"; @@ -27,7 +28,13 @@ const requiredEnvironment = (name: string) => { return value; }; -test("exact-SHA browser tracker preserves sessions, retries, unloads, and its main-thread budget", async ({ +const percentile = (samples: readonly number[], value: number) => { + if (samples.length === 0) throw new Error("Performance samples are required"); + const sorted = [...samples].sort((left, right) => left - right); + return sorted[Math.ceil((value / 100) * sorted.length) - 1]; +}; + +test("exact-SHA browser tracker preserves sessions, retries, unloads, and matched-control performance", async ({ browser, }) => { const previewUrl = requiredEnvironment("ANALYTICS_PREVIEW_URL"); @@ -49,22 +56,33 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and its ma }); const captured: CapturedEvent[] = []; const acceptedEventIds = new Set(); + const benchmarkEventIds = new Set(); let acceptedRequests = 0; let failedRequests = 0; + let benchmarking = false; context.on("request", (request) => { if (!request.url().endsWith("/api/events") || request.method() !== "POST") { return; } - captured.push(...requestEvents(request)); + const events = requestEvents(request); + if ( + benchmarking || + events.some((event) => benchmarkEventIds.has(event.eventId)) + ) { + return; + } + captured.push(...events); }); context.on("response", (response) => { if ( response.url().endsWith("/api/events") && response.request().method() === "POST" ) { + const events = requestEvents(response.request()); + if (events.some((event) => benchmarkEventIds.has(event.eventId))) return; if (response.ok()) { acceptedRequests += 1; - for (const event of requestEvents(response.request())) { + for (const event of events) { acceptedEventIds.add(event.eventId); } } @@ -215,6 +233,158 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and its ma process.env.ANALYTICS_BROWSER_TASK_BUDGET_MS ?? 1_500, ); expect(interactionTaskDurationMs).toBeLessThanOrEqual(taskDurationBudgetMs); + + const captureSampleCount = 30; + let benchmarkCapturedEvents = 0; + let benchmarkCapturedEngagementEvents = 0; + await context.route("**/api/events", async (route) => { + const events = requestEvents(route.request()); + benchmarkCapturedEvents += events.length; + benchmarkCapturedEngagementEvents += events.filter( + (event) => event.eventName === "page_engagement", + ).length; + for (const event of events) benchmarkEventIds.add(event.eventId); + await route.fulfill({ + body: JSON.stringify({ accepted: events.length }), + contentType: "application/json", + status: 200, + }); + }); + benchmarking = true; + const longTaskSupported = await page.evaluate(() => { + const benchmarkWindow = window as typeof window & { + __capAnalyticsLongTaskObserver?: PerformanceObserver; + __capAnalyticsLongTasks?: Array<{ duration: number; startTime: number }>; + }; + benchmarkWindow.__capAnalyticsLongTaskObserver?.disconnect(); + benchmarkWindow.__capAnalyticsLongTasks = []; + if (!PerformanceObserver.supportedEntryTypes.includes("longtask")) { + return false; + } + const observer = new PerformanceObserver((entries) => { + for (const entry of entries.getEntries()) { + benchmarkWindow.__capAnalyticsLongTasks?.push({ + duration: entry.duration, + startTime: entry.startTime, + }); + } + }); + observer.observe({ type: "longtask" }); + benchmarkWindow.__capAnalyticsLongTaskObserver = observer; + Object.defineProperty(navigator, "sendBeacon", { + configurable: true, + value: () => false, + }); + window.addEventListener("cap-analytics-control-pointerdown", () => {}, { + passive: true, + }); + window.addEventListener("cap-analytics-control-pagehide", () => {}, { + passive: true, + }); + return true; + }); + expect(longTaskSupported).toBe(true); + type DispatchMeasurement = { + durationMs: number; + endedAt: number; + startedAt: number; + }; + const dispatchMeasurement = async ( + control: boolean, + event: "pagehide" | "pointerdown", + ): Promise => + page.evaluate( + ({ control, event }) => { + const startedAt = performance.now(); + if (control) { + window.dispatchEvent( + event === "pagehide" + ? new PageTransitionEvent("cap-analytics-control-pagehide") + : new Event("cap-analytics-control-pointerdown"), + ); + } else if (event === "pagehide") { + window.dispatchEvent(new PageTransitionEvent("pagehide")); + } else { + window.dispatchEvent(new Event("pointerdown")); + } + const endedAt = performance.now(); + return { durationMs: endedAt - startedAt, endedAt, startedAt }; + }, + { control, event }, + ); + const captureMeasurements: number[] = []; + const controlMeasurements: number[] = []; + const captureWindows: Array<{ endedAt: number; startedAt: number }> = []; + const controlWindows: Array<{ endedAt: number; startedAt: number }> = []; + const measureCapture = async (control: boolean) => { + const pointer = await dispatchMeasurement(control, "pointerdown"); + await page.waitForTimeout(5); + const pagehide = await dispatchMeasurement(control, "pagehide"); + const windows = control ? controlWindows : captureWindows; + windows.push(pointer, pagehide); + return pointer.durationMs + pagehide.durationMs; + }; + for (let index = 0; index < captureSampleCount; index += 1) { + if (index % 2 === 0) { + controlMeasurements.push(await measureCapture(true)); + captureMeasurements.push(await measureCapture(false)); + } else { + captureMeasurements.push(await measureCapture(false)); + controlMeasurements.push(await measureCapture(true)); + } + await page.waitForTimeout(10); + } + await expect + .poll(() => benchmarkCapturedEngagementEvents, { timeout: 15_000 }) + .toBeGreaterThanOrEqual(captureSampleCount); + await page.waitForTimeout(100); + const longTasks = await page.evaluate(() => { + const benchmarkWindow = window as typeof window & { + __capAnalyticsLongTaskObserver?: PerformanceObserver; + __capAnalyticsLongTasks?: Array<{ duration: number; startTime: number }>; + }; + benchmarkWindow.__capAnalyticsLongTaskObserver?.disconnect(); + return benchmarkWindow.__capAnalyticsLongTasks ?? []; + }); + benchmarking = false; + await context.unroute("**/api/events"); + const overlapsWindow = ( + entry: { duration: number; startTime: number }, + windows: ReadonlyArray<{ endedAt: number; startedAt: number }>, + ) => + windows.some( + (window) => + entry.startTime < window.endedAt && + entry.startTime + entry.duration > window.startedAt, + ); + const captureLongTasks = longTasks.filter((entry) => + overlapsWindow(entry, captureWindows), + ); + const controlLongTasks = longTasks.filter((entry) => + overlapsWindow(entry, controlWindows), + ); + const perEventDeltas = captureMeasurements.map((duration, index) => + Math.max(0, duration - controlMeasurements[index]), + ); + const captureP50Ms = percentile(perEventDeltas, 50); + const captureP95Ms = percentile(perEventDeltas, 95); + const captureP99Ms = percentile(perEventDeltas, 99); + const captureP95BudgetMs = Number( + process.env.ANALYTICS_BROWSER_CAPTURE_P95_BUDGET_MS ?? 2, + ); + const captureP99BudgetMs = Number( + process.env.ANALYTICS_BROWSER_CAPTURE_P99_BUDGET_MS ?? 5, + ); + const additionalLongTaskBudget = Number( + process.env.ANALYTICS_BROWSER_LONG_TASK_BUDGET ?? 0, + ); + expect(captureMeasurements).toHaveLength(captureSampleCount); + expect(controlMeasurements).toHaveLength(captureSampleCount); + expect(captureP95Ms).toBeLessThanOrEqual(captureP95BudgetMs); + expect(captureP99Ms).toBeLessThanOrEqual(captureP99BudgetMs); + expect(captureLongTasks.length).toBeLessThanOrEqual( + controlLongTasks.length + additionalLongTaskBudget, + ); const uniqueEventIds = new Set(captured.map((event) => event.eventId)); expect(uniqueEventIds.size).toBeGreaterThanOrEqual(6); expect( @@ -223,12 +393,21 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and its ma expect(captured.some((event) => event.eventName === "page_engagement")).toBe( true, ); + const anonymousId = await page.evaluate(() => + localStorage.getItem("cap_analytics_anonymous_id_v1"), + ); + if (!anonymousId) { + throw new Error("Browser analytics anonymous identity was not persisted"); + } const state = JSON.parse(fs.readFileSync(statePath, "utf8")) as Record< string, unknown >; state.browserExpectedEvents = acceptedEventIds.size; + state.browserAnonymousIdentityHash = createHash("sha256") + .update(`anonymous\0${anonymousId}`) + .digest("hex"); fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600, }); @@ -253,6 +432,32 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and its ma unloadPassed: true, interactionTaskDurationMs, taskDurationBudgetMs, + capturePerformance: { + capturedEngagementEvents: benchmarkCapturedEngagementEvents, + capturedEvents: benchmarkCapturedEvents, + captureP50Ms, + captureP95BudgetMs, + captureP95Ms, + captureP99BudgetMs, + captureP99Ms, + captureSamples: captureMeasurements, + controlP50Ms: percentile(controlMeasurements, 50), + controlP95Ms: percentile(controlMeasurements, 95), + controlP99Ms: percentile(controlMeasurements, 99), + controlSamples: controlMeasurements, + additionalLongTaskBudget, + captureLongTaskCount: captureLongTasks.length, + captureLongTaskMaxDurationMs: Math.max( + 0, + ...captureLongTasks.map((entry) => entry.duration), + ), + controlLongTaskCount: controlLongTasks.length, + controlLongTaskMaxDurationMs: Math.max( + 0, + ...controlLongTasks.map((entry) => entry.duration), + ), + sampleCount: captureSampleCount, + }, }; artifact.assertions = { ...(artifact.assertions ?? {}), diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts index 2d551fe26b8..9b83d3e02c8 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -1,4 +1,16 @@ -import { createHash, timingSafeEqual } from "node:crypto"; +import { createHash, createHmac, timingSafeEqual } from "node:crypto"; +import { + productAnalyticsEventIdHash, + productAnalyticsIdentityHash, +} from "@cap/analytics"; +import { db } from "@cap/database"; +import { + productAnalyticsErasureRequests, + productAnalyticsEventReceipts, + productAnalyticsIdentityLinks, + productAnalyticsIdentityState, + productAnalyticsOutbox, +} from "@cap/database/schema"; import { Tinybird } from "@cap/web-backend"; import { HttpApi, @@ -8,6 +20,7 @@ import { HttpApiGroup, HttpServerRequest, } from "@effect/platform"; +import { inArray, or, sql } from "drizzle-orm"; import { Effect, Layer, Schema } from "effect"; import type Stripe from "stripe"; import { queueServerProductEvent } from "@/lib/analytics/server"; @@ -51,6 +64,88 @@ class Api extends HttpApi.make("AnalyticsStagingTestApi").add( .addError(HttpApiError.Unauthorized) .addError(HttpApiError.NotFound) .addError(HttpApiError.ServiceUnavailable), + ) + .add( + HttpApiEndpoint.post("health", "/api/analytics/staging-test/health") + .setPayload( + Schema.Struct({ + runId: Schema.String, + sha: Schema.String, + }), + ) + .addSuccess( + Schema.Struct({ + activeEvents: Schema.Number, + deadLetterEvents: Schema.Number, + healthy: Schema.Boolean, + oldestActiveAgeSeconds: Schema.Number, + payloadConflictEvents: Schema.Number, + provider429Retries: Schema.Number, + provider503Retries: Schema.Number, + providerRejectedDeadLetters: Schema.Number, + receiptPayloadConflictAttempts: Schema.Number, + receiptPayloadConflictEvents: Schema.Number, + timeoutAfterAcceptRetries: Schema.Number, + }), + ) + .addError(HttpApiError.BadRequest) + .addError(HttpApiError.Unauthorized) + .addError(HttpApiError.NotFound) + .addError(HttpApiError.ServiceUnavailable), + ) + .add( + HttpApiEndpoint.post( + "cleanupDatabase", + "/api/analytics/staging-test/cleanup-database", + ) + .setPayload( + Schema.Struct({ + anonymousIdentityHashes: Schema.Array(Schema.String), + runId: Schema.String, + scopeRunIds: Schema.Array(Schema.String), + sha: Schema.String, + }), + ) + .addSuccess( + Schema.Struct({ + cleaned: Schema.Boolean, + identityHashes: Schema.Number, + remaining: Schema.Number, + runIds: Schema.Number, + }), + ) + .addError(HttpApiError.BadRequest) + .addError(HttpApiError.Unauthorized) + .addError(HttpApiError.NotFound) + .addError(HttpApiError.ServiceUnavailable), + ) + .add( + HttpApiEndpoint.post("attest", "/api/analytics/staging-test/attest") + .setPayload( + Schema.Struct({ + runId: Schema.String, + sha: Schema.String, + }), + ) + .addSuccess( + Schema.Struct({ + databaseFingerprint: Schema.String, + databaseSchema: Schema.Literal("0042_lying_sharon_ventura"), + host: Schema.String, + sha: Schema.String, + workspaces: Schema.Array( + Schema.Struct({ + name: Schema.String, + tokenHash: Schema.String, + workspaceId: Schema.String, + }), + ), + }), + ) + .addError(HttpApiError.BadRequest) + .addError(HttpApiError.Unauthorized) + .addError(HttpApiError.NotFound) + .addError(HttpApiError.ServiceUnavailable), ), ) {} @@ -70,6 +165,434 @@ const boundedRunId = (value: string) => const draftSha = (value: string) => /^[0-9a-f]{40}$/.test(value); +const syntheticIdentities = (runId: string) => { + const hash = createHash("sha256").update(runId).digest("hex"); + return { + anonymousId: `synthetic_${hash.slice(0, 24)}`, + hash, + organizationId: `synthetic_org_${hash.slice(24, 48)}`, + userId: `synthetic_user_${hash.slice(0, 24)}`, + }; +}; + +const syntheticEventIds = (runId: string) => { + const { hash } = syntheticIdentities(runId); + return [ + `staging_signup_${hash.slice(0, 24)}`, + `staging_retry_429_${hash.slice(0, 24)}`, + `staging_retry_503_${hash.slice(0, 24)}`, + `staging_ambiguous_${hash.slice(0, 24)}`, + `staging_reject_400_${hash.slice(0, 24)}`, + `staging_erasure_replay_${hash.slice(0, 24)}`, + ]; +}; + +const syntheticIdentityHashes = (runId: string) => { + const { anonymousId, organizationId, userId } = syntheticIdentities(runId); + return [ + productAnalyticsIdentityHash("anonymous", anonymousId), + productAnalyticsIdentityHash("organization", organizationId), + productAnalyticsIdentityHash("user", userId), + ]; +}; + +const scopedDatabaseHealth = async (runId: string) => { + const eventIds = syntheticEventIds(runId); + const eventIdHashes = eventIds.map(productAnalyticsEventIdHash); + const [outboxRows, receiptRows] = await Promise.all([ + db() + .select({ + createdAt: productAnalyticsOutbox.createdAt, + lastErrorCode: productAnalyticsOutbox.lastErrorCode, + payloadConflict: productAnalyticsOutbox.payloadConflict, + status: productAnalyticsOutbox.status, + }) + .from(productAnalyticsOutbox) + .where(inArray(productAnalyticsOutbox.eventId, eventIds)), + db() + .select({ conflictCount: productAnalyticsEventReceipts.conflictCount }) + .from(productAnalyticsEventReceipts) + .where(inArray(productAnalyticsEventReceipts.eventIdHash, eventIdHashes)), + ]); + const activeRows = outboxRows.filter((row) => + ["pending", "workflow_started"].includes(row.status), + ); + const oldestActiveAt = activeRows.reduce( + (oldest, row) => Math.min(oldest, row.createdAt.getTime()), + Date.now(), + ); + const countError = (code: string) => + outboxRows.filter((row) => row.lastErrorCode === code).length; + const receiptPayloadConflictEvents = receiptRows.filter( + (row) => row.conflictCount > 0, + ).length; + const receiptPayloadConflictAttempts = receiptRows.reduce( + (sum, row) => sum + row.conflictCount, + 0, + ); + const deadLetterEvents = outboxRows.filter( + (row) => row.status === "dead_letter", + ).length; + const payloadConflictEvents = outboxRows.filter( + (row) => row.payloadConflict, + ).length; + return { + activeEvents: activeRows.length, + deadLetterEvents, + healthy: + activeRows.length === 0 && + deadLetterEvents === 0 && + payloadConflictEvents === 0 && + receiptPayloadConflictEvents === 0, + oldestActiveAgeSeconds: + activeRows.length === 0 + ? 0 + : Math.max(0, Math.floor((Date.now() - oldestActiveAt) / 1_000)), + payloadConflictEvents, + provider429Retries: countError("staging_provider_429"), + provider503Retries: countError("staging_provider_503"), + providerRejectedDeadLetters: outboxRows.filter( + (row) => + row.status === "dead_letter" && + row.lastErrorCode === "provider_rejected", + ).length, + receiptPayloadConflictAttempts, + receiptPayloadConflictEvents, + timeoutAfterAcceptRetries: countError("staging_timeout_after_accept"), + }; +}; + +const cleanupSyntheticDatabaseState = async ({ + anonymousIdentityHashes, + runIds, +}: { + anonymousIdentityHashes: readonly string[]; + runIds: readonly string[]; +}) => { + const identities = runIds.map(syntheticIdentities); + const eventIds = runIds.flatMap(syntheticEventIds); + const eventIdHashes = eventIds.map(productAnalyticsEventIdHash); + const identityHashes = [ + ...new Set([ + ...anonymousIdentityHashes, + ...runIds.flatMap(syntheticIdentityHashes), + ]), + ]; + const anonymousIds = identities.map(({ anonymousId }) => anonymousId); + const organizationIds = identities.map( + ({ organizationId }) => organizationId, + ); + const userIds = identities.map(({ userId }) => userId); + await db().transaction(async (tx) => { + await tx + .delete(productAnalyticsIdentityLinks) + .where( + or( + inArray( + productAnalyticsIdentityLinks.anonymousIdentityHash, + identityHashes, + ), + inArray( + productAnalyticsIdentityLinks.userIdentityHash, + identityHashes, + ), + inArray( + productAnalyticsIdentityLinks.organizationIdentityHash, + identityHashes, + ), + ), + ); + await tx + .delete(productAnalyticsEventReceipts) + .where( + or( + inArray(productAnalyticsEventReceipts.eventIdHash, eventIdHashes), + inArray( + productAnalyticsEventReceipts.anonymousIdentityHash, + identityHashes, + ), + inArray( + productAnalyticsEventReceipts.userIdentityHash, + identityHashes, + ), + inArray( + productAnalyticsEventReceipts.organizationIdentityHash, + identityHashes, + ), + ), + ); + await tx + .delete(productAnalyticsOutbox) + .where( + or( + inArray(productAnalyticsOutbox.eventId, eventIds), + inArray(productAnalyticsOutbox.anonymousId, anonymousIds), + inArray(productAnalyticsOutbox.userId, userIds), + inArray(productAnalyticsOutbox.organizationId, organizationIds), + ), + ); + await tx + .delete(productAnalyticsErasureRequests) + .where( + or( + inArray(productAnalyticsErasureRequests.userId, userIds), + inArray( + productAnalyticsErasureRequests.organizationId, + organizationIds, + ), + ), + ); + await tx + .delete(productAnalyticsIdentityState) + .where( + inArray(productAnalyticsIdentityState.identityHash, identityHashes), + ); + }); + const [remainingOutbox, remainingReceipts, remainingLinks, remainingStates] = + await Promise.all([ + db() + .select({ eventId: productAnalyticsOutbox.eventId }) + .from(productAnalyticsOutbox) + .where(inArray(productAnalyticsOutbox.eventId, eventIds)), + db() + .select({ eventIdHash: productAnalyticsEventReceipts.eventIdHash }) + .from(productAnalyticsEventReceipts) + .where( + inArray(productAnalyticsEventReceipts.eventIdHash, eventIdHashes), + ), + db() + .select({ + anonymousIdentityHash: + productAnalyticsIdentityLinks.anonymousIdentityHash, + }) + .from(productAnalyticsIdentityLinks) + .where( + or( + inArray( + productAnalyticsIdentityLinks.anonymousIdentityHash, + identityHashes, + ), + inArray( + productAnalyticsIdentityLinks.userIdentityHash, + identityHashes, + ), + ), + ), + db() + .select({ identityHash: productAnalyticsIdentityState.identityHash }) + .from(productAnalyticsIdentityState) + .where( + inArray(productAnalyticsIdentityState.identityHash, identityHashes), + ), + ]); + return { + identityHashes: identityHashes.length, + remaining: + remainingOutbox.length + + remainingReceipts.length + + remainingLinks.length + + remainingStates.length, + runIds: runIds.length, + }; +}; + +const tinybirdTokenNames = [ + "PRODUCT_ANALYTICS_TINYBIRD_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN", +] as const; +const TINYBIRD_STAGING_ORIGIN = "https://api.us-east.aws.tinybird.co"; +const TINYBIRD_STAGING_WORKSPACE_ID = "37b8fef9-817f-4c3c-b21f-218c36a6077d"; +const STAGING_DATABASE_FINGERPRINT = + "fff37a9b160f31bfb82b8c5585829b8ee08f70b3645169dca6e7cb29033a039a"; + +const tokenWorkspaceId = (token: string) => { + const segment = token.split(".")[1]; + if (!segment) return undefined; + try { + const payload: unknown = JSON.parse( + Buffer.from(segment, "base64url").toString("utf8"), + ); + if (!payload || typeof payload !== "object") return undefined; + const record = payload as Record; + const workspaceId = record.u ?? record.workspace_id ?? record.workspaceId; + return typeof workspaceId === "string" ? workspaceId : undefined; + } catch { + return undefined; + } +}; + +const configurationAttestation = (runId: string) => { + const host = process.env.PRODUCT_ANALYTICS_TINYBIRD_HOST; + const sha = process.env.VERCEL_GIT_COMMIT_SHA; + const secret = process.env.CAP_ANALYTICS_STAGING_TEST_SECRET; + const databaseUrl = process.env.DATABASE_URL; + if (!host || !sha || !secret || !databaseUrl) return undefined; + const databaseFingerprint = createHash("sha256") + .update(databaseUrl) + .digest("hex"); + if (databaseFingerprint !== STAGING_DATABASE_FINGERPRINT) return undefined; + let origin: string; + try { + origin = new URL(host).origin; + } catch { + return undefined; + } + const workspaces = tinybirdTokenNames.map((name) => { + const token = process.env[name]; + if (!token) return undefined; + const workspaceId = tokenWorkspaceId(token); + if (!workspaceId) return undefined; + return { + name, + tokenHash: createHmac("sha256", `${secret}:${runId}`) + .update(token) + .digest("hex"), + workspaceId, + }; + }); + if (workspaces.some((workspace) => !workspace)) return undefined; + if ( + origin !== TINYBIRD_STAGING_ORIGIN || + workspaces.some( + (workspace) => workspace?.workspaceId !== TINYBIRD_STAGING_WORKSPACE_ID, + ) + ) { + return undefined; + } + return { + databaseFingerprint, + host: origin, + sha, + workspaces: workspaces.filter( + (workspace): workspace is NonNullable => + Boolean(workspace), + ), + }; +}; + +const databaseResultRows = (result: unknown) => { + if (!Array.isArray(result)) return []; + const rows = Array.isArray(result[0]) ? result[0] : result; + return rows.filter( + (row): row is Record => + typeof row === "object" && row !== null && !Array.isArray(row), + ); +}; + +const attestDatabaseSchema = async () => { + const tableNames = [ + "comments", + "organization_invites", + "product_analytics_erasure_requests", + "product_analytics_event_receipts", + "product_analytics_identity_links", + "product_analytics_ingestion_leases", + "product_analytics_outbox", + "product_analytics_refresh_leases", + "product_analytics_reconciliation_failures", + "users", + "videos", + ]; + const tableNamesSql = sql.join( + tableNames.map((tableName) => sql`${tableName}`), + sql`, `, + ); + const [columnResult, indexResult] = await Promise.all([ + db().execute(sql` + SELECT TABLE_NAME AS tableName, COLUMN_NAME AS columnName, + COLUMN_TYPE AS columnType, IS_NULLABLE AS isNullable, + COLUMN_DEFAULT AS columnDefault, COLUMN_KEY AS columnKey + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME IN (${tableNamesSql}) + `), + db().execute(sql` + SELECT TABLE_NAME AS tableName, INDEX_NAME AS indexName, + NON_UNIQUE AS nonUnique, SEQ_IN_INDEX AS sequence, + COLUMN_NAME AS columnName + FROM information_schema.STATISTICS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME IN (${tableNamesSql}) + `), + ]); + const columnSignatures = new Set( + databaseResultRows(columnResult).map((row) => + [ + row.tableName, + row.columnName, + row.columnType, + row.isNullable, + row.columnDefault ?? "NULL", + row.columnKey ?? "", + ] + .map(String) + .join(":"), + ), + ); + const indexSignatures = new Set( + databaseResultRows(indexResult).map((row) => + [ + row.tableName, + row.indexName, + Number(row.nonUnique), + Number(row.sequence), + row.columnName, + ].join(":"), + ), + ); + const requiredColumns = [ + "organization_invites:emailDeliveryState:varchar(32):NO:legacy:MUL", + "product_analytics_event_receipts:eventIdHash:varchar(64):NO:NULL:PRI", + "product_analytics_ingestion_leases:fencingToken:bigint unsigned:NO:NULL:", + "product_analytics_outbox:eventId:varchar(128):NO:NULL:PRI", + "product_analytics_refresh_leases:name:varchar(64):NO:NULL:PRI", + "product_analytics_reconciliation_failures:attemptCount:int:NO:1:", + "product_analytics_reconciliation_failures:sourceHash:varchar(64):NO:NULL:PRI", + ]; + const requiredIndexes = [ + "comments:analytics_reconciliation_idx:1:1:createdAt", + "comments:analytics_reconciliation_idx:1:2:id", + "organization_invites:email_delivery_idx:1:1:emailDeliveryState", + "organization_invites:email_delivery_idx:1:2:emailDeliveryNextAttemptAt", + "organization_invites:normalized_email_idx:0:1:organizationId", + "organization_invites:normalized_email_idx:0:2:invitedEmailNormalized", + "product_analytics_erasure_requests:scope_hash_idx:0:1:scopeHash", + "product_analytics_event_receipts:PRIMARY:0:1:eventIdHash", + "product_analytics_event_receipts:conflict_idx:1:1:conflictCount", + "product_analytics_identity_links:PRIMARY:0:1:anonymousIdentityHash", + "product_analytics_identity_links:PRIMARY:0:2:userIdentityHash", + "product_analytics_ingestion_leases:PRIMARY:0:1:id", + "product_analytics_ingestion_leases:expiry_idx:1:1:expiresAt", + "product_analytics_outbox:PRIMARY:0:1:eventId", + "product_analytics_outbox:delivery_key_idx:0:1:deliveryKey", + "product_analytics_outbox:delivery_idx:1:1:status", + "product_analytics_outbox:delivery_idx:1:2:nextAttemptAt", + "product_analytics_outbox:delivery_idx:1:3:createdAt", + "product_analytics_refresh_leases:PRIMARY:0:1:name", + "product_analytics_refresh_leases:expiry_idx:1:1:leaseExpiresAt", + "product_analytics_refresh_leases:status_idx:1:1:status", + "product_analytics_reconciliation_failures:PRIMARY:0:1:sourceHash", + "product_analytics_reconciliation_failures:source_type_idx:1:1:sourceType", + "product_analytics_reconciliation_failures:source_type_idx:1:2:lastSeenAt", + "users:analytics_reconciliation_idx:1:1:created_at", + "users:analytics_reconciliation_idx:1:2:id", + "videos:analytics_created_at_idx:1:1:createdAt", + "videos:analytics_created_at_idx:1:2:id", + "videos:analytics_first_view_at_idx:1:1:firstExternalViewAt", + "videos:analytics_first_view_at_idx:1:2:id", + ]; + if ( + requiredColumns.some((signature) => !columnSignatures.has(signature)) || + requiredIndexes.some((signature) => !indexSignatures.has(signature)) + ) { + throw new Error("The staging database schema signature is incomplete"); + } +}; + const authorize = (payload: { runId: string; sha: string }) => Effect.gen(function* () { if (process.env.VERCEL_ENV !== "preview") { @@ -89,7 +612,8 @@ const authorize = (payload: { runId: string; sha: string }) => if ( !runId || !draftSha(payload.sha) || - payload.sha !== process.env.VERCEL_GIT_COMMIT_SHA + payload.sha !== process.env.VERCEL_GIT_COMMIT_SHA || + !configurationAttestation(runId) ) { return yield* Effect.fail(new HttpApiError.BadRequest()); } @@ -102,16 +626,33 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( Effect.gen(function* () { const tinybird = yield* Tinybird; return handlers + .handle("attest", ({ payload }) => + Effect.gen(function* () { + const runId = yield* authorize(payload); + const attestation = configurationAttestation(runId); + if (!attestation) { + return yield* Effect.fail( + new HttpApiError.ServiceUnavailable(), + ); + } + yield* Effect.tryPromise({ + try: attestDatabaseSchema, + catch: () => new HttpApiError.ServiceUnavailable(), + }); + return { + ...attestation, + databaseSchema: "0042_lying_sharon_ventura" as const, + }; + }), + ) .handle("run", ({ payload }) => Effect.gen(function* () { const runId = yield* authorize(payload); - const hash = createHash("sha256").update(runId).digest("hex"); + const { anonymousId, hash, organizationId, userId } = + syntheticIdentities(runId); const occurredAt = new Date().toISOString(); - const userId = `staging_user_${hash.slice(0, 20)}`; - const organizationId = `staging_org_${hash.slice(20, 40)}`; - const anonymousId = `staging_anon_${hash.slice(40, 60)}`; const purchase = subscriptionCheckoutProductEvent({ - eventId: `staging_${hash.slice(0, 24)}`, + eventId: `staging_ambiguous_${hash.slice(0, 24)}`, occurredAt, session: { amount_subtotal: 2_500, @@ -151,7 +692,7 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( { _syntheticRunId: runId, anonymousId, - eventId: `staging_share_${hash.slice(0, 24)}`, + eventId: `staging_retry_429_${hash.slice(0, 24)}`, eventName: "share_link_created", occurredAt, organizationId, @@ -165,7 +706,7 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( { _syntheticRunId: runId, anonymousId, - eventId: `staging_checkout_${hash.slice(0, 24)}`, + eventId: `staging_retry_503_${hash.slice(0, 24)}`, eventName: "checkout_started", occurredAt, organizationId, @@ -178,32 +719,117 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( userId, }, { ...purchase, _syntheticRunId: runId }, - { ...purchase, _syntheticRunId: runId }, + { + _syntheticRunId: runId, + anonymousId, + eventId: `staging_reject_400_${hash.slice(0, 24)}`, + eventName: "organization_member_joined", + occurredAt, + organizationId, + platform: "web", + properties: { + assigned_pro_seat: false, + role: "member", + }, + userId, + }, ]; - const workflowRuns = yield* Effect.tryPromise({ + const deliveries = yield* Effect.tryPromise({ try: () => Promise.all(events.map(queueServerProductEvent)), catch: () => new HttpApiError.ServiceUnavailable(), }); + if ( + deliveries.some((delivery) => delivery.status !== "started") + ) { + return yield* Effect.fail( + new HttpApiError.ServiceUnavailable(), + ); + } return { accepted: events.length, uniqueEvents: new Set(events.map((event) => event.eventId)) .size, - workflowRuns: workflowRuns.map(({ runId: id }) => id), + workflowRuns: deliveries.map(({ runId }) => runId), }; }), ) + .handle("health", ({ payload }) => + Effect.gen(function* () { + const runId = yield* authorize(payload); + const health = yield* Effect.tryPromise({ + try: () => scopedDatabaseHealth(runId), + catch: () => new HttpApiError.ServiceUnavailable(), + }); + return health; + }), + ) + .handle("cleanupDatabase", ({ payload }) => + Effect.gen(function* () { + yield* authorize(payload); + const runIds = [ + ...new Set(payload.scopeRunIds.map(boundedRunId)), + ].filter((runId) => runId !== undefined); + const anonymousIdentityHashes = [ + ...new Set(payload.anonymousIdentityHashes), + ].filter((identityHash) => /^[0-9a-f]{64}$/.test(identityHash)); + if ( + runIds.length === 0 || + runIds.length !== payload.scopeRunIds.length || + runIds.length > 8 || + anonymousIdentityHashes.length !== + payload.anonymousIdentityHashes.length || + anonymousIdentityHashes.length > 16 + ) { + return yield* Effect.fail(new HttpApiError.BadRequest()); + } + const cleanup = yield* Effect.tryPromise({ + try: () => + cleanupSyntheticDatabaseState({ + anonymousIdentityHashes, + runIds, + }), + catch: () => new HttpApiError.ServiceUnavailable(), + }); + if (cleanup.remaining !== 0) { + return yield* Effect.fail( + new HttpApiError.ServiceUnavailable(), + ); + } + return { ...cleanup, cleaned: true }; + }), + ) .handle("erase", ({ payload }) => Effect.gen(function* () { const runId = yield* authorize(payload); - const hash = createHash("sha256").update(runId).digest("hex"); + const { anonymousId, hash, organizationId, userId } = + syntheticIdentities(runId); yield* tinybird .eraseProductAnalytics({ - userId: `synthetic_user_${hash.slice(0, 24)}`, - organizationId: `synthetic_org_${hash.slice(24, 48)}`, + userId, + organizationId, }) .pipe( Effect.mapError(() => new HttpApiError.ServiceUnavailable()), ); + const replay = yield* Effect.tryPromise({ + try: () => + queueServerProductEvent({ + _syntheticRunId: runId, + anonymousId, + eventId: `staging_erasure_replay_${hash.slice(0, 24)}`, + eventName: "user_signed_up", + occurredAt: new Date().toISOString(), + organizationId, + platform: "web", + userId, + }), + catch: () => new HttpApiError.ServiceUnavailable(), + }); + if (replay.status !== "suppressed") { + return yield* Effect.fail( + new HttpApiError.ServiceUnavailable(), + ); + } return { erased: true }; }), ); diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index a255f061fc7..b980fc0e44d 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -38,14 +38,16 @@ The deployed datafiles create four runtime tokens: - `product_events_ingest`: append-only access to `product_events_v1`. - `product_events_agent_read`: read-only access to privacy-safe product endpoints. It has no raw identity datasource or Copy Pipe scope. -- `product_events_copy_runner`: execution-only access to the eight reviewed Copy Pipes. It cannot query endpoints or raw identity data. +- `product_events_copy_runner`: execution-only access to the reviewed Copy Pipes enumerated by the staging runner. It cannot query endpoints or raw identity data. - `product_events_erasure_lookup`: read-only access to `product_events_v1` and `product_events_canonical_v1` for bounded identity erasure. It cannot query decision endpoints, append rows, or execute Copy Pipes. Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. -Staging also requires `TINYBIRD_STAGING_SCHEDULER_TOKEN`, a protected token with schedule-control access only to the eight reviewed Copy Pipes. The destructive erasure phase cancels and pauses those schedules before deleting rows, runs each replacement Copy exactly once, proves completion with scoped state transitions or copy-run markers, and resumes every schedule in an always-run recovery step. A failed or ambiguous Copy submission is never retried automatically. +Staging also requires `TINYBIRD_STAGING_SCHEDULER_TOKEN`, a protected token with schedule-control access only to the Copy Pipes enumerated by the staging runner. The destructive erasure phase cancels and pauses those schedules before deleting rows, runs each replacement Copy exactly once, proves completion with scoped state transitions or copy-run markers, and resumes every schedule in an always-run recovery step. A failed or ambiguous Copy submission is never retried automatically. -The staging workflow validates candidate ingestion, duplicate/conflict visibility, live isolation, and every typed endpoint through the exact numeric deployment ID before promotion. Tinybird's Copy service does not reliably route an on-demand Copy mutation to a deployment candidate, so CI refuses that path. After the exact candidate is promoted inside the staging workspace, CI triggers the reviewed Copy Pipes through Tinybird's direct Copy API with the protected `product_events_copy_runner` token. `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. Tinybird does not grant either the resource token or the deployment token access to the resulting Jobs API record, so CI proves terminal completion from canonical, daily, and health state transitions plus a unique zero-valued marker written by every other aggregate Copy. A real browser drives the exact-SHA preview through reload, shared-tab, inactivity, SPA retry, and unload behavior. A preview-only authenticated route starts durable server workflows for deduplicated activation and paid-purchase facts. Registry-validated synthetic fixtures separately prove exact anonymous-to-authenticated, guest checkout, cross-device checkout, lifecycle revenue, and public endpoint values while remaining excluded from normal queries. Performance compares shared endpoints with the retained deployment, applies absolute budgets to a newly introduced endpoint that has no honest historical baseline, and validates mixed 1,000-row and 10,000-row traffic, activation, retention, identity, adoption, and revenue corpora before timing them. Before the prior deployment can be deleted, CI switches it live, executes the shared typed data plane, proves the exact-SHA admin client gracefully degrades only the identity endpoint absent from that rollback target, restores the candidate, and repeats the full synthetic business and health assertions. +The staging workflow validates candidate ingestion, duplicate/conflict visibility, live isolation, and every typed endpoint through the exact numeric deployment ID before promotion. Tinybird's Copy service does not reliably route an on-demand Copy mutation to a deployment candidate, so CI refuses that path. After the exact candidate is promoted inside the staging workspace, CI triggers the reviewed Copy Pipes through Tinybird's direct Copy API with the protected `product_events_copy_runner` token. `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. CI polls both exact event-state jobs to terminal success before rebuilding canonical state, then proves every downstream Copy with canonical, daily, and health transitions plus unique zero-valued markers. A real browser drives the exact-SHA preview through reload, shared-tab, inactivity, SPA retry, unload, and matched-control main-thread measurements. A preview-only authenticated route starts durable server workflows for deduplicated activation and paid-purchase facts. Registry-validated synthetic fixtures separately prove exact anonymous-to-authenticated, guest checkout, cross-device checkout, lifecycle revenue, explicit experiment outcomes, and public endpoint values while remaining excluded from normal queries. Performance compares shared endpoints with the retained deployment, applies absolute budgets when no independent baseline exists, and validates mixed 1,000-row and 100,000-row traffic, activation, retention, identity, attribution, experimentation, adoption, and revenue corpora spread across 30 and 80 UTC days. Direct provider ingestion, the exact-SHA application collector, browser capture, desktop encrypted journal writes, every typed endpoint, and full-dashboard fanout have separate budgets and redacted measurements. Before the prior deployment can be deleted, CI switches it live, executes every endpoint available on that retained schema, proves the exact-SHA admin client degrades only for explicitly optional candidate-only endpoints that return `404`, restores the candidate, and repeats the full synthetic business and health assertions. + +Immutable workflow artifacts checkpoint the exact deployment inventory before creation, the synthetic scope before ingestion, the prior read plane before promotion, and the ready-to-finalize state before cleanup. The same workflow has an always-run recovery job that resolves uncertain creation by exact deployment-ID set difference, restores and validates the prior deployment, resumes paused Copies, cleans the owned synthetic scope, and discards only the owned candidate. A force-cancel or GitHub control-plane outage can prevent that recovery job from starting; use the last immutable checkpoint for the manual recovery procedure. Set `TINYBIRD_AGENT_TOKEN` and `TINYBIRD_URL` for the query command. Set a separate `TINYBIRD_READ_TOKEN` with workspace metadata access when running `pnpm analytics:check`; the append and deployment tokens are intentionally rejected for that task. @@ -63,6 +65,10 @@ The Analytics GitHub workflow runs static tests, Docker Compose validation, a co - Monthly partitions and a shared 800-day raw, canonical, and decision horizon support complete year-over-year windows while keeping erasure rebuilds complete for every retained contribution. Health detail retains 90 days. - Common event trends use `product_events_daily_exact`; they do not scan raw events. - Daily counts use unique event states, so retried deliveries cannot inflate the rollup. +- The bounded v2 hot/cold resources are unscheduled side-by-side infrastructure. Public endpoints remain on the exact aggregate tables until cold backfill, common-generation publication, parity, physical erasure rebuilding, and cutover are proven. +- Copy execution records submission-to-marker visibility p50/p95/p99 and full pipeline wall-clock. The first successful phase establishes the run baseline; later phases fail above the absolute limits or a twofold/30-second regression allowance. +- Default Copy limits are 120 seconds for visibility p95 and 600 seconds for the complete sequential pipeline. Environment overrides remain explicit in the redacted artifact. +- Tinybird's scoped resource and deployment tokens cannot read Copy Jobs API resource metrics, so CI cannot report provider CPU, memory, or bytes scanned. Visibility and wall-clock are enforced as the available black-box budgets. - Raw health queries require explicit start and end times. - Event properties are stored as JSON strings and fixtures enforce a 16 KiB ceiling. - The infrastructure contains no autocapture or session-replay path. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 3dcc329d21c..8e09302271a 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -1,9 +1,22 @@ import { createHash } from "node:crypto"; export const STAGING_WORKSPACE_ID = "37b8fef9-817f-4c3c-b21f-218c36a6077d"; +export const STAGING_DATABASE_FINGERPRINT = + "fff37a9b160f31bfb82b8c5585829b8ee08f70b3645169dca6e7cb29033a039a"; +export const STAGING_DATABASE_SCHEMA = "0042_lying_sharon_ventura"; export const FEATURE_BRANCH = "codex/first-party-analytics"; export const FEATURE_PULL_REQUEST = 2003; +export const PREVIEW_TINYBIRD_TOKEN_NAMES = [ + "PRODUCT_ANALYTICS_TINYBIRD_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN", + "PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN", +]; export const COPY_PIPES = [ + "snapshot_product_event_id_states_v2", + "snapshot_product_event_day_states_v2", "snapshot_product_events_canonical_v1", "snapshot_product_events_daily_exact", "snapshot_product_traffic_daily_exact", @@ -11,6 +24,8 @@ export const COPY_PIPES = [ "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", "snapshot_product_identity_funnel_exact", + "snapshot_product_attribution_daily_exact", + "snapshot_product_experiment_outcomes_exact", "snapshot_product_events_health_hourly", ]; const COPY_MARKER_PIPES = new Set([ @@ -20,6 +35,8 @@ const COPY_MARKER_PIPES = new Set([ "snapshot_product_activation_daily_exact", "snapshot_product_creator_retention_exact", "snapshot_product_identity_funnel_exact", + "snapshot_product_attribution_daily_exact", + "snapshot_product_experiment_outcomes_exact", "snapshot_product_events_health_hourly", ]); @@ -90,6 +107,7 @@ export const applyCopyScheduleAction = async ({ export const copyScheduleMatchesAction = (value, action) => { const payload = value?.data ?? value; + if (!payload?.schedule) return true; const status = String(payload?.schedule?.status ?? "").toLowerCase(); return action === "pause" ? status === "paused" @@ -185,6 +203,50 @@ export const validateTinybirdCredentials = ({ url, tokens }) => { return parsedUrl.origin; }; +export const assertPreviewTinybirdAttestation = ({ + attestation, + expectedOrigin, + expectedSha, + expectedTokenHashes, +}) => { + if ( + !attestation || + attestation.sha !== expectedSha || + attestation.host !== expectedOrigin || + attestation.databaseFingerprint !== STAGING_DATABASE_FINGERPRINT || + attestation.databaseSchema !== STAGING_DATABASE_SCHEMA || + !Array.isArray(attestation.workspaces) + ) { + throw new Error( + "The exact-SHA preview did not attest its Tinybird staging configuration", + ); + } + const workspaces = new Map( + attestation.workspaces.map(({ name, tokenHash, workspaceId }) => [ + name, + { tokenHash, workspaceId }, + ]), + ); + if ( + workspaces.size !== PREVIEW_TINYBIRD_TOKEN_NAMES.length || + PREVIEW_TINYBIRD_TOKEN_NAMES.some((name) => { + const workspace = workspaces.get(name); + return ( + !workspace || + typeof workspace.workspaceId !== "string" || + workspace.workspaceId.toLowerCase() !== + STAGING_WORKSPACE_ID.toLowerCase() || + !expectedTokenHashes || + workspace.tokenHash !== expectedTokenHashes[name] + ); + }) + ) { + throw new Error( + "The exact-SHA preview is not bound to the verified analytics staging tokens", + ); + } +}; + const deploymentId = (deployment) => deployment.id ?? deployment.ID ?? deployment.deployment_id; @@ -207,6 +269,88 @@ const deploymentsFromResponse = (value) => { return deployments; }; +export const createDeploymentBoundary = (value) => { + const deployments = deploymentsFromResponse(value); + const liveDeployments = deployments.filter(isLiveDeployment); + if (liveDeployments.length !== 1) { + throw new Error( + "Tinybird deployment recovery requires exactly one current live deployment", + ); + } + const liveDeploymentId = String(deploymentId(liveDeployments[0])); + if (!DEPLOYMENT_ID_PATTERN.test(liveDeploymentId)) { + throw new Error("Tinybird returned an invalid live deployment ID"); + } + const deploymentIds = deployments.map((deployment) => + String(deploymentId(deployment)), + ); + if ( + deploymentIds.some((id) => !DEPLOYMENT_ID_PATTERN.test(id)) || + new Set(deploymentIds).size !== deploymentIds.length + ) { + throw new Error("Tinybird returned invalid or duplicate deployment IDs"); + } + return { + deploymentIds: deploymentIds.sort((left, right) => + left.localeCompare(right, "en", { numeric: true }), + ), + liveDeploymentId, + }; +}; + +export const resolveDeploymentCreatedAfterBoundary = ( + value, + boundary, + { allowNone = false } = {}, +) => { + if ( + !boundary || + typeof boundary !== "object" || + !Array.isArray(boundary.deploymentIds) || + !DEPLOYMENT_ID_PATTERN.test(String(boundary.liveDeploymentId ?? "")) || + boundary.deploymentIds.some((id) => !DEPLOYMENT_ID_PATTERN.test(String(id))) + ) { + throw new Error("Tinybird recovery boundary is invalid"); + } + const deployments = deploymentsFromResponse(value); + const liveDeployments = deployments.filter(isLiveDeployment); + if ( + liveDeployments.length !== 1 || + String(deploymentId(liveDeployments[0])) !== + String(boundary.liveDeploymentId) + ) { + throw new Error( + "Tinybird live deployment changed after the create boundary", + ); + } + const previousIds = new Set(boundary.deploymentIds.map(String)); + const created = deployments.filter((deployment) => { + const id = String(deploymentId(deployment)); + const state = deploymentState(deployment); + return ( + !previousIds.has(id) && + !isLiveDeployment(deployment) && + !state.includes("deleted") && + (isStagingDeployment(deployment) || + isPendingDeployment(deployment) || + state.includes("failed")) + ); + }); + if (created.length === 0 && allowNone) return undefined; + if (created.length !== 1) { + throw new Error( + "Tinybird uncertain create did not resolve to exactly one new deployment", + ); + } + const id = String(deploymentId(created[0])); + if (!DEPLOYMENT_ID_PATTERN.test(id)) { + throw new Error( + "Tinybird uncertain create returned an invalid deployment ID", + ); + } + return { id, needsPromotion: true }; +}; + const isLiveDeployment = (deployment) => deployment.live === true || deploymentState(deployment).includes("live"); @@ -503,12 +647,16 @@ export const submitTinybirdCopyJobs = async ({ now = () => Date.now(), pipes = COPY_PIPES, copyRunId = "", + sourceCutoff = "", assertMutationOwnership, }) => { if (!DEPLOYMENT_ID_PATTERN.test(deploymentId)) { throw new Error("Tinybird copy jobs require a numeric deployment ID"); } if (copyRunId) validateSyntheticRunId(copyRunId); + if (sourceCutoff && !Number.isFinite(Date.parse(sourceCutoff))) { + throw new Error("Tinybird copy source cutoff must be an ISO timestamp"); + } if (typeof assertMutationOwnership !== "function") { throw new Error("Tinybird copies require an ownership check"); } @@ -523,6 +671,7 @@ export const submitTinybirdCopyJobs = async ({ origin, ); copyUrl.searchParams.set("_mode", "replace"); + if (sourceCutoff) copyUrl.searchParams.set("source_cutoff", sourceCutoff); if (COPY_MARKER_PIPES.has(pipe)) { if (!copyRunId) { throw new Error(`Tinybird copy marker is required for ${pipe}`); @@ -556,6 +705,67 @@ export const submitTinybirdCopyJobs = async ({ return results; }; +export const waitForTinybirdCopyJob = async ({ + origin, + token, + jobId, + request, + assertMutationOwnership, + now = () => Date.now(), + wait = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)), + timeoutMs = 900_000, + pollIntervalMs = 2_000, +}) => { + if (!COPY_JOB_ID_PATTERN.test(jobId)) { + throw new Error("Tinybird Copy status requires a valid job ID"); + } + if (typeof assertMutationOwnership !== "function") { + throw new Error("Tinybird Copy status requires an ownership check"); + } + if ( + !Number.isFinite(timeoutMs) || + timeoutMs <= 0 || + !Number.isFinite(pollIntervalMs) || + pollIntervalMs <= 0 + ) { + throw new Error("Tinybird Copy status polling bounds are invalid"); + } + const startedAt = now(); + const deadline = startedAt + timeoutMs; + let polls = 0; + while (now() < deadline) { + const job = await request( + new URL(`/v0/jobs/${encodeURIComponent(jobId)}`, origin), + { + token, + attempts: 3, + beforeAttempt: assertMutationOwnership, + }, + ); + polls += 1; + const status = String( + job.data.status ?? + job.data.state ?? + job.data.job?.status ?? + job.data.job?.state ?? + "", + ).toLowerCase(); + if (["done", "success", "finished", "completed"].includes(status)) { + return { + status, + polls, + completionMs: Math.max(0, now() - startedAt), + }; + } + if (["failed", "error", "cancelled", "canceled"].includes(status)) { + throw new Error(`Tinybird Copy job ended in ${status}`); + } + await wait(pollIntervalMs); + } + throw new Error("Timed out waiting for Tinybird Copy job"); +}; + export const validateSyntheticRunId = (runId) => { if (!SYNTHETIC_RUN_PATTERN.test(runId)) { throw new Error("Synthetic run ID has an unsafe format"); @@ -654,6 +864,7 @@ export const createSyntheticEvents = ({ runId, now = new Date() }) => { properties: JSON.stringify({ hostname: "preview.cap.so", is_session_entry: true, + session_started_at: now.toISOString(), }), }; const duplicateId = `synthetic_duplicate_${runHash.slice(0, 24)}`; @@ -677,6 +888,7 @@ export const createSyntheticEvents = ({ runId, now = new Date() }) => { properties: JSON.stringify({ hostname: "preview.cap.so", is_session_entry: false, + session_started_at: now.toISOString(), }), }, ], @@ -703,6 +915,7 @@ export const createSyntheticErasureControl = ({ runId, now = new Date() }) => { properties: JSON.stringify({ hostname: fixture.rows[0].hostname, is_session_entry: true, + session_started_at: now.toISOString(), }), }, }; @@ -723,6 +936,8 @@ export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { const abandonedGuestSessionId = `synthetic_abandoned_session_${runHash.slice(0, 16)}`; const sharedAnonymousId = `synthetic_shared_${runHash.slice(0, 20)}`; const sharedSessionId = `synthetic_shared_session_${runHash.slice(0, 16)}`; + const sessionStartedAt = (index) => + new Date(now.getTime() + index * 1_000).toISOString(); const event = ({ anonymousId = fixture.anonymousId, channel = "direct", @@ -790,10 +1005,17 @@ export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { properties: { hostname, is_session_entry: true, + session_started_at: sessionStartedAt(0), + first_touch_source: "first-touch", + first_touch_medium: "first", + first_touch_campaign: "first-campaign", session_touch_source: "google", session_touch_medium: "cpc", session_touch_campaign: "synthetic-campaign", session_touch_gclid: "synthetic-click-id", + last_touch_source: "last-touch", + last_touch_medium: "last", + last_touch_campaign: "last-campaign", }, source: "client", }), @@ -807,6 +1029,7 @@ export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { page_view_id: pageViewId, engaged_ms: 15_000, max_scroll_depth: 75, + session_started_at: sessionStartedAt(0), }, source: "client", }), @@ -857,10 +1080,17 @@ export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { properties: { hostname, is_session_entry: true, + session_started_at: sessionStartedAt(6), + first_touch_source: "first-touch", + first_touch_medium: "first", + first_touch_campaign: "first-campaign", session_touch_source: "google", session_touch_medium: "cpc", session_touch_campaign: "synthetic-campaign", session_touch_gclid: "synthetic-guest-click-id", + last_touch_source: "last-touch", + last_touch_medium: "last", + last_touch_campaign: "last-campaign", }, sessionId: guestSessionId, source: "client", @@ -1066,8 +1296,15 @@ export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { properties: { hostname, is_session_entry: true, + session_started_at: sessionStartedAt(20), + first_touch_source: "first-touch", + first_touch_medium: "first", + first_touch_campaign: "first-campaign", session_touch_source: "synthetic-partner", session_touch_medium: "referral", + last_touch_source: "last-touch", + last_touch_medium: "last", + last_touch_campaign: "last-campaign", }, referrer: "https://synthetic-partner.example/path", sessionId: sharedSessionId, @@ -1148,6 +1385,16 @@ export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { }, source: "client", }), + event({ + eventName: "share_link_created", + index: 27, + platform: "server", + properties: { + asset_type: "recording", + recording_mode: "screen", + }, + source: "server", + }), ], }; }; @@ -1155,16 +1402,32 @@ export const createSyntheticDecisionEvents = ({ runId, now = new Date() }) => { export const createSyntheticLoadEvents = ({ runId, count, + dimensionBucketCount = 64, + daySpan = 1, now = new Date(), }) => { if ( !Number.isInteger(count) || count < 100 || - count > 10_000 || + count > 100_000 || count % 10 !== 0 ) { throw new Error( - "Synthetic load size must be a multiple of 10 between 100 and 10000 rows", + "Synthetic load size must be a multiple of 10 between 100 and 100000 rows", + ); + } + if ( + !Number.isInteger(dimensionBucketCount) || + dimensionBucketCount < 1 || + dimensionBucketCount > 100 + ) { + throw new Error( + "Synthetic load dimension buckets must be an integer between 1 and 100", + ); + } + if (!Number.isInteger(daySpan) || daySpan < 1 || daySpan > 400) { + throw new Error( + "Synthetic load day span must be an integer between 1 and 400", ); } const fixture = createSyntheticEvents({ runId, now }); @@ -1174,25 +1437,38 @@ export const createSyntheticLoadEvents = ({ .replace(/[0-9]/g, (digit) => String.fromCharCode(103 + Number(digit))); const appVersion = `staging-load-${runHash.slice(0, 12)}`; const loadRunId = `${runId}_load`; + const effectiveDimensionBucketCount = Math.min( + dimensionBucketCount, + count / 10, + ); validateSyntheticRunId(loadRunId); return { appVersion, + daySpan, + dimensionBucketCount: effectiveDimensionBucketCount, runId: loadRunId, rows: Array.from({ length: count }, (_, index) => { const cohort = Math.floor(index / 10); + const dayOffset = cohort % daySpan; + const cohortOccurredAt = + now.getTime() - dayOffset * 86_400_000 - (cohort % 86_400) * 1_000; + const dimensionBucket = cohort % effectiveDimensionBucketCount; const eventKind = index % 10; const eventId = `synthetic_load_${eventNamespace}_${cohort}_${eventKind}`; const pageViewId = `synthetic_load_${eventNamespace}_${cohort}_0`; - const hostname = `load-${runHash.slice(0, 8)}-${cohort}.preview.cap.so`; + const hostname = `load-${runHash.slice(0, 8)}-${dimensionBucket}.preview.cap.so`; const anonymousId = `synthetic_load_${runHash.slice(0, 8)}_${cohort}`; const userId = `synthetic_load_user_${runHash.slice(0, 8)}_${cohort}`; const organizationId = `synthetic_load_org_${runHash.slice(0, 8)}_${cohort}`; const sessionId = `synthetic_load_session_${runHash.slice(0, 8)}_${cohort}`; - const checkoutPlatform = ["web", "desktop", "mobile"][cohort % 3]; - const occurredAt = new Date(now.getTime() + index * 10) + const checkoutPlatform = ["web", "desktop", "mobile"][ + dimensionBucket % 3 + ]; + const occurredAt = new Date(cohortOccurredAt) .toISOString() .replace("T", " ") .replace("Z", ""); + const sessionStartedAt = new Date(cohortOccurredAt).toISOString(); const shapes = [ { eventName: "page_view", @@ -1201,9 +1477,10 @@ export const createSyntheticLoadEvents = ({ properties: { hostname, is_session_entry: true, - session_touch_source: `source-${cohort}`, + session_started_at: sessionStartedAt, + session_touch_source: `source-${dimensionBucket}`, session_touch_medium: "cpc", - session_touch_campaign: `campaign-${cohort}`, + session_touch_campaign: `campaign-${dimensionBucket}`, }, channel: "paid_other", }, @@ -1215,6 +1492,7 @@ export const createSyntheticLoadEvents = ({ page_view_id: pageViewId, engaged_ms: 5_000, max_scroll_depth: 60, + session_started_at: sessionStartedAt, }, }, { @@ -1239,15 +1517,13 @@ export const createSyntheticLoadEvents = ({ }, }, { - eventName: "recording_completed", + eventName: "experiment_exposed", source: "client", - platform: "desktop", + platform: "web", properties: { - mode: "screen", - status: "success", - duration_secs: 30, - segment_count: 1, - track_failure_count: 0, + experiment_id: `synthetic-load-${dimensionBucket}`, + variant: dimensionBucket % 2 === 0 ? "control" : "treatment", + assignment_version: "v1", }, }, { @@ -1255,7 +1531,7 @@ export const createSyntheticLoadEvents = ({ source: "server", platform: checkoutPlatform, properties: { - price_id: `price_load_${cohort}`, + price_id: `price_load_${dimensionBucket}`, quantity: 1, is_onboarding: false, }, @@ -1266,7 +1542,7 @@ export const createSyntheticLoadEvents = ({ platform: checkoutPlatform, properties: { subscription_status: "trialing", - price_id: `price_load_${cohort}`, + price_id: `price_load_${dimensionBucket}`, quantity: 1, currency: "usd", unit_amount_minor: 1_000, @@ -1291,7 +1567,7 @@ export const createSyntheticLoadEvents = ({ billing_interval: "month", billing_interval_count: 1, invite_quota: 1, - price_id: `price_load_${cohort}`, + price_id: `price_load_${dimensionBucket}`, quantity: 1, is_first_purchase: true, is_guest_checkout: false, @@ -1305,7 +1581,7 @@ export const createSyntheticLoadEvents = ({ properties: { amount_paid_minor: 1_000, currency: "usd", - price_id: `price_load_${cohort}`, + price_id: `price_load_${dimensionBucket}`, billing_reason: "subscription_cycle", }, }, @@ -1327,7 +1603,7 @@ export const createSyntheticLoadEvents = ({ organization_id: eventKind < 2 ? "" : organizationId, app_version: appVersion, hostname, - pathname: `/analytics-load/${cohort}`, + pathname: `/analytics-load/${dimensionBucket}`, country: "US", device: shape.platform === "mobile" ? "mobile" : "desktop", browser: "Chrome", @@ -1538,6 +1814,19 @@ export const assertSyntheticEndpointDecisions = ({ engaged_ms: 15_000, }, ); + assertEndpointFields( + "product_traffic_totals", + singleEndpointRow(payloads, "product_traffic_totals"), + { + visitors: 3, + visits: 3, + pageviews: 3, + views_per_visit: 1, + bounce_rate: 66.67, + visit_duration_ms: 5_000, + engaged_ms: 15_000, + }, + ); assertEndpointFields( "product_traffic_pages", singleEndpointRow(payloads, "product_traffic_pages"), @@ -1572,6 +1861,64 @@ export const assertSyntheticEndpointDecisions = ({ bounce_rate: 50, }, ); + const attributionRows = endpointRows(payloads, "product_attribution"); + if (attributionRows.length !== 4) { + throw new Error( + `Synthetic endpoint product_attribution returned ${attributionRows.length} rows, expected 4`, + ); + } + for (const [model, source, medium, campaign] of [ + ["first", "first-touch", "first", "first-campaign"], + ["last", "last-touch", "last", "last-campaign"], + ]) { + assertEndpointFields( + "product_attribution", + attributionRows.find( + (row) => row.attribution_model === model && row.source === source, + ) ?? {}, + { + attribution_model: model, + source, + medium, + campaign, + visitors: 3, + visits: 3, + pageviews: 3, + }, + ); + } + assertEndpointFields( + "product_attribution", + attributionRows.find( + (row) => row.attribution_model === "session" && row.source === "google", + ) ?? {}, + { + attribution_model: "session", + source: "google", + medium: "cpc", + campaign: "synthetic-campaign", + visitors: 2, + visits: 2, + pageviews: 2, + }, + ); + assertEndpointFields( + "product_attribution", + attributionRows.find( + (row) => + row.attribution_model === "session" && + row.source === "synthetic-partner", + ) ?? {}, + { + attribution_model: "session", + source: "synthetic-partner", + medium: "referral", + campaign: "", + visitors: 1, + visits: 1, + pageviews: 1, + }, + ); assertEndpointFields( "product_traffic_sources", sourceRows.find((row) => row.channel === "referral") ?? {}, @@ -1708,11 +2055,17 @@ export const assertSyntheticEndpointDecisions = ({ users: 2, }, { eventName: "user_signed_up", source: "server", platform: "web" }, - { eventName: "share_link_created", source: "server", platform: "server" }, + { + eventName: "share_link_created", + source: "server", + platform: "server", + events: 2, + }, { eventName: "recording_completed", source: "client", platform: "desktop", + recordingStatus: "success", }, { eventName: "guest_checkout_started", @@ -1932,6 +2285,7 @@ export const assertSyntheticEndpointDecisions = ({ users: expected.users ?? 1, organizations: expected.organizations ?? 1, plan_id: expected.planId ?? "", + recording_status: expected.recordingStatus ?? "", payment_status: expected.paymentStatus ?? "", subscription_status: expected.subscriptionStatus ?? "", currency: expected.currency ?? "", @@ -1967,7 +2321,7 @@ export const assertSyntheticEndpointDecisions = ({ ["page_engagement", 1, 0], ["identity_linked", 2, 2, 1], ["user_signed_up", 1, 1], - ["share_link_created", 1, 1], + ["share_link_created", 2, 1], ["recording_completed", 1, 1], ["guest_checkout_started", 2, 0, 0, 2], ["checkout_started", 3, 1], @@ -2011,6 +2365,33 @@ export const assertSyntheticEndpointDecisions = ({ organization_days: expected.organizations, }); } + const experimentRows = endpointRows(payloads, "product_experiment_outcomes"); + if (experimentRows.length !== 3) { + throw new Error( + `Synthetic endpoint product_experiment_outcomes returned ${experimentRows.length} rows, expected 3`, + ); + } + for (const [outcomeName, convertedActors] of [ + ["signup", 0], + ["share_created", 1], + ["paid_purchase", 0], + ]) { + assertEndpointFields( + "product_experiment_outcomes", + experimentRows.find((row) => row.outcome_name === outcomeName) ?? {}, + { + experiment_id: "synthetic-checkout-copy", + assignment_version: "v1", + variant: "treatment", + platform: "web", + app_version: appVersion, + outcome_name: outcomeName, + exposed_actors: 1, + converted_actors: convertedActors, + conversion_rate: convertedActors * 100, + }, + ); + } const freshness = singleEndpointRow(payloads, "product_analytics_freshness"); for (const field of [ "latest_received_hour", @@ -2018,6 +2399,8 @@ export const assertSyntheticEndpointDecisions = ({ "traffic_calculated_at", "retention_calculated_at", "identity_calculated_at", + "attribution_calculated_at", + "experiment_calculated_at", ]) { if (typeof freshness[field] !== "string" || freshness[field].length === 0) { throw new Error( @@ -2028,24 +2411,29 @@ export const assertSyntheticEndpointDecisions = ({ }; export const assertRepresentativeEndpointCoverage = ({ + dimensionBucketCount, expectedEvents, payloads, }) => { const cohorts = expectedEvents / 10; + const boundedDimensions = dimensionBucketCount ?? cohorts; const sum = (rows, field) => rows.reduce((total, row) => total + Number(row[field] ?? 0), 0); const expectedRows = { product_traffic_overview: 1, - product_traffic_pages: cohorts, - product_traffic_sources: cohorts, + product_traffic_totals: 1, + product_traffic_pages: boundedDimensions, + product_traffic_sources: boundedDimensions, + product_attribution: boundedDimensions + 2, product_traffic_countries: 1, product_traffic_technology: 1, product_activation: 1, product_creator_activity: 1, product_creator_retention: 1, product_identity_funnel: 1, - product_events_daily: Math.min(expectedEvents, 1_000), + product_events_daily: boundedDimensions * 10, product_feature_adoption: 10, + product_experiment_outcomes: boundedDimensions * 3, product_analytics_freshness: 1, }; for (const [name, count] of Object.entries(expectedRows)) { @@ -2058,8 +2446,10 @@ export const assertRepresentativeEndpointCoverage = ({ } const exactTotals = [ ["product_traffic_overview", "pageviews", cohorts], + ["product_traffic_totals", "pageviews", cohorts], ["product_traffic_pages", "pageviews", cohorts], ["product_traffic_sources", "pageviews", cohorts], + ["product_attribution", "pageviews", cohorts * 3], ["product_activation", "signups", cohorts], ["product_creator_activity", "dau", cohorts], ["product_creator_retention", "creators", cohorts], @@ -2068,6 +2458,8 @@ export const assertRepresentativeEndpointCoverage = ({ ["product_events_daily", "events", expectedEvents], ["product_events_daily", "revenue_minor", cohorts * 2_000], ["product_feature_adoption", "events", expectedEvents], + ["product_experiment_outcomes", "exposed_actors", cohorts * 3], + ["product_experiment_outcomes", "converted_actors", cohorts], ]; for (const [name, field, expected] of exactTotals) { const actual = sum(endpointRows(payloads, name), field); @@ -2367,6 +2759,8 @@ export const normalizeCopyAssertions = (payload) => { activationMarkers: number(row.activation_markers), retentionMarkers: number(row.retention_markers), identityMarkers: number(row.identity_markers), + attributionMarkers: number(row.attribution_markers), + experimentMarkers: number(row.experiment_markers), healthMarkers: number(row.health_markers), }; }; @@ -2428,19 +2822,109 @@ export const percentile = (samples, quantile) => { return sorted[Math.ceil(quantile * sorted.length) - 1]; }; -export const latencySummary = (samples) => ({ - count: samples.length, - minMs: Math.min(...samples), - maxMs: Math.max(...samples), - p50Ms: percentile(samples, 0.5), - p95Ms: percentile(samples, 0.95), - p99Ms: percentile(samples, 0.99), -}); +export const latencySummary = (samples) => { + if (samples.length === 0) { + throw new Error("At least one latency sample is required"); + } + const meanMs = + samples.reduce((total, sample) => total + sample, 0) / samples.length; + const variance = + samples.reduce((total, sample) => total + (sample - meanMs) ** 2, 0) / + samples.length; + const standardDeviationMs = Math.sqrt(variance); + return { + count: samples.length, + minMs: Math.min(...samples), + maxMs: Math.max(...samples), + meanMs: Number(meanMs.toFixed(3)), + standardDeviationMs: Number(standardDeviationMs.toFixed(3)), + coefficientOfVariation: + meanMs === 0 ? 0 : Number((standardDeviationMs / meanMs).toFixed(4)), + p50Ms: percentile(samples, 0.5), + p95Ms: percentile(samples, 0.95), + p99Ms: percentile(samples, 0.99), + }; +}; + +export const evaluateCopyPerformanceBudget = ({ + absolutePipelineMs, + absoluteVisibilityP95Ms, + baseline, + measured, + regressionFactor, + regressionFloorMs, +}) => { + const baselineValues = baseline + ? [baseline.pipelineWallClockMs, baseline.visibility.p95Ms] + : []; + if ( + ![ + absolutePipelineMs, + absoluteVisibilityP95Ms, + measured.pipelineWallClockMs, + measured.visibility.p95Ms, + regressionFactor, + regressionFloorMs, + ...baselineValues, + ].every(Number.isFinite) || + absolutePipelineMs <= 0 || + absoluteVisibilityP95Ms <= 0 || + measured.pipelineWallClockMs < 0 || + measured.visibility.p95Ms < 0 || + regressionFactor < 1 || + regressionFloorMs < 0 || + baselineValues.some((value) => value < 0) + ) { + throw new Error("Copy performance budget inputs are invalid"); + } + const pipelineRegressionLimitMs = baseline + ? Math.ceil( + Math.max( + baseline.pipelineWallClockMs * regressionFactor, + baseline.pipelineWallClockMs + regressionFloorMs, + ), + ) + : null; + const visibilityRegressionLimitMs = baseline + ? Math.ceil( + Math.max( + baseline.visibility.p95Ms * regressionFactor, + baseline.visibility.p95Ms + regressionFloorMs, + ), + ) + : null; + return { + mode: baseline ? "baseline_comparison" : "baseline_capture", + absolutePipelineMs, + absoluteVisibilityP95Ms, + regressionFactor, + regressionFloorMs, + pipelineRegressionLimitMs, + visibilityRegressionLimitMs, + pipelineRegressionRatio: + baseline && baseline.pipelineWallClockMs > 0 + ? measured.pipelineWallClockMs / baseline.pipelineWallClockMs + : null, + visibilityRegressionRatio: + baseline && baseline.visibility.p95Ms > 0 + ? measured.visibility.p95Ms / baseline.visibility.p95Ms + : null, + passed: + measured.pipelineWallClockMs <= absolutePipelineMs && + measured.visibility.p95Ms <= absoluteVisibilityP95Ms && + (pipelineRegressionLimitMs === null || + measured.pipelineWallClockMs <= pipelineRegressionLimitMs) && + (visibilityRegressionLimitMs === null || + measured.visibility.p95Ms <= visibilityRegressionLimitMs), + }; +}; const DECISION_ENDPOINT_NAMES = [ "product_traffic_overview", + "product_traffic_totals", "product_traffic_pages", "product_traffic_sources", + "product_attribution", "product_traffic_countries", "product_traffic_technology", "product_activation", @@ -2449,6 +2933,7 @@ const DECISION_ENDPOINT_NAMES = [ "product_identity_funnel", "product_events_daily", "product_feature_adoption", + "product_experiment_outcomes", "product_analytics_freshness", ]; @@ -2456,11 +2941,14 @@ export const decisionEndpointQueries = ({ startDate, endDate, deploymentId = "", + excludedEndpointNames = [], includeIdentityFunnel = true, syntheticRunId = "", }) => DECISION_ENDPOINT_NAMES.filter( - (name) => includeIdentityFunnel || name !== "product_identity_funnel", + (name) => + (includeIdentityFunnel || name !== "product_identity_funnel") && + !excludedEndpointNames.includes(name), ).map((name) => ({ name, parameters: { @@ -2531,6 +3019,7 @@ export const assertWorkflowSafety = (workflow) => { "TINYBIRD_STAGING_CLEANUP_TOKEN", "staging-ci.js run-copies", "staging-ci.js set-copy-schedules", + "attest-preview", "probe-preview", "verify-promoted", ]) { diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 66a6153ba90..61ece82117e 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -1,9 +1,11 @@ +import { createHmac } from "node:crypto"; import fs from "node:fs"; import process from "node:process"; import { applyCopyScheduleAction, assertExecutionScope, + assertPreviewTinybirdAttestation, assertRepresentativeEndpointCoverage, assertSyntheticBusinessDecisions, assertSyntheticDecisions, @@ -15,6 +17,7 @@ import { assertSyntheticMonetizationFilters, COPY_PIPES, copyScheduleMatchesAction, + createDeploymentBoundary, createSyntheticDecisionEvents, createSyntheticErasureControl, createSyntheticEvents, @@ -22,6 +25,7 @@ import { dataMutationDeploymentParameters, decisionEndpointQueries, evaluateBundleBudget, + evaluateCopyPerformanceBudget, evaluateLatencyBudget, extractSameOriginNextScriptUrls, hashIdentifier, @@ -30,6 +34,7 @@ import { normalizeCopyAssertions, normalizeHealth, reconcileCleanupTarget, + resolveDeploymentCreatedAfterBoundary, resolveDeploymentState, resolveExactDeploymentLifecycle, resolveExactPromotionPlan, @@ -89,6 +94,50 @@ const writeJson = (filePath, value, mode = 0o644) => { }); }; +const exactSha = (value, label) => { + if (!/^[0-9a-f]{40}$/.test(value)) { + throw new Error(`${label} must be an exact 40-character Git SHA`); + } + return value; +}; + +const recoveryIdentity = () => ({ + eventName: environment("GITHUB_EVENT_NAME"), + expectedSha: exactSha(environment("EXPECTED_SHA"), "EXPECTED_SHA"), + headRef: process.env.HEAD_REF ?? "", + pullRequest: process.env.EVENT_NUMBER ?? "", + ref: environment("GITHUB_REF"), + repository: environment("GITHUB_REPOSITORY"), + runAttempt: environment("GITHUB_RUN_ATTEMPT"), + runId: environment("GITHUB_RUN_ID"), + workspaceId: environment("TINYBIRD_WORKSPACE_ID"), +}); + +const assertRecoveryIdentity = (value) => { + const expected = recoveryIdentity(); + for (const [name, expectedValue] of Object.entries(expected)) { + if (String(value?.[name] ?? "") !== expectedValue) { + throw new Error(`Recovery checkpoint ${name} does not match this run`); + } + } + const pullRequestScope = + expected.eventName === "pull_request" && + expected.pullRequest === "2003" && + expected.headRef === "codex/first-party-analytics"; + const dispatchScope = + expected.eventName === "workflow_dispatch" && + expected.ref === "refs/heads/codex/first-party-analytics"; + if ( + (!pullRequestScope && !dispatchScope) || + expected.workspaceId !== STAGING_WORKSPACE_ID + ) { + throw new Error( + "Recovery is restricted to the analytics staging pull request", + ); + } + return expected; +}; + const TINYBIRD_TOKEN_NAMES = [ "TINYBIRD_STAGING_DEPLOY_TOKEN", "TINYBIRD_STAGING_COPY_TOKEN", @@ -99,6 +148,17 @@ const TINYBIRD_TOKEN_NAMES = [ "TINYBIRD_STAGING_CLEANUP_TOKEN", ]; +const PREVIEW_TINYBIRD_TOKEN_ENV = { + PRODUCT_ANALYTICS_TINYBIRD_TOKEN: "TINYBIRD_STAGING_INGEST_TOKEN", + PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN: "TINYBIRD_STAGING_READ_TOKEN", + PRODUCT_ANALYTICS_TINYBIRD_ERASURE_TOKEN: "TINYBIRD_STAGING_CLEANUP_TOKEN", + PRODUCT_ANALYTICS_TINYBIRD_ERASURE_LOOKUP_TOKEN: + "TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN", + PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN: "TINYBIRD_STAGING_COPY_TOKEN", + PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN: + "TINYBIRD_STAGING_SCHEDULER_TOKEN", +}; + const tinybirdEnvironment = (requiredTokenNames = TINYBIRD_TOKEN_NAMES) => { if (environment("TINYBIRD_WORKSPACE_ID") !== STAGING_WORKSPACE_ID) { throw new Error( @@ -192,16 +252,56 @@ const healthQuery = async ({ state, deploymentId = "", appVersion }) => { appVersion === state.previewAppVersion && state.previewStartTime && state.previewEndTime; - return request( - tinybirdUrl(origin, "/v0/pipes/product_events_health.json", { - start_time: previewWindow ? state.previewStartTime : state.startTime, - end_time: previewWindow ? state.previewEndTime : state.endTime, - platform: "web", - app_version: appVersion ?? state.appVersion, - __tb__deployment: deploymentId, - }), - { token: tokens.TINYBIRD_STAGING_READ_TOKEN, attempts: 3 }, + const allPlatformCorpus = [ + state.loadAppVersion, + state.largeLoadAppVersion, + ].includes(appVersion); + const startTime = previewWindow ? state.previewStartTime : state.startTime; + const endTime = previewWindow ? state.previewEndTime : state.endTime; + const windows = []; + const endMs = Date.parse(endTime); + for (let startMs = Date.parse(startTime); startMs <= endMs; ) { + const endExclusiveMs = Math.min(startMs + 30 * 86_400_000, endMs + 1); + windows.push({ + startTime: new Date(startMs).toISOString(), + endTime: new Date(endExclusiveMs - 1).toISOString(), + }); + startMs = endExclusiveMs; + } + const startedAt = performance.now(); + const results = await Promise.all( + windows.map((window) => + request( + tinybirdUrl(origin, "/v0/pipes/product_events_health.json", { + start_time: window.startTime, + end_time: window.endTime, + platform: allPlatformCorpus ? undefined : "web", + app_version: appVersion ?? state.appVersion, + __tb__deployment: deploymentId, + }), + { token: tokens.TINYBIRD_STAGING_READ_TOKEN, attempts: 3 }, + ), + ), ); + if (results.length === 1) return results[0]; + const total = { + received_rows: 0, + unique_events: 0, + unique_payloads: 0, + duplicate_rows: 0, + payload_conflicts: 0, + }; + for (const result of results) { + const row = result.data?.data?.[0] ?? {}; + for (const name of Object.keys(total)) { + total[name] += Number(row[name] ?? 0); + } + } + return { + data: { data: [total] }, + latencyMs: Math.round(performance.now() - startedAt), + attempt: Math.max(...results.map((result) => result.attempt)), + }; }; const ciAssertionsQuery = async ({ @@ -266,20 +366,103 @@ const deploymentList = async ({ origin, token }) => attempts: 3, }); +const prepareDeploymentBoundary = async () => { + const outputPath = option("output"); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + ]); + const deployments = await deploymentList({ + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + }); + const boundary = createDeploymentBoundary(deployments.data); + writeJson( + outputPath, + { + schemaVersion: 1, + identity: recoveryIdentity(), + phase: "precreate", + preview: { + deploymentId: environment("VERCEL_DEPLOYMENT_ID"), + url: new URL(environment("VERCEL_PREVIEW_URL")).origin, + }, + tinybird: { + ...boundary, + createStartedAt: new Date().toISOString(), + }, + }, + 0o600, + ); +}; + const exactDeployment = async ({ origin, token, deploymentId }) => request( tinybirdUrl(origin, `/v1/deployments/${encodeURIComponent(deploymentId)}`), { token, attempts: 3 }, ); +const settledDeploymentLifecycle = async ({ origin, token, deploymentId }) => { + const deadline = + Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); + while (Date.now() < deadline) { + try { + const lifecycle = resolveExactDeploymentLifecycle( + (await exactDeployment({ origin, token, deploymentId })).data, + deploymentId, + ); + if (lifecycle !== "deleting") return lifecycle; + } catch (error) { + if (error instanceof Error && error.status === 404) return "deleted"; + throw error; + } + await delay(2_000); + } + throw new Error("Timed out resolving a deleting Tinybird deployment"); +}; + +const prepareOwnedPromotion = async () => { + const deploymentId = option("deployment-id"); + const statePath = option("state"); + const state = readJson(statePath); + if ( + String(state.deploymentId) !== deploymentId || + state.needsPromotion !== true + ) { + throw new Error("Tinybird promotion checkpoint does not own the candidate"); + } + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + ]); + const current = await deploymentList({ + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + }); + const plan = resolveExactPromotionPlan(current.data, deploymentId); + state.previousLiveDeploymentId = plan.previousLiveDeploymentId; + state.recoveryPhase = "prepromote"; + writeJson(statePath, state, 0o600); + writeOutput("previous_live_id", plan.previousLiveDeploymentId); +}; + const promoteOwnedDeployment = async () => { const deploymentId = option("deployment-id"); + const state = readJson(option("state")); + if ( + String(state.deploymentId) !== deploymentId || + state.recoveryPhase !== "prepromote" || + !/^\d+$/.test(String(state.previousLiveDeploymentId ?? "")) + ) { + throw new Error("Tinybird promotion requires a persisted promotion plan"); + } const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); const token = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; const initial = await deploymentList({ origin, token }); const plan = resolveExactPromotionPlan(initial.data, deploymentId); + if (plan.previousLiveDeploymentId !== state.previousLiveDeploymentId) { + throw new Error("The persisted Tinybird promotion plan is stale"); + } writeOutput("previous_live_id", plan.previousLiveDeploymentId); const promotionDeadline = Date.now() + Number(process.env.DEPLOYMENT_WAIT_MS ?? 300_000); @@ -564,18 +747,17 @@ const drillOwnedRollback = async () => { } let rollbackEndpointSuite; let rollbackProbeError; - let retainedIdentityFunnelAvailable = false; + let excludedRollbackEndpoints = []; try { - retainedIdentityFunnelAvailable = await decisionEndpointAvailable({ + excludedRollbackEndpoints = await unavailableDecisionEndpoints({ deploymentId: previousLiveDeploymentId, - name: "product_identity_funnel", origin, state, token: tokens.TINYBIRD_STAGING_READ_TOKEN, }); rollbackEndpointSuite = await queryDecisionEndpointSuite({ deploymentId: previousLiveDeploymentId, - includeIdentityFunnel: retainedIdentityFunnelAvailable, + excludedEndpointNames: excludedRollbackEndpoints, origin, state, token: tokens.TINYBIRD_STAGING_READ_TOKEN, @@ -677,7 +859,7 @@ const drillOwnedRollback = async () => { restoredEndpointLatencyMs: restoredEndpointSuite.latencyMs, restoredMonetizationFilterLatencyMs: restoredMonetizationFilters.latencyMs, restoredIdentityFilterLatencyMs: restoredIdentityFilters.latencyMs, - retainedIdentityFunnelAvailable, + excludedRollbackEndpoints, verifiedAt: new Date().toISOString(), }; writeJson(artifactPath, artifact); @@ -717,18 +899,17 @@ const rollbackOwnedPromotion = async () => { }); } let rollbackEndpointSuite; - let retainedIdentityFunnelAvailable = false; + let excludedRollbackEndpoints = []; try { - retainedIdentityFunnelAvailable = await decisionEndpointAvailable({ + excludedRollbackEndpoints = await unavailableDecisionEndpoints({ deploymentId: previousLiveDeploymentId, - name: "product_identity_funnel", origin, state, token: tokens.TINYBIRD_STAGING_READ_TOKEN, }); rollbackEndpointSuite = await queryDecisionEndpointSuite({ deploymentId: previousLiveDeploymentId, - includeIdentityFunnel: retainedIdentityFunnelAvailable, + excludedEndpointNames: excludedRollbackEndpoints, origin, state, token: tokens.TINYBIRD_STAGING_READ_TOKEN, @@ -766,16 +947,16 @@ const rollbackOwnedPromotion = async () => { restoredDeploymentId: previousLiveDeploymentId, rejectedDeploymentId: deploymentId, endpointLatencyMs: rollbackEndpointSuite.latencyMs, - retainedIdentityFunnelAvailable, + excludedRollbackEndpoints, verifiedAt: new Date().toISOString(), }; writeJson(artifactPath, artifact); } }; -const discardOwnedDeployment = async () => { - const deploymentId = option("deployment-id"); - const artifactPath = options.get("artifact"); +const discardOwnedDeployment = async (parameters = {}) => { + const deploymentId = parameters.deploymentId ?? option("deployment-id"); + const artifactPath = parameters.artifactPath ?? options.get("artifact"); const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); @@ -846,6 +1027,13 @@ const decisionEndpointQuery = async ({ origin, token, name, parameters }) => attempts: 3, }); +const ROLLBACK_OPTIONAL_DECISION_ENDPOINTS = [ + "product_traffic_totals", + "product_attribution", + "product_identity_funnel", + "product_experiment_outcomes", +]; + const decisionEndpointAvailable = async ({ deploymentId, name, @@ -868,8 +1056,32 @@ const decisionEndpointAvailable = async ({ } }; +const unavailableDecisionEndpoints = async ({ + deploymentId, + origin, + state, + token, +}) => { + const availability = await Promise.all( + ROLLBACK_OPTIONAL_DECISION_ENDPOINTS.map(async (name) => ({ + available: await decisionEndpointAvailable({ + deploymentId, + name, + origin, + state, + token, + }), + name, + })), + ); + return availability + .filter(({ available }) => !available) + .map(({ name }) => name); +}; + const queryDecisionEndpointSuite = async ({ deploymentId, + excludedEndpointNames = [], includeIdentityFunnel = true, origin, state, @@ -883,6 +1095,7 @@ const queryDecisionEndpointSuite = async ({ startDate: state.startTime.slice(0, 10), endDate, deploymentId, + excludedEndpointNames, includeIdentityFunnel, syntheticRunId, }); @@ -1050,6 +1263,100 @@ const waitForVercel = async () => { ); }; +const verifyFreshPullRequestHead = async () => { + const repository = environment("GITHUB_REPOSITORY"); + const token = environment("GITHUB_TOKEN"); + const apiOrigin = environment("GITHUB_API_URL"); + const expectedSha = environment("EXPECTED_SHA"); + const pullRequestNumber = process.env.EVENT_NUMBER?.trim() || "2003"; + if (pullRequestNumber !== "2003") { + throw new Error("Analytics staging promotion is restricted to PR 2003"); + } + const response = await fetch( + new URL(`/repos/${repository}/pulls/${pullRequestNumber}`, apiOrigin), + { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28", + }, + signal: AbortSignal.timeout(15_000), + }, + ); + if (!response.ok) { + throw new Error( + `GitHub pull request lookup failed with HTTP ${response.status}`, + ); + } + const pullRequest = await response.json(); + if ( + pullRequest.state !== "open" || + pullRequest.head?.ref !== "codex/first-party-analytics" || + pullRequest.head?.sha !== expectedSha + ) { + throw new Error( + "The analytics pull request head changed after this staging run started", + ); + } +}; + +const attestPreviewTinybird = async () => { + const secret = environment("CAP_ANALYTICS_STAGING_TEST_SECRET"); + const previewOrigin = new URL(environment("ANALYTICS_PREVIEW_URL")).origin; + const runId = validateSyntheticRunId(environment("ANALYTICS_TEST_RUN_ID")); + const { origin: expectedOrigin, tokens } = tinybirdEnvironment( + Object.values(PREVIEW_TINYBIRD_TOKEN_ENV), + ); + const expectedTokenHashes = Object.fromEntries( + Object.entries(PREVIEW_TINYBIRD_TOKEN_ENV).map( + ([runtimeName, environmentName]) => [ + runtimeName, + createHmac("sha256", `${secret}:${runId}`) + .update(tokens[environmentName]) + .digest("hex"), + ], + ), + ); + const url = new URL("/api/analytics/staging-test/attest", previewOrigin); + const body = (sha) => JSON.stringify({ runId, sha }); + const send = (authorization, sha = environment("EXPECTED_SHA")) => + previewRequest(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(authorization ? { Authorization: authorization } : {}), + }, + body: body(sha), + }); + const unauthorized = await send(); + if (unauthorized.status !== 401) { + throw new Error( + `The preview configuration attestation accepted missing authorization with HTTP ${unauthorized.status}`, + ); + } + const wrongSha = await send( + `Bearer ${secret}`, + "0000000000000000000000000000000000000000", + ); + if (wrongSha.status !== 400) { + throw new Error( + `The preview configuration attestation accepted a wrong SHA with HTTP ${wrongSha.status}`, + ); + } + const response = await send(`Bearer ${secret}`); + if (!response.ok) { + throw new Error( + `The preview configuration attestation failed with HTTP ${response.status}`, + ); + } + assertPreviewTinybirdAttestation({ + attestation: await response.json(), + expectedOrigin, + expectedSha: environment("EXPECTED_SHA"), + expectedTokenHashes, + }); +}; + const previewCookies = (headers) => { const values = typeof headers.getSetCookie === "function" @@ -1078,6 +1385,52 @@ const previewRequest = async (url, init = {}) => { }); }; +const cleanupPreviewDatabaseState = async ({ + anonymousIdentityHashes, + artifact, + runIds, + secret, +}) => { + const url = new URL( + "/api/analytics/staging-test/cleanup-database", + artifact.vercel.url, + ); + const response = await previewRequest(url, { + method: "POST", + headers: { + Authorization: `Bearer ${secret}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + anonymousIdentityHashes, + runId: runIds[0], + scopeRunIds: runIds, + sha: artifact.sha, + }), + signal: AbortSignal.timeout(60_000), + }); + if (!response.ok) { + throw new Error( + `The scoped staging database cleanup failed with HTTP ${response.status}`, + ); + } + const result = await response.json(); + if ( + result.cleaned !== true || + Number(result.remaining) !== 0 || + Number(result.runIds) !== runIds.length || + Number(result.identityHashes) < anonymousIdentityHashes.length + ) { + throw new Error("The scoped staging database cleanup was incomplete"); + } + return { + cleaned: true, + identityHashes: Number(result.identityHashes), + remaining: Number(result.remaining), + runIds: Number(result.runIds), + }; +}; + const measurePageBundle = async ({ landing, requestImpl }) => { const origin = new URL(landing.url).origin; const urls = extractSameOriginNextScriptUrls(await landing.text(), origin); @@ -1178,6 +1531,7 @@ const probePreview = async () => { properties: { hostname: new URL(previewOrigin).hostname, is_session_entry: true, + session_started_at: occurredAt, }, }; const body = JSON.stringify({ @@ -1192,8 +1546,7 @@ const probePreview = async () => { contract_rejected: 0, }, }); - const collectorLatencySamples = []; - const post = async (cookieHeader = cookies) => { + const post = async ({ cookieHeader = cookies, requestBody = body } = {}) => { const startedAt = performance.now(); const response = await previewRequest( new URL("/api/events", previewOrigin), @@ -1207,38 +1560,137 @@ const probePreview = async () => { "User-Agent": "Cap-Analytics-Staging-E2E/1.0", "x-cap-analytics-test-run": previewRunId, }, - body, + body: requestBody, }, ); - collectorLatencySamples.push(Math.round(performance.now() - startedAt)); - return response; + return { + latencyMs: Math.round(performance.now() - startedAt), + response, + }; }; - const missingToken = await post(""); - if (missingToken.status !== 400) { + const missingToken = await post({ cookieHeader: "" }); + if (missingToken.response.status !== 400) { throw new Error( - `The preview collector accepted a missing browser token with HTTP ${missingToken.status}`, + `The preview collector accepted a missing browser token with HTTP ${missingToken.response.status}`, ); } - const expiredToken = await post( - `cap_analytics_anonymous_id=${anonymousId}; cap_analytics_browser_token=v1.0.${anonymousId}.expired`, - ); - if (expiredToken.status !== 400) { + const expiredToken = await post({ + cookieHeader: `cap_analytics_anonymous_id=${anonymousId}; cap_analytics_browser_token=v1.0.${anonymousId}.expired`, + }); + if (expiredToken.response.status !== 400) { throw new Error( - `The preview collector accepted an expired browser token with HTTP ${expiredToken.status}`, + `The preview collector accepted an expired browser token with HTTP ${expiredToken.response.status}`, ); } const duplicateResponses = await Promise.all([post(), post()]); - if (duplicateResponses.some((response) => !response.ok)) { + if (duplicateResponses.some(({ response }) => !response.ok)) { throw new Error( "The preview collector rejected a valid concurrent duplicate", ); } + const collectorRequestCount = Number( + process.env.COLLECTOR_PERFORMANCE_REQUESTS ?? 20, + ); + const collectorConcurrency = Number( + process.env.COLLECTOR_PERFORMANCE_CONCURRENCY ?? 4, + ); + if ( + !Number.isInteger(collectorRequestCount) || + collectorRequestCount < 10 || + collectorRequestCount > 40 || + !Number.isInteger(collectorConcurrency) || + collectorConcurrency < 1 || + collectorConcurrency > 8 + ) { + throw new Error("Collector performance configuration is invalid"); + } + const collectorBatches = Array.from( + { length: collectorRequestCount }, + (_, batchIndex) => + JSON.stringify({ + events: Array.from({ length: 20 }, (_, eventIndex) => ({ + ...event, + eventId: `synthetic_preview_load_${runHash.slice(0, 16)}_${batchIndex}_${eventIndex}`, + occurredAt: new Date( + Date.parse(occurredAt) + batchIndex * 20 + eventIndex, + ).toISOString(), + pathname: `/analytics-synthetic/load-${eventIndex}`, + properties: { + ...event.properties, + is_session_entry: batchIndex === 0 && eventIndex === 0, + }, + })), + delivery: { + attempted: 20, + accepted: 0, + retried: 0, + dropped: 0, + queue_overflow: 0, + oversize: 0, + contract_rejected: 0, + }, + }), + ); + const collectorBenchmarkStartedAt = performance.now(); + const collectorLatencySamples = []; + let collectorAcceptedEvents = 0; + let collectorErrors = 0; + for ( + let offset = 0; + offset < collectorBatches.length; + offset += collectorConcurrency + ) { + const wave = await Promise.all( + collectorBatches + .slice(offset, offset + collectorConcurrency) + .map((requestBody) => post({ requestBody })), + ); + for (const result of wave) { + collectorLatencySamples.push(result.latencyMs); + if (!result.response.ok) { + collectorErrors += 1; + continue; + } + const payload = await result.response.json(); + if (Number(payload.accepted) !== 20) { + collectorErrors += 1; + continue; + } + collectorAcceptedEvents += 20; + } + } + const collectorElapsedMs = Math.max( + 1, + Math.round(performance.now() - collectorBenchmarkStartedAt), + ); + const collectorLatency = latencySummary(collectorLatencySamples); + const collectorP95BudgetMs = Number( + process.env.COLLECTOR_P95_BUDGET_MS ?? 2_500, + ); + const collectorP99BudgetMs = Number( + process.env.COLLECTOR_P99_BUDGET_MS ?? 3_000, + ); + const collectorMinimumEventsPerSecond = Number( + process.env.COLLECTOR_MINIMUM_EVENTS_PER_SECOND ?? 50, + ); + const collectorEventsPerSecond = Math.round( + (collectorAcceptedEvents * 1_000) / collectorElapsedMs, + ); + if ( + collectorErrors !== 0 || + collectorAcceptedEvents !== collectorBatches.length * 20 || + collectorLatency.p95Ms > collectorP95BudgetMs || + collectorLatency.p99Ms > collectorP99BudgetMs || + collectorEventsPerSecond < collectorMinimumEventsPerSecond + ) { + throw new Error("The exact-SHA collector performance budget failed"); + } const minimumAccepted = Number(process.env.RATE_LIMIT_MIN_ACCEPTED ?? 20); - const maximumAccepted = Number(process.env.RATE_LIMIT_MAX_ACCEPTED ?? 80); + const maximumAccepted = Number(process.env.RATE_LIMIT_MAX_ACCEPTED ?? 120); let replayAccepted = 0; let rateLimited = false; for (let index = 0; index <= maximumAccepted; index += 1) { - const response = await post(); + const { response } = await post(); if (response.status === 429) { rateLimited = true; break; @@ -1255,17 +1707,13 @@ const probePreview = async () => { `The preview rate limit accepted ${replayAccepted} replay requests before ${rateLimited ? "limiting too aggressively" : "failing to limit"}`, ); } - const collectorLatency = latencySummary(collectorLatencySamples); - const collectorP95BudgetMs = Number( - process.env.COLLECTOR_P95_BUDGET_MS ?? 3_000, + state.previewAcceptedRows = + duplicateResponses.length + replayAccepted + collectorAcceptedEvents; + state.previewExpectedEvents = + Number(state.browserExpectedEvents ?? 0) + 1 + collectorAcceptedEvents; + state.previewAnonymousIdentityHash = hashIdentifier( + `anonymous\0${anonymousId}`, ); - if (collectorLatency.p95Ms > collectorP95BudgetMs) { - throw new Error( - `The exact-SHA collector p95 was ${collectorLatency.p95Ms}ms, over ${collectorP95BudgetMs}ms`, - ); - } - state.previewAcceptedRows = duplicateResponses.length + replayAccepted; - state.previewExpectedEvents = Number(state.browserExpectedEvents ?? 0) + 1; state.previewStartTime = new Date( new Date(occurredAt).getTime() - 120_000, ).toISOString(); @@ -1276,10 +1724,21 @@ const probePreview = async () => { missingTokenRejected: true, expiredTokenRejected: true, concurrentDuplicateAccepted: true, + collectorPerformance: { + acceptedEvents: collectorAcceptedEvents, + concurrency: collectorConcurrency, + elapsedMs: collectorElapsedMs, + errorCount: collectorErrors, + eventsPerSecond: collectorEventsPerSecond, + latency: collectorLatency, + requestCount: collectorRequestCount, + }, replayAcceptedBeforeRateLimit: replayAccepted, rateLimitPassed: true, collectorLatency, collectorP95BudgetMs, + collectorP99BudgetMs, + collectorMinimumEventsPerSecond, }; artifact.bundle = { baselineOrigin, @@ -1346,7 +1805,7 @@ const probeDurableServerPath = async () => { const result = await response.json(); if ( Number(result.accepted) !== 5 || - Number(result.uniqueEvents) !== 4 || + Number(result.uniqueEvents) !== 5 || !Array.isArray(result.workflowRuns) || result.workflowRuns.length !== 5 ) { @@ -1382,33 +1841,133 @@ const probeDurableServerPath = async () => { } }, }); + const healthUrl = new URL( + "/api/analytics/staging-test/health", + artifact.vercel.url, + ); + const healthResponse = await previewRequest(healthUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${secret}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ runId: serverRunId, sha: artifact.sha }), + }); + if (!healthResponse.ok) { + throw new Error( + `The durable outbox health probe failed with HTTP ${healthResponse.status}`, + ); + } + const outboxHealth = await healthResponse.json(); + if ( + outboxHealth.healthy !== false || + Number(outboxHealth.activeEvents) !== 0 || + Number(outboxHealth.deadLetterEvents) !== 1 || + Number(outboxHealth.payloadConflictEvents) !== 0 || + Number(outboxHealth.receiptPayloadConflictEvents) !== 0 || + Number(outboxHealth.receiptPayloadConflictAttempts) !== 0 || + Number(outboxHealth.provider429Retries) !== 1 || + Number(outboxHealth.provider503Retries) !== 1 || + Number(outboxHealth.timeoutAfterAcceptRetries) !== 1 || + Number(outboxHealth.providerRejectedDeadLetters) !== 1 + ) { + throw new Error( + "The durable staging outbox did not expose the expected retry and dead-letter evidence", + ); + } + const databaseCleanup = await cleanupPreviewDatabaseState({ + anonymousIdentityHashes: [], + artifact, + runIds: [serverRunId], + secret, + }); + const cleanHealthResponse = await previewRequest(healthUrl, { + method: "POST", + headers: { + Authorization: `Bearer ${secret}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ runId: serverRunId, sha: artifact.sha }), + }); + if (!cleanHealthResponse.ok) { + throw new Error( + `The durable outbox cleanup health probe failed with HTTP ${cleanHealthResponse.status}`, + ); + } + const cleanHealth = await cleanHealthResponse.json(); + if ( + cleanHealth.healthy !== true || + Number(cleanHealth.activeEvents) !== 0 || + Number(cleanHealth.deadLetterEvents) !== 0 || + Number(cleanHealth.payloadConflictEvents) !== 0 || + Number(cleanHealth.receiptPayloadConflictEvents) !== 0 || + Number(cleanHealth.receiptPayloadConflictAttempts) !== 0 + ) { + throw new Error("The durable staging database cleanup was incomplete"); + } artifact.serverDelivery = { - acceptedRows: 5, + acceptedRows: visibility.value.receivedRows, uniqueEvents: 4, duplicateRows: visibility.value.duplicateRows, workflowRuns: 5, + provider429RetryPassed: true, + provider503RetryPassed: true, + providerRejectionDeadLetterPassed: true, + lostAcknowledgementRetryPassed: true, visibilityMs: visibility.visibilityMs, + databaseCleanup, + outboxHealth: { + activeEvents: Number(outboxHealth.activeEvents), + deadLetterEvents: Number(outboxHealth.deadLetterEvents), + oldestActiveAgeSeconds: Number(outboxHealth.oldestActiveAgeSeconds), + payloadConflictEvents: Number(outboxHealth.payloadConflictEvents), + receiptPayloadConflictAttempts: Number( + outboxHealth.receiptPayloadConflictAttempts, + ), + receiptPayloadConflictEvents: Number( + outboxHealth.receiptPayloadConflictEvents, + ), + }, unauthorizedRejected: true, wrongShaRejected: true, }; artifact.assertions = { ...artifact.assertions, durableServerPathPassed: true, + durableOutboxHealthPassed: true, + durableDatabaseCleanupPassed: true, serverDuplicateDeliveryPassed: true, }; writeJson(artifactPath, artifact); }; -const seed = async () => { +const prepareSeed = async () => { const runId = validateSyntheticRunId(option("run-id")); const deploymentId = option("deployment-id"); const statePath = option("state"); const artifactPath = option("artifact"); - const sha = environment("EXPECTED_SHA"); - const { origin, tokens } = tinybirdEnvironment([ - "TINYBIRD_STAGING_COPY_TOKEN", - "TINYBIRD_STAGING_INGEST_TOKEN", - ]); + const boundary = readJson(option("boundary")); + assertRecoveryIdentity(boundary.identity); + const needsPromotionValue = option("needs-promotion"); + if (!["true", "false"].includes(needsPromotionValue)) { + throw new Error("--needs-promotion must be true or false"); + } + const needsPromotion = needsPromotionValue === "true"; + if ( + needsPromotion && + String(boundary.tinybird?.liveDeploymentId ?? "") === deploymentId + ) { + throw new Error( + "A Tinybird candidate cannot equal the prior live deployment", + ); + } + if ( + !needsPromotion && + String(boundary.tinybird?.liveDeploymentId ?? "") !== deploymentId + ) { + throw new Error("A Tinybird no-op must retain the prior live deployment"); + } + const sha = exactSha(environment("EXPECTED_SHA"), "EXPECTED_SHA"); const startedAt = new Date(); const fixture = createSyntheticEvents({ runId, now: startedAt }); const erasureControl = createSyntheticErasureControl({ @@ -1422,11 +1981,19 @@ const seed = async () => { const loadFixture = createSyntheticLoadEvents({ runId, count: Number(process.env.PERFORMANCE_EVENT_COUNT ?? 1_000), + daySpan: Number(process.env.PERFORMANCE_DAY_SPAN ?? 30), + dimensionBucketCount: Number( + process.env.PERFORMANCE_DIMENSION_BUCKETS ?? 32, + ), now: startedAt, }); const largeLoadFixture = createSyntheticLoadEvents({ runId: `${runId}_large`, - count: Number(process.env.LARGE_PERFORMANCE_EVENT_COUNT ?? 10_000), + count: Number(process.env.LARGE_PERFORMANCE_EVENT_COUNT ?? 100_000), + daySpan: Number(process.env.LARGE_PERFORMANCE_DAY_SPAN ?? 80), + dimensionBucketCount: Number( + process.env.LARGE_PERFORMANCE_DIMENSION_BUCKETS ?? 64, + ), now: startedAt, }); if (largeLoadFixture.rows.length <= loadFixture.rows.length) { @@ -1437,17 +2004,28 @@ const seed = async () => { const previewRunId = validateSyntheticRunId(`${runId}_preview`); const previewAppVersion = `staging-preview-${hashIdentifier(runId).slice(0, 12)}`; const state = { + recoveryIdentity: boundary.identity, + recoveryPhase: "preseed", runId, previewRunId, previewAppVersion, + previewAnonymousIdentityHash: hashIdentifier( + `anonymous\0synthetic_${previewRunId}`, + ), deploymentId, + liveBeforeDeploymentId: boundary.tinybird.liveDeploymentId, + needsPromotion, appVersion: fixture.appVersion, loadAppVersion: loadFixture.appVersion, + loadDaySpan: loadFixture.daySpan, loadRunId: loadFixture.runId, loadEventCount: loadFixture.rows.length, + loadDimensionBucketCount: loadFixture.dimensionBucketCount, largeLoadAppVersion: largeLoadFixture.appVersion, + largeLoadDaySpan: largeLoadFixture.daySpan, largeLoadRunId: largeLoadFixture.runId, largeLoadEventCount: largeLoadFixture.rows.length, + largeLoadDimensionBucketCount: largeLoadFixture.dimensionBucketCount, decisionRunId: decisionFixture.runId, decisionAppVersion: decisionFixture.appVersion, decisionEventCount: decisionFixture.rows.length, @@ -1467,7 +2045,15 @@ const seed = async () => { erasureControlRunId: erasureControl.runId, erasureControlAppVersion: erasureControl.appVersion, startedAt: startedAt.toISOString(), - startTime: new Date(startedAt.getTime() - 120_000).toISOString(), + startTime: new Date( + Math.floor( + (startedAt.getTime() - + (Math.max(loadFixture.daySpan, largeLoadFixture.daySpan) - 1) * + 86_400_000 - + 12 * 60 * 60 * 1_000) / + 3_600_000, + ) * 3_600_000, + ).toISOString(), endTime: new Date(startedAt.getTime() + 300_000).toISOString(), }; writeJson(statePath, state, 0o600); @@ -1492,11 +2078,15 @@ const seed = async () => { }, load: { rowsPlanned: loadFixture.rows.length, + daySpan: loadFixture.daySpan, + dimensionBucketCount: loadFixture.dimensionBucketCount, rowsAttempted: 0, rowsAccepted: 0, }, largeLoad: { rowsPlanned: largeLoadFixture.rows.length, + daySpan: largeLoadFixture.daySpan, + dimensionBucketCount: largeLoadFixture.dimensionBucketCount, rowsAttempted: 0, rowsAccepted: 0, }, @@ -1516,6 +2106,56 @@ const seed = async () => { assertions: { seedAccepted: false }, }; writeJson(artifactPath, artifact); +}; + +const seed = async () => { + const runId = validateSyntheticRunId(option("run-id")); + const deploymentId = option("deployment-id"); + const statePath = option("state"); + const artifactPath = option("artifact"); + const state = readJson(statePath); + const artifact = readJson(artifactPath); + assertRecoveryIdentity(state.recoveryIdentity); + if ( + state.recoveryPhase !== "preseed" || + state.runId !== runId || + String(state.deploymentId) !== deploymentId || + artifact.sha !== environment("EXPECTED_SHA") || + String(artifact.tinybird?.deploymentId ?? "") !== deploymentId + ) { + throw new Error("Tinybird seed does not match its persisted checkpoint"); + } + const startedAt = new Date(state.startedAt); + if (!Number.isFinite(startedAt.getTime())) { + throw new Error("Tinybird seed checkpoint has an invalid start time"); + } + const fixture = createSyntheticEvents({ runId, now: startedAt }); + const erasureControl = createSyntheticErasureControl({ + runId, + now: startedAt, + }); + const decisionFixture = createSyntheticDecisionEvents({ + runId, + now: startedAt, + }); + const loadFixture = createSyntheticLoadEvents({ + runId, + count: state.loadEventCount, + daySpan: state.loadDaySpan, + dimensionBucketCount: state.loadDimensionBucketCount, + now: startedAt, + }); + const largeLoadFixture = createSyntheticLoadEvents({ + runId: `${runId}_large`, + count: state.largeLoadEventCount, + daySpan: state.largeLoadDaySpan, + dimensionBucketCount: state.largeLoadDimensionBucketCount, + now: startedAt, + }); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_COPY_TOKEN", + "TINYBIRD_STAGING_INGEST_TOKEN", + ]); const deliver = async (row, fixtureRow = false) => { if (fixtureRow) { artifact.delivery.rowsAttempted += 1; @@ -1553,48 +2193,83 @@ const seed = async () => { const deliveries = [...concurrentDeliveries, ...separateBatchDeliveries]; const sendLoadFixture = async (fixture, artifactKey) => { const batchSize = Number(process.env.PERFORMANCE_INGEST_BATCH_SIZE ?? 500); + const concurrency = Number(process.env.PERFORMANCE_INGEST_CONCURRENCY ?? 4); if (!Number.isInteger(batchSize) || batchSize < 100 || batchSize > 1_000) { throw new Error("Performance ingestion batch size must be 100 to 1000"); } + if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 8) { + throw new Error("Performance ingestion concurrency must be 1 to 8"); + } artifact[artifactKey].rowsAttempted = fixture.rows.length; writeJson(artifactPath, artifact); const started = performance.now(); const latencies = []; let accepted = 0; + let errorCount = 0; let retryAttempts = 0; + const batches = []; for (let offset = 0; offset < fixture.rows.length; offset += batchSize) { - const batch = fixture.rows.slice(offset, offset + batchSize); - const delivery = await request( - tinybirdUrl(origin, "/v0/events", { - name: "product_events_v1", - wait: "true", - __tb__min_deployment: deploymentId, + batches.push(fixture.rows.slice(offset, offset + batchSize)); + } + for (let offset = 0; offset < batches.length; offset += concurrency) { + const wave = await Promise.all( + batches.slice(offset, offset + concurrency).map(async (batch) => { + const batchStartedAt = performance.now(); + try { + const delivery = await request( + tinybirdUrl(origin, "/v0/events", { + name: "product_events_v1", + wait: "true", + __tb__min_deployment: deploymentId, + }), + { + token: tokens.TINYBIRD_STAGING_INGEST_TOKEN, + method: "POST", + body: `${batch.map((row) => JSON.stringify(row)).join("\n")}\n`, + headers: { "Content-Type": "application/x-ndjson" }, + attempts: 1, + }, + ); + return { + accepted: batch.length, + error: false, + latencyMs: delivery.latencyMs, + retryAttempts: delivery.attempt - 1, + }; + } catch { + return { + accepted: 0, + error: true, + latencyMs: Math.round(performance.now() - batchStartedAt), + retryAttempts: 0, + }; + } }), - { - token: tokens.TINYBIRD_STAGING_INGEST_TOKEN, - method: "POST", - body: `${batch.map((row) => JSON.stringify(row)).join("\n")}\n`, - headers: { "Content-Type": "application/x-ndjson" }, - attempts: 1, - }, ); - latencies.push(delivery.latencyMs); - retryAttempts += delivery.attempt - 1; - accepted += batch.length; + for (const result of wave) { + latencies.push(result.latencyMs); + accepted += result.accepted; + retryAttempts += result.retryAttempts; + if (result.error) errorCount += 1; + } } const elapsedMs = Math.max(1, Math.round(performance.now() - started)); artifact[artifactKey] = { + ...artifact[artifactKey], rows: fixture.rows.length, rowsPlanned: fixture.rows.length, rowsAttempted: fixture.rows.length, rowsAccepted: accepted, batchSize, - batches: latencies.length, + concurrency, + batches: batches.length, batchLatency: latencySummary(latencies), - errorRate: 0, + errorCount, + errorRate: errorCount / batches.length, retryAttempts, - rowsPerSecond: Math.round((fixture.rows.length * 1_000) / elapsedMs), + rowsPerSecond: Math.round((accepted * 1_000) / elapsedMs), }; + writeJson(artifactPath, artifact); }; await sendLoadFixture(loadFixture, "load"); await sendLoadFixture(largeLoadFixture, "largeLoad"); @@ -1683,6 +2358,28 @@ const waitForCopyVisibility = async ({ label, read, assert }) => { ); }; +const waitForCopyJobs = async ({ + jobs, + origin, + token, + assertMutationOwnership, +}) => { + const completions = []; + for (const job of jobs) { + completions.push({ + pipe: job.pipe, + ...(await waitForTinybirdCopyJob({ + origin, + token, + jobId: job.jobId, + request, + assertMutationOwnership, + })), + }); + } + return completions; +}; + const phaseRunExpectations = ({ state, phase }) => { const expectations = [ { @@ -1735,9 +2432,9 @@ const phaseRunExpectations = ({ state, phase }) => { phase === "cleanup" ? 0 : Number(state.previewExpectedEvents ?? 1), }); } - if (state.serverRunId && phase !== "staged") { + if ((state.serverRunId || phase === "cleanup") && phase !== "staged") { expectations.push({ - runId: state.serverRunId, + runId: state.serverRunId ?? `${state.runId}_server`, canonicalEvents: phase === "cleanup" ? 0 : Number(state.serverExpectedEvents), decisionEvents: @@ -1858,12 +2555,14 @@ const readAndAssertPhaseHealth = async ({ state, phase, deploymentId }) => { return health; }; -const runCopies = async () => { - const state = readJson(option("state")); - const artifactPath = option("artifact"); +const runCopies = async (parameters = {}) => { + const state = + parameters.state ?? readJson(parameters.statePath ?? option("state")); + const artifactPath = parameters.artifactPath ?? option("artifact"); const artifact = readJson(artifactPath); - const phase = option("phase"); - const requestedTarget = option("target"); + const phase = parameters.phase ?? option("phase"); + const requestedTarget = parameters.target ?? option("target"); + const deploymentId = parameters.deploymentId ?? option("deployment-id"); if (!["staged", "promoted", "erasure", "cleanup"].includes(phase)) { throw new Error("Tinybird copy phase is invalid"); } @@ -1872,7 +2571,7 @@ const runCopies = async () => { "Tinybird Copy mutations are allowed only after staging promotion", ); } - if (String(state.deploymentId) !== option("deployment-id")) { + if (String(state.deploymentId) !== deploymentId) { throw new Error("Tinybird copy deployment does not match the seeded run"); } const { origin, tokens } = tinybirdEnvironment([ @@ -1882,6 +2581,8 @@ const runCopies = async () => { const copyRunId = validateSyntheticRunId(`${state.runId}_${phase}`); const expectations = phaseRunExpectations({ state, phase }); const executeCopies = async (target) => { + const pipelineStartedAt = performance.now(); + const sourceCutoff = new Date().toISOString(); const assertMutationOwnership = async () => { if ( (await ownedMutationTarget({ @@ -1893,12 +2594,42 @@ const runCopies = async () => { throw new Error("The owned Tinybird deployment target changed"); } }; + const stateJobs = await submitTinybirdCopyJobs({ + origin, + token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + deploymentId: state.deploymentId, + request, + pipes: [ + "snapshot_product_event_id_states_v2", + "snapshot_product_event_day_states_v2", + ], + sourceCutoff, + assertMutationOwnership, + }); + artifact.copyJobs = { + ...artifact.copyJobs, + [phase]: { + status: "state_in_progress", + target, + copyRunHash: hashIdentifier(copyRunId), + sourceCutoff, + jobs: stateJobs, + }, + }; + writeJson(artifactPath, artifact); + const stateJobCompletions = await waitForCopyJobs({ + jobs: stateJobs, + origin, + token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + assertMutationOwnership, + }); const canonicalJobs = await submitTinybirdCopyJobs({ origin, token: tokens.TINYBIRD_STAGING_COPY_TOKEN, deploymentId: state.deploymentId, request, pipes: ["snapshot_product_events_canonical_v1"], + sourceCutoff, assertMutationOwnership, }); artifact.copyJobs = { @@ -1907,7 +2638,9 @@ const runCopies = async () => { status: "in_progress", target, copyRunHash: hashIdentifier(copyRunId), - jobs: canonicalJobs, + sourceCutoff, + jobs: [...stateJobs, ...canonicalJobs], + stateJobCompletions, }, }; writeJson(artifactPath, artifact); @@ -1949,6 +2682,14 @@ const runCopies = async () => { pipe: "snapshot_product_identity_funnel_exact", marker: "identityMarkers", }, + { + pipe: "snapshot_product_attribution_daily_exact", + marker: "attributionMarkers", + }, + { + pipe: "snapshot_product_experiment_outcomes_exact", + marker: "experimentMarkers", + }, { pipe: "snapshot_product_events_health_hourly", marker: "healthMarkers", @@ -1963,6 +2704,7 @@ const runCopies = async () => { request, pipes: [copyStep.pipe], copyRunId, + sourceCutoff, assertMutationOwnership, })), ); @@ -1996,16 +2738,28 @@ const runCopies = async () => { }; } await assertMutationOwnership(); + const visibility = latencySummary([ + ...stateJobCompletions.map((completion) => completion.completionMs), + canonicalVisibility.visibilityMs, + ...Object.values(downstreamVisibility).map( + (copyVisibility) => copyVisibility.visibilityMs, + ), + ]); return { status: "passed", target, copyRunHash: hashIdentifier(copyRunId), - jobs: [...canonicalJobs, ...downstreamJobs], + jobs: [...stateJobs, ...canonicalJobs, ...downstreamJobs], + stateJobCompletions, canonicalVisibility: { polls: canonicalVisibility.polls, visibilityMs: canonicalVisibility.visibilityMs, }, downstreamVisibility: { copies: downstreamVisibility }, + performance: { + pipelineWallClockMs: Math.round(performance.now() - pipelineStartedAt), + visibility, + }, }; }; let target = requestedTarget; @@ -2015,12 +2769,56 @@ const runCopies = async () => { transitionAttempt += 1 ) { try { + const result = await executeCopies(target); + const baseline = artifact.copyPerformance?.baseline ?? null; + const budget = evaluateCopyPerformanceBudget({ + absolutePipelineMs: Number( + process.env.COPY_PIPELINE_WALL_CLOCK_BUDGET_MS ?? 600_000, + ), + absoluteVisibilityP95Ms: Number( + process.env.COPY_VISIBILITY_P95_BUDGET_MS ?? 120_000, + ), + baseline, + measured: result.performance, + regressionFactor: Number(process.env.COPY_REGRESSION_FACTOR ?? 2), + regressionFloorMs: Number( + process.env.COPY_REGRESSION_FLOOR_MS ?? 30_000, + ), + }); + const phasePerformance = { + ...result.performance, + budget, + }; + artifact.copyPerformance = { + providerResourceMetrics: { + available: false, + limitation: + "Tinybird Copy job completion is polled for the prerequisite state rebuild, but provider resource metrics remain unavailable; CI measures job completion, marker visibility, and end-to-end pipeline wall-clock.", + }, + baseline: baseline ?? { + phase, + pipelineWallClockMs: result.performance.pipelineWallClockMs, + visibility: result.performance.visibility, + }, + phases: { + ...artifact.copyPerformance?.phases, + [phase]: phasePerformance, + }, + }; artifact.copyJobs = { ...artifact.copyJobs, - [phase]: await executeCopies(target), + [phase]: { + ...result, + performance: phasePerformance, + }, }; if (phase === "cleanup") writeOutput("target", target); writeJson(artifactPath, artifact); + if (!budget.passed) { + throw new Error( + `Tinybird Copy performance budget failed for ${phase}: pipeline ${result.performance.pipelineWallClockMs}ms, visibility p95 ${result.performance.visibility.p95Ms}ms`, + ); + } return; } catch (error) { if (phase === "cleanup" && target === "staging") { @@ -2053,11 +2851,11 @@ const runCopies = async () => { throw new Error("Tinybird cleanup copies changed target more than once"); }; -const setCopySchedules = async () => { - const state = readJson(option("state")); - const artifactPath = option("artifact"); +const setCopySchedules = async (parameters = {}) => { + const state = parameters.state ?? readJson(option("state")); + const artifactPath = parameters.artifactPath ?? option("artifact"); const artifact = readJson(artifactPath); - const action = option("action"); + const action = parameters.action ?? option("action"); if (!["pause", "resume"].includes(action)) { throw new Error("Tinybird Copy schedule action must be pause or resume"); } @@ -2380,21 +3178,23 @@ const verify = async () => { "Tinybird performance baseline must be a numeric deployment", ); } - const retainedIdentityFunnelAvailable = - baselineDeploymentId === state.deploymentId || - (await decisionEndpointAvailable({ - deploymentId: baselineDeploymentId, - name: "product_identity_funnel", - origin, - state, - token: tokens.TINYBIRD_STAGING_READ_TOKEN, - })); - const baselineQueries = decisionEndpointQueries({ - startDate: state.startTime.slice(0, 10), - endDate: state.endTime.slice(0, 10), - deploymentId: baselineDeploymentId, - includeIdentityFunnel: retainedIdentityFunnelAvailable, - }); + const hasIndependentBaseline = baselineDeploymentId !== state.deploymentId; + const excludedBaselineEndpoints = hasIndependentBaseline + ? await unavailableDecisionEndpoints({ + deploymentId: baselineDeploymentId, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }) + : []; + const baselineQueries = hasIndependentBaseline + ? decisionEndpointQueries({ + startDate: state.startTime.slice(0, 10), + endDate: state.endTime.slice(0, 10), + deploymentId: baselineDeploymentId, + excludedEndpointNames: excludedBaselineEndpoints, + }) + : []; const measuredQueries = decisionEndpointQueries({ startDate: state.startTime.slice(0, 10), endDate: state.endTime.slice(0, 10), @@ -2420,6 +3220,7 @@ const verify = async () => { token: tokens.TINYBIRD_STAGING_READ_TOKEN, }); assertRepresentativeEndpointCoverage({ + dimensionBucketCount: state.loadDimensionBucketCount, expectedEvents: state.loadEventCount, payloads: representativeCoverage.payloads, }); @@ -2431,6 +3232,7 @@ const verify = async () => { token: tokens.TINYBIRD_STAGING_READ_TOKEN, }); assertRepresentativeEndpointCoverage({ + dimensionBucketCount: state.largeLoadDimensionBucketCount, expectedEvents: state.largeLoadEventCount, payloads: largeCoverage.payloads, }); @@ -2470,29 +3272,40 @@ const verify = async () => { endpointSamples[queries[index].name].push(results[index].latencyMs); } }; - for (let index = 0; index < 10; index += 1) { - await sampleDecisionRound( - baselineQueries, - baselineSamples, - baselineFanoutSamples, - ); - } - for (let index = 0; index < 10; index += 1) { - await sampleDecisionRound(largeQueries, largeSamples, largeFanoutSamples); - } - for (let index = 0; index < 15; index += 1) { - await sampleDecisionRound( - measuredQueries, - measuredSamples, - measuredFanoutSamples, - ); + const samplingWorkloads = [ + { + queries: measuredQueries, + samples: measuredSamples, + fanout: measuredFanoutSamples, + }, + { + queries: representativeQueries, + samples: representativeSamples, + fanout: representativeFanoutSamples, + }, + { + queries: largeQueries, + samples: largeSamples, + fanout: largeFanoutSamples, + }, + ]; + if (hasIndependentBaseline) { + samplingWorkloads.push({ + queries: baselineQueries, + samples: baselineSamples, + fanout: baselineFanoutSamples, + }); } - for (let index = 0; index < 10; index += 1) { - await sampleDecisionRound( - representativeQueries, - representativeSamples, - representativeFanoutSamples, - ); + for (let round = 0; round < 30; round += 1) { + for (let offset = 0; offset < samplingWorkloads.length; offset += 1) { + const workload = + samplingWorkloads[(round + offset) % samplingWorkloads.length]; + await sampleDecisionRound( + workload.queries, + workload.samples, + workload.fanout, + ); + } } const regressionFactor = Number(process.env.ENDPOINT_REGRESSION_FACTOR ?? 2); const regressionFloorMs = Number( @@ -2511,6 +3324,9 @@ const verify = async () => { large.p95Ms <= Math.max(representative.p95Ms * 2, 250), }; if (!baselineSamples[name]) { + const noBaselineMode = hasIndependentBaseline + ? "endpoint_unavailable_on_baseline" + : "absolute_only_no_independent_baseline"; return [ name, { @@ -2519,12 +3335,12 @@ const verify = async () => { representative, large, budget: { - mode: "new_endpoint_no_baseline", + mode: noBaselineMode, absoluteP95Ms: endpointP95BudgetMs, passed: measured.p95Ms <= endpointP95BudgetMs, }, representativeBudget: { - mode: "new_endpoint_no_baseline", + mode: noBaselineMode, absoluteP95Ms: endpointP95BudgetMs, passed: representative.p95Ms <= endpointP95BudgetMs, }, @@ -2562,24 +3378,38 @@ const verify = async () => { const fanoutP95BudgetMs = Number( process.env.DASHBOARD_FANOUT_P95_BUDGET_MS ?? 3_500, ); - const dashboardBaseline = latencySummary(baselineFanoutSamples); + const dashboardBaseline = hasIndependentBaseline + ? latencySummary(baselineFanoutSamples) + : null; const dashboardMeasured = latencySummary(measuredFanoutSamples); const dashboardRepresentative = latencySummary(representativeFanoutSamples); const dashboardLarge = latencySummary(largeFanoutSamples); - const dashboardBudget = evaluateLatencyBudget({ - baseline: dashboardBaseline, - measured: dashboardMeasured, - absoluteP95Ms: fanoutP95BudgetMs, - regressionFactor, - regressionFloorMs, - }); - const dashboardRepresentativeBudget = evaluateLatencyBudget({ - baseline: dashboardBaseline, - measured: dashboardRepresentative, - absoluteP95Ms: fanoutP95BudgetMs, - regressionFactor, - regressionFloorMs, - }); + const dashboardBudget = dashboardBaseline + ? evaluateLatencyBudget({ + baseline: dashboardBaseline, + measured: dashboardMeasured, + absoluteP95Ms: fanoutP95BudgetMs, + regressionFactor, + regressionFloorMs, + }) + : { + mode: "absolute_only_no_independent_baseline", + absoluteP95Ms: fanoutP95BudgetMs, + passed: dashboardMeasured.p95Ms <= fanoutP95BudgetMs, + }; + const dashboardRepresentativeBudget = dashboardBaseline + ? evaluateLatencyBudget({ + baseline: dashboardBaseline, + measured: dashboardRepresentative, + absoluteP95Ms: fanoutP95BudgetMs, + regressionFactor, + regressionFloorMs, + }) + : { + mode: "absolute_only_no_independent_baseline", + absoluteP95Ms: fanoutP95BudgetMs, + passed: dashboardRepresentative.p95Ms <= fanoutP95BudgetMs, + }; const dashboardLargeBudget = { absoluteP95Ms: fanoutP95BudgetMs, maximumRepresentativeFactor: 2, @@ -2609,16 +3439,13 @@ const verify = async () => { artifact.endpointLatency = endpointLatency; artifact.decisionEndpointLatency = decisionEndpointLatency; artifact.performanceBaseline = { - deploymentId: baselineDeploymentId, - mode: - baselineDeploymentId === state.deploymentId - ? "same_deployment_noop" - : "retained_deployment", + deploymentId: hasIndependentBaseline ? baselineDeploymentId : null, + mode: hasIndependentBaseline + ? "retained_deployment" + : "absolute_only_no_independent_baseline", representativeRows: state.loadEventCount, largeRows: state.largeLoadEventCount, - newEndpointsWithoutBaseline: retainedIdentityFunnelAvailable - ? [] - : ["product_identity_funnel"], + newEndpointsWithoutBaseline: excludedBaselineEndpoints, representativeCoverageLatencyMs: representativeCoverage.latencyMs, largeCoverageLatencyMs: largeCoverage.latencyMs, }; @@ -2982,13 +3809,17 @@ const verifySyntheticIdentityErasure = async () => { writeJson(artifactPath, artifact); }; -const cleanup = async () => { - const state = readJson(option("state")); - const artifactPath = option("artifact"); - const requestedTarget = option("target"); +const cleanup = async (parameters = {}) => { + const state = parameters.state ?? readJson(option("state")); + const artifactPath = parameters.artifactPath ?? option("artifact"); + const requestedTarget = parameters.target ?? option("target"); + const deploymentId = parameters.deploymentId ?? option("deployment-id"); if (!["live", "staging"].includes(requestedTarget)) { throw new Error("Tinybird cleanup target is invalid"); } + if (String(state.deploymentId) !== deploymentId) { + throw new Error("Tinybird cleanup does not match the seeded deployment"); + } const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_CLEANUP_TOKEN", "TINYBIRD_STAGING_DEPLOY_TOKEN", @@ -3006,7 +3837,9 @@ const cleanup = async () => { validateSyntheticRunId(state.largeLoadRunId); validateSyntheticRunId(state.decisionRunId); validateSyntheticRunId(state.erasureControlRunId); - if (state.serverRunId) validateSyntheticRunId(state.serverRunId); + const serverRunId = validateSyntheticRunId( + state.serverRunId ?? `${state.runId}_server`, + ); const runIds = [ state.runId, state.loadRunId, @@ -3017,7 +3850,32 @@ const cleanup = async () => { if (state.previewRunId) { runIds.push(validateSyntheticRunId(state.previewRunId)); } - if (state.serverRunId) runIds.push(state.serverRunId); + runIds.push(serverRunId); + { + const databaseArtifact = readJson(artifactPath); + const anonymousIdentityHashes = [ + state.browserAnonymousIdentityHash, + state.previewAnonymousIdentityHash, + ].filter( + (identityHash) => + typeof identityHash === "string" && /^[0-9a-f]{64}$/.test(identityHash), + ); + const databaseCleanup = await cleanupPreviewDatabaseState({ + anonymousIdentityHashes, + artifact: databaseArtifact, + runIds: [state.runId, serverRunId], + secret: environment("CAP_ANALYTICS_STAGING_TEST_SECRET"), + }); + databaseArtifact.cleanup = { + ...databaseArtifact.cleanup, + database: databaseCleanup, + }; + databaseArtifact.assertions = { + ...databaseArtifact.assertions, + syntheticDatabaseCleanupPassed: true, + }; + writeJson(artifactPath, databaseArtifact); + } if (target === "staging") { writeOutput("target", target); writeOutput("requires_copies", "false"); @@ -3039,7 +3897,7 @@ const cleanup = async () => { ) { const deploymentParameters = dataMutationDeploymentParameters({ target, - deploymentId: option("deployment-id"), + deploymentId, expectedDeploymentId: String(state.deploymentId), }); const assertMutationOwnership = async () => { @@ -3626,6 +4484,276 @@ const verifyTokenScopes = async () => { writeJson(artifactPath, artifact); }; +const checkpointFiles = (directory, fileName) => { + const matches = []; + for (const entry of fs.readdirSync(directory, { withFileTypes: true })) { + const entryPath = `${directory}/${entry.name}`; + if (entry.isDirectory()) { + matches.push(...checkpointFiles(entryPath, fileName)); + } else if (entry.isFile() && entry.name === fileName) { + matches.push(entryPath); + } + } + return matches; +}; + +const markRecoveryReadyToFinalize = async () => { + const statePath = option("state"); + const artifact = readJson(option("artifact")); + const state = readJson(statePath); + assertRecoveryIdentity(state.recoveryIdentity); + if ( + state.needsPromotion !== true || + state.recoveryPhase !== "prepromote" || + artifact.assertions?.cleanupPassed !== true || + artifact.rollbackDrill?.passed !== true || + artifact.copySchedule?.resume?.status !== "passed" + ) { + throw new Error( + "Tinybird finalization checkpoint requires cleanup, rollback, and schedule proof", + ); + } + state.recoveryPhase = "ready_to_finalize"; + writeJson(statePath, state, 0o600); +}; + +const recoveryCheckpoint = (directory) => { + const stateCandidates = checkpointFiles( + directory, + "analytics-staging-state.json", + ).map((statePath) => { + const state = readJson(statePath); + assertRecoveryIdentity(state.recoveryIdentity); + const phaseRank = { + preseed: 1, + prepromote: 2, + ready_to_finalize: 3, + }[state.recoveryPhase]; + if (!phaseRank) { + throw new Error("Recovery checkpoint has an unsupported phase"); + } + const artifactPath = `${statePath.slice(0, -"analytics-staging-state.json".length)}analytics-staging-report.json`; + if (!fs.existsSync(artifactPath)) { + throw new Error("Recovery checkpoint is missing its staging report"); + } + return { artifactPath, phaseRank, state, statePath }; + }); + if (stateCandidates.length > 0) { + stateCandidates.sort((left, right) => right.phaseRank - left.phaseRank); + const selected = stateCandidates[0]; + for (const candidate of stateCandidates.slice(1)) { + if ( + candidate.state.runId !== selected.state.runId || + String(candidate.state.deploymentId) !== + String(selected.state.deploymentId) + ) { + throw new Error("Recovery checkpoints disagree on mutation ownership"); + } + } + return { kind: "seeded", ...selected }; + } + const boundaries = checkpointFiles( + directory, + "analytics-deployment-boundary.json", + ); + if (boundaries.length !== 1) { + throw new Error("Recovery requires exactly one deployment boundary"); + } + const boundary = readJson(boundaries[0]); + assertRecoveryIdentity(boundary.identity); + return { boundary, kind: "boundary" }; +}; + +const recoverStaging = async () => { + const checkpoint = recoveryCheckpoint(option("checkpoint-directory")); + const recoveryArtifactPath = option("artifact"); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_CLEANUP_TOKEN", + "TINYBIRD_STAGING_COPY_TOKEN", + "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_READ_TOKEN", + "TINYBIRD_STAGING_SCHEDULER_TOKEN", + ]); + const deployToken = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; + const currentDeployments = await deploymentList({ + origin, + token: deployToken, + }); + if (checkpoint.kind === "boundary") { + const candidate = resolveDeploymentCreatedAfterBoundary( + currentDeployments.data, + checkpoint.boundary.tinybird, + { allowNone: true }, + ); + if (candidate) { + await discardOwnedDeployment({ deploymentId: candidate.id }); + } + writeJson(recoveryArtifactPath, { + recovered: true, + strategy: candidate ? "discard_uncertain_create" : "no_mutation", + workspaceId: STAGING_WORKSPACE_ID, + verifiedAt: new Date().toISOString(), + }); + return; + } + const state = checkpoint.state; + const artifact = readJson(checkpoint.artifactPath); + if ( + artifact.sha !== state.recoveryIdentity.expectedSha || + String(artifact.tinybird?.deploymentId ?? "") !== String(state.deploymentId) + ) { + throw new Error("Recovery report does not match its exact-SHA checkpoint"); + } + const candidateId = String(state.deploymentId); + const previousLiveDeploymentId = String( + state.previousLiveDeploymentId ?? state.liveBeforeDeploymentId ?? "", + ); + let retainedDeploymentId = candidateId; + let strategy = "clean_noop_live"; + const candidateLifecycle = await settledDeploymentLifecycle({ + origin, + token: deployToken, + deploymentId: candidateId, + }); + if (state.needsPromotion === true) { + if (!/^\d+$/.test(previousLiveDeploymentId)) { + throw new Error("Recovery is missing the exact prior live deployment"); + } + if (candidateLifecycle === "live") { + const previousLifecycle = await settledDeploymentLifecycle({ + origin, + token: deployToken, + deploymentId: previousLiveDeploymentId, + }); + if (previousLifecycle === "ready") { + await switchLiveDeployment({ + origin, + token: deployToken, + fromDeploymentId: candidateId, + toDeploymentId: previousLiveDeploymentId, + }); + try { + const excludedEndpoints = await unavailableDecisionEndpoints({ + deploymentId: previousLiveDeploymentId, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertDecisionEndpointSuiteReadable( + ( + await queryDecisionEndpointSuite({ + deploymentId: previousLiveDeploymentId, + excludedEndpointNames: excludedEndpoints, + origin, + state, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }) + ).payloads, + ); + } catch (error) { + await switchLiveDeployment({ + origin, + token: deployToken, + fromDeploymentId: previousLiveDeploymentId, + toDeploymentId: candidateId, + }); + throw new Error( + "The prior Tinybird deployment failed recovery validation", + { cause: error }, + ); + } + retainedDeploymentId = previousLiveDeploymentId; + strategy = "rollback_and_discard"; + } else if ( + previousLifecycle === "deleted" && + state.recoveryPhase === "ready_to_finalize" + ) { + strategy = "retain_finalized_candidate"; + } else { + throw new Error( + "The prior Tinybird deployment cannot safely receive rollback", + ); + } + } else if (["ready", "failed", "pending"].includes(candidateLifecycle)) { + retainedDeploymentId = previousLiveDeploymentId; + strategy = "discard_rejected_candidate"; + } else if (candidateLifecycle === "deleted") { + retainedDeploymentId = previousLiveDeploymentId; + strategy = "already_discarded"; + } else { + throw new Error("The exact Tinybird candidate is not recoverable"); + } + } + const serverRunId = validateSyntheticRunId(`${state.runId}_server`); + let databaseCleanup; + if ( + state.needsPromotion !== true || + strategy === "retain_finalized_candidate" + ) { + await setCopySchedules({ + action: "pause", + artifactPath: checkpoint.artifactPath, + state, + }); + try { + await cleanup({ + artifactPath: checkpoint.artifactPath, + deploymentId: candidateId, + state, + target: "live", + }); + databaseCleanup = readJson(checkpoint.artifactPath).cleanup?.database; + if (!databaseCleanup?.cleaned) { + throw new Error("Recovery did not attest staging database cleanup"); + } + await runCopies({ + artifactPath: checkpoint.artifactPath, + deploymentId: candidateId, + phase: "cleanup", + state, + target: "live", + }); + } finally { + await setCopySchedules({ + action: "resume", + artifactPath: checkpoint.artifactPath, + state, + }); + } + } else { + const retainedState = { ...state, deploymentId: retainedDeploymentId }; + await setCopySchedules({ + action: "resume", + artifactPath: checkpoint.artifactPath, + state: retainedState, + }); + if (candidateLifecycle !== "deleted") { + await discardOwnedDeployment({ deploymentId: candidateId }); + } + databaseCleanup = await cleanupPreviewDatabaseState({ + anonymousIdentityHashes: [ + state.browserAnonymousIdentityHash, + state.previewAnonymousIdentityHash, + ].filter( + (identityHash) => + typeof identityHash === "string" && + /^[0-9a-f]{64}$/.test(identityHash), + ), + artifact, + runIds: [state.runId, serverRunId], + secret: environment("CAP_ANALYTICS_STAGING_TEST_SECRET"), + }); + } + writeJson(recoveryArtifactPath, { + databaseCleanup, + recovered: true, + retainedDeploymentId, + strategy, + workspaceId: STAGING_WORKSPACE_ID, + verifiedAt: new Date().toISOString(), + }); +}; + const handlers = { "verify-scope": async () => assertExecutionScope({ @@ -3638,6 +4766,9 @@ const handlers = { }), "verify-credentials": async () => tinybirdEnvironment(), "verify-token-scopes": verifyTokenScopes, + "prepare-deployment-boundary": prepareDeploymentBoundary, + "prepare-promotion": prepareOwnedPromotion, + "prepare-seed": prepareSeed, "promote-deployment": promoteOwnedDeployment, "drill-rollback": drillOwnedRollback, "finalize-promotion": finalizeOwnedPromotion, @@ -3650,14 +4781,22 @@ const handlers = { /Deployment URL:\s+\S+\/deployments\/(\d+)/, )?.[1]; const noOpConfirmed = + Number(createOutput.exitCode) === 0 && output.includes("No changes to be deployed") && output.includes("Not deploying. No changes."); - const selection = selectStagingDeployment( - readJson(option("input")), - option("minimum-created-at"), - createdDeploymentId, - noOpConfirmed, - ); + const deployments = readJson(option("input")); + const selection = + createdDeploymentId || noOpConfirmed + ? selectStagingDeployment( + deployments, + option("minimum-created-at"), + createdDeploymentId, + noOpConfirmed, + ) + : resolveDeploymentCreatedAfterBoundary( + deployments, + readJson(option("boundary")).tinybird, + ); writeOutput("id", selection.id); writeOutput("needs_promotion", String(selection.needsPromotion)); }, @@ -3681,6 +4820,8 @@ const handlers = { writeOutput("pending_recovery", resolution.pending); }, "wait-vercel": waitForVercel, + "verify-pr-head": verifyFreshPullRequestHead, + "attest-preview": attestPreviewTinybird, seed, "run-copies": runCopies, "set-copy-schedules": setCopySchedules, @@ -3690,6 +4831,8 @@ const handlers = { "verify-promoted": verifyPromoted, "erase-synthetic-identity": eraseSyntheticIdentity, "verify-synthetic-identity-erasure": verifySyntheticIdentityErasure, + "mark-recovery-ready": markRecoveryReadyToFinalize, + recover: recoverStaging, cleanup, "verify-cleanup": verifyCleanup, }; diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 4c0c9ce92ab..b8bcc5217b9 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -5,6 +5,7 @@ import test from "node:test"; import { applyCopyScheduleAction, assertExecutionScope, + assertPreviewTinybirdAttestation, assertRepresentativeEndpointCoverage, assertSyntheticBusinessDecisions, assertSyntheticDecisions, @@ -16,6 +17,7 @@ import { assertSyntheticMonetizationFilters, assertWorkflowSafety, copyScheduleMatchesAction, + createDeploymentBoundary, createSyntheticDecisionEvents, createSyntheticErasureControl, createSyntheticEvents, @@ -23,6 +25,7 @@ import { dataMutationDeploymentParameters, decisionEndpointQueries, evaluateBundleBudget, + evaluateCopyPerformanceBudget, evaluateLatencyBudget, extractSameOriginNextScriptUrls, FEATURE_BRANCH, @@ -31,12 +34,16 @@ import { normalizeCiAssertions, normalizeCopyAssertions, normalizeHealth, + PREVIEW_TINYBIRD_TOKEN_NAMES, reconcileCleanupTarget, + resolveDeploymentCreatedAfterBoundary, resolveDeploymentState, resolveExactDeploymentLifecycle, resolveExactPromotionPlan, resolveOwnedDiscardTarget, resolveOwnedMutationTarget, + STAGING_DATABASE_FINGERPRINT, + STAGING_DATABASE_SCHEMA, STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, @@ -45,6 +52,7 @@ import { tokenWorkspaceId, validateSyntheticRunId, validateTinybirdCredentials, + waitForTinybirdCopyJob, } from "../staging-ci-lib.js"; const SHA = "1234567890abcdef1234567890abcdef12345678"; @@ -109,6 +117,8 @@ test("schedule state attestation distinguishes paused from active copies", () => copyScheduleMatchesAction({ schedule: { status: "paused" } }, "resume"), false, ); + assert.equal(copyScheduleMatchesAction({}, "pause"), true); + assert.equal(copyScheduleMatchesAction({}, "resume"), true); }); test("execution scope accepts only PR 2003 or manual feature branch runs", () => { @@ -195,6 +205,67 @@ test("Tinybird credentials must all decode to the hard-coded staging workspace", ); }); +test("preview Tinybird attestation requires the exact SHA, host, and staging workspace", () => { + const expectedTokenHashes = Object.fromEntries( + PREVIEW_TINYBIRD_TOKEN_NAMES.map((name, index) => [ + name, + String(index).padStart(64, "0"), + ]), + ); + const attestation = { + databaseFingerprint: STAGING_DATABASE_FINGERPRINT, + databaseSchema: STAGING_DATABASE_SCHEMA, + host: "https://api.us-east.aws.tinybird.co", + sha: SHA, + workspaces: PREVIEW_TINYBIRD_TOKEN_NAMES.map((name) => ({ + name, + tokenHash: expectedTokenHashes[name], + workspaceId: STAGING_WORKSPACE_ID, + })), + }; + assert.doesNotThrow(() => + assertPreviewTinybirdAttestation({ + attestation, + expectedOrigin: attestation.host, + expectedSha: SHA, + expectedTokenHashes, + }), + ); + for (const invalid of [ + { ...attestation, sha: "0".repeat(40) }, + { ...attestation, host: "https://api.tinybird.co" }, + { ...attestation, databaseFingerprint: "f".repeat(64) }, + { ...attestation, databaseSchema: "0039_bumpy_phil_sheldon" }, + { ...attestation, workspaces: attestation.workspaces.slice(1) }, + { + ...attestation, + workspaces: attestation.workspaces.map((workspace, index) => + index === 0 + ? { + ...workspace, + workspaceId: "00000000-0000-4000-8000-000000000000", + } + : workspace, + ), + }, + { + ...attestation, + workspaces: attestation.workspaces.map((workspace, index) => + index === 0 ? { ...workspace, tokenHash: "f".repeat(64) } : workspace, + ), + }, + ]) { + assert.throws(() => + assertPreviewTinybirdAttestation({ + attestation: invalid, + expectedOrigin: attestation.host, + expectedSha: SHA, + expectedTokenHashes, + }), + ); + } +}); + test("deployment selection rejects stale or ambiguous staging deployments", () => { const minimum = "2026-07-31T10:00:00.000Z"; assert.deepEqual( @@ -291,6 +362,61 @@ test("deployment selection rejects stale or ambiguous staging deployments", () = ); }); +test("deployment recovery resolves only one candidate created after its boundary", () => { + const before = { + deployments: [ + { id: "6", status: "Live" }, + { id: "5", status: "Deleted" }, + ], + }; + const boundary = createDeploymentBoundary(before); + assert.deepEqual(boundary, { + deploymentIds: ["5", "6"], + liveDeploymentId: "6", + }); + assert.deepEqual( + resolveDeploymentCreatedAfterBoundary( + { + deployments: [ + ...before.deployments, + { id: "7", status: "creating_schema" }, + ], + }, + boundary, + ), + { id: "7", needsPromotion: true }, + ); + assert.equal( + resolveDeploymentCreatedAfterBoundary(before, boundary, { + allowNone: true, + }), + undefined, + ); + assert.throws(() => + resolveDeploymentCreatedAfterBoundary( + { + deployments: [ + ...before.deployments, + { id: "7", status: "Staging" }, + { id: "8", status: "failed" }, + ], + }, + boundary, + ), + ); + assert.throws(() => + resolveDeploymentCreatedAfterBoundary( + { + deployments: [ + { id: "6", status: "Staging" }, + { id: "7", status: "Live" }, + ], + }, + boundary, + ), + ); +}); + test("data mutations validate the exact deployment before using the staging selector", () => { assert.deepEqual( dataMutationDeploymentParameters({ @@ -599,6 +725,79 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker ); }); +test("Copy prerequisite polling waits for a successful terminal job", async () => { + let now = 1_000; + let ownershipChecks = 0; + const requests = []; + const responses = [ + { data: { status: "working" } }, + { data: { job: { state: "completed" } } }, + ]; + const result = await waitForTinybirdCopyJob({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + jobId: "copy_job_state", + request: async (url, options) => { + await options.beforeAttempt(); + requests.push(String(url)); + return responses.shift(); + }, + assertMutationOwnership: async () => { + ownershipChecks += 1; + }, + now: () => now, + wait: async (milliseconds) => { + now += milliseconds; + }, + timeoutMs: 10_000, + pollIntervalMs: 2_000, + }); + assert.deepEqual(result, { + status: "completed", + polls: 2, + completionMs: 2_000, + }); + assert.equal(ownershipChecks, 2); + assert.equal(requests.length, 2); + assert.ok(requests.every((url) => url.endsWith("/v0/jobs/copy_job_state"))); +}); + +test("Copy prerequisite polling fails closed on rejection and timeout", async () => { + await assert.rejects( + waitForTinybirdCopyJob({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + jobId: "copy_job_failed", + request: async (_url, options) => { + await options.beforeAttempt(); + return { data: { status: "failed" } }; + }, + assertMutationOwnership: async () => undefined, + }), + /Tinybird Copy job ended in failed/, + ); + let now = 1_000; + await assert.rejects( + waitForTinybirdCopyJob({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + jobId: "copy_job_timeout", + request: async (_url, options) => { + await options.beforeAttempt(); + return { data: { status: "working" } }; + }, + assertMutationOwnership: async () => undefined, + now: () => now, + wait: async (milliseconds) => { + now += milliseconds; + }, + timeoutMs: 2_000, + pollIntervalMs: 2_000, + }), + /Timed out waiting for Tinybird Copy job/, + ); +}); + test("synthetic fixtures are deterministic, isolated, and model duplicates and conflicts", () => { const runId = "run_12345678"; const fixture = createSyntheticEvents({ @@ -626,6 +825,50 @@ test("synthetic fixtures are deterministic, isolated, and model duplicates and c assert.equal(new Set(load.rows.map((row) => row.event_id)).size, 100); assert.ok(load.rows.every((row) => row.app_version === load.appVersion)); assert.ok(load.rows.every((row) => row.synthetic_run_id === load.runId)); + const retainedLoad = createSyntheticLoadEvents({ + runId: `${runId}_retained`, + count: 10_000, + daySpan: 30, + dimensionBucketCount: 8, + now: new Date("2026-07-31T10:00:00.000Z"), + }); + assert.equal(retainedLoad.dimensionBucketCount, 8); + assert.equal(retainedLoad.daySpan, 30); + assert.equal( + new Set(retainedLoad.rows.map((row) => row.occurred_at.slice(0, 10))).size, + 30, + ); + assert.ok( + retainedLoad.rows.every( + (row) => + Date.parse(`${row.occurred_at}Z`) <= + Date.parse("2026-07-31T10:00:00.000Z"), + ), + ); + assert.equal(new Set(retainedLoad.rows.map((row) => row.hostname)).size, 8); + assert.equal(new Set(retainedLoad.rows.map((row) => row.pathname)).size, 8); + assert.equal( + new Set(retainedLoad.rows.map((row) => row.event_id)).size, + 10_000, + ); + const retainedPrices = new Set( + retainedLoad.rows + .map((row) => JSON.parse(row.properties).price_id) + .filter(Boolean), + ); + assert.equal(retainedPrices.size, 8); + const retainedEndpointDimensions = new Set( + retainedLoad.rows.map((row) => { + const properties = JSON.parse(row.properties); + return [ + row.event_name, + row.platform, + row.hostname, + properties.price_id ?? "", + ].join(":"); + }), + ); + assert.ok(retainedEndpointDimensions.size <= 80); const control = createSyntheticErasureControl({ runId, now: new Date("2026-07-31T10:00:00.000Z"), @@ -638,9 +881,9 @@ test("synthetic fixtures are deterministic, isolated, and model duplicates and c runId, now: new Date("2026-07-31T10:00:00.000Z"), }); - assert.equal(decisions.rows.length, 27); + assert.equal(decisions.rows.length, 28); assert.equal(decisions.runId, `${runId}_decisions`); - assert.equal(new Set(decisions.rows.map((row) => row.event_id)).size, 27); + assert.equal(new Set(decisions.rows.map((row) => row.event_id)).size, 28); assert.deepEqual( decisions.rows.map((row) => row.event_name), [ @@ -671,6 +914,7 @@ test("synthetic fixtures are deterministic, isolated, and model duplicates and c "subscription_renewed", "experiment_exposed", "analytics_delivery_loss", + "share_link_created", ], ); assert.equal(decisions.rows[0].user_id, ""); @@ -700,6 +944,17 @@ test("synthetic fixtures are deterministic, isolated, and model duplicates and c ); assert.match(decisions.pathname, /^\/analytics-synthetic-[0-9a-f]{12}$/); assert.throws(() => createSyntheticLoadEvents({ runId, count: 99 })); + assert.throws(() => createSyntheticLoadEvents({ runId, count: 100_010 })); + assert.throws(() => + createSyntheticLoadEvents({ runId, count: 100, daySpan: 0 }), + ); + assert.throws(() => + createSyntheticLoadEvents({ + runId, + count: 100, + dimensionBucketCount: 101, + }), + ); }); test("health normalization and latency percentiles use decision-facing assertions", () => { @@ -723,6 +978,9 @@ test("health normalization and latency percentiles use decision-facing assertion count: 5, minMs: 1, maxMs: 5, + meanMs: 3, + standardDeviationMs: Number(Math.SQRT2.toFixed(3)), + coefficientOfVariation: 0.4714, p50Ms: 3, p95Ms: 5, p99Ms: 5, @@ -870,8 +1128,13 @@ test("representative endpoint coverage requires mixed funnel and revenue data", Array.from({ length: count }, () => ({ ...value })); const payloads = { product_traffic_overview: row([{ pageviews: cohorts }]), + product_traffic_totals: row([{ pageviews: cohorts }]), product_traffic_pages: row(repeated(cohorts, { pageviews: 1 })), product_traffic_sources: row(repeated(cohorts, { pageviews: 1 })), + product_attribution: row([ + ...repeated(cohorts + 1, { pageviews: 0 }), + { pageviews: cohorts * 3 }, + ]), product_traffic_countries: row([{}]), product_traffic_technology: row([{}]), product_activation: row([{ signups: cohorts }]), @@ -885,6 +1148,13 @@ test("representative endpoint coverage requires mixed funnel and revenue data", { events: 1, revenue_minor: 20_000 }, ]), product_feature_adoption: row(repeated(10, { events: 10 })), + product_experiment_outcomes: row([ + ...repeated(cohorts * 3 - 1, { + exposed_actors: 1, + converted_actors: 0, + }), + { exposed_actors: 1, converted_actors: cohorts }, + ]), product_analytics_freshness: row([{}]), }; assert.doesNotThrow(() => @@ -981,6 +1251,7 @@ test("typed endpoint assertions require exact public response semantics", () => hostname, channel: "direct", plan_id: "", + recording_status: "", payment_status: "", subscription_status: "", currency: "", @@ -1036,8 +1307,10 @@ test("typed endpoint assertions require exact public response semantics", () => users: 2, }), event("user_signed_up", "server", "web"), - event("share_link_created", "server", "server"), - event("recording_completed", "client", "desktop"), + event("share_link_created", "server", "server", { events: 2 }), + event("recording_completed", "client", "desktop", { + recording_status: "success", + }), event("guest_checkout_started", "server", "web", { plan_id: "price_pro_annual", quantity: 1, @@ -1164,7 +1437,7 @@ test("typed endpoint assertions require exact public response semantics", () => ["page_engagement", 1, 1, 0, 0], ["identity_linked", 2, 2, 2, 1], ["user_signed_up", 1, 1, 1, 1], - ["share_link_created", 1, 1, 1, 1], + ["share_link_created", 2, 1, 1, 1], ["recording_completed", 1, 1, 1, 1], ["guest_checkout_started", 2, 2, 0, 0], ["checkout_started", 3, 1, 1, 1], @@ -1190,6 +1463,15 @@ test("typed endpoint assertions require exact public response semantics", () => visit_duration_ms: 5_000, engaged_ms: 15_000, }), + product_traffic_totals: row({ + visitors: 3, + visits: 3, + pageviews: 3, + views_per_visit: 1, + bounce_rate: 66.67, + visit_duration_ms: 5_000, + engaged_ms: 15_000, + }), product_traffic_pages: row({ pathname, visitors: 3, @@ -1224,6 +1506,46 @@ test("typed endpoint assertions require exact public response semantics", () => }, ], }, + product_attribution: { + data: [ + { + attribution_model: "first", + source: "first-touch", + medium: "first", + campaign: "first-campaign", + visitors: 3, + visits: 3, + pageviews: 3, + }, + { + attribution_model: "last", + source: "last-touch", + medium: "last", + campaign: "last-campaign", + visitors: 3, + visits: 3, + pageviews: 3, + }, + { + attribution_model: "session", + source: "google", + medium: "cpc", + campaign: "synthetic-campaign", + visitors: 2, + visits: 2, + pageviews: 2, + }, + { + attribution_model: "session", + source: "synthetic-partner", + medium: "referral", + campaign: "", + visitors: 1, + visits: 1, + pageviews: 1, + }, + ], + }, product_traffic_countries: row({ country: "US", visitors: 3, @@ -1295,12 +1617,31 @@ test("typed endpoint assertions require exact public response semantics", () => }), ), }, + product_experiment_outcomes: { + data: [ + ["signup", 0], + ["share_created", 1], + ["paid_purchase", 0], + ].map(([outcomeName, convertedActors]) => ({ + experiment_id: "synthetic-checkout-copy", + assignment_version: "v1", + variant: "treatment", + platform: "web", + app_version: appVersion, + outcome_name: outcomeName, + exposed_actors: 1, + converted_actors: convertedActors, + conversion_rate: convertedActors * 100, + })), + }, product_analytics_freshness: row({ latest_received_hour: "2026-07-31 10:00:00", product_calculated_at: "2026-07-31 10:01:00", traffic_calculated_at: "2026-07-31 10:01:00", retention_calculated_at: "2026-07-31 10:01:00", identity_calculated_at: "2026-07-31 10:01:00", + attribution_calculated_at: "2026-07-31 10:01:00", + experiment_calculated_at: "2026-07-31 10:01:00", }), }; const input = { appVersion, date, hostname, pathname, payloads }; @@ -1325,7 +1666,7 @@ test("staging performance covers every typed decision endpoint", () => { endDate: "2026-07-31", deploymentId: "deployment-1", }); - assert.equal(queries.length, 12); + assert.equal(queries.length, 15); assert.equal(new Set(queries.map(({ name }) => name)).size, queries.length); assert.ok( queries.every( @@ -1336,12 +1677,24 @@ test("staging performance covers every typed decision endpoint", () => { startDate: "2026-07-01", endDate: "2026-07-31", deploymentId: "deployment-0", - includeIdentityFunnel: false, + excludedEndpointNames: [ + "product_traffic_totals", + "product_attribution", + "product_identity_funnel", + "product_experiment_outcomes", + ], }); assert.equal(retainedQueries.length, 11); - assert.equal( - retainedQueries.some(({ name }) => name === "product_identity_funnel"), - false, + assert.ok( + retainedQueries.every( + ({ name }) => + !{ + product_traffic_totals: true, + product_attribution: true, + product_identity_funnel: true, + product_experiment_outcomes: true, + }[name], + ), ); assert.deepEqual( queries.find(({ name }) => name === "product_creator_activity")?.parameters, @@ -1435,6 +1788,66 @@ test("latency budgets require both absolute and measured-baseline gates", () => ); }); +test("Copy performance budgets gate visibility and pipeline regressions", () => { + const baseline = { + pipelineWallClockMs: 80_000, + visibility: latencySummary([5_000, 10_000, 20_000]), + }; + assert.deepEqual( + evaluateCopyPerformanceBudget({ + absolutePipelineMs: 600_000, + absoluteVisibilityP95Ms: 120_000, + baseline, + measured: { + pipelineWallClockMs: 100_000, + visibility: latencySummary([10_000, 20_000, 30_000]), + }, + regressionFactor: 2, + regressionFloorMs: 30_000, + }), + { + mode: "baseline_comparison", + absolutePipelineMs: 600_000, + absoluteVisibilityP95Ms: 120_000, + regressionFactor: 2, + regressionFloorMs: 30_000, + pipelineRegressionLimitMs: 160_000, + visibilityRegressionLimitMs: 50_000, + pipelineRegressionRatio: 1.25, + visibilityRegressionRatio: 1.5, + passed: true, + }, + ); + assert.equal( + evaluateCopyPerformanceBudget({ + absolutePipelineMs: 600_000, + absoluteVisibilityP95Ms: 120_000, + baseline, + measured: { + pipelineWallClockMs: 200_000, + visibility: latencySummary([60_000]), + }, + regressionFactor: 2, + regressionFloorMs: 30_000, + }).passed, + false, + ); + assert.equal( + evaluateCopyPerformanceBudget({ + absolutePipelineMs: 600_000, + absoluteVisibilityP95Ms: 120_000, + baseline: null, + measured: { + pipelineWallClockMs: 80_000, + visibility: latencySummary([20_000]), + }, + regressionFactor: 2, + regressionFloorMs: 30_000, + }).mode, + "baseline_capture", + ); +}); + test("CI assertion normalization proves decision deduplication and conflict quarantine", () => { const assertions = normalizeCiAssertions({ data: [ @@ -1466,6 +1879,8 @@ test("copy assertion normalization exposes every marker independently", () => { activation_markers: "1", retention_markers: "1", identity_markers: "1", + attribution_markers: "1", + experiment_markers: "1", health_markers: "1", }, ], @@ -1477,6 +1892,8 @@ test("copy assertion normalization exposes every marker independently", () => { activationMarkers: 1, retentionMarkers: 1, identityMarkers: 1, + attributionMarkers: 1, + experimentMarkers: 1, healthMarkers: 1, }, ); @@ -1503,9 +1920,17 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.match( /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 14, + 16, ); assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); + assert.ok( + workflow.indexOf("Refuse a preview bound outside Tinybird staging") < + workflow.indexOf("Record Tinybird deployment boundary"), + ); + assert.match( + workflow, + /Refuse a preview bound outside Tinybird staging[\s\S]*CAP_ANALYTICS_STAGING_TEST_SECRET[\s\S]*staging-ci\.js attest-preview/, + ); assert.ok( workflow.indexOf( "Prove promoted delivery, business values, and decision deduplication", @@ -1650,26 +2075,108 @@ test("the analytics workflow is statically restricted to staging", () => { workflow, /Upload redacted staging evidence\n {8}if: always\(\) && steps\.cleanup\.outputs\.required == 'true'/, ); + assert.ok( + workflow.indexOf("Upload the immutable pre-create recovery boundary") < + workflow.indexOf("Create isolated Tinybird staging deployment"), + ); + assert.ok( + workflow.indexOf("Upload the immutable pre-ingestion recovery checkpoint") < + workflow.indexOf("Seed bounded duplicate and conflict probes"), + ); + assert.ok( + workflow.indexOf("Upload the immutable pre-promotion recovery checkpoint") < + workflow.indexOf("Promote the verified staging deployment"), + ); + assert.match( + workflow, + /recover-staging:[\s\S]*needs: deploy-staging[\s\S]*if: always\(\) && needs\.deploy-staging\.result != 'success'/, + ); + assert.match(workflow, /permissions:[\s\S]*actions: read/); + assert.match( + workflow, + /actions\/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093/, + ); + assert.doesNotMatch(workflow, /actions\/(?:upload|download)-artifact@v\d/); + assert.match(workflow, /staging-ci\.js recover/); }); -test("the seed persists cleanup state and partial evidence before ingestion", () => { +test("the preview mutation route independently enforces Tinybird staging", () => { + const route = fs.readFileSync( + new URL( + "../../../apps/web/app/api/analytics/staging-test/route.ts", + import.meta.url, + ), + "utf8", + ); + assert.match( + route, + /const TINYBIRD_STAGING_ORIGIN = "https:\/\/api\.us-east\.aws\.tinybird\.co"/, + ); + assert.match( + route, + /const TINYBIRD_STAGING_WORKSPACE_ID =\s*"37b8fef9-817f-4c3c-b21f-218c36a6077d"/, + ); + assert.match( + route, + /const authorize = [\s\S]*!configurationAttestation\(runId\)/, + ); + assert.match( + route, + /const STAGING_DATABASE_FINGERPRINT =\s*"fff37a9b160f31bfb82b8c5585829b8ee08f70b3645169dca6e7cb29033a039a"/, + ); + assert.match( + route, + /databaseSchema: Schema\.Literal\("0042_lying_sharon_ventura"\)/, + ); + assert.match(route, /HttpApiEndpoint\.post\("health"/); + assert.match(route, /const scopedDatabaseHealth = async/); + assert.match(route, /FROM information_schema\.STATISTICS/); + assert.match(route, /product_analytics_ingestion_leases:fencingToken/); + assert.match(route, /product_analytics_refresh_leases:name/); + assert.match(route, /product_analytics_outbox:delivery_idx:1:3:createdAt/); + assert.match(route, /\/api\/analytics\/staging-test\/cleanup-database/); + + const runner = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const serverProbe = runner.slice( + runner.indexOf("const probeDurableServerPath = async () => {"), + runner.indexOf("const seed = async () => {"), + ); + assert.match(serverProbe, /\/api\/analytics\/staging-test\/health/); + assert.match(serverProbe, /Number\(outboxHealth\.activeEvents\) !== 0/); + assert.match(serverProbe, /Number\(outboxHealth\.deadLetterEvents\) !== 1/); + assert.match(serverProbe, /durableOutboxHealthPassed: true/); +}); + +test("the seed checkpoint is persisted before ingestion", () => { const source = fs.readFileSync( new URL("../staging-ci.js", import.meta.url), "utf8", ); + const prepareSource = source.slice( + source.indexOf("const prepareSeed = async () => {"), + source.indexOf("const seed = async () => {"), + ); const seedSource = source.slice( source.indexOf("const seed = async () => {"), source.indexOf("const waitForCopyVisibility"), ); - const stateWrite = seedSource.indexOf("writeJson(statePath, state, 0o600)"); - const artifactWrite = seedSource.indexOf("writeJson(artifactPath, artifact)"); + const stateWrite = prepareSource.indexOf( + "writeJson(statePath, state, 0o600)", + ); + const artifactWrite = prepareSource.indexOf( + "writeJson(artifactPath, artifact)", + ); const firstDelivery = seedSource.indexOf("const concurrentDeliveries"); assert.ok(stateWrite >= 0); assert.ok(artifactWrite > stateWrite); - assert.ok(firstDelivery > artifactWrite); - assert.match(seedSource, /assertions: \{ seedAccepted: false \}/); - assert.match(seedSource, /rowsPlanned: fixture\.rows\.length/); - assert.match(seedSource, /rowsAttempted: 0/); + assert.ok(firstDelivery >= 0); + assert.match(prepareSource, /recoveryPhase: "preseed"/); + assert.match(prepareSource, /assertions: \{ seedAccepted: false \}/); + assert.match(prepareSource, /rowsPlanned: fixture\.rows\.length/); + assert.match(prepareSource, /rowsAttempted: 0/); assert.ok( seedSource.indexOf("artifact.delivery.rowsAttempted += 1") < seedSource.indexOf("const result = await request"), @@ -1680,6 +2187,31 @@ test("the seed persists cleanup state and partial evidence before ingestion", () ); }); +test("event state reaches terminal success before canonical rebuilding", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const runCopiesSource = source.slice( + source.indexOf("const runCopies = async"), + source.indexOf("const verify = async"), + ); + const stateCopy = runCopiesSource.indexOf( + '"snapshot_product_event_id_states_v2",\n\t\t\t\t"snapshot_product_event_day_states_v2",', + ); + const stateWait = runCopiesSource.indexOf("await waitForCopyJobs"); + const canonicalCopy = runCopiesSource.indexOf( + 'pipes: ["snapshot_product_events_canonical_v1"]', + ); + assert.ok(stateCopy >= 0); + assert.ok(stateCopy < stateWait); + assert.ok(stateWait < canonicalCopy); + assert.match( + runCopiesSource, + /jobs: \[\.\.\.stateJobs, \.\.\.canonicalJobs, \.\.\.downstreamJobs\]/, + ); +}); + test("rollback drill proves old and restored deployment data planes", () => { const source = fs.readFileSync( new URL("../staging-ci.js", import.meta.url), @@ -1702,8 +2234,12 @@ test("rollback drill proves old and restored deployment data planes", () => { ); assert.match(drillSource, /previousLiveDeploymentId/); assert.match(drillSource, /assertDecisionEndpointSuiteReadable/); - assert.match(drillSource, /retainedIdentityFunnelAvailable/); - assert.match(drillSource, /decisionEndpointAvailable/); + assert.match(drillSource, /excludedRollbackEndpoints/); + assert.match(drillSource, /unavailableDecisionEndpoints/); + assert.match( + source, + /unavailableDecisionEndpoints[\s\S]*await decisionEndpointAvailable[\s\S]*filter\(\(\{ available \}\) => !available\)/, + ); assert.match(drillSource, /assertSyntheticEndpointDecisions/); assert.match(drillSource, /assertSyntheticBusinessDecisions/); assert.match(drillSource, /readAndAssertPhaseHealth/); @@ -1744,9 +2280,23 @@ test("performance compares the retained deployment and a larger synthetic volume assert.match(verifySource, /representativePerformancePassed/); assert.match(verifySource, /assertRepresentativeEndpointCoverage/); assert.match(verifySource, /representativeBudget/); - assert.match(verifySource, /new_endpoint_no_baseline/); - assert.match(verifySource, /retainedIdentityFunnelAvailable/); - assert.match(verifySource, /decisionEndpointAvailable/); + assert.match(verifySource, /absolute_only_no_independent_baseline/); + assert.match(verifySource, /round < 30/); + assert.match(verifySource, /excludedBaselineEndpoints/); + assert.match(verifySource, /unavailableDecisionEndpoints/); + assert.match( + verifySource, + /newEndpointsWithoutBaseline: excludedBaselineEndpoints/, + ); + assert.match(source, /LARGE_PERFORMANCE_EVENT_COUNT \?\? 100_000/); + assert.match(source, /PERFORMANCE_DAY_SPAN \?\? 30/); + assert.match(source, /LARGE_PERFORMANCE_DAY_SPAN \?\? 80/); + assert.match(source, /COLLECTOR_PERFORMANCE_REQUESTS \?\? 20/); + assert.match(source, /COLLECTOR_PERFORMANCE_CONCURRENCY \?\? 4/); + assert.match(source, /LARGE_PERFORMANCE_DIMENSION_BUCKETS \?\? 64/); + assert.match(source, /COPY_PIPELINE_WALL_CLOCK_BUDGET_MS \?\? 600_000/); + assert.match(source, /COPY_VISIBILITY_P95_BUDGET_MS \?\? 120_000/); + assert.match(source, /providerResourceMetrics/); }); test("synthetic deletion targets the deployment used for ingestion", () => { From 409d18a97eebe90fd0c893e583ce635c47efe021 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:41:56 +0100 Subject: [PATCH 085/110] fix: use a valid analytics workflow path glob --- .github/workflows/analytics.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index df9681e4e57..b2b947c4ce8 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -37,7 +37,7 @@ on: - "apps/web/app/api/settings/billing/**" - "apps/web/app/api/webhooks/stripe/route.ts" - "apps/web/app/mobile/checkout/**" - - "apps/web/app/api/mobile/[...route]/route.ts" + - "apps/web/app/api/mobile/**" - "apps/web/app/utils/analytics.ts" - "apps/web/app/utils/product-analytics.ts" - "apps/web/lib/analytics/**" From 340e03e48147983db06377fa6157520fc513a14c Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:43:00 +0100 Subject: [PATCH 086/110] fix: keep analytics workflow runtime portable --- apps/web/workflows/refresh-product-analytics.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/apps/web/workflows/refresh-product-analytics.ts b/apps/web/workflows/refresh-product-analytics.ts index 28da7e3c109..1541e9d272e 100644 --- a/apps/web/workflows/refresh-product-analytics.ts +++ b/apps/web/workflows/refresh-product-analytics.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { serverEnv } from "@cap/env"; import { FatalError } from "workflow"; import { @@ -154,10 +153,7 @@ export async function refreshProductAnalyticsWorkflow(input: { const sourceCutoff = new Date(input.scheduledAt).toISOString(); const lease = await acquireProductAnalyticsRefreshStep(sourceCutoff); if (!lease) return { refreshed: false as const, reason: "lease_unavailable" }; - const copyRunId = `refresh_${createHash("sha256") - .update(sourceCutoff) - .digest("hex") - .slice(0, 24)}`; + const copyRunId = `refresh_${sourceCutoff.replace(/\D/g, "")}`; try { const jobs = []; for (const [pipe, marker] of REFRESH_COPIES) { From c707ffcb1bf28505b4b776776fd44e55ceb6bdae Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 13:49:40 +0100 Subject: [PATCH 087/110] fix: prevent Tinybird resource node collisions --- scripts/analytics/datafiles.js | 3 +++ scripts/analytics/tests/datafiles.test.js | 17 +++++++++++++++++ .../pipes/product_events_canonical_current.pipe | 2 +- scripts/analytics/tooling.js | 11 +++++++++++ 4 files changed, 32 insertions(+), 1 deletion(-) diff --git a/scripts/analytics/datafiles.js b/scripts/analytics/datafiles.js index 86d208aadc3..8a346b183cd 100644 --- a/scripts/analytics/datafiles.js +++ b/scripts/analytics/datafiles.js @@ -89,6 +89,9 @@ const parsePipe = (filePath) => { return { name: path.basename(filePath, ".pipe"), filePath, + nodeNames: [...contents.matchAll(/^NODE\s+(\S+)\s*$/gim)].map( + ([, name]) => name, + ), type: (readDirective(contents, "TYPE") ?? "generic").toLowerCase(), targetDatasource: readDirective(contents, "DATASOURCE") ?? diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index 8276d1dabc8..8f5d69462c8 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -132,6 +132,23 @@ test("product datasource matches the runtime event contract", () => { ); }); +test("Tinybird node names do not conflict with resource names", () => { + const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); + const resourceNames = new Set([ + ...project.datasources.map((datasource) => datasource.name), + ...project.pipes.map((pipe) => pipe.name), + ]); + for (const pipe of project.pipes) { + for (const nodeName of pipe.nodeNames) { + assert.equal( + resourceNames.has(nodeName), + false, + `${pipe.name} node ${nodeName} conflicts with a resource name`, + ); + } + } +}); + test("existing viewer resources remain in the Tinybird project", () => { const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); const names = project.datasources.map(({ name }) => name); diff --git a/scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe b/scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe index 9fb3ec0ec74..d5bed993f34 100644 --- a/scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_canonical_current.pipe @@ -3,7 +3,7 @@ DESCRIPTION > TOKEN product_events_copy_runner READ -NODE product_events_canonical_current +NODE canonical_current SQL > % SELECT diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index a802f921a30..cc98cbcf510 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -458,10 +458,21 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { const datasourceNames = new Set( project.datasources.map((datasource) => datasource.name), ); + const resourceNames = new Set([ + ...datasourceNames, + ...project.pipes.map((pipe) => pipe.name), + ]); for (const pipe of project.pipes) { if (datasourceNames.has(pipe.name)) { issues.push(`Tinybird resource name ${pipe.name} is not unique`); } + for (const nodeName of pipe.nodeNames) { + if (resourceNames.has(nodeName)) { + issues.push( + `Tinybird node ${nodeName} in ${pipe.name} conflicts with a resource name`, + ); + } + } } for (const name of [ "analytics_events", From 23e6ae9d5cebeb450d0a30ec99f25d0ffd6e3df4 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:20:43 +0100 Subject: [PATCH 088/110] fix: harden analytics event admission --- .../src-tauri/src/product_analytics.rs | 150 +++++++++++++++++- .../product-analytics-client.test.ts | 56 ++++++- .../src/analytics/product-analytics-client.ts | 67 +++++++- .../e2e/product-analytics-mysql-e2e.test.ts | 131 +++++++++++++-- .../unit/product-analytics-queue.test.ts | 20 +++ apps/web/app/api/events/route.ts | 33 +++- apps/web/app/utils/product-analytics.ts | 53 ++++++- packages/analytics/src/index.test.ts | 7 +- packages/analytics/src/index.ts | 48 +++++- .../web-backend/src/ProductAnalytics/index.ts | 47 +++--- 10 files changed, 550 insertions(+), 62 deletions(-) diff --git a/apps/desktop/src-tauri/src/product_analytics.rs b/apps/desktop/src-tauri/src/product_analytics.rs index 4817bc84174..2c88c2bbb9e 100644 --- a/apps/desktop/src-tauri/src/product_analytics.rs +++ b/apps/desktop/src-tauri/src/product_analytics.rs @@ -424,6 +424,20 @@ struct DeliveryError { status: Option, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ProductEventAdmissionResponse { + accepted: usize, + accepted_event_ids: Option>, + rejected_event_ids: Option>, +} + +#[derive(Debug, Eq, PartialEq)] +struct ProductEventAdmission { + accepted_event_ids: HashSet, + rejected_event_ids: HashSet, +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ProductEventBatchKind { LossReport, @@ -469,9 +483,12 @@ struct ProductEventOutbox { async fn send_product_batch_once( app: &AppHandle, events: &[ProductEvent], -) -> Result<(), DeliveryError> { +) -> Result { if !live_telemetry_enabled(app) { - return Ok(()); + return Ok(ProductEventAdmission { + accepted_event_ids: events.iter().map(|event| event.event_id.clone()).collect(), + rejected_event_ids: HashSet::new(), + }); } let auth_token = product_auth_token(app); @@ -493,7 +510,18 @@ async fn send_product_batch_once( })?; if response.status().is_success() { - Ok(()) + let status = response.status().as_u16(); + let payload = response + .json::() + .await + .map_err(|_| DeliveryError { + retryable: true, + status: Some(status), + })?; + parse_product_event_admission(events, payload).ok_or(DeliveryError { + retryable: true, + status: Some(status), + }) } else { let status = response.status().as_u16(); Err(DeliveryError { @@ -503,8 +531,42 @@ async fn send_product_batch_once( } } +fn parse_product_event_admission( + events: &[ProductEvent], + response: ProductEventAdmissionResponse, +) -> Option { + let requested_event_ids = events + .iter() + .map(|event| event.event_id.clone()) + .collect::>(); + match (response.accepted_event_ids, response.rejected_event_ids) { + (Some(accepted_event_ids), Some(rejected_event_ids)) => { + let accepted_event_ids = accepted_event_ids.into_iter().collect::>(); + let rejected_event_ids = rejected_event_ids.into_iter().collect::>(); + let accounted_event_ids = accepted_event_ids + .union(&rejected_event_ids) + .cloned() + .collect::>(); + (response.accepted == accepted_event_ids.len() + && accepted_event_ids.is_disjoint(&rejected_event_ids) + && accounted_event_ids == requested_event_ids) + .then_some(ProductEventAdmission { + accepted_event_ids, + rejected_event_ids, + }) + } + (None, None) if response.accepted == requested_event_ids.len() => { + Some(ProductEventAdmission { + accepted_event_ids: requested_event_ids, + rejected_event_ids: HashSet::new(), + }) + } + _ => None, + } +} + fn should_retry_product_status(status: u16) -> bool { - status == 429 || status >= 500 + status == 404 || status == 410 || status == 429 || status >= 500 } fn decode_outbox_encryption_key(encoded: &str) -> Result<[u8; 32], String> { @@ -1416,8 +1478,39 @@ async fn run_product_event_worker(app: AppHandle, mut receiver: mpsc::Receiver<( } match send_product_batch_once(&app, &events).await { - Ok(()) => { - remove_delivered_events(&app, &events); + Ok(admission) => { + let accepted_events = events + .iter() + .filter(|event| admission.accepted_event_ids.contains(&event.event_id)) + .cloned() + .collect::>(); + let rejected_events = events + .iter() + .filter(|event| admission.rejected_event_ids.contains(&event.event_id)) + .cloned() + .collect::>(); + remove_delivered_events(&app, &accepted_events); + if !rejected_events.is_empty() { + let error = DeliveryError { + retryable: false, + status: Some(409), + }; + match batch_kind { + ProductEventBatchKind::Pending => { + move_to_dead_letters(&app, &rejected_events, &error); + } + ProductEventBatchKind::DeadLetterRetry => { + record_dead_letter_retry_rejection(&app, &rejected_events, &error); + } + ProductEventBatchKind::LossReport => { + warn!( + event_count = rejected_events.len(), + "Product analytics loss reports remain encrypted after selective rejection" + ); + break; + } + } + } retry_attempt = 0; } Err(error) if error.retryable => { @@ -1806,6 +1899,13 @@ mod tests { } } + fn product_event_with_id(event_id: &str) -> ProductEvent { + let mut event = product_event(&event_data(recording_started()), "install-id".to_string()) + .expect("recording event should be registered"); + event.event_id = event_id.to_string(); + event + } + #[test] fn core_product_event_catalog_is_intentionally_small() { let included = [ @@ -2385,7 +2485,45 @@ mod tests { fn product_status_retry_policy_matches_the_browser() { assert!(!should_retry_product_status(400)); assert!(!should_retry_product_status(401)); + assert!(should_retry_product_status(404)); + assert!(should_retry_product_status(410)); assert!(should_retry_product_status(429)); assert!(should_retry_product_status(503)); } + + #[test] + fn product_admission_is_selective_and_fail_closed() { + let first = product_event_with_id("first"); + let second = product_event_with_id("second"); + let events = [first, second]; + let admission = parse_product_event_admission( + &events, + ProductEventAdmissionResponse { + accepted: 1, + accepted_event_ids: Some(vec!["first".to_string()]), + rejected_event_ids: Some(vec!["second".to_string()]), + }, + ) + .expect("selective admission should be valid"); + assert_eq!( + admission.accepted_event_ids, + HashSet::from(["first".to_string()]) + ); + assert_eq!( + admission.rejected_event_ids, + HashSet::from(["second".to_string()]) + ); + + assert!( + parse_product_event_admission( + &events, + ProductEventAdmissionResponse { + accepted: 1, + accepted_event_ids: Some(vec!["first".to_string()]), + rejected_event_ids: Some(Vec::new()), + }, + ) + .is_none() + ); + } } diff --git a/apps/mobile/src/analytics/product-analytics-client.test.ts b/apps/mobile/src/analytics/product-analytics-client.test.ts index 3eea70dae71..24011d7536c 100644 --- a/apps/mobile/src/analytics/product-analytics-client.test.ts +++ b/apps/mobile/src/analytics/product-analytics-client.test.ts @@ -126,10 +126,42 @@ describe("MobileProductAnalyticsClient", () => { ]); }); - it.each([429, 500])("retries a retryable %s response", async (status) => { + it.each([404, 410, 429, 500])( + "retries a retryable %s response", + async (status) => { + const harness = createHarness({ + fetchImpl: vi.fn( + async () => new Response(null, { status }), + ), + }); + await harness.client.configure({ + apiKey: null, + credentialScope: "scope_1", + baseUrl: "https://cap.so", + }); + const eventId = await harness.client.track("user_signed_in"); + await harness.client.configure({ + apiKey: "mobile_key", + credentialScope: "scope_1", + baseUrl: "https://cap.so", + }); + const snapshot = await harness.client.snapshot(); + expect(snapshot.pending[0]?.event.eventId).toBe(eventId); + expect(snapshot.pending[0]?.attempts).toBe(1); + expect(snapshot.delivery.retried).toBe(1); + }, + ); + + it("finalizes accepted rows without losing a rejected batch neighbor", async () => { + let acceptedEventId = ""; + let rejectedEventId = ""; const harness = createHarness({ - fetchImpl: vi.fn( - async () => new Response(null, { status }), + fetchImpl: vi.fn(async () => + Response.json({ + accepted: 1, + acceptedEventIds: [acceptedEventId], + rejectedEventIds: [rejectedEventId], + }), ), }); await harness.client.configure({ @@ -137,16 +169,26 @@ describe("MobileProductAnalyticsClient", () => { credentialScope: "scope_1", baseUrl: "https://cap.so", }); - const eventId = await harness.client.track("user_signed_in"); + acceptedEventId = await harness.client.track("user_signed_in"); + rejectedEventId = await harness.client.track("user_signed_in"); await harness.client.configure({ apiKey: "mobile_key", credentialScope: "scope_1", baseUrl: "https://cap.so", }); const snapshot = await harness.client.snapshot(); - expect(snapshot.pending[0]?.event.eventId).toBe(eventId); - expect(snapshot.pending[0]?.attempts).toBe(1); - expect(snapshot.delivery.retried).toBe(1); + expect(snapshot.pending).toEqual([]); + expect(snapshot.delivery).toMatchObject({ + accepted: 1, + contract_rejected: 1, + }); + expect(snapshot.deadLetters).toEqual([ + expect.objectContaining({ + eventId: rejectedEventId, + reason: "contract", + status: 409, + }), + ]); }); it("aborts and retries a request that exceeds the timeout", async () => { diff --git a/apps/mobile/src/analytics/product-analytics-client.ts b/apps/mobile/src/analytics/product-analytics-client.ts index 7cba5f215cf..1a5eed7b93d 100644 --- a/apps/mobile/src/analytics/product-analytics-client.ts +++ b/apps/mobile/src/analytics/product-analytics-client.ts @@ -504,21 +504,70 @@ export class MobileProductAnalyticsClient { } if (response.ok) { const payload = (await response.json().catch(() => null)) as unknown; - if (!isRecord(payload) || payload.accepted !== batch.length) { + if (!isRecord(payload)) { await this.#retry(batch); return; } - const ids = new Set(batch.map((entry) => entry.event.eventId)); + const requestedIds = new Set(batch.map((entry) => entry.event.eventId)); + const rawAcceptedEventIds = Array.isArray(payload.acceptedEventIds) + ? payload.acceptedEventIds + : undefined; + const rawRejectedEventIds = Array.isArray(payload.rejectedEventIds) + ? payload.rejectedEventIds + : undefined; + const hasSelectiveResult = Boolean( + rawAcceptedEventIds && rawRejectedEventIds, + ); + const acceptedEventIds = hasSelectiveResult + ? (rawAcceptedEventIds ?? []).filter( + (eventId: unknown): eventId is string => + typeof eventId === "string" && requestedIds.has(eventId), + ) + : payload.accepted === batch.length + ? [...requestedIds] + : []; + const rejectedEventIds = hasSelectiveResult + ? (rawRejectedEventIds ?? []).filter( + (eventId: unknown): eventId is string => + typeof eventId === "string" && requestedIds.has(eventId), + ) + : []; + if ( + payload.accepted !== acceptedEventIds.length || + acceptedEventIds.length + rejectedEventIds.length !== + requestedIds.size || + new Set([...acceptedEventIds, ...rejectedEventIds]).size !== + requestedIds.size + ) { + await this.#retry(batch); + return; + } + const ids = new Set([...acceptedEventIds, ...rejectedEventIds]); state.pending = state.pending.filter( (entry) => !ids.has(entry.event.eventId), ); - state.delivery.accepted += batch.length; + state.delivery.accepted += acceptedEventIds.length; + state.delivery.contract_rejected += rejectedEventIds.length; for (const entry of batch) { - this.#recordFinalizedEvent( - entry.event, - entry.credentialScope, - "accepted", - ); + if (acceptedEventIds.includes(entry.event.eventId)) { + this.#recordFinalizedEvent( + entry.event, + entry.credentialScope, + "accepted", + ); + } else { + this.#deadLetter( + entry.event, + "contract", + 409, + entry.credentialScope, + ); + this.#recordFinalizedEvent( + entry.event, + entry.credentialScope, + "dead_letter", + ); + } } await this.#persist(); continue; @@ -526,6 +575,8 @@ export class MobileProductAnalyticsClient { if ( response.status === 401 || response.status === 403 || + response.status === 404 || + response.status === 410 || response.status === 429 || response.status >= 500 ) { diff --git a/apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts b/apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts index 3c1cd69d755..36344351a2a 100644 --- a/apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts +++ b/apps/web/__tests__/e2e/product-analytics-mysql-e2e.test.ts @@ -344,7 +344,7 @@ analyticsMysqlE2e("product analytics MySQL concurrency", () => { expect(ingestionRows).toHaveLength(0); }); - it("forwards exact retries and rejects event ID payload conflicts", async () => { + it("isolates payload conflicts without poisoning valid batch neighbors", async () => { const { createProductEventRows, productAnalyticsEventIdHash } = await import("@cap/analytics"); const appendRequests: string[] = []; @@ -388,12 +388,25 @@ analyticsMysqlE2e("product analytics MySQL concurrency", () => { pathname: "/different", payload_hash: "f".repeat(32), }; - const conflict = await appendCollectorRowsEither([conflicting]); - expect(conflict._tag).toBe("Left"); - if (conflict._tag === "Left") { - expect(conflict.left).toMatchObject({ retryable: false, status: 409 }); - } - expect(appendRequests).toHaveLength(2); + const validNeighbor = { + ...original, + event_id: `${original.event_id}_neighbor`, + payload_hash: "e".repeat(32), + }; + const admission = await appendCollectorRowsEither([ + conflicting, + validNeighbor, + ]); + expect(admission).toMatchObject({ + _tag: "Right", + right: { + acceptedEventIds: [validNeighbor.event_id], + rejectedEventIds: [original.event_id], + }, + }); + expect(appendRequests).toHaveLength(3); + expect(appendRequests[2]).toContain(validNeighbor.event_id); + expect(appendRequests[2]).not.toContain(conflicting.payload_hash); const verifier = await mysql.createConnection(databaseUrl); const [receipts] = await verifier.query< Array< @@ -411,8 +424,79 @@ analyticsMysqlE2e("product analytics MySQL concurrency", () => { payloadHash: original.payload_hash, }); await verifier.query( - "DELETE FROM product_analytics_event_receipts WHERE eventIdHash = ?", - [productAnalyticsEventIdHash(original.event_id)], + "DELETE FROM product_analytics_event_receipts WHERE eventIdHash IN (?, ?)", + [ + productAnalyticsEventIdHash(original.event_id), + productAnalyticsEventIdHash(validNeighbor.event_id), + ], + ); + await verifier.end(); + }); + + it("keeps untrusted client IDs disjoint from authoritative server IDs", async () => { + const { createProductEventRows, productAnalyticsEventIdHash } = + await import("@cap/analytics"); + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response(null, { status: 202 })), + ); + const rawEventId = "share_link_created:known-video-id"; + const [clientRow] = createProductEventRows( + [ + { + anonymousId: "preclaim-anonymous", + eventId: rawEventId, + eventName: "page_view", + occurredAt: "2026-07-31T12:00:00.000Z", + platform: "web", + properties: { + hostname: "cap.test", + is_session_entry: true, + session_started_at: "2026-07-31T12:00:00.000Z", + }, + }, + ], + { + hostname: "cap.test", + receivedAt: "2026-07-31T12:00:01.000Z", + source: "client", + }, + ); + const [serverRow] = createProductEventRows( + [ + { + anonymousId: "user:preclaim-user", + eventId: rawEventId, + eventName: "share_link_created", + occurredAt: "2026-07-31T12:00:02.000Z", + platform: "server", + properties: { + asset_type: "recording", + recording_mode: "instant", + }, + }, + ], + { + organizationId: "preclaim-org", + receivedAt: "2026-07-31T12:00:03.000Z", + source: "server", + userId: "preclaim-user", + }, + ); + if (!clientRow || !serverRow) throw new Error("Expected preclaim fixtures"); + expect(clientRow.event_id).not.toBe(serverRow.event_id); + const clientAdmission = await appendCollectorRows([clientRow]); + const serverAdmission = await appendCollectorRows([serverRow]); + expect(clientAdmission.acceptedEventIds).toEqual([clientRow.event_id]); + expect(serverAdmission.acceptedEventIds).toEqual([serverRow.event_id]); + + const verifier = await mysql.createConnection(databaseUrl); + await verifier.query( + "DELETE FROM product_analytics_event_receipts WHERE eventIdHash IN (?, ?)", + [ + productAnalyticsEventIdHash(clientRow.event_id), + productAnalyticsEventIdHash(serverRow.event_id), + ], ); await verifier.end(); }); @@ -449,6 +533,7 @@ analyticsMysqlE2e("product analytics MySQL concurrency", () => { }); it("does not spread a deleted user's tombstone to a live organization", async () => { + const { createProductEventRows } = await import("@cap/analytics"); const { db } = await import("@cap/database"); const { productAnalyticsIdentityHash } = await import("@cap/web-backend"); const { persistProductAnalyticsEvent } = await import( @@ -469,6 +554,34 @@ analyticsMysqlE2e("product analytics MySQL concurrency", () => { }), ); expect(suppressed.status).toBe("suppressed"); + const [blockedCollectorRow] = createProductEventRows( + [ + { + anonymousId: "deleted-user:new-alias", + eventId: "deleted-user:late-client-event", + eventName: "page_view", + occurredAt: "2026-07-31T12:00:00.000Z", + platform: "web", + properties: { + hostname: "cap.test", + is_session_entry: true, + session_started_at: "2026-07-31T12:00:00.000Z", + }, + }, + ], + { + hostname: "cap.test", + receivedAt: "2026-07-31T12:00:01.000Z", + source: "client", + userId: "deleted-user", + }, + ); + if (!blockedCollectorRow) + throw new Error("Expected a blocked collector row"); + await expect(appendCollectorRows([blockedCollectorRow])).resolves.toEqual({ + acceptedEventIds: [], + rejectedEventIds: [blockedCollectorRow.event_id], + }); const accepted = await db().transaction((tx) => persistProductAnalyticsEvent(tx, { eventId: "live-user:signup", diff --git a/apps/web/__tests__/unit/product-analytics-queue.test.ts b/apps/web/__tests__/unit/product-analytics-queue.test.ts index 14ba6284917..3e39984502e 100644 --- a/apps/web/__tests__/unit/product-analytics-queue.test.ts +++ b/apps/web/__tests__/unit/product-analytics-queue.test.ts @@ -436,7 +436,27 @@ describe("browser product analytics transport", () => { expect(fetchImpl.mock.calls[0]?.[1]).toMatchObject({ keepalive: true }); }); + it("returns per-event admission for a mixed conflict batch", async () => { + const fetchImpl = vi.fn().mockResolvedValue( + Response.json({ + accepted: 1, + acceptedEventIds: ["event-1"], + rejectedEventIds: ["event-2"], + }), + ); + await expect( + sendBrowserProductAnalytics([makeEvent(1), makeEvent(2)], "normal", { + fetchImpl, + }), + ).resolves.toEqual({ + acceptedEventIds: ["event-1"], + rejectedEventIds: ["event-2"], + }); + }); + it.each([ + [404, "retry"], + [410, "retry"], [429, "retry"], [503, "retry"], [400, "drop"], diff --git a/apps/web/app/api/events/route.ts b/apps/web/app/api/events/route.ts index f3db66da80f..f32a4e77c1d 100644 --- a/apps/web/app/api/events/route.ts +++ b/apps/web/app/api/events/route.ts @@ -75,7 +75,13 @@ class Api extends HttpApi.make("ProductAnalyticsApi").add( ), }), ) - .addSuccess(Schema.Struct({ accepted: Schema.Number })) + .addSuccess( + Schema.Struct({ + accepted: Schema.Number, + acceptedEventIds: Schema.Array(Schema.String), + rejectedEventIds: Schema.Array(Schema.String), + }), + ) .addError(HttpApiError.BadRequest) .addError(HttpApiError.ServiceUnavailable) .addError(RateLimited), @@ -226,7 +232,7 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( syntheticRunId, }); - yield* analytics + const admission = yield* analytics .appendWithIdentityFence(rows) .pipe( Effect.catchAll((error) => @@ -249,7 +255,28 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( }); } - return { accepted: rows.length }; + const clientEventIds = new Map( + rows.map((row, index) => [row.event_id, events[index]?.eventId]), + ); + const acceptedEventIds = [ + ...new Set( + admission.acceptedEventIds.flatMap( + (eventId) => clientEventIds.get(eventId) ?? [], + ), + ), + ]; + const rejectedEventIds = [ + ...new Set( + admission.rejectedEventIds.flatMap( + (eventId) => clientEventIds.get(eventId) ?? [], + ), + ), + ]; + return { + accepted: acceptedEventIds.length, + acceptedEventIds, + rejectedEventIds, + }; }), ); }), diff --git a/apps/web/app/utils/product-analytics.ts b/apps/web/app/utils/product-analytics.ts index f4405bf87c7..c8030a1330f 100644 --- a/apps/web/app/utils/product-analytics.ts +++ b/apps/web/app/utils/product-analytics.ts @@ -21,7 +21,14 @@ const REQUEST_TIMEOUT_MS = 3_000; const ANONYMOUS_ID_KEY = "cap_analytics_anonymous_id_v1"; const QUEUE_STORAGE_KEY = "cap_analytics_queue_v1"; -type TransportResult = "success" | "retry" | "drop"; +type TransportResult = + | "success" + | "retry" + | "drop" + | { + acceptedEventIds: string[]; + rejectedEventIds: string[]; + }; type TransportMode = "normal" | "unload"; type QueuedEvent = { event: ProductEventInput; attempts: number }; type QueueStorage = Pick; @@ -214,6 +221,14 @@ export class ProductAnalyticsQueue { this.persist(); return false; } + if (typeof result === "object") { + this.delivery.accepted += result.acceptedEventIds.length; + this.delivery.dropped += result.rejectedEventIds.length; + this.delivery.contract_rejected += result.rejectedEventIds.length; + this.persistedInFlight = []; + this.persist(); + return false; + } if (result === "drop") { this.delivery.dropped += batch.length; this.persistedInFlight = []; @@ -641,8 +656,40 @@ export const sendBrowserProductAnalytics = async ( keepalive: mode === "unload", signal: controller.signal, }).finally(() => clearTimeout(timeout)); - if (response.ok) return "success"; - return response.status === 429 || response.status >= 500 ? "retry" : "drop"; + if (response.ok) { + const payload = (await response.json().catch(() => null)) as unknown; + if (payload === null && response.status === 202) return "success"; + if ( + isRecord(payload) && + Array.isArray(payload.acceptedEventIds) && + Array.isArray(payload.rejectedEventIds) + ) { + const requestedIds = new Set(events.map((event) => event.eventId)); + const acceptedEventIds = payload.acceptedEventIds.filter( + (eventId): eventId is string => + typeof eventId === "string" && requestedIds.has(eventId), + ); + const rejectedEventIds = payload.rejectedEventIds.filter( + (eventId): eventId is string => + typeof eventId === "string" && requestedIds.has(eventId), + ); + if ( + acceptedEventIds.length + rejectedEventIds.length === + requestedIds.size && + new Set([...acceptedEventIds, ...rejectedEventIds]).size === + requestedIds.size + ) { + return { acceptedEventIds, rejectedEventIds }; + } + } + return "retry"; + } + return response.status === 404 || + response.status === 410 || + response.status === 429 || + response.status >= 500 + ? "retry" + : "drop"; } catch { return "retry"; } diff --git a/packages/analytics/src/index.test.ts b/packages/analytics/src/index.test.ts index c74c4ddeaf9..ec92b146363 100644 --- a/packages/analytics/src/index.test.ts +++ b/packages/analytics/src/index.test.ts @@ -424,7 +424,7 @@ describe("createProductEventRows", () => { }); }); - it("conflict-hashes every decision and exclusion dimension", () => { + it("keeps client retry hashes stable across mutable server enrichment", () => { const event = { eventId: "page-1", eventName: "page_view" as const, @@ -457,7 +457,8 @@ describe("createProductEventRows", () => { country: "US", }); - expect(external?.payload_hash).not.toBe(synthetic?.payload_hash); - expect(external?.payload_hash).not.toBe(differentCountry?.payload_hash); + expect(external?.payload_hash).toBe(synthetic?.payload_hash); + expect(external?.payload_hash).toBe(differentCountry?.payload_hash); + expect(external?.event_id).toMatch(/^client:[0-9a-f]{64}$/); }); }); diff --git a/packages/analytics/src/index.ts b/packages/analytics/src/index.ts index 9e68c8ed1ca..3500d62f7b5 100644 --- a/packages/analytics/src/index.ts +++ b/packages/analytics/src/index.ts @@ -89,6 +89,13 @@ export function productAnalyticsEventIdHash(eventId: string) { return bytesToHex(sha256(new TextEncoder().encode(`event\0${eventId}`))); } +export function canonicalClientProductEventId( + anonymousId: string, + eventId: string, +) { + return `client:${productAnalyticsEventIdHash(`${anonymousId}\0${eventId}`)}`; +} + export interface ProductEventInput { eventId: string; eventName: Name; @@ -419,8 +426,25 @@ export function createProductEventRows( context: ProductEventContext, ): ProductEventRow[] { return events.map((event) => { + const eventId = + context.source === "client" + ? canonicalClientProductEventId(event.anonymousId, event.eventId) + : event.eventId; + const properties = + context.source === "client" && + event.properties && + "page_view_id" in event.properties && + typeof event.properties.page_view_id === "string" + ? { + ...event.properties, + page_view_id: canonicalClientProductEventId( + event.anonymousId, + event.properties.page_view_id, + ), + } + : event.properties; const channel = normalizeAcquisitionChannel( - event.properties as ProductEventProperties | undefined, + properties as ProductEventProperties | undefined, event.referrer, context.hostname, ); @@ -447,11 +471,27 @@ export function createProductEventRows( channel, traffic_class: context.trafficClass ?? "external", synthetic_run_id: context.syntheticRunId ?? "", - properties: event.properties ? JSON.stringify(event.properties) : "{}", + properties: properties ? JSON.stringify(properties) : "{}", + }; + const immutableClientPayload = { + event_id: eventId, + event_name: payload.event_name, + schema_version: payload.schema_version, + source: payload.source, + platform: payload.platform, + occurred_at: payload.occurred_at, + anonymous_id: payload.anonymous_id, + session_id: payload.session_id, + app_version: payload.app_version, + pathname: payload.pathname, + referrer: payload.referrer, + properties: payload.properties, }; return { - event_id: event.eventId, - payload_hash: createProductEventPayloadHash(payload), + event_id: eventId, + payload_hash: createProductEventPayloadHash( + context.source === "client" ? immutableClientPayload : payload, + ), received_at: context.receivedAt, ...payload, }; diff --git a/packages/web-backend/src/ProductAnalytics/index.ts b/packages/web-backend/src/ProductAnalytics/index.ts index 9395b7c4986..80042e4cb73 100644 --- a/packages/web-backend/src/ProductAnalytics/index.ts +++ b/packages/web-backend/src/ProductAnalytics/index.ts @@ -232,7 +232,13 @@ export class ProductAnalytics extends Effect.Service()( }); const appendWithIdentityFence = ( rows: readonly ProductEventRow[], - ): Effect.Effect => { + ): Effect.Effect< + { + acceptedEventIds: string[]; + rejectedEventIds: string[]; + }, + ProductAnalyticsError | DatabaseError + > => { const identities = productAnalyticsRowIdentities(rows); return database .use(async (db) => { @@ -331,7 +337,11 @@ export class ProductAnalytics extends Effect.Service()( ), ); } - return undefined; + return { + acceptedEventIds: [], + rejectedEventIds: rows.map((row) => row.event_id), + rows: [] as ProductEventRow[], + }; } const anonymousIdentity = identities.find( (identity) => identity.identityKind === "anonymous", @@ -368,7 +378,7 @@ export class ProductAnalytics extends Effect.Service()( now.getTime() + PRODUCT_ANALYTICS_RECEIPT_RETENTION_MS, ); const admittedRows: ProductEventRow[] = []; - let conflicts = 0; + const rejectedEventIds: string[] = []; for (const row of rows) { const eventIdHash = productAnalyticsEventIdHash( row.event_id, @@ -422,28 +432,23 @@ export class ProductAnalytics extends Effect.Service()( if (receipt?.payloadHash === row.payload_hash) { admittedRows.push(row); } else { - conflicts += 1; + rejectedEventIds.push(row.event_id); } } - return { conflicts, rows: admittedRows }; + return { + acceptedEventIds: admittedRows.map((row) => row.event_id), + rejectedEventIds, + rows: admittedRows, + }; }), ) .pipe( Effect.flatMap((admission) => - !admission - ? Effect.void - : admission.conflicts > 0 - ? Effect.fail( - new ProductAnalyticsError({ - cause: - "Product analytics event ID was reused with a different payload", - retryable: false, - status: 409, - }), - ) - : admission.rows.length === 0 - ? Effect.void - : service.append(admission.rows), + admission.rows.length === 0 + ? Effect.succeed(admission) + : service + .append(admission.rows) + .pipe(Effect.as(admission)), ), Effect.ensuring( database @@ -459,6 +464,10 @@ export class ProductAnalytics extends Effect.Service()( ) .pipe(Effect.catchAll(() => Effect.void)), ), + Effect.map(({ acceptedEventIds, rejectedEventIds }) => ({ + acceptedEventIds, + rejectedEventIds, + })), ), ), ); From c3788deff20710b4628eed1c3fcc26dea44abb2a Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:20:46 +0100 Subject: [PATCH 089/110] fix: expose durable analytics health --- apps/web/app/admin/analytics/page.tsx | 95 +++++++++++++++---- apps/web/app/admin/analytics/tinybird.ts | 13 +-- .../drain-product-analytics-outbox/route.ts | 5 +- 3 files changed, 90 insertions(+), 23 deletions(-) diff --git a/apps/web/app/admin/analytics/page.tsx b/apps/web/app/admin/analytics/page.tsx index 848819c4886..340719c5320 100644 --- a/apps/web/app/admin/analytics/page.tsx +++ b/apps/web/app/admin/analytics/page.tsx @@ -1,5 +1,6 @@ import { buildEnv } from "@cap/env"; import { notFound } from "next/navigation"; +import { getProductAnalyticsOutboxHealth } from "@/lib/analytics/product-event-outbox"; import { getViewerContext } from "@/lib/messenger/data"; import { AdminAnalyticsConfigurationError, @@ -10,6 +11,9 @@ import { } from "./tinybird"; type SearchParams = Record; +type ProductAnalyticsOutboxHealth = Awaited< + ReturnType +>; type MetricCardProps = { label: string; @@ -148,7 +152,8 @@ function parseFilters(params: SearchParams): AdminAnalyticsFilters { endDate, platform: optionalFilter(firstParam(params, "platform")), appVersion: optionalFilter(firstParam(params, "appVersion")), - source: optionalFilter(firstParam(params, "source")), + acquisitionSource: optionalFilter(firstParam(params, "acquisitionSource")), + eventOrigin: optionalFilter(firstParam(params, "eventOrigin")), country: optionalFilter(firstParam(params, "country"))?.toUpperCase(), plan: optionalFilter(firstParam(params, "plan")), organizationCohort: isDate(firstParam(params, "organizationCohort")) @@ -316,7 +321,10 @@ function featureAdoptionRows(data: AdminAnalyticsDashboard) { return data.featureAdoption; } -function qualityStatus(data: AdminAnalyticsDashboard) { +function qualityStatus( + data: AdminAnalyticsDashboard, + outboxHealth: ProductAnalyticsOutboxHealth, +) { const freshness = data.freshness[0]; const health = data.health[0]; if (!freshness || !health) { @@ -328,7 +336,8 @@ function qualityStatus(data: AdminAnalyticsDashboard) { if ( freshness.healthFreshnessMs > 7_200_000 || health.payloadConflicts > 0 || - health.missingIdentityEvents > 0 + health.missingIdentityEvents > 0 || + !outboxHealth.healthy ) { return { label: "Attention required", @@ -389,14 +398,26 @@ function AnalyticsFilters({ filters }: { filters: AdminAnalyticsFilters }) { /> +

- Platform and app version affect product, retention, and health. Source, - plan, and country affect endpoints that expose those dimensions. - Organization cohort selects one first-value UTC date for the retention - view. + Platform and app version affect product, retention, and health. + Acquisition source filters traffic and attribution; event origin filters + product facts. Plan and country affect endpoints that expose those + dimensions. Organization cohort selects one first-value UTC date for the + retention view.

); @@ -833,8 +855,9 @@ function ProductSection({
row.deliveryLossCount, ); + const outboxPending = sumBy( + outboxHealth.partitions, + (row) => row.pending + row.workflowStarted, + ); + const outboxDeadLetters = sumBy( + outboxHealth.partitions, + (row) => row.deadLetter, + ); + const reconciliationFailures = sumBy( + outboxHealth.reconciliationFailures, + (row) => row.events, + ); return (
+ + + + @@ -1295,7 +1354,7 @@ function Definitions() { ], [ "Filter coverage", - "Date applies throughout. Platform and app version apply to traffic, attribution, product, experiment, creator, retention, and health aggregates where those dimensions exist. Source and plan apply to product aggregates; source also filters exact traffic totals and attribution. Country applies to product and supported traffic aggregates. Organization cohort selects a first-value UTC date for creator and organization retention.", + "Date applies throughout. Platform and app version apply to traffic, attribution, product, experiment, creator, retention, and health aggregates where those dimensions exist. Acquisition source filters traffic and attribution, while event origin filters client versus server product facts. Plan and country apply where those dimensions exist. Organization cohort selects a first-value UTC date for creator and organization retention.", ], ]; return ( @@ -1329,8 +1388,12 @@ export default async function AdminAnalyticsPage({ const filters = parseFilters(await searchParams); let data: AdminAnalyticsDashboard; + let outboxHealth: ProductAnalyticsOutboxHealth; try { - data = await fetchAdminAnalyticsDashboard(filters); + [data, outboxHealth] = await Promise.all([ + fetchAdminAnalyticsDashboard(filters), + getProductAnalyticsOutboxHealth(), + ]); } catch (error) { return (
@@ -1371,7 +1434,7 @@ export default async function AdminAnalyticsPage({

- + diff --git a/apps/web/app/admin/analytics/tinybird.ts b/apps/web/app/admin/analytics/tinybird.ts index 793581e644f..30fff55d662 100644 --- a/apps/web/app/admin/analytics/tinybird.ts +++ b/apps/web/app/admin/analytics/tinybird.ts @@ -29,7 +29,8 @@ export type AdminAnalyticsFilters = { endDate: string; platform?: string; appVersion?: string; - source?: string; + acquisitionSource?: string; + eventOrigin?: string; country?: string; plan?: string; organizationCohort?: string; @@ -820,7 +821,7 @@ export async function fetchAdminAnalyticsDashboard( ...dateParams, platform: filters.platform, app_version: filters.appVersion, - source: filters.source, + source: filters.acquisitionSource, country: filters.country, }, decodeTrafficTotalsRow, @@ -853,7 +854,7 @@ export async function fetchAdminAnalyticsDashboard( ...dateParams, platform: filters.platform, app_version: filters.appVersion, - source: filters.source, + source: filters.acquisitionSource, country: filters.country, limit: 300, }, @@ -898,7 +899,7 @@ export async function fetchAdminAnalyticsDashboard( "product_identity_funnel", { ...dateParams, - source: filters.source, + source: filters.acquisitionSource, country: filters.country, }, decodeIdentityFunnelRow, @@ -909,7 +910,7 @@ export async function fetchAdminAnalyticsDashboard( ...dateParams, platform: filters.platform, app_version: filters.appVersion, - source: filters.source, + source: filters.eventOrigin, country: filters.country, plan_id: filters.plan, limit: 1000, @@ -922,7 +923,7 @@ export async function fetchAdminAnalyticsDashboard( ...dateParams, platform: filters.platform, app_version: filters.appVersion, - source: filters.source, + source: filters.eventOrigin, country: filters.country, plan_id: filters.plan, }, diff --git a/apps/web/app/api/cron/drain-product-analytics-outbox/route.ts b/apps/web/app/api/cron/drain-product-analytics-outbox/route.ts index ea172bfc271..965fd4fa629 100644 --- a/apps/web/app/api/cron/drain-product-analytics-outbox/route.ts +++ b/apps/web/app/api/cron/drain-product-analytics-outbox/route.ts @@ -29,5 +29,8 @@ export async function GET(request: Request) { start(drainProductAnalyticsOutboxWorkflow, []), getProductAnalyticsOutboxHealth(), ]); - return NextResponse.json({ accepted: true, runId: run.runId, health }); + return NextResponse.json( + { accepted: true, runId: run.runId, health }, + { status: health.healthy ? 200 : 503 }, + ); } From 1dd669fb36f1ffbc78963002984981fa79c15e49 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:22:08 +0100 Subject: [PATCH 090/110] ci: harden Tinybird staging lifecycle --- .github/workflows/analytics.yml | 6 +- scripts/analytics/staging-ci-lib.js | 73 +++++++++++++- scripts/analytics/staging-ci.js | 67 ++++++++++++- scripts/analytics/tests/staging-ci.test.js | 96 ++++++++++++++++++- .../product_events_daily_current_v2.pipe | 39 ++++---- .../product_traffic_daily_current_v2.pipe | 39 ++++---- ...roduct_traffic_pages_daily_current_v2.pipe | 39 ++++---- 7 files changed, 296 insertions(+), 63 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index b2b947c4ce8..0eb49e6d98d 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -518,6 +518,7 @@ jobs: if: always() && steps.seed.outcome != 'skipped' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} run: >- @@ -527,7 +528,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Delete strictly scoped synthetic raw rows id: cleanup - if: always() && steps.seed.outcome != 'skipped' + if: always() && steps.seed.outcome != 'skipped' && steps.pause-copies.outcome == 'success' env: CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} @@ -577,6 +578,7 @@ jobs: if: always() && steps.pause-copies.outcome != 'skipped' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} run: >- @@ -586,7 +588,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Persist finalization eligibility id: recovery-ready - if: steps.verify-cleanup.outcome == 'success' && steps.rollback-drill.outcome == 'success' && steps.resume-copies.outcome == 'success' + if: steps.pause-copies.outcome == 'success' && steps.verify-cleanup.outcome == 'success' && steps.rollback-drill.outcome == 'success' && steps.resume-copies.outcome == 'success' run: >- node scripts/analytics/staging-ci.js mark-recovery-ready --state "$RUNNER_TEMP/analytics-staging-state.json" diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 8e09302271a..4ccb4f2cadd 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -107,13 +107,77 @@ export const applyCopyScheduleAction = async ({ export const copyScheduleMatchesAction = (value, action) => { const payload = value?.data ?? value; - if (!payload?.schedule) return true; + if (!payload?.schedule) return action === "resume"; const status = String(payload?.schedule?.status ?? "").toLowerCase(); return action === "pause" ? status === "paused" : status === "scheduled" || status === "active"; }; +export const waitForTinybirdCopyPipesQuiescent = async ({ + origin, + token, + pipes = COPY_PIPES, + request, + assertMutationOwnership, + requiredVisibleJobIds = [], + now = () => Date.now(), + wait = (milliseconds) => + new Promise((resolve) => setTimeout(resolve, milliseconds)), + timeoutMs = 120_000, + pollIntervalMs = 2_000, +}) => { + if (typeof assertMutationOwnership !== "function") { + throw new Error("Tinybird Copy quiescence requires an ownership check"); + } + const startedAt = now(); + const deadline = startedAt + timeoutMs; + let polls = 0; + const visibleJobIds = new Set(); + while (now() < deadline) { + await assertMutationOwnership(); + const activeJobs = []; + for (const pipe of pipes) { + const url = new URL("/v0/jobs", origin); + url.searchParams.set("kind", "copy"); + url.searchParams.set("pipe_name", pipe); + const response = await request(url, { token, attempts: 3 }); + if (!Array.isArray(response.data?.jobs)) { + throw new Error("Tinybird Jobs API returned an invalid Copy job list"); + } + const jobs = response.data.jobs; + for (const job of jobs) { + if (typeof job?.id === "string") visibleJobIds.add(job.id); + const status = String(job?.status ?? "").toLowerCase(); + if ( + ["waiting", "working", "cancelling", "canceling"].includes(status) + ) { + activeJobs.push({ id: String(job?.id ?? ""), pipe, status }); + } + } + } + polls += 1; + if (activeJobs.length === 0) { + const missingJobIds = requiredVisibleJobIds.filter( + (jobId) => !visibleJobIds.has(jobId), + ); + if (missingJobIds.length > 0) { + throw new Error( + "Tinybird Jobs API could not attest the Copy jobs created by this run", + ); + } + return { + activeJobs: 0, + polls, + quiescenceMs: Math.max(0, now() - startedAt), + visibleRequiredJobs: requiredVisibleJobIds.length, + }; + } + await wait(pollIntervalMs); + } + throw new Error("Timed out waiting for Tinybird Copy jobs to quiesce"); +}; + export const assertExecutionScope = ({ eventName, eventNumber, @@ -692,7 +756,12 @@ export const submitTinybirdCopyJobs = async ({ cause: error, }); } - const jobId = String(created.data.id ?? created.data.job_id ?? ""); + const jobId = String( + created.data?.job?.id ?? + created.data?.job?.job_id ?? + created.data?.job_id ?? + "", + ); if (!COPY_JOB_ID_PATTERN.test(jobId)) { throw new Error(`Tinybird did not return a valid copy job for ${pipe}`); } diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 61ece82117e..ce1beec7a60 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -47,6 +47,7 @@ import { syntheticMonetizationFilterQueries, validateSyntheticRunId, validateTinybirdCredentials, + waitForTinybirdCopyPipesQuiescent, } from "./staging-ci-lib.js"; const args = process.argv.slice(2); @@ -2861,6 +2862,7 @@ const setCopySchedules = async (parameters = {}) => { } const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_COPY_TOKEN", "TINYBIRD_STAGING_SCHEDULER_TOKEN", ]); if ( @@ -2874,6 +2876,19 @@ const setCopySchedules = async (parameters = {}) => { "Copy schedules can change only on the owned live deployment", ); } + const assertLiveOwnership = async () => { + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== "live" + ) { + throw new Error( + "The owned Tinybird deployment changed during Copy control", + ); + } + }; await applyCopyScheduleAction({ pipes: COPY_PIPES, action, @@ -2894,6 +2909,23 @@ const setCopySchedules = async (parameters = {}) => { } catch (error) { mutationError = error; } + if (scheduleAction === "pause") { + if (!mutationError) return; + if ( + mutationError instanceof Error && + [400, 404].includes(mutationError.status) + ) { + await waitForTinybirdCopyPipesQuiescent({ + origin, + token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + pipes: [pipe], + request, + assertMutationOwnership: assertLiveOwnership, + }); + return; + } + throw mutationError; + } const pipeState = await request( tinybirdUrl(origin, `/v0/pipes/${encodeURIComponent(pipe)}`), { @@ -2901,19 +2933,40 @@ const setCopySchedules = async (parameters = {}) => { attempts: 4, }, ); - if (copyScheduleMatchesAction(pipeState, scheduleAction)) return; + if ( + copyScheduleMatchesAction(pipeState, scheduleAction) && + (!mutationError || + (mutationError instanceof Error && + [400, 404].includes(mutationError.status))) + ) { + return; + } if (mutationError) throw mutationError; throw new Error( `Tinybird did not attest the ${scheduleAction} state for ${pipe}`, ); }, }); + const quiescence = + action === "pause" + ? await waitForTinybirdCopyPipesQuiescent({ + origin, + token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + request, + requiredVisibleJobIds: Object.values(artifact.copyJobs ?? {}) + .flatMap((phase) => (Array.isArray(phase?.jobs) ? phase.jobs : [])) + .map((job) => job?.jobId) + .filter((jobId) => typeof jobId === "string"), + assertMutationOwnership: assertLiveOwnership, + }) + : undefined; artifact.copySchedule = { ...(artifact.copySchedule ?? {}), [action]: { status: "passed", deploymentId: String(state.deploymentId), pipeCount: COPY_PIPES.length, + ...(quiescence ? { quiescence } : {}), }, }; writeJson(artifactPath, artifact); @@ -4365,6 +4418,15 @@ const verifyTokenScopes = async () => { }), { token: tokens.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN }, ); + const copyJobsProbe = await tokenScopeProbe( + tinybirdUrl(origin, "/v0/jobs", { kind: "copy" }), + tokens.TINYBIRD_STAGING_COPY_TOKEN, + ); + if (!copyJobsProbe.ok) { + throw new Error( + `The copy-runner token cannot attest Copy job quiescence: HTTP ${copyJobsProbe.status}`, + ); + } await assertScopeDenied( "The aggregate read token raw identity query", tokenScopeProbe( @@ -4472,6 +4534,7 @@ const verifyTokenScopes = async () => { ingestTokenAggregateReadDenied: true, cleanupTokenAggregateReadDenied: true, copyTokenAggregateReadDenied: true, + copyTokenJobsReadPassed: true, erasureLookupRawReadPassed: true, erasureLookupAppendDenied: true, erasureLookupCopyMutationDenied: true, @@ -4507,6 +4570,8 @@ const markRecoveryReadyToFinalize = async () => { state.recoveryPhase !== "prepromote" || artifact.assertions?.cleanupPassed !== true || artifact.rollbackDrill?.passed !== true || + artifact.copySchedule?.pause?.status !== "passed" || + artifact.copySchedule?.pause?.quiescence?.activeJobs !== 0 || artifact.copySchedule?.resume?.status !== "passed" ) { throw new Error( diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index b8bcc5217b9..427744b0274 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -53,6 +53,7 @@ import { validateSyntheticRunId, validateTinybirdCredentials, waitForTinybirdCopyJob, + waitForTinybirdCopyPipesQuiescent, } from "../staging-ci-lib.js"; const SHA = "1234567890abcdef1234567890abcdef12345678"; @@ -117,10 +118,72 @@ test("schedule state attestation distinguishes paused from active copies", () => copyScheduleMatchesAction({ schedule: { status: "paused" } }, "resume"), false, ); - assert.equal(copyScheduleMatchesAction({}, "pause"), true); + assert.equal(copyScheduleMatchesAction({}, "pause"), false); assert.equal(copyScheduleMatchesAction({}, "resume"), true); }); +test("Copy quiescence waits until every approved pipe has no active jobs", async () => { + let now = 1_000; + let round = 0; + const result = await waitForTinybirdCopyPipesQuiescent({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + pipes: ["copy_one", "copy_two"], + request: async (url) => { + const pipe = new URL(url).searchParams.get("pipe_name"); + return { + data: { + jobs: + round === 0 && pipe === "copy_one" + ? [{ id: "copy_job_active", status: "working" }] + : [], + }, + }; + }, + assertMutationOwnership: async () => undefined, + now: () => now, + wait: async (milliseconds) => { + now += milliseconds; + round += 1; + }, + timeoutMs: 10_000, + pollIntervalMs: 2_000, + }); + assert.deepEqual(result, { + activeJobs: 0, + polls: 2, + quiescenceMs: 2_000, + visibleRequiredJobs: 0, + }); +}); + +test("Copy quiescence rejects a malformed Jobs API payload", async () => { + await assert.rejects( + waitForTinybirdCopyPipesQuiescent({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + pipes: ["copy_one"], + request: async () => ({ data: {} }), + assertMutationOwnership: async () => undefined, + }), + /invalid Copy job list/, + ); +}); + +test("Copy quiescence proves visibility of this run's completed jobs", async () => { + await assert.rejects( + waitForTinybirdCopyPipesQuiescent({ + origin: "https://api.us-east.aws.tinybird.co", + token: token(), + pipes: ["copy_one"], + requiredVisibleJobIds: ["copy_job_expected"], + request: async () => ({ data: { jobs: [] } }), + assertMutationOwnership: async () => undefined, + }), + /could not attest the Copy jobs created by this run/, + ); +}); + test("execution scope accepts only PR 2003 or manual feature branch runs", () => { assert.doesNotThrow(() => assertExecutionScope({ @@ -633,8 +696,18 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker const requests = []; const resourceToken = token(); const responses = [ - { data: { id: "copy_job_canonical" } }, - { data: { id: "copy_job_traffic" } }, + { + data: { + id: "t_canonical_pipe_node", + job: { id: "copy_job_canonical", job_id: "copy_job_canonical" }, + }, + }, + { + data: { + id: "t_traffic_pipe_node", + job: { job_id: "copy_job_traffic" }, + }, + }, ]; let now = 1_000; let ownershipChecks = 0; @@ -687,7 +760,12 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker request: async (url, options) => { await options.beforeAttempt(); liveRequests.push({ url: String(url), options }); - return { data: { id: "copy_job_live" } }; + return { + data: { + id: "t_live_pipe_node", + job: { id: "copy_job_live" }, + }, + }; }, assertMutationOwnership: async () => undefined, }); @@ -700,6 +778,16 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker request: async () => ({ data: {} }), }), ); + await assert.rejects(() => + submitTinybirdCopyJobs({ + origin: "https://api.us-east.aws.tinybird.co", + token: resourceToken, + deploymentId: "6", + pipes: ["snapshot_product_events_canonical_v1"], + request: async () => ({ data: { id: "t_pipe_node_is_not_a_job" } }), + assertMutationOwnership: async () => undefined, + }), + ); await assert.rejects(() => submitTinybirdCopyJobs({ origin: "https://api.us-east.aws.tinybird.co", diff --git a/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe index c29e7abdee8..7f424eb0857 100644 --- a/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe @@ -1,36 +1,39 @@ DESCRIPTION > Expose the active bounded event-decision generation and latest settled UTC partitions. +NODE product_events_daily_latest_hot_generation_v2_node +SQL > + SELECT + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'hot' + +NODE product_events_daily_latest_cold_generations_v2_node +SQL > + SELECT + settled_date AS date, + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'cold' + GROUP BY settled_date + NODE product_events_daily_current_v2_node SQL > - WITH hot_generation AS ( - SELECT - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'hot' - ), cold_generations AS ( - SELECT - settled_date AS date, - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'cold' - GROUP BY settled_date - ) SELECT hot.* FROM product_events_daily_hot_v2 AS hot - INNER JOIN hot_generation AS generation + INNER JOIN product_events_daily_latest_hot_generation_v2_node AS generation ON hot.generation_id = generation.generation_id AND hot.source_cutoff = generation.source_cutoff WHERE hot.is_marker = 0 UNION ALL SELECT cold.* FROM product_events_daily_cold_v2 AS cold - INNER JOIN cold_generations AS generation + INNER JOIN product_events_daily_latest_cold_generations_v2_node AS generation ON cold.date = generation.date AND cold.generation_id = generation.generation_id AND cold.source_cutoff = generation.source_cutoff - CROSS JOIN hot_generation AS hot_generation + CROSS JOIN product_events_daily_latest_hot_generation_v2_node AS hot_generation WHERE cold.is_marker = 0 AND cold.date < toDate(hot_generation.source_cutoff, 'UTC') - INTERVAL 8 DAY diff --git a/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe index 538ebf50935..4aeba08dbb4 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe @@ -1,36 +1,39 @@ DESCRIPTION > Expose the active bounded traffic generation and latest settled UTC partitions. +NODE product_traffic_daily_latest_hot_generation_v2_node +SQL > + SELECT + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'hot' + +NODE product_traffic_daily_latest_cold_generations_v2_node +SQL > + SELECT + settled_date AS date, + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'cold' + GROUP BY settled_date + NODE product_traffic_daily_current_v2_node SQL > - WITH hot_generation AS ( - SELECT - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'hot' - ), cold_generations AS ( - SELECT - settled_date AS date, - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'cold' - GROUP BY settled_date - ) SELECT hot.* FROM product_traffic_daily_hot_v2 AS hot - INNER JOIN hot_generation AS generation + INNER JOIN product_traffic_daily_latest_hot_generation_v2_node AS generation ON hot.generation_id = generation.generation_id AND hot.source_cutoff = generation.source_cutoff WHERE hot.is_marker = 0 UNION ALL SELECT cold.* FROM product_traffic_daily_cold_v2 AS cold - INNER JOIN cold_generations AS generation + INNER JOIN product_traffic_daily_latest_cold_generations_v2_node AS generation ON cold.date = generation.date AND cold.generation_id = generation.generation_id AND cold.source_cutoff = generation.source_cutoff - CROSS JOIN hot_generation AS hot_generation + CROSS JOIN product_traffic_daily_latest_hot_generation_v2_node AS hot_generation WHERE cold.is_marker = 0 AND cold.date < toDate(hot_generation.source_cutoff, 'UTC') - INTERVAL 8 DAY diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe index 3cd7559f723..7f4c4954e1b 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe @@ -1,36 +1,39 @@ DESCRIPTION > Expose the active bounded page generation and latest settled UTC partitions. +NODE product_traffic_pages_daily_latest_hot_generation_v2_node +SQL > + SELECT + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'hot' + +NODE product_traffic_pages_daily_latest_cold_generations_v2_node +SQL > + SELECT + settled_date AS date, + argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, + argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 + WHERE generation_kind = 'cold' + GROUP BY settled_date + NODE product_traffic_pages_daily_current_v2_node SQL > - WITH hot_generation AS ( - SELECT - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'hot' - ), cold_generations AS ( - SELECT - settled_date AS date, - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'cold' - GROUP BY settled_date - ) SELECT hot.* FROM product_traffic_pages_daily_hot_v2 AS hot - INNER JOIN hot_generation AS generation + INNER JOIN product_traffic_pages_daily_latest_hot_generation_v2_node AS generation ON hot.generation_id = generation.generation_id AND hot.source_cutoff = generation.source_cutoff WHERE hot.is_marker = 0 UNION ALL SELECT cold.* FROM product_traffic_pages_daily_cold_v2 AS cold - INNER JOIN cold_generations AS generation + INNER JOIN product_traffic_pages_daily_latest_cold_generations_v2_node AS generation ON cold.date = generation.date AND cold.generation_id = generation.generation_id AND cold.source_cutoff = generation.source_cutoff - CROSS JOIN hot_generation AS hot_generation + CROSS JOIN product_traffic_pages_daily_latest_hot_generation_v2_node AS hot_generation WHERE cold.is_marker = 0 AND cold.date < toDate(hot_generation.source_cutoff, 'UTC') - INTERVAL 8 DAY From a183e23bb23941f854bd03c7792db56999eb52ce Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:22:21 +0100 Subject: [PATCH 091/110] docs: tighten analytics rollout contract --- .../first-party-analytics-production-readiness.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index e33e72b2bfb..0cff92f748c 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -2,7 +2,7 @@ ## Decision contract -Cap analytics is at-least-once at the delivery boundary and exactly once for decision metrics. Every event carries a stable `event_id` and a SHA-256-derived payload fingerprint. Retry deliveries retain both values. The raw Tinybird datasource is append-only; health metrics preserve every accepted delivery. Canonical and decision snapshots group by `event_id`, count one event only when every accepted copy has the same fingerprint, and quarantine an ID when different fingerprints arrive. +Cap analytics is at-least-once at the delivery boundary and exactly once for decision metrics. Every event carries a stable `event_id` and a SHA-256-derived payload fingerprint. Untrusted client IDs are rewritten into an anonymous-identity-bound hashed `client:` trust domain, while authoritative server IDs retain their established stable values, so a client cannot preclaim a server-owned business fact and in-flight server events keep their deployment-spanning deduplication identity. Client relationship IDs such as `page_view_id` are rewritten into the same client domain. Client retry fingerprints cover immutable normalized client input and exclude request-time user, organization, geography, device, network, traffic, and synthetic enrichment, so a retry after sign-in or a network change remains the same event. Retry deliveries retain the same event ID and fingerprint. The raw Tinybird datasource is append-only; health metrics preserve every accepted delivery. Canonical and decision snapshots group by `event_id`, count one event only when every accepted copy has the same fingerprint, and quarantine an ID when different fingerprints arrive. A conflicting member of a client batch is rejected independently while valid neighbors continue to ingestion. The central registry is `packages/analytics/src/event-registry.ts`. It defines the version, authority, delivery class, permitted platforms, semantic, and property schema for every event. Contract CI rejects unknown literal names, templates, invalid property keys, missing emitters, and drift between the Rust desktop catalogue and the TypeScript registry. Invalid properties reject the whole event rather than being silently removed. @@ -77,7 +77,7 @@ The preview-only `/api/analytics/staging-test` route must not receive a producti 3. Create least-privilege Tinybird tokens: - append-only token for `product_events_v1`; - aggregate endpoint read token with no raw or canonical datasource access; - - resource-scoped copy token for the reviewed copy pipes enumerated by the staging runner, with no raw identity datasource access; + - resource-scoped copy token for the reviewed copy pipes enumerated by the staging runner, with access to read the status and recent Jobs API records for only its own Copy jobs and no raw identity datasource access; - erasure-lookup token limited to read access on `product_events_v1` and `product_events_canonical_v1`, protected from all agent and admin surfaces; - schedule-controller token limited to cancelling, pausing, and resuming the reviewed Copy Pipes enumerated by the staging runner; - dedicated erasure token with Tinybird's required `DATASOURCES:CREATE` scope, no read/deploy scopes, protected as a high-impact operational secret until Tinybird offers resource-scoped row deletion; @@ -95,11 +95,12 @@ The preview-only `/api/analytics/staging-test` route must not receive a producti - `NEXTAUTH_SECRET` (existing application secret used to sign the short-lived anonymous browser token; do not create an analytics-specific duplicate) For a self-hosted release only, set `CAP_ANALYTICS_TRUST_PROXY=1` after verifying that the fronting reverse proxy overwrites `x-forwarded-for`. Omit it on Vercel. 5. Configure the Vercel firewall rule referenced by the collector so `/api/events` has the verified per-IP limit. The collector also applies the same rule by anonymous-ID key. Verify normal navigation is usable, the documented burst test receives `429`, forged forwarding headers do not bypass classification, and preview hostnames are excluded from decision metrics. -6. Deploy the web application through the normal production release process. Confirm the Vercel deployment Git SHA exactly matches the reviewed commit, the database schema attestation matches, and the configured Tinybird workspace and deployment IDs match the reviewed release before sending any test event. -7. Run a uniquely tagged production smoke test only if separately approved. Do not use customer identifiers. Confirm ingestion, durable outbox drain, health, decision deduplication, admin freshness, and strictly scoped cleanup. -8. Enable dashboards only after freshness, duplicates, conflicts, missing identity, clock skew, late events, queue drops, dead letters, and ingestion lag are healthy. Reconcile signup and billing totals against database and Stripe source-of-truth counts before using conversion or revenue decisions. +6. Deploy the web application and `/api/events` collector through the normal production release process before releasing any desktop, mobile, or CLI build that emits this contract. Confirm the Vercel deployment Git SHA exactly matches the reviewed commit, the database schema attestation matches, and the configured Tinybird workspace and deployment IDs match the reviewed release before sending any test event. +7. Run a uniquely tagged production smoke test only if separately approved. Do not use customer identifiers. Confirm ingestion, durable outbox drain, health, decision deduplication, admin freshness, and strictly scoped cleanup. Record this collector-capable Vercel deployment as the minimum safe web rollback target. +8. Release desktop, mobile, and CLI clients only after the collector smoke test passes. Their durable queues treat collector-absent `404` and retired-contract `410` responses as retryable migration states, retaining the original event and ID across restart rather than dead-lettering it. +9. Enable dashboards only after freshness, duplicates, conflicts, missing identity, clock skew, late events, queue drops, dead letters, and ingestion lag are healthy. Reconcile signup and billing totals against database and Stripe source-of-truth counts before using conversion or revenue decisions. -Rollback is fail closed: disable collection by removing the app append token or route feature switch, roll Vercel back to the recorded prior deployment, and promote the recorded prior Tinybird deployment if schema/query behavior is implicated. Do not delete raw data during rollback. Keep reconciliation paused until the prior collector contract is confirmed compatible, retain health visibility, and record the rollback SHA and deployment IDs. Token rotation follows containment, and erasure credentials must remain available for pending deletion requests. +Rollback is fail closed. Before any native client release, Vercel may roll back to the recorded pre-collector deployment. After a desktop, mobile, or CLI release can emit the new contract, a pre-collector Vercel deployment is no longer a valid rollback target: Vercel must roll back only to a recorded collector-capable deployment. The native `404` and `410` retry policy protects brief routing races, but bounded queues and the seven-day event-age contract mean it is not authorization for an extended collector outage. Disable provider delivery by removing the app append token when containment is required, and promote the recorded prior Tinybird deployment only when its schema and endpoints remain compatible with the active collector. Do not delete raw data during rollback. Keep reconciliation paused until the retained collector contract is confirmed compatible, retain MySQL and Tinybird health visibility, and record the rollback SHA and deployment IDs. Token rotation follows containment, and erasure credentials must remain available for pending deletion requests. ## Honest limitations From 131b29be643694694b6a6523c46a2ba346cd9f04 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 14:50:35 +0100 Subject: [PATCH 092/110] fix: make Tinybird staging aggregates executable --- scripts/analytics/staging-ci-lib.js | 7 +- scripts/analytics/tests/datafiles.test.js | 12 ++-- scripts/analytics/tests/staging-ci.test.js | 5 ++ scripts/analytics/tests/tooling.test.js | 5 ++ ...terialize_product_event_day_states_v2.pipe | 62 +++++++++--------- ...aterialize_product_event_id_states_v2.pipe | 60 ++++++++--------- .../product_events_daily_bounded_v2.pipe | 3 +- .../product_events_daily_current_v2.pipe | 20 +++--- .../pipes/product_experiment_outcomes.pipe | 26 ++++---- .../product_traffic_daily_bounded_v2.pipe | 1 - .../product_traffic_daily_current_v2.pipe | 20 +++--- ...roduct_traffic_pages_daily_bounded_v2.pipe | 1 - ...roduct_traffic_pages_daily_current_v2.pipe | 20 +++--- .../pipes/product_traffic_totals.pipe | 18 +++--- .../snapshot_product_event_day_states_v2.pipe | 64 +++++++++---------- .../snapshot_product_event_id_states_v2.pipe | 62 +++++++++--------- ...hot_product_experiment_outcomes_exact.pipe | 42 ++++++------ ...napshot_product_identity_funnel_exact.pipe | 30 ++++----- scripts/analytics/tooling.js | 24 ++++++- scripts/analytics/verify-local.js | 2 + 20 files changed, 260 insertions(+), 224 deletions(-) diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 4ccb4f2cadd..092410b0028 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -721,6 +721,9 @@ export const submitTinybirdCopyJobs = async ({ if (sourceCutoff && !Number.isFinite(Date.parse(sourceCutoff))) { throw new Error("Tinybird copy source cutoff must be an ISO timestamp"); } + const tinybirdSourceCutoff = sourceCutoff + ? new Date(sourceCutoff).toISOString().replace("T", " ").replace("Z", "") + : ""; if (typeof assertMutationOwnership !== "function") { throw new Error("Tinybird copies require an ownership check"); } @@ -735,7 +738,9 @@ export const submitTinybirdCopyJobs = async ({ origin, ); copyUrl.searchParams.set("_mode", "replace"); - if (sourceCutoff) copyUrl.searchParams.set("source_cutoff", sourceCutoff); + if (tinybirdSourceCutoff) { + copyUrl.searchParams.set("source_cutoff", tinybirdSourceCutoff); + } if (COPY_MARKER_PIPES.has(pipe)) { if (!copyRunId) { throw new Error(`Tinybird copy marker is required for ${pipe}`); diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index 8f5d69462c8..d572601f1c3 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -211,7 +211,7 @@ test("traffic totals merge visitor states across the selected range", () => { path.join(TINYBIRD_PROJECT_DIR, "pipes", "product_traffic_totals.pipe"), "utf8", ); - assert.match(contents, /uniqExactMerge\(visitors\) AS visitors/); + assert.match(contents, /uniqExactMerge\(traffic\.visitors\) AS visitors/); assert.match(contents, /FROM product_traffic_daily_exact/); assert.match(contents, /platform = \{\{String\(platform\)\}\}/); assert.match(contents, /app_version = \{\{String\(app_version\)\}\}/); @@ -333,8 +333,8 @@ test("experiment outcomes are aggregate-only and anchored to explicit exposures" "utf8", ); assert.match(snapshot, /event_name = 'experiment_exposed'/); - assert.match(snapshot, /HAVING uniqExact\(user_id\) = 1/); - assert.match(snapshot, /HAVING uniqExact\(variant\) = 1/); + assert.match(snapshot, /HAVING uniqExact\(candidates\.user_id\) = 1/); + assert.match(snapshot, /HAVING uniqExact\(resolved\.variant\) = 1/); assert.match(snapshot, /outcome_candidates AS/); assert.match( snapshot, @@ -347,7 +347,7 @@ test("experiment outcomes are aggregate-only and anchored to explicit exposures" /JSONExtractString\(properties, 'payment_status'\) = 'paid'/, ); assert.match(endpoint, /FROM product_experiment_outcomes_exact/); - assert.match(endpoint, /sum\(converted_actors\)/); + assert.match(endpoint, /sum\(outcomes\.converted_actors\)/); assert.doesNotMatch( endpoint, /anonymous_id|session_id|user_id|organization_id/, @@ -438,8 +438,8 @@ test("daily snapshot quarantines payload conflicts and rebuilds exact metrics", ), "utf8", ); - assert.match(stateContents, /uniqExactState\(payload_hash\)/); - assert.match(stateContents, /GROUP BY event_id/); + assert.match(stateContents, /uniqExactState\(raw\.payload_hash\)/); + assert.match(stateContents, /GROUP BY raw\.event_id/); assert.match(stateContents, /^TYPE MATERIALIZED$/m); const currentContents = fs.readFileSync( path.join( diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 427744b0274..1c6c2c7fccb 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -727,6 +727,7 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker }, now: () => now, copyRunId: "run_12345678_staged", + sourceCutoff: "2026-08-01T13:34:45.197Z", assertMutationOwnership: async () => { ownershipChecks += 1; }, @@ -745,6 +746,10 @@ test("copy jobs use only approved resource-scoped submissions and bounded marker }, ]); assert.match(requests[0].url, /_mode=replace/); + assert.equal( + new URL(requests[0].url).searchParams.get("source_cutoff"), + "2026-08-01 13:34:45.197", + ); assert.doesNotMatch(requests[0].url, /__tb__deployment/); assert.doesNotMatch(requests[0].url, /copy_run_id/); assert.equal(requests[0].options.method, "POST"); diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index ffc6b4165f3..72d68cb98c4 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -74,12 +74,17 @@ test("local setup builds, verifies copied endpoints and writes its deterministic const appendIndex = steps.findIndex((step) => step.args?.join(" ").includes("datasource append product_events_v1"), ); + const resetIndex = steps.findIndex( + (step) => step.type === "reset-local-fixture", + ); const pauseIndexes = steps .map((step, index) => ({ index, command: step.args?.join(" ") ?? "" })) .filter(({ command }) => command.includes("--local copy pause")) .map(({ index }) => index); assert.equal(pauseIndexes.length, 19); assert.ok(pauseIndexes.every((index) => index < appendIndex)); + assert.ok(resetIndex > Math.max(...pauseIndexes)); + assert.ok(resetIndex < appendIndex); const copyCommands = commands.filter((command) => command.includes("--local copy run"), ); diff --git a/scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe b/scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe index b1136d2459f..a26a44cf475 100644 --- a/scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe +++ b/scripts/analytics/tinybird/pipes/materialize_product_event_day_states_v2.pipe @@ -4,37 +4,37 @@ DESCRIPTION > NODE materialize_product_event_day_states_v2_node SQL > SELECT - toDate(occurred_at, 'UTC') AS occurred_date, - event_id, - uniqExactState(payload_hash) AS payload_hashes, - argMinState(payload_hash, received_at) AS payload_hash, - argMinState(occurred_at, received_at) AS occurred_at, - min(received_at) AS first_received_at, - max(received_at) AS received_at, - argMinState(toString(event_name), received_at) AS event_name, - argMinState(schema_version, received_at) AS schema_version, - argMinState(toString(source), received_at) AS source, - argMinState(toString(platform), received_at) AS platform, - argMinState(anonymous_id, received_at) AS anonymous_id, - argMinState(session_id, received_at) AS session_id, - argMinState(user_id, received_at) AS user_id, - argMinState(organization_id, received_at) AS organization_id, - argMinState(toString(app_version), received_at) AS app_version, - argMinState(pathname, received_at) AS pathname, - argMinState(referrer, received_at) AS referrer, - argMinState(toString(country), received_at) AS country, - argMinState(toString(region), received_at) AS region, - argMinState(toString(city), received_at) AS city, - argMinState(toString(hostname), received_at) AS hostname, - argMinState(toString(browser), received_at) AS browser, - argMinState(toString(device), received_at) AS device, - argMinState(toString(os), received_at) AS os, - argMinState(toString(channel), received_at) AS channel, - argMinState(toString(traffic_class), received_at) AS traffic_class, - argMinState(synthetic_run_id, received_at) AS synthetic_run_id, - argMinState(properties, received_at) AS properties - FROM product_events_v1 - GROUP BY occurred_date, event_id + toDate(raw.occurred_at, 'UTC') AS occurred_date, + raw.event_id, + uniqExactState(raw.payload_hash) AS payload_hashes, + argMinState(raw.payload_hash, raw.received_at) AS payload_hash, + argMinState(raw.occurred_at, raw.received_at) AS occurred_at, + min(raw.received_at) AS first_received_at, + max(raw.received_at) AS received_at, + argMinState(toString(raw.event_name), raw.received_at) AS event_name, + argMinState(raw.schema_version, raw.received_at) AS schema_version, + argMinState(toString(raw.source), raw.received_at) AS source, + argMinState(toString(raw.platform), raw.received_at) AS platform, + argMinState(raw.anonymous_id, raw.received_at) AS anonymous_id, + argMinState(raw.session_id, raw.received_at) AS session_id, + argMinState(raw.user_id, raw.received_at) AS user_id, + argMinState(raw.organization_id, raw.received_at) AS organization_id, + argMinState(toString(raw.app_version), raw.received_at) AS app_version, + argMinState(raw.pathname, raw.received_at) AS pathname, + argMinState(raw.referrer, raw.received_at) AS referrer, + argMinState(toString(raw.country), raw.received_at) AS country, + argMinState(toString(raw.region), raw.received_at) AS region, + argMinState(toString(raw.city), raw.received_at) AS city, + argMinState(toString(raw.hostname), raw.received_at) AS hostname, + argMinState(toString(raw.browser), raw.received_at) AS browser, + argMinState(toString(raw.device), raw.received_at) AS device, + argMinState(toString(raw.os), raw.received_at) AS os, + argMinState(toString(raw.channel), raw.received_at) AS channel, + argMinState(toString(raw.traffic_class), raw.received_at) AS traffic_class, + argMinState(raw.synthetic_run_id, raw.received_at) AS synthetic_run_id, + argMinState(raw.properties, raw.received_at) AS properties + FROM product_events_v1 AS raw + GROUP BY toDate(raw.occurred_at, 'UTC'), raw.event_id TYPE MATERIALIZED DATASOURCE product_event_day_states_v2 diff --git a/scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe b/scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe index aa26d0d922d..a9b219e008c 100644 --- a/scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe +++ b/scripts/analytics/tinybird/pipes/materialize_product_event_id_states_v2.pipe @@ -4,36 +4,36 @@ DESCRIPTION > NODE materialize_product_event_id_states_v2_node SQL > SELECT - event_id, - uniqExactState(payload_hash) AS payload_hashes, - argMinState(payload_hash, received_at) AS payload_hash, - argMinState(occurred_at, received_at) AS occurred_at, - min(received_at) AS first_received_at, - max(received_at) AS received_at, - argMinState(toString(event_name), received_at) AS event_name, - argMinState(schema_version, received_at) AS schema_version, - argMinState(toString(source), received_at) AS source, - argMinState(toString(platform), received_at) AS platform, - argMinState(anonymous_id, received_at) AS anonymous_id, - argMinState(session_id, received_at) AS session_id, - argMinState(user_id, received_at) AS user_id, - argMinState(organization_id, received_at) AS organization_id, - argMinState(toString(app_version), received_at) AS app_version, - argMinState(pathname, received_at) AS pathname, - argMinState(referrer, received_at) AS referrer, - argMinState(toString(country), received_at) AS country, - argMinState(toString(region), received_at) AS region, - argMinState(toString(city), received_at) AS city, - argMinState(toString(hostname), received_at) AS hostname, - argMinState(toString(browser), received_at) AS browser, - argMinState(toString(device), received_at) AS device, - argMinState(toString(os), received_at) AS os, - argMinState(toString(channel), received_at) AS channel, - argMinState(toString(traffic_class), received_at) AS traffic_class, - argMinState(synthetic_run_id, received_at) AS synthetic_run_id, - argMinState(properties, received_at) AS properties - FROM product_events_v1 - GROUP BY event_id + raw.event_id, + uniqExactState(raw.payload_hash) AS payload_hashes, + argMinState(raw.payload_hash, raw.received_at) AS payload_hash, + argMinState(raw.occurred_at, raw.received_at) AS occurred_at, + min(raw.received_at) AS first_received_at, + max(raw.received_at) AS received_at, + argMinState(toString(raw.event_name), raw.received_at) AS event_name, + argMinState(raw.schema_version, raw.received_at) AS schema_version, + argMinState(toString(raw.source), raw.received_at) AS source, + argMinState(toString(raw.platform), raw.received_at) AS platform, + argMinState(raw.anonymous_id, raw.received_at) AS anonymous_id, + argMinState(raw.session_id, raw.received_at) AS session_id, + argMinState(raw.user_id, raw.received_at) AS user_id, + argMinState(raw.organization_id, raw.received_at) AS organization_id, + argMinState(toString(raw.app_version), raw.received_at) AS app_version, + argMinState(raw.pathname, raw.received_at) AS pathname, + argMinState(raw.referrer, raw.received_at) AS referrer, + argMinState(toString(raw.country), raw.received_at) AS country, + argMinState(toString(raw.region), raw.received_at) AS region, + argMinState(toString(raw.city), raw.received_at) AS city, + argMinState(toString(raw.hostname), raw.received_at) AS hostname, + argMinState(toString(raw.browser), raw.received_at) AS browser, + argMinState(toString(raw.device), raw.received_at) AS device, + argMinState(toString(raw.os), raw.received_at) AS os, + argMinState(toString(raw.channel), raw.received_at) AS channel, + argMinState(toString(raw.traffic_class), raw.received_at) AS traffic_class, + argMinState(raw.synthetic_run_id, raw.received_at) AS synthetic_run_id, + argMinState(raw.properties, raw.received_at) AS properties + FROM product_events_v1 AS raw + GROUP BY raw.event_id TYPE MATERIALIZED DATASOURCE product_event_id_states_v2 diff --git a/scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe b/scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe index 63053a09f90..14771905666 100644 --- a/scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_daily_bounded_v2.pipe @@ -99,7 +99,7 @@ SQL > now64(3), {% if defined(copy_run_id) %} {{String(copy_run_id)}} {% else %} '' {% end %}, {% if defined(settled_date) %} toDate({{Date(settled_date)}}) {% else %} toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) {% end %}, - '', toUInt16(0), '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', + '', toUInt16(0), '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', toInt64(0), toInt64(0), toInt64(0), toInt64(0), '', '', '', '', '', toInt64(0), toInt64(0), toInt64(0), toInt64(0), @@ -109,4 +109,3 @@ SQL > toInt64(0) WHERE throwIf(length({{String(generation_id)}}) < 8 OR length({{String(generation_id)}}) > 128, 'generation_id has an invalid length') = 0 {% if defined(settled_date) %} AND throwIf(toDate({{Date(settled_date)}}) > toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY, 'settled_date is inside the mutable window') = 0 {% end %} - diff --git a/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe index 7f424eb0857..16cb3769293 100644 --- a/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_daily_current_v2.pipe @@ -4,20 +4,20 @@ DESCRIPTION > NODE product_events_daily_latest_hot_generation_v2_node SQL > SELECT - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'hot' + argMax(generations.generation_id, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS generation_id, + argMax(generations.source_cutoff, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 AS generations + WHERE generations.generation_kind = 'hot' NODE product_events_daily_latest_cold_generations_v2_node SQL > SELECT - settled_date AS date, - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'cold' - GROUP BY settled_date + generations.settled_date AS date, + argMax(generations.generation_id, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS generation_id, + argMax(generations.source_cutoff, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 AS generations + WHERE generations.generation_kind = 'cold' + GROUP BY generations.settled_date NODE product_events_daily_current_v2_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe b/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe index 2a3469d9dd0..843120aeb9a 100644 --- a/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe +++ b/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe @@ -7,17 +7,17 @@ NODE product_experiment_outcomes_node SQL > % SELECT - experiment_id, - assignment_version, - variant, - platform, - app_version, - outcome_name, - toUInt64(sum(exposed_actors)) AS exposed_actors, - toUInt64(sum(converted_actors)) AS converted_actors, - round(if(sum(exposed_actors) = 0, 0, 100 * sum(converted_actors) / sum(exposed_actors)), 2) AS conversion_rate - FROM product_experiment_outcomes_exact - WHERE copy_run_id = '' + outcomes.experiment_id, + outcomes.assignment_version, + outcomes.variant, + outcomes.platform, + outcomes.app_version, + outcomes.outcome_name, + toUInt64(sum(outcomes.exposed_actors)) AS exposed_actors, + toUInt64(sum(outcomes.converted_actors)) AS converted_actors, + round(if(sum(outcomes.exposed_actors) = 0, 0, 100 * sum(outcomes.converted_actors) / sum(outcomes.exposed_actors)), 2) AS conversion_rate + FROM product_experiment_outcomes_exact AS outcomes + WHERE outcomes.copy_run_id = '' {% if defined(synthetic_run_id) %} AND synthetic_run_id = {{String(synthetic_run_id)}} AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 @@ -35,8 +35,8 @@ SQL > AND outcome_name = {{String(outcome_name)}} AND throwIf(NOT has(['signup', 'share_created', 'paid_purchase'], {{String(outcome_name)}}), 'outcome_name is invalid') = 0 {% end %} - GROUP BY experiment_id, assignment_version, variant, platform, app_version, outcome_name - ORDER BY experiment_id ASC, assignment_version ASC, variant ASC, outcome_name ASC, platform ASC, app_version ASC + GROUP BY outcomes.experiment_id, outcomes.assignment_version, outcomes.variant, outcomes.platform, outcomes.app_version, outcomes.outcome_name + ORDER BY outcomes.experiment_id ASC, outcomes.assignment_version ASC, outcomes.variant ASC, outcomes.outcome_name ASC, outcomes.platform ASC, outcomes.app_version ASC LIMIT greatest(1, least({{Int32(limit, 1000)}}, 1000)) TYPE ENDPOINT diff --git a/scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe index 474c925f36d..466213e3213 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_daily_bounded_v2.pipe @@ -95,4 +95,3 @@ SQL > toUInt64(0), toUInt64(0), toUInt64(0), toUInt64(0), toUInt64(0) WHERE throwIf(length({{String(generation_id)}}) < 8 OR length({{String(generation_id)}}) > 128, 'generation_id has an invalid length') = 0 {% if defined(settled_date) %} AND throwIf(toDate({{Date(settled_date)}}) > toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY, 'settled_date is inside the mutable window') = 0 {% end %} - diff --git a/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe index 4aeba08dbb4..6ec17c7d1a3 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_daily_current_v2.pipe @@ -4,20 +4,20 @@ DESCRIPTION > NODE product_traffic_daily_latest_hot_generation_v2_node SQL > SELECT - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'hot' + argMax(generations.generation_id, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS generation_id, + argMax(generations.source_cutoff, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 AS generations + WHERE generations.generation_kind = 'hot' NODE product_traffic_daily_latest_cold_generations_v2_node SQL > SELECT - settled_date AS date, - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'cold' - GROUP BY settled_date + generations.settled_date AS date, + argMax(generations.generation_id, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS generation_id, + argMax(generations.source_cutoff, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 AS generations + WHERE generations.generation_kind = 'cold' + GROUP BY generations.settled_date NODE product_traffic_daily_current_v2_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe index aaaf2d1b7a5..d0be5cb693f 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_bounded_v2.pipe @@ -102,4 +102,3 @@ SQL > toUInt64(0), toUInt64(0), toUInt64(0), toUInt64(0), toFloat64(0) WHERE throwIf(length({{String(generation_id)}}) < 8 OR length({{String(generation_id)}}) > 128, 'generation_id has an invalid length') = 0 {% if defined(settled_date) %} AND throwIf(toDate({{Date(settled_date)}}) > toDate(toDateTime64({{DateTime64(source_cutoff)}}, 3)) - INTERVAL 8 DAY, 'settled_date is inside the mutable window') = 0 {% end %} - diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe index 7f4c4954e1b..948a3ee2af7 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages_daily_current_v2.pipe @@ -4,20 +4,20 @@ DESCRIPTION > NODE product_traffic_pages_daily_latest_hot_generation_v2_node SQL > SELECT - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'hot' + argMax(generations.generation_id, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS generation_id, + argMax(generations.source_cutoff, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 AS generations + WHERE generations.generation_kind = 'hot' NODE product_traffic_pages_daily_latest_cold_generations_v2_node SQL > SELECT - settled_date AS date, - argMax(generation_id, tuple(source_cutoff, published_at, generation_id)) AS generation_id, - argMax(source_cutoff, tuple(source_cutoff, published_at, generation_id)) AS source_cutoff - FROM product_analytics_generations_v2 - WHERE generation_kind = 'cold' - GROUP BY settled_date + generations.settled_date AS date, + argMax(generations.generation_id, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS generation_id, + argMax(generations.source_cutoff, tuple(generations.source_cutoff, generations.published_at, generations.generation_id)) AS source_cutoff + FROM product_analytics_generations_v2 AS generations + WHERE generations.generation_kind = 'cold' + GROUP BY generations.settled_date NODE product_traffic_pages_daily_current_v2_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe b/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe index c08588fd565..c1021bd0505 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe @@ -7,15 +7,15 @@ NODE product_traffic_totals_node SQL > % SELECT - uniqExactMerge(visitors) AS visitors, - toUInt64(sum(visits)) AS visits, - toUInt64(sum(pageviews)) AS pageviews, - round(if(sum(visits) = 0, 0, sum(pageviews) / sum(visits)), 2) AS views_per_visit, - round(if(sum(visits) = 0, 0, 100 * sum(bounces) / sum(visits)), 2) AS bounce_rate, - toUInt64(if(sum(visits) = 0, 0, sum(visit_duration_ms) / sum(visits))) AS visit_duration_ms, - toUInt64(sum(engaged_ms)) AS engaged_ms - FROM product_traffic_daily_exact - WHERE copy_run_id = '' + uniqExactMerge(traffic.visitors) AS visitors, + toUInt64(sum(traffic.visits)) AS visits, + toUInt64(sum(traffic.pageviews)) AS pageviews, + round(if(sum(traffic.visits) = 0, 0, sum(traffic.pageviews) / sum(traffic.visits)), 2) AS views_per_visit, + round(if(sum(traffic.visits) = 0, 0, 100 * sum(traffic.bounces) / sum(traffic.visits)), 2) AS bounce_rate, + toUInt64(if(sum(traffic.visits) = 0, 0, sum(traffic.visit_duration_ms) / sum(traffic.visits))) AS visit_duration_ms, + toUInt64(sum(traffic.engaged_ms)) AS engaged_ms + FROM product_traffic_daily_exact AS traffic + WHERE traffic.copy_run_id = '' {% if defined(synthetic_run_id) %} AND synthetic_run_id = {{String(synthetic_run_id)}} AND throwIf(length({{String(synthetic_run_id)}}) < 8 OR length({{String(synthetic_run_id)}}) > 128, 'synthetic_run_id has an invalid length') = 0 diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe index 57cc6025f07..0d431146f26 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_event_day_states_v2.pipe @@ -10,40 +10,40 @@ SQL > {{max_threads(Int32(copy_max_threads))}} {% end %} SELECT - toDate(occurred_at, 'UTC') AS occurred_date, - event_id, - uniqExactState(payload_hash) AS payload_hashes, - argMinState(payload_hash, received_at) AS payload_hash, - argMinState(occurred_at, received_at) AS occurred_at, - min(received_at) AS first_received_at, - max(received_at) AS received_at, - argMinState(toString(event_name), received_at) AS event_name, - argMinState(schema_version, received_at) AS schema_version, - argMinState(toString(source), received_at) AS source, - argMinState(toString(platform), received_at) AS platform, - argMinState(anonymous_id, received_at) AS anonymous_id, - argMinState(session_id, received_at) AS session_id, - argMinState(user_id, received_at) AS user_id, - argMinState(organization_id, received_at) AS organization_id, - argMinState(toString(app_version), received_at) AS app_version, - argMinState(pathname, received_at) AS pathname, - argMinState(referrer, received_at) AS referrer, - argMinState(toString(country), received_at) AS country, - argMinState(toString(region), received_at) AS region, - argMinState(toString(city), received_at) AS city, - argMinState(toString(hostname), received_at) AS hostname, - argMinState(toString(browser), received_at) AS browser, - argMinState(toString(device), received_at) AS device, - argMinState(toString(os), received_at) AS os, - argMinState(toString(channel), received_at) AS channel, - argMinState(toString(traffic_class), received_at) AS traffic_class, - argMinState(synthetic_run_id, received_at) AS synthetic_run_id, - argMinState(properties, received_at) AS properties - FROM product_events_v1 + toDate(raw.occurred_at, 'UTC') AS occurred_date, + raw.event_id, + uniqExactState(raw.payload_hash) AS payload_hashes, + argMinState(raw.payload_hash, raw.received_at) AS payload_hash, + argMinState(raw.occurred_at, raw.received_at) AS occurred_at, + min(raw.received_at) AS first_received_at, + max(raw.received_at) AS received_at, + argMinState(toString(raw.event_name), raw.received_at) AS event_name, + argMinState(raw.schema_version, raw.received_at) AS schema_version, + argMinState(toString(raw.source), raw.received_at) AS source, + argMinState(toString(raw.platform), raw.received_at) AS platform, + argMinState(raw.anonymous_id, raw.received_at) AS anonymous_id, + argMinState(raw.session_id, raw.received_at) AS session_id, + argMinState(raw.user_id, raw.received_at) AS user_id, + argMinState(raw.organization_id, raw.received_at) AS organization_id, + argMinState(toString(raw.app_version), raw.received_at) AS app_version, + argMinState(raw.pathname, raw.received_at) AS pathname, + argMinState(raw.referrer, raw.received_at) AS referrer, + argMinState(toString(raw.country), raw.received_at) AS country, + argMinState(toString(raw.region), raw.received_at) AS region, + argMinState(toString(raw.city), raw.received_at) AS city, + argMinState(toString(raw.hostname), raw.received_at) AS hostname, + argMinState(toString(raw.browser), raw.received_at) AS browser, + argMinState(toString(raw.device), raw.received_at) AS device, + argMinState(toString(raw.os), raw.received_at) AS os, + argMinState(toString(raw.channel), raw.received_at) AS channel, + argMinState(toString(raw.traffic_class), raw.received_at) AS traffic_class, + argMinState(raw.synthetic_run_id, raw.received_at) AS synthetic_run_id, + argMinState(raw.properties, raw.received_at) AS properties + FROM product_events_v1 AS raw {% if defined(source_cutoff) %} - WHERE received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + WHERE raw.received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) {% end %} - GROUP BY occurred_date, event_id + GROUP BY toDate(raw.occurred_at, 'UTC'), raw.event_id TYPE COPY TARGET_DATASOURCE product_event_day_states_v2 diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe index 7af8f1c1ac3..7d976299049 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_event_id_states_v2.pipe @@ -10,39 +10,39 @@ SQL > {{max_threads(Int32(copy_max_threads))}} {% end %} SELECT - event_id, - uniqExactState(payload_hash) AS payload_hashes, - argMinState(payload_hash, received_at) AS payload_hash, - argMinState(occurred_at, received_at) AS occurred_at, - min(received_at) AS first_received_at, - max(received_at) AS received_at, - argMinState(toString(event_name), received_at) AS event_name, - argMinState(schema_version, received_at) AS schema_version, - argMinState(toString(source), received_at) AS source, - argMinState(toString(platform), received_at) AS platform, - argMinState(anonymous_id, received_at) AS anonymous_id, - argMinState(session_id, received_at) AS session_id, - argMinState(user_id, received_at) AS user_id, - argMinState(organization_id, received_at) AS organization_id, - argMinState(toString(app_version), received_at) AS app_version, - argMinState(pathname, received_at) AS pathname, - argMinState(referrer, received_at) AS referrer, - argMinState(toString(country), received_at) AS country, - argMinState(toString(region), received_at) AS region, - argMinState(toString(city), received_at) AS city, - argMinState(toString(hostname), received_at) AS hostname, - argMinState(toString(browser), received_at) AS browser, - argMinState(toString(device), received_at) AS device, - argMinState(toString(os), received_at) AS os, - argMinState(toString(channel), received_at) AS channel, - argMinState(toString(traffic_class), received_at) AS traffic_class, - argMinState(synthetic_run_id, received_at) AS synthetic_run_id, - argMinState(properties, received_at) AS properties - FROM product_events_v1 + raw.event_id, + uniqExactState(raw.payload_hash) AS payload_hashes, + argMinState(raw.payload_hash, raw.received_at) AS payload_hash, + argMinState(raw.occurred_at, raw.received_at) AS occurred_at, + min(raw.received_at) AS first_received_at, + max(raw.received_at) AS received_at, + argMinState(toString(raw.event_name), raw.received_at) AS event_name, + argMinState(raw.schema_version, raw.received_at) AS schema_version, + argMinState(toString(raw.source), raw.received_at) AS source, + argMinState(toString(raw.platform), raw.received_at) AS platform, + argMinState(raw.anonymous_id, raw.received_at) AS anonymous_id, + argMinState(raw.session_id, raw.received_at) AS session_id, + argMinState(raw.user_id, raw.received_at) AS user_id, + argMinState(raw.organization_id, raw.received_at) AS organization_id, + argMinState(toString(raw.app_version), raw.received_at) AS app_version, + argMinState(raw.pathname, raw.received_at) AS pathname, + argMinState(raw.referrer, raw.received_at) AS referrer, + argMinState(toString(raw.country), raw.received_at) AS country, + argMinState(toString(raw.region), raw.received_at) AS region, + argMinState(toString(raw.city), raw.received_at) AS city, + argMinState(toString(raw.hostname), raw.received_at) AS hostname, + argMinState(toString(raw.browser), raw.received_at) AS browser, + argMinState(toString(raw.device), raw.received_at) AS device, + argMinState(toString(raw.os), raw.received_at) AS os, + argMinState(toString(raw.channel), raw.received_at) AS channel, + argMinState(toString(raw.traffic_class), raw.received_at) AS traffic_class, + argMinState(raw.synthetic_run_id, raw.received_at) AS synthetic_run_id, + argMinState(raw.properties, raw.received_at) AS properties + FROM product_events_v1 AS raw {% if defined(source_cutoff) %} - WHERE received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) + WHERE raw.received_at <= toDateTime64({{DateTime64(source_cutoff)}}, 3) {% end %} - GROUP BY event_id + GROUP BY raw.event_id TYPE COPY TARGET_DATASOURCE product_event_id_states_v2 diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe index a953a9bdaec..161523dedea 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_experiment_outcomes_exact.pipe @@ -25,12 +25,12 @@ SQL > AND user_id != '' ), identity_links AS ( SELECT - synthetic_run_id, - anonymous_id, - argMin(user_id, tuple(occurred_at, event_id)) AS user_id - FROM identity_candidates - GROUP BY synthetic_run_id, anonymous_id - HAVING uniqExact(user_id) = 1 + candidates.synthetic_run_id, + candidates.anonymous_id, + argMin(candidates.user_id, tuple(candidates.occurred_at, candidates.event_id)) AS user_id + FROM identity_candidates AS candidates + GROUP BY candidates.synthetic_run_id, candidates.anonymous_id + HAVING uniqExact(candidates.user_id) = 1 ), resolved_exposures AS ( SELECT exposures.synthetic_run_id AS synthetic_run_id, @@ -54,21 +54,21 @@ SQL > AND (exposures.user_id != '' OR exposures.anonymous_id != '') ), exposures AS ( SELECT - synthetic_run_id, - actor_id, - argMin(resolved_user_id, tuple(occurred_at, event_id)) AS user_id, - min(occurred_at) AS exposed_at, - experiment_id, - assignment_version, - argMin(variant, tuple(occurred_at, event_id)) AS variant, - argMin(platform, tuple(occurred_at, event_id)) AS platform, - argMin(app_version, tuple(occurred_at, event_id)) AS app_version - FROM resolved_exposures - WHERE experiment_id != '' - AND assignment_version != '' - AND variant != '' - GROUP BY synthetic_run_id, actor_id, experiment_id, assignment_version - HAVING uniqExact(variant) = 1 + resolved.synthetic_run_id, + resolved.actor_id, + argMin(resolved.resolved_user_id, tuple(resolved.occurred_at, resolved.event_id)) AS user_id, + min(resolved.occurred_at) AS exposed_at, + resolved.experiment_id, + resolved.assignment_version, + argMin(resolved.variant, tuple(resolved.occurred_at, resolved.event_id)) AS variant, + argMin(resolved.platform, tuple(resolved.occurred_at, resolved.event_id)) AS platform, + argMin(resolved.app_version, tuple(resolved.occurred_at, resolved.event_id)) AS app_version + FROM resolved_exposures AS resolved + WHERE resolved.experiment_id != '' + AND resolved.assignment_version != '' + AND resolved.variant != '' + GROUP BY resolved.synthetic_run_id, resolved.actor_id, resolved.experiment_id, resolved.assignment_version + HAVING uniqExact(resolved.variant) = 1 ), outcome_candidates AS ( SELECT synthetic_run_id, diff --git a/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe b/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe index dce54b6623b..f0cf811b349 100644 --- a/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe +++ b/scripts/analytics/tinybird/pipes/snapshot_product_identity_funnel_exact.pipe @@ -29,13 +29,13 @@ SQL > SELECT * FROM paid_guest_links ), first_links AS ( SELECT - synthetic_run_id, - user_id, - argMin(anonymous_id, tuple(occurred_at, event_id)) AS anonymous_id, - argMinIf(organization_id, tuple(occurred_at, event_id), organization_id != '') AS organization_id, - min(occurred_at) AS linked_at - FROM link_candidates - GROUP BY synthetic_run_id, user_id + candidates.synthetic_run_id, + candidates.user_id, + argMin(candidates.anonymous_id, tuple(candidates.occurred_at, candidates.event_id)) AS anonymous_id, + argMinIf(candidates.organization_id, tuple(candidates.occurred_at, candidates.event_id), candidates.organization_id != '') AS organization_id, + min(candidates.occurred_at) AS linked_at + FROM link_candidates AS candidates + GROUP BY candidates.synthetic_run_id, candidates.user_id ), organization_first_links AS ( SELECT synthetic_run_id, @@ -107,14 +107,14 @@ SQL > FROM paid_guest_links ), guest_paths AS ( SELECT - synthetic_run_id, - user_id, - argMin(anonymous_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at, anonymous_id)) AS anonymous_id, - argMin(organization_id, tuple(if(paid_guest_purchase > 0, 0, 1), path_at, anonymous_id)) AS organization_id, - argMin(path_at, tuple(if(paid_guest_purchase > 0, 0, 1), path_at, anonymous_id)) AS linked_at, - toUInt64(max(paid_guest_purchase) > 0) AS guest_purchased - FROM guest_path_candidates - GROUP BY synthetic_run_id, user_id + candidates.synthetic_run_id, + candidates.user_id, + argMin(candidates.anonymous_id, tuple(if(candidates.paid_guest_purchase > 0, 0, 1), candidates.path_at, candidates.anonymous_id)) AS anonymous_id, + argMin(candidates.organization_id, tuple(if(candidates.paid_guest_purchase > 0, 0, 1), candidates.path_at, candidates.anonymous_id)) AS organization_id, + argMin(candidates.path_at, tuple(if(candidates.paid_guest_purchase > 0, 0, 1), candidates.path_at, candidates.anonymous_id)) AS linked_at, + toUInt64(max(candidates.paid_guest_purchase) > 0) AS guest_purchased + FROM guest_path_candidates AS candidates + GROUP BY candidates.synthetic_run_id, candidates.user_id ), user_linked_outcomes AS ( SELECT first_links.synthetic_run_id AS synthetic_run_id, diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index cc98cbcf510..d1188d81397 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -189,7 +189,10 @@ const cloudCliStep = (...args) => ({ }); const operationPlan = (operation) => { - const localSourceCutoff = new Date().toISOString(); + const localSourceCutoff = new Date() + .toISOString() + .replace("T", " ") + .replace("Z", ""); const localGenerationId = `local_${localSourceCutoff.replaceAll(/[^0-9]/g, "")}`; const localCopyStep = (name) => { const parameters = [ @@ -245,6 +248,7 @@ const operationPlan = (operation) => { ...PRODUCT_COPY_PIPES.map((name) => localCliStep("--local", "copy", "pause", name), ), + { type: "reset-local-fixture" }, localCliStep( "--local", "datasource", @@ -276,6 +280,7 @@ const operationPlan = (operation) => { ...PRODUCT_COPY_PIPES.map((name) => localCliStep("--local", "copy", "pause", name), ), + { type: "reset-local-fixture" }, localCliStep( "--local", "datasource", @@ -1128,6 +1133,23 @@ const runAnalyticsCommand = async (operation) => { prepareLocalFixture(); continue; } + if (step.type === "reset-local-fixture") { + await runProcess( + "docker", + composeArgs( + "run", + "--rm", + "tinybird-cli", + "--local", + "datasource", + "truncate", + "product_events_v1", + "--yes", + ), + { env: localEnvironment() }, + ); + continue; + } if (step.type === "verify-cloud-workspace") { verifyCloudWorkspace(); continue; diff --git a/scripts/analytics/verify-local.js b/scripts/analytics/verify-local.js index 3757d5b66f0..5873b9389a7 100644 --- a/scripts/analytics/verify-local.js +++ b/scripts/analytics/verify-local.js @@ -139,6 +139,8 @@ assert.deepEqual(copyAssertions, [ activation_markers: 1, retention_markers: 1, identity_markers: 1, + attribution_markers: 1, + experiment_markers: 1, health_markers: 1, }, ]); From ea74b61745bc3d83757ec31a7429a85f784a9a13 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:10:00 +0100 Subject: [PATCH 093/110] ci: authorize protected analytics previews --- .github/workflows/analytics.yml | 16 ++- .../e2e/analytics-staging.spec.ts | 27 ++++ scripts/analytics/staging-ci.js | 119 +++++++++++++++--- scripts/analytics/tests/staging-ci.test.js | 33 +++++ 4 files changed, 178 insertions(+), 17 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 0eb49e6d98d..29e87bc7cc3 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -170,6 +170,7 @@ jobs: EVENT_NUMBER: ${{ github.event.pull_request.number || '' }} HEAD_REF: ${{ github.event.pull_request.head.ref || '' }} ANALYTICS_TEST_RUN_ID: run_${{ github.run_id }}_${{ github.run_attempt }}_${{ github.event.pull_request.head.sha || github.sha }} + ANALYTICS_PREVIEW_ACCESS_URL: https://cap-web-git-codex-first-party-analytics-mc-ilroy.vercel.app steps: - name: Check out exact requested SHA uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 @@ -197,7 +198,7 @@ jobs: run: node scripts/analytics/staging-ci.js wait-vercel - name: Refuse a preview bound outside Tinybird staging env: - ANALYTICS_PREVIEW_URL: ${{ steps.vercel.outputs.url }} + ANALYTICS_PREVIEW_URL: ${{ env.ANALYTICS_PREVIEW_ACCESS_URL }} CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} @@ -207,10 +208,12 @@ jobs: TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: node scripts/analytics/staging-ci.js attest-preview - name: Persist the pre-create Tinybird recovery boundary env: VERCEL_DEPLOYMENT_ID: ${{ steps.vercel.outputs.deployment_id }} + VERCEL_PREVIEW_ACCESS_URL: ${{ env.ANALYTICS_PREVIEW_ACCESS_URL }} VERCEL_PREVIEW_URL: ${{ steps.vercel.outputs.url }} TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} @@ -263,6 +266,7 @@ jobs: - name: Prepare immutable synthetic cleanup state before ingestion env: VERCEL_DEPLOYMENT_ID: ${{ steps.vercel.outputs.deployment_id }} + VERCEL_PREVIEW_ACCESS_URL: ${{ env.ANALYTICS_PREVIEW_ACCESS_URL }} VERCEL_PREVIEW_URL: ${{ steps.vercel.outputs.url }} run: >- node scripts/analytics/staging-ci.js prepare-seed @@ -419,18 +423,22 @@ jobs: - name: Prove the exact-SHA deployed browser tracker id: browser-e2e env: - ANALYTICS_PREVIEW_URL: ${{ steps.vercel.outputs.url }} + ANALYTICS_PREVIEW_URL: ${{ env.ANALYTICS_PREVIEW_ACCESS_URL }} ANALYTICS_BROWSER_RUN_ID: ${{ env.ANALYTICS_TEST_RUN_ID }}_preview ANALYTICS_STATE_PATH: ${{ runner.temp }}/analytics-staging-state.json ANALYTICS_ARTIFACT_PATH: ${{ runner.temp }}/analytics-staging-report.json + CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: | pnpm --dir apps/chrome-extension exec playwright install --with-deps chromium pnpm --dir apps/chrome-extension exec playwright test e2e/analytics-staging.spec.ts - name: Probe the exact-SHA Vercel browser collector and staging rate limit id: preview-api env: + CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: >- node scripts/analytics/staging-ci.js probe-preview --state "$RUNNER_TEMP/analytics-staging-state.json" @@ -442,6 +450,7 @@ jobs: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: >- node scripts/analytics/staging-ci.js probe-server --state "$RUNNER_TEMP/analytics-staging-state.json" @@ -499,6 +508,7 @@ jobs: env: CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: >- node scripts/analytics/staging-ci.js erase-synthetic-identity --state "$RUNNER_TEMP/analytics-staging-state.json" @@ -535,6 +545,7 @@ jobs: TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: | if [[ ! -f "$RUNNER_TEMP/analytics-staging-state.json" ]]; then echo "required=false" >> "$GITHUB_OUTPUT" @@ -701,6 +712,7 @@ jobs: TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} + VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: >- node scripts/analytics/staging-ci.js recover --checkpoint-directory "$RUNNER_TEMP/analytics-recovery-checkpoints" diff --git a/apps/chrome-extension/e2e/analytics-staging.spec.ts b/apps/chrome-extension/e2e/analytics-staging.spec.ts index 7718ad3ff1b..0c9ce5c9ecc 100644 --- a/apps/chrome-extension/e2e/analytics-staging.spec.ts +++ b/apps/chrome-extension/e2e/analytics-staging.spec.ts @@ -41,7 +41,12 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and matche const runId = requiredEnvironment("ANALYTICS_BROWSER_RUN_ID"); const statePath = requiredEnvironment("ANALYTICS_STATE_PATH"); const artifactPath = requiredEnvironment("ANALYTICS_ARTIFACT_PATH"); + const expectedSha = requiredEnvironment("EXPECTED_SHA"); + const stagingSecret = requiredEnvironment( + "CAP_ANALYTICS_STAGING_TEST_SECRET", + ); const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim(); + const shareSecret = process.env.VERCEL_PREVIEW_SHARE_SECRET?.trim(); const context = await browser.newContext({ baseURL: previewUrl, extraHTTPHeaders: { @@ -54,6 +59,27 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and matche : {}), }, }); + if (shareSecret) { + const shareUrl = new URL("/api/analytics/staging-test/attest", previewUrl); + shareUrl.searchParams.set("_vercel_share", shareSecret); + const bootstrap = await context.request.get(shareUrl.toString(), { + maxRedirects: 0, + }); + expect([302, 303, 307, 308]).toContain(bootstrap.status()); + } + const attestExactSha = async () => { + const response = await context.request.post( + "/api/analytics/staging-test/attest", + { + data: { runId, sha: expectedSha }, + headers: { Authorization: `Bearer ${stagingSecret}` }, + }, + ); + expect(response.ok()).toBe(true); + const payload = (await response.json()) as { sha?: string }; + expect(payload.sha).toBe(expectedSha); + }; + await attestExactSha(); const captured: CapturedEvent[] = []; const acceptedEventIds = new Set(); const benchmarkEventIds = new Set(); @@ -465,5 +491,6 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and matche browserMainThreadBudgetPassed: true, }; fs.writeFileSync(artifactPath, `${JSON.stringify(artifact, null, 2)}\n`); + await attestExactSha(); await context.close(); }); diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index ce1beec7a60..e8c5caaf15a 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -160,6 +160,9 @@ const PREVIEW_TINYBIRD_TOKEN_ENV = { "TINYBIRD_STAGING_SCHEDULER_TOKEN", }; +const STAGING_PREVIEW_ACCESS_ORIGIN = + "https://cap-web-git-codex-first-party-analytics-mc-ilroy.vercel.app"; + const tinybirdEnvironment = (requiredTokenNames = TINYBIRD_TOKEN_NAMES) => { if (environment("TINYBIRD_WORKSPACE_ID") !== STAGING_WORKSPACE_ID) { throw new Error( @@ -385,6 +388,7 @@ const prepareDeploymentBoundary = async () => { phase: "precreate", preview: { deploymentId: environment("VERCEL_DEPLOYMENT_ID"), + accessUrl: new URL(environment("VERCEL_PREVIEW_ACCESS_URL")).origin, url: new URL(environment("VERCEL_PREVIEW_URL")).origin, }, tinybird: { @@ -1304,6 +1308,11 @@ const verifyFreshPullRequestHead = async () => { const attestPreviewTinybird = async () => { const secret = environment("CAP_ANALYTICS_STAGING_TEST_SECRET"); const previewOrigin = new URL(environment("ANALYTICS_PREVIEW_URL")).origin; + if (previewOrigin !== STAGING_PREVIEW_ACCESS_ORIGIN) { + throw new Error( + "The preview access URL is not the analytics staging alias", + ); + } const runId = validateSyntheticRunId(environment("ANALYTICS_TEST_RUN_ID")); const { origin: expectedOrigin, tokens } = tinybirdEnvironment( Object.values(PREVIEW_TINYBIRD_TOKEN_ENV), @@ -1369,21 +1378,86 @@ const previewCookies = (headers) => { .join("; "); }; +let previewShareCookie; + const previewRequest = async (url, init = {}) => { const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim(); - return fetch(url, { + const shareSecret = process.env.VERCEL_PREVIEW_SHARE_SECRET?.trim(); + const headers = new Headers({ + ...(bypass + ? { + "x-vercel-protection-bypass": bypass, + "x-vercel-set-bypass-cookie": "true", + } + : {}), + ...init.headers, + }); + if (previewShareCookie) { + headers.set( + "Cookie", + [headers.get("Cookie"), previewShareCookie].filter(Boolean).join("; "), + ); + } + const requestInit = { ...init, - headers: { - ...(bypass - ? { - "x-vercel-protection-bypass": bypass, - "x-vercel-set-bypass-cookie": "true", - } - : {}), - ...init.headers, - }, + headers, signal: init.signal ?? AbortSignal.timeout(20_000), + }; + if (!shareSecret || previewShareCookie) return fetch(url, requestInit); + + const shareUrl = new URL(url); + shareUrl.searchParams.set("_vercel_share", shareSecret); + const handshake = await fetch(shareUrl, { + ...requestInit, + redirect: "manual", }); + if (![302, 303, 307, 308].includes(handshake.status)) return handshake; + const location = handshake.headers.get("location"); + const cookie = previewCookies(handshake.headers) + .split("; ") + .find((value) => value.startsWith("_vercel_jwt=")); + if (!location || !cookie) { + throw new Error("The staging alias did not issue a Vercel share cookie"); + } + const redirectUrl = new URL(location, shareUrl); + if (redirectUrl.origin !== shareUrl.origin) { + throw new Error("The Vercel share bootstrap left the staging alias"); + } + previewShareCookie = cookie; + headers.set( + "Cookie", + [headers.get("Cookie"), previewShareCookie].filter(Boolean).join("; "), + ); + return fetch(redirectUrl, requestInit); +}; + +const artifactPreviewUrl = (artifact) => + artifact.vercel.accessUrl ?? artifact.vercel.url; + +const attestExactPreviewSha = async ({ previewOrigin, runId }) => { + const response = await previewRequest( + new URL("/api/analytics/staging-test/attest", previewOrigin), + { + method: "POST", + headers: { + Authorization: `Bearer ${environment("CAP_ANALYTICS_STAGING_TEST_SECRET")}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + runId: validateSyntheticRunId(runId), + sha: exactSha(environment("EXPECTED_SHA"), "EXPECTED_SHA"), + }), + }, + ); + if (!response.ok) { + throw new Error( + `The staging alias failed exact-SHA attestation with HTTP ${response.status}`, + ); + } + const payload = await response.json(); + if (payload.sha !== environment("EXPECTED_SHA")) { + throw new Error("The staging alias moved to a different Vercel SHA"); + } }; const cleanupPreviewDatabaseState = async ({ @@ -1394,7 +1468,7 @@ const cleanupPreviewDatabaseState = async ({ }) => { const url = new URL( "/api/analytics/staging-test/cleanup-database", - artifact.vercel.url, + artifactPreviewUrl(artifact), ); const response = await previewRequest(url, { method: "POST", @@ -1466,7 +1540,11 @@ const probePreview = async () => { const artifactPath = option("artifact"); const state = readJson(statePath); const artifact = readJson(artifactPath); - const previewOrigin = new URL(artifact.vercel.url).origin; + const previewOrigin = new URL(artifactPreviewUrl(artifact)).origin; + await attestExactPreviewSha({ + previewOrigin, + runId: `${state.runId}_preview_api`, + }); const landing = await previewRequest(previewOrigin); if (!landing.ok) { throw new Error( @@ -1756,6 +1834,10 @@ const probePreview = async () => { bundleBudgetPassed: true, collectorBudgetPassed: true, }; + await attestExactPreviewSha({ + previewOrigin, + runId: `${state.runId}_preview_api_final`, + }); writeJson(artifactPath, artifact); }; @@ -1766,7 +1848,10 @@ const probeDurableServerPath = async () => { const artifact = readJson(artifactPath); const secret = environment("CAP_ANALYTICS_STAGING_TEST_SECRET"); const serverRunId = validateSyntheticRunId(`${state.runId}_server`); - const url = new URL("/api/analytics/staging-test", artifact.vercel.url); + const url = new URL( + "/api/analytics/staging-test", + artifactPreviewUrl(artifact), + ); const body = (sha) => JSON.stringify({ scenario: "business_lifecycle", @@ -1844,7 +1929,7 @@ const probeDurableServerPath = async () => { }); const healthUrl = new URL( "/api/analytics/staging-test/health", - artifact.vercel.url, + artifactPreviewUrl(artifact), ); const healthResponse = await previewRequest(healthUrl, { method: "POST", @@ -2067,6 +2152,7 @@ const prepareSeed = async () => { }, vercel: { deploymentId: environment("VERCEL_DEPLOYMENT_ID"), + accessUrl: environment("VERCEL_PREVIEW_ACCESS_URL"), url: environment("VERCEL_PREVIEW_URL"), }, tinybird: { deploymentId }, @@ -3616,7 +3702,10 @@ const eraseSyntheticIdentity = async () => { const artifactPath = option("artifact"); const artifact = readJson(artifactPath); const secret = environment("CAP_ANALYTICS_STAGING_TEST_SECRET"); - const url = new URL("/api/analytics/staging-test/erase", artifact.vercel.url); + const url = new URL( + "/api/analytics/staging-test/erase", + artifactPreviewUrl(artifact), + ); const body = JSON.stringify({ runId: state.runId, sha: artifact.sha }); const send = (authorization) => previewRequest(url, { diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 1c6c2c7fccb..0d60f31233c 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2024,6 +2024,14 @@ test("the analytics workflow is statically restricted to staging", () => { workflow, /Refuse a preview bound outside Tinybird staging[\s\S]*CAP_ANALYTICS_STAGING_TEST_SECRET[\s\S]*staging-ci\.js attest-preview/, ); + assert.match( + workflow, + /ANALYTICS_PREVIEW_ACCESS_URL: https:\/\/cap-web-git-codex-first-party-analytics-mc-ilroy\.vercel\.app/, + ); + assert.match( + workflow, + /Refuse a preview bound outside Tinybird staging[\s\S]*VERCEL_PREVIEW_SHARE_SECRET[\s\S]*staging-ci\.js attest-preview/, + ); assert.ok( workflow.indexOf( "Prove promoted delivery, business values, and decision deduplication", @@ -2233,6 +2241,31 @@ test("the preview mutation route independently enforces Tinybird staging", () => new URL("../staging-ci.js", import.meta.url), "utf8", ); + assert.match( + runner, + /const STAGING_PREVIEW_ACCESS_ORIGIN =\s*"https:\/\/cap-web-git-codex-first-party-analytics-mc-ilroy\.vercel\.app"/, + ); + assert.match( + runner, + /shareUrl\.searchParams\.set\("_vercel_share", shareSecret\)/, + ); + assert.match(runner, /value\.startsWith\("_vercel_jwt="\)/); + assert.match( + runner, + /artifact\.vercel\.accessUrl \?\? artifact\.vercel\.url/, + ); + const browserProbe = fs.readFileSync( + new URL( + "../../../apps/chrome-extension/e2e/analytics-staging.spec.ts", + import.meta.url, + ), + "utf8", + ); + assert.equal(browserProbe.match(/await attestExactSha\(\)/g)?.length, 2); + assert.match( + browserProbe, + /shareUrl\.searchParams\.set\("_vercel_share", shareSecret\)/, + ); const serverProbe = runner.slice( runner.indexOf("const probeDurableServerPath = async () => {"), runner.indexOf("const seed = async () => {"), From 9be88621e950e9c3f78bb805106926c64c943488 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:23:46 +0100 Subject: [PATCH 094/110] fix: attest branch-scoped analytics previews --- apps/web/app/api/analytics/staging-test/route.ts | 5 ++++- scripts/analytics/tests/staging-ci.test.js | 4 ++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts index 9b83d3e02c8..dedd555fbfa 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -595,7 +595,10 @@ const attestDatabaseSchema = async () => { const authorize = (payload: { runId: string; sha: string }) => Effect.gen(function* () { - if (process.env.VERCEL_ENV !== "preview") { + if ( + process.env.VERCEL_ENV === "production" || + process.env.CAP_ANALYTICS_STAGING_PREVIEW !== "true" + ) { return yield* Effect.fail(new HttpApiError.NotFound()); } const secret = process.env.CAP_ANALYTICS_STAGING_TEST_SECRET; diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 0d60f31233c..37bd33a2cd0 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2221,6 +2221,10 @@ test("the preview mutation route independently enforces Tinybird staging", () => route, /const authorize = [\s\S]*!configurationAttestation\(runId\)/, ); + assert.match( + route, + /process\.env\.VERCEL_ENV === "production"[\s\S]*process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"/, + ); assert.match( route, /const STAGING_DATABASE_FINGERPRINT =\s*"fff37a9b160f31bfb82b8c5585829b8ee08f70b3645169dca6e7cb29033a039a"/, From cde2ccc2c23aed5a3e160b5499b925bbbea2f0b5 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:34:23 +0100 Subject: [PATCH 095/110] fix: restrict staging analytics by preview host --- apps/web/app/api/analytics/staging-test/route.ts | 12 ++++++++---- scripts/analytics/tests/staging-ci.test.js | 6 +++++- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts index dedd555fbfa..9cac6c983ac 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -151,8 +151,12 @@ class Api extends HttpApi.make("AnalyticsStagingTestApi").add( const RequestHeaders = Schema.Struct({ authorization: Schema.optional(Schema.String), + host: Schema.String, }); +const STAGING_PREVIEW_HOST = + "cap-web-git-codex-first-party-analytics-mc-ilroy.vercel.app"; + const safeEqual = (actual: string | undefined, expected: string) => Boolean( actual && @@ -595,9 +599,12 @@ const attestDatabaseSchema = async () => { const authorize = (payload: { runId: string; sha: string }) => Effect.gen(function* () { + const headers = yield* HttpServerRequest.schemaHeaders(RequestHeaders).pipe( + Effect.mapError(() => new HttpApiError.BadRequest()), + ); if ( process.env.VERCEL_ENV === "production" || - process.env.CAP_ANALYTICS_STAGING_PREVIEW !== "true" + headers.host !== STAGING_PREVIEW_HOST ) { return yield* Effect.fail(new HttpApiError.NotFound()); } @@ -605,9 +612,6 @@ const authorize = (payload: { runId: string; sha: string }) => if (!secret) { return yield* Effect.fail(new HttpApiError.ServiceUnavailable()); } - const headers = yield* HttpServerRequest.schemaHeaders(RequestHeaders).pipe( - Effect.mapError(() => new HttpApiError.BadRequest()), - ); if (!safeEqual(headers.authorization, `Bearer ${secret}`)) { return yield* Effect.fail(new HttpApiError.Unauthorized()); } diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 37bd33a2cd0..9f9508c744e 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2223,7 +2223,11 @@ test("the preview mutation route independently enforces Tinybird staging", () => ); assert.match( route, - /process\.env\.VERCEL_ENV === "production"[\s\S]*process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"/, + /process\.env\.VERCEL_ENV === "production"[\s\S]*headers\.host !== STAGING_PREVIEW_HOST/, + ); + assert.match( + route, + /const STAGING_PREVIEW_HOST =\s*"cap-web-git-codex-first-party-analytics-mc-ilroy\.vercel\.app"/, ); assert.match( route, From f3572017d2cca0def694b62260825fb37fee12f9 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:47:55 +0100 Subject: [PATCH 096/110] fix: trust scoped analytics preview marker --- apps/web/app/api/analytics/staging-test/route.ts | 2 +- scripts/analytics/tests/staging-ci.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts index 9cac6c983ac..fe6cc2beb2d 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -603,7 +603,7 @@ const authorize = (payload: { runId: string; sha: string }) => Effect.mapError(() => new HttpApiError.BadRequest()), ); if ( - process.env.VERCEL_ENV === "production" || + process.env.CAP_ANALYTICS_STAGING_PREVIEW !== "true" || headers.host !== STAGING_PREVIEW_HOST ) { return yield* Effect.fail(new HttpApiError.NotFound()); diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 9f9508c744e..c057aafcd2e 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2223,7 +2223,7 @@ test("the preview mutation route independently enforces Tinybird staging", () => ); assert.match( route, - /process\.env\.VERCEL_ENV === "production"[\s\S]*headers\.host !== STAGING_PREVIEW_HOST/, + /process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"[\s\S]*headers\.host !== STAGING_PREVIEW_HOST/, ); assert.match( route, From f55226ba13d0c69e109f7a4954f24b593400e95e Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 15:57:42 +0100 Subject: [PATCH 097/110] fix: accept Vercel deployment routing host --- apps/web/app/api/analytics/staging-test/route.ts | 4 +++- scripts/analytics/tests/staging-ci.test.js | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts index fe6cc2beb2d..1f8f42b23e0 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -152,6 +152,7 @@ class Api extends HttpApi.make("AnalyticsStagingTestApi").add( const RequestHeaders = Schema.Struct({ authorization: Schema.optional(Schema.String), host: Schema.String, + "x-vercel-deployment-url": Schema.optional(Schema.String), }); const STAGING_PREVIEW_HOST = @@ -604,7 +605,8 @@ const authorize = (payload: { runId: string; sha: string }) => ); if ( process.env.CAP_ANALYTICS_STAGING_PREVIEW !== "true" || - headers.host !== STAGING_PREVIEW_HOST + (headers.host !== STAGING_PREVIEW_HOST && + headers.host !== headers["x-vercel-deployment-url"]) ) { return yield* Effect.fail(new HttpApiError.NotFound()); } diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index c057aafcd2e..6559e9bebd6 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2223,7 +2223,7 @@ test("the preview mutation route independently enforces Tinybird staging", () => ); assert.match( route, - /process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"[\s\S]*headers\.host !== STAGING_PREVIEW_HOST/, + /process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"[\s\S]*headers\.host !== STAGING_PREVIEW_HOST[\s\S]*headers\.host !== headers\["x-vercel-deployment-url"\]/, ); assert.match( route, From d8c622c91e8f2a3a0786226705d9437fb4ebf8a2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:07:41 +0100 Subject: [PATCH 098/110] fix: gate staging analytics by branch marker --- .../web/app/api/analytics/staging-test/route.ts | 17 ++++------------- scripts/analytics/tests/staging-ci.test.js | 10 ++-------- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts index 1f8f42b23e0..b0eccff6fe3 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -151,13 +151,8 @@ class Api extends HttpApi.make("AnalyticsStagingTestApi").add( const RequestHeaders = Schema.Struct({ authorization: Schema.optional(Schema.String), - host: Schema.String, - "x-vercel-deployment-url": Schema.optional(Schema.String), }); -const STAGING_PREVIEW_HOST = - "cap-web-git-codex-first-party-analytics-mc-ilroy.vercel.app"; - const safeEqual = (actual: string | undefined, expected: string) => Boolean( actual && @@ -600,20 +595,16 @@ const attestDatabaseSchema = async () => { const authorize = (payload: { runId: string; sha: string }) => Effect.gen(function* () { - const headers = yield* HttpServerRequest.schemaHeaders(RequestHeaders).pipe( - Effect.mapError(() => new HttpApiError.BadRequest()), - ); - if ( - process.env.CAP_ANALYTICS_STAGING_PREVIEW !== "true" || - (headers.host !== STAGING_PREVIEW_HOST && - headers.host !== headers["x-vercel-deployment-url"]) - ) { + if (process.env.CAP_ANALYTICS_STAGING_PREVIEW !== "true") { return yield* Effect.fail(new HttpApiError.NotFound()); } const secret = process.env.CAP_ANALYTICS_STAGING_TEST_SECRET; if (!secret) { return yield* Effect.fail(new HttpApiError.ServiceUnavailable()); } + const headers = yield* HttpServerRequest.schemaHeaders(RequestHeaders).pipe( + Effect.mapError(() => new HttpApiError.BadRequest()), + ); if (!safeEqual(headers.authorization, `Bearer ${secret}`)) { return yield* Effect.fail(new HttpApiError.Unauthorized()); } diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 6559e9bebd6..2043892040c 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2221,14 +2221,8 @@ test("the preview mutation route independently enforces Tinybird staging", () => route, /const authorize = [\s\S]*!configurationAttestation\(runId\)/, ); - assert.match( - route, - /process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"[\s\S]*headers\.host !== STAGING_PREVIEW_HOST[\s\S]*headers\.host !== headers\["x-vercel-deployment-url"\]/, - ); - assert.match( - route, - /const STAGING_PREVIEW_HOST =\s*"cap-web-git-codex-first-party-analytics-mc-ilroy\.vercel\.app"/, - ); + assert.match(route, /process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"/); + assert.doesNotMatch(route, /process\.env\.VERCEL_ENV/); assert.match( route, /const STAGING_DATABASE_FINGERPRINT =\s*"fff37a9b160f31bfb82b8c5585829b8ee08f70b3645169dca6e7cb29033a039a"/, From 224c011ff46b5b0b277388a930aec66eac9cc1d4 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:20:29 +0100 Subject: [PATCH 099/110] fix: sign staging analytics requests --- .../e2e/analytics-staging.spec.ts | 10 ++++- .../app/api/analytics/staging-test/route.ts | 20 ++++++--- scripts/analytics/staging-ci.js | 45 ++++++++++++++++++- scripts/analytics/tests/staging-ci.test.js | 15 ++++++- 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/apps/chrome-extension/e2e/analytics-staging.spec.ts b/apps/chrome-extension/e2e/analytics-staging.spec.ts index 0c9ce5c9ecc..0abca91f074 100644 --- a/apps/chrome-extension/e2e/analytics-staging.spec.ts +++ b/apps/chrome-extension/e2e/analytics-staging.spec.ts @@ -1,4 +1,4 @@ -import { createHash } from "node:crypto"; +import { createHash, createHmac } from "node:crypto"; import fs from "node:fs"; import { expect, test } from "@playwright/test"; @@ -68,11 +68,17 @@ test("exact-SHA browser tracker preserves sessions, retries, unloads, and matche expect([302, 303, 307, 308]).toContain(bootstrap.status()); } const attestExactSha = async () => { + const stagingSignature = createHmac("sha256", stagingSecret) + .update(`${runId}:${expectedSha}`) + .digest("hex"); const response = await context.request.post( "/api/analytics/staging-test/attest", { data: { runId, sha: expectedSha }, - headers: { Authorization: `Bearer ${stagingSecret}` }, + headers: { + Authorization: `Bearer ${stagingSecret}`, + "x-cap-analytics-staging-signature": stagingSignature, + }, }, ); expect(response.ok()).toBe(true); diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/route.ts index b0eccff6fe3..2571ee0bc67 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/route.ts @@ -151,6 +151,7 @@ class Api extends HttpApi.make("AnalyticsStagingTestApi").add( const RequestHeaders = Schema.Struct({ authorization: Schema.optional(Schema.String), + "x-cap-analytics-staging-signature": Schema.optional(Schema.String), }); const safeEqual = (actual: string | undefined, expected: string) => @@ -595,9 +596,6 @@ const attestDatabaseSchema = async () => { const authorize = (payload: { runId: string; sha: string }) => Effect.gen(function* () { - if (process.env.CAP_ANALYTICS_STAGING_PREVIEW !== "true") { - return yield* Effect.fail(new HttpApiError.NotFound()); - } const secret = process.env.CAP_ANALYTICS_STAGING_TEST_SECRET; if (!secret) { return yield* Effect.fail(new HttpApiError.ServiceUnavailable()); @@ -609,9 +607,21 @@ const authorize = (payload: { runId: string; sha: string }) => return yield* Effect.fail(new HttpApiError.Unauthorized()); } const runId = boundedRunId(payload.runId); + if (!runId || !draftSha(payload.sha)) { + return yield* Effect.fail(new HttpApiError.BadRequest()); + } + const expectedSignature = createHmac("sha256", secret) + .update(`${runId}:${payload.sha}`) + .digest("hex"); + if ( + !safeEqual( + headers["x-cap-analytics-staging-signature"], + expectedSignature, + ) + ) { + return yield* Effect.fail(new HttpApiError.Unauthorized()); + } if ( - !runId || - !draftSha(payload.sha) || payload.sha !== process.env.VERCEL_GIT_COMMIT_SHA || !configurationAttestation(runId) ) { diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index e8c5caaf15a..638f8dd55e4 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -1329,12 +1329,17 @@ const attestPreviewTinybird = async () => { ); const url = new URL("/api/analytics/staging-test/attest", previewOrigin); const body = (sha) => JSON.stringify({ runId, sha }); - const send = (authorization, sha = environment("EXPECTED_SHA")) => + const send = ( + authorization, + sha = environment("EXPECTED_SHA"), + headers = {}, + ) => previewRequest(url, { method: "POST", headers: { "Content-Type": "application/json", ...(authorization ? { Authorization: authorization } : {}), + ...headers, }, body: body(sha), }); @@ -1344,6 +1349,14 @@ const attestPreviewTinybird = async () => { `The preview configuration attestation accepted missing authorization with HTTP ${unauthorized.status}`, ); } + const invalidSignature = await send(`Bearer ${secret}`, undefined, { + "x-cap-analytics-staging-signature": "0".repeat(64), + }); + if (invalidSignature.status !== 401) { + throw new Error( + `The preview configuration attestation accepted an invalid request signature with HTTP ${invalidSignature.status}`, + ); + } const wrongSha = await send( `Bearer ${secret}`, "0000000000000000000000000000000000000000", @@ -1383,6 +1396,35 @@ let previewShareCookie; const previewRequest = async (url, init = {}) => { const bypass = process.env.VERCEL_AUTOMATION_BYPASS_SECRET?.trim(); const shareSecret = process.env.VERCEL_PREVIEW_SHARE_SECRET?.trim(); + const stagingSignatureHeaders = (() => { + const pathname = new URL(url).pathname; + if ( + pathname !== "/api/analytics/staging-test" && + !pathname.startsWith("/api/analytics/staging-test/") + ) { + return {}; + } + if (typeof init.body !== "string") return {}; + let payload; + try { + payload = JSON.parse(init.body); + } catch { + return {}; + } + if ( + typeof payload?.runId !== "string" || + typeof payload?.sha !== "string" + ) { + return {}; + } + const secret = process.env.CAP_ANALYTICS_STAGING_TEST_SECRET; + if (!secret) return {}; + return { + "x-cap-analytics-staging-signature": createHmac("sha256", secret) + .update(`${payload.runId}:${payload.sha}`) + .digest("hex"), + }; + })(); const headers = new Headers({ ...(bypass ? { @@ -1390,6 +1432,7 @@ const previewRequest = async (url, init = {}) => { "x-vercel-set-bypass-cookie": "true", } : {}), + ...stagingSignatureHeaders, ...init.headers, }); if (previewShareCookie) { diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 2043892040c..ca1018e3f32 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2221,8 +2221,13 @@ test("the preview mutation route independently enforces Tinybird staging", () => route, /const authorize = [\s\S]*!configurationAttestation\(runId\)/, ); - assert.match(route, /process\.env\.CAP_ANALYTICS_STAGING_PREVIEW !== "true"/); + assert.doesNotMatch(route, /CAP_ANALYTICS_STAGING_PREVIEW/); assert.doesNotMatch(route, /process\.env\.VERCEL_ENV/); + assert.match(route, /"x-cap-analytics-staging-signature"/); + assert.match( + route, + /createHmac\("sha256", secret\)[\s\S]*\.update\(`\$\{runId\}:\$\{payload\.sha\}`\)/, + ); assert.match( route, /const STAGING_DATABASE_FINGERPRINT =\s*"fff37a9b160f31bfb82b8c5585829b8ee08f70b3645169dca6e7cb29033a039a"/, @@ -2256,6 +2261,12 @@ test("the preview mutation route independently enforces Tinybird staging", () => runner, /artifact\.vercel\.accessUrl \?\? artifact\.vercel\.url/, ); + assert.match(runner, /"x-cap-analytics-staging-signature"/); + assert.match( + runner, + /pathname !== "\/api\/analytics\/staging-test" &&[\s\S]*!pathname\.startsWith\("\/api\/analytics\/staging-test\/"\)/, + ); + assert.match(runner, /accepted an invalid request signature with HTTP/); const browserProbe = fs.readFileSync( new URL( "../../../apps/chrome-extension/e2e/analytics-staging.spec.ts", @@ -2264,6 +2275,8 @@ test("the preview mutation route independently enforces Tinybird staging", () => "utf8", ); assert.equal(browserProbe.match(/await attestExactSha\(\)/g)?.length, 2); + assert.match(browserProbe, /createHmac\("sha256", stagingSecret\)/); + assert.match(browserProbe, /"x-cap-analytics-staging-signature"/); assert.match( browserProbe, /shareUrl\.searchParams\.set\("_vercel_share", shareSecret\)/, From 8838813ba3d9af8e7b0814e4466af284ff5fc7d1 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:30:10 +0100 Subject: [PATCH 100/110] fix: bootstrap protected analytics preview --- scripts/analytics/staging-ci.js | 4 +++- scripts/analytics/tests/staging-ci.test.js | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 638f8dd55e4..3c67ac1e29c 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -1451,8 +1451,10 @@ const previewRequest = async (url, init = {}) => { const shareUrl = new URL(url); shareUrl.searchParams.set("_vercel_share", shareSecret); const handshake = await fetch(shareUrl, { - ...requestInit, + headers: { Accept: "text/html" }, + method: "GET", redirect: "manual", + signal: requestInit.signal, }); if (![302, 303, 307, 308].includes(handshake.status)) return handshake; const location = handshake.headers.get("location"); diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index ca1018e3f32..bf8a81332a4 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2256,6 +2256,10 @@ test("the preview mutation route independently enforces Tinybird staging", () => runner, /shareUrl\.searchParams\.set\("_vercel_share", shareSecret\)/, ); + assert.match( + runner, + /const handshake = await fetch\(shareUrl, \{[\s\S]*method: "GET",[\s\S]*redirect: "manual"/, + ); assert.match(runner, /value\.startsWith\("_vercel_jwt="\)/); assert.match( runner, From 217a95d9d3a96bf62c686c34a0e34654099c6344 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:43:09 +0100 Subject: [PATCH 101/110] fix: route analytics staging subpaths --- .../api/analytics/staging-test/{ => [[...route]]}/route.ts | 4 ++-- scripts/analytics/tests/staging-ci.test.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) rename apps/web/app/api/analytics/staging-test/{ => [[...route]]}/route.ts (99%) diff --git a/apps/web/app/api/analytics/staging-test/route.ts b/apps/web/app/api/analytics/staging-test/[[...route]]/route.ts similarity index 99% rename from apps/web/app/api/analytics/staging-test/route.ts rename to apps/web/app/api/analytics/staging-test/[[...route]]/route.ts index 2571ee0bc67..8fc22bd442b 100644 --- a/apps/web/app/api/analytics/staging-test/route.ts +++ b/apps/web/app/api/analytics/staging-test/[[...route]]/route.ts @@ -848,5 +848,5 @@ const ApiLive = HttpApiBuilder.api(Api).pipe( ), ); -const handler = apiToHandler(ApiLive); -export const POST = handler; +const postHandler = apiToHandler(ApiLive); +export const POST = postHandler; diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index bf8a81332a4..bd396dac2dd 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2204,7 +2204,7 @@ test("the analytics workflow is statically restricted to staging", () => { test("the preview mutation route independently enforces Tinybird staging", () => { const route = fs.readFileSync( new URL( - "../../../apps/web/app/api/analytics/staging-test/route.ts", + "../../../apps/web/app/api/analytics/staging-test/[[...route]]/route.ts", import.meta.url, ), "utf8", From c9e8b678a76090dd1b5ca1fbbf7dae9a022e0327 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 16:53:17 +0100 Subject: [PATCH 102/110] improve: report staging credential drift --- scripts/analytics/staging-ci-lib.js | 25 +++++++++++----------- scripts/analytics/tests/staging-ci.test.js | 17 +++++++++++++++ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 092410b0028..f7f17372409 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -291,22 +291,23 @@ export const assertPreviewTinybirdAttestation = ({ { tokenHash, workspaceId }, ]), ); + const mismatchedTokens = PREVIEW_TINYBIRD_TOKEN_NAMES.filter((name) => { + const workspace = workspaces.get(name); + return ( + !workspace || + typeof workspace.workspaceId !== "string" || + workspace.workspaceId.toLowerCase() !== + STAGING_WORKSPACE_ID.toLowerCase() || + !expectedTokenHashes || + workspace.tokenHash !== expectedTokenHashes[name] + ); + }); if ( workspaces.size !== PREVIEW_TINYBIRD_TOKEN_NAMES.length || - PREVIEW_TINYBIRD_TOKEN_NAMES.some((name) => { - const workspace = workspaces.get(name); - return ( - !workspace || - typeof workspace.workspaceId !== "string" || - workspace.workspaceId.toLowerCase() !== - STAGING_WORKSPACE_ID.toLowerCase() || - !expectedTokenHashes || - workspace.tokenHash !== expectedTokenHashes[name] - ); - }) + mismatchedTokens.length > 0 ) { throw new Error( - "The exact-SHA preview is not bound to the verified analytics staging tokens", + `The exact-SHA preview is not bound to the verified analytics staging tokens: ${mismatchedTokens.join(", ") || "unexpected token names"}`, ); } }; diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index bd396dac2dd..433a85c4733 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -327,6 +327,23 @@ test("preview Tinybird attestation requires the exact SHA, host, and staging wor }), ); } + assert.throws( + () => + assertPreviewTinybirdAttestation({ + attestation: { + ...attestation, + workspaces: attestation.workspaces.map((workspace, index) => + index === 0 + ? { ...workspace, tokenHash: "f".repeat(64) } + : workspace, + ), + }, + expectedOrigin: attestation.host, + expectedSha: SHA, + expectedTokenHashes, + }), + new RegExp(PREVIEW_TINYBIRD_TOKEN_NAMES[0]), + ); }); test("deployment selection rejects stale or ambiguous staging deployments", () => { From 65a968e426be8a996e4bef6072b3ea75d6878a4c Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 17:35:22 +0100 Subject: [PATCH 103/110] fix: harden analytics staging recovery --- .github/workflows/analytics.yml | 6 +-- scripts/analytics/staging-ci-lib.js | 2 + scripts/analytics/staging-ci.js | 31 ++++++++---- scripts/analytics/tests/staging-ci.test.js | 56 +++++++++++++++++++++- 4 files changed, 82 insertions(+), 13 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 29e87bc7cc3..02bece1a394 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -525,7 +525,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Quiesce scheduled and active Copy jobs before final cleanup id: pause-copies - if: always() && steps.seed.outcome != 'skipped' + if: always() && steps.seed.outcome != 'skipped' && steps.deployment-state.outputs.promoted == 'true' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} @@ -538,7 +538,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Delete strictly scoped synthetic raw rows id: cleanup - if: always() && steps.seed.outcome != 'skipped' && steps.pause-copies.outcome == 'success' + if: always() && steps.seed.outcome != 'skipped' && (steps.deployment-state.outputs.target == 'staging' || steps.pause-copies.outcome == 'success') env: CAP_ANALYTICS_STAGING_TEST_SECRET: ${{ secrets.CAP_ANALYTICS_STAGING_TEST_SECRET }} TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} @@ -586,7 +586,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Resume reviewed Copy schedules id: resume-copies - if: always() && steps.pause-copies.outcome != 'skipped' + if: always() && steps.pause-copies.outcome == 'success' env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index f7f17372409..303a1541191 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -3094,6 +3094,8 @@ export const assertWorkflowSafety = (workflow) => { "TINYBIRD_STAGING_CLEANUP_TOKEN", "staging-ci.js run-copies", "staging-ci.js set-copy-schedules", + "steps.deployment-state.outputs.promoted == 'true'", + "steps.deployment-state.outputs.target == 'staging' || steps.pause-copies.outcome == 'success'", "attest-preview", "probe-preview", "verify-promoted", diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 3c67ac1e29c..c7d014b69e3 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -2285,7 +2285,6 @@ const seed = async () => { now: startedAt, }); const { origin, tokens } = tinybirdEnvironment([ - "TINYBIRD_STAGING_COPY_TOKEN", "TINYBIRD_STAGING_INGEST_TOKEN", ]); const deliver = async (row, fixtureRow = false) => { @@ -4016,8 +4015,8 @@ const cleanup = async (parameters = {}) => { origin, token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, }); - if (requestedTarget === "live" && target !== "live") { - throw new Error("Tinybird cleanup target regressed from live to staging"); + if (target !== requestedTarget) { + throw new Error("Tinybird cleanup target changed before scoped cleanup"); } validateSyntheticRunId(state.runId); validateSyntheticRunId(state.loadRunId); @@ -4764,8 +4763,14 @@ const recoveryCheckpoint = (directory) => { }; const recoverStaging = async () => { - const checkpoint = recoveryCheckpoint(option("checkpoint-directory")); const recoveryArtifactPath = option("artifact"); + writeJson(recoveryArtifactPath, { + recovered: false, + strategy: "incomplete", + workspaceId: STAGING_WORKSPACE_ID, + verifiedAt: new Date().toISOString(), + }); + const checkpoint = recoveryCheckpoint(option("checkpoint-directory")); const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_CLEANUP_TOKEN", "TINYBIRD_STAGING_COPY_TOKEN", @@ -4921,11 +4926,19 @@ const recoverStaging = async () => { } } else { const retainedState = { ...state, deploymentId: retainedDeploymentId }; - await setCopySchedules({ - action: "resume", - artifactPath: checkpoint.artifactPath, - state: retainedState, - }); + const pausedDeploymentId = String( + artifact.copySchedule?.pause?.deploymentId ?? "", + ); + if ( + artifact.copySchedule?.pause?.status === "passed" && + pausedDeploymentId === retainedDeploymentId + ) { + await setCopySchedules({ + action: "resume", + artifactPath: checkpoint.artifactPath, + state: retainedState, + }); + } if (candidateLifecycle !== "deleted") { await discardOwnedDeployment({ deploymentId: candidateId }); } diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 433a85c4733..4289c0e3cb9 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2184,7 +2184,18 @@ test("the analytics workflow is statically restricted to staging", () => { workflow, /--baseline-deployment-id "\$\{\{ steps\.promote\.outputs\.previous_live_id \|\| steps\.tinybird\.outputs\.id \}\}"/, ); - assert.match(workflow, /steps\.seed\.outcome != 'skipped'/); + assert.match( + workflow, + /Quiesce scheduled and active Copy jobs before final cleanup\n {8}id: pause-copies\n {8}if: always\(\) && steps\.seed\.outcome != 'skipped' && steps\.deployment-state\.outputs\.promoted == 'true'/, + ); + assert.match( + workflow, + /Delete strictly scoped synthetic raw rows\n {8}id: cleanup\n {8}if: always\(\) && steps\.seed\.outcome != 'skipped' && \(steps\.deployment-state\.outputs\.target == 'staging' \|\| steps\.pause-copies\.outcome == 'success'\)/, + ); + assert.match( + workflow, + /Resume reviewed Copy schedules\n {8}id: resume-copies\n {8}if: always\(\) && steps\.pause-copies\.outcome == 'success'/, + ); assert.match(workflow, /steps\.cleanup\.outputs\.requires_copies == 'true'/); assert.doesNotMatch(workflow, /steps\.seed\.outcome == 'success'/); assert.match(workflow, /echo "required=false" >> "\$GITHUB_OUTPUT"/); @@ -2339,6 +2350,11 @@ test("the seed checkpoint is persisted before ingestion", () => { assert.match(prepareSource, /assertions: \{ seedAccepted: false \}/); assert.match(prepareSource, /rowsPlanned: fixture\.rows\.length/); assert.match(prepareSource, /rowsAttempted: 0/); + assert.match( + seedSource, + /tinybirdEnvironment\(\[\s*"TINYBIRD_STAGING_INGEST_TOKEN",?\s*\]\)/, + ); + assert.doesNotMatch(seedSource, /TINYBIRD_STAGING_COPY_TOKEN/); assert.ok( seedSource.indexOf("artifact.delivery.rowsAttempted += 1") < seedSource.indexOf("const result = await request"), @@ -2349,6 +2365,44 @@ test("the seed checkpoint is persisted before ingestion", () => { ); }); +test("recovery avoids unowned schedule changes and publishes failure evidence", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const recoverySource = source.slice( + source.indexOf("const recoverStaging = async () => {"), + source.indexOf("const handlers ="), + ); + assert.ok( + recoverySource.indexOf('strategy: "incomplete"') < + recoverySource.indexOf("recoveryCheckpoint("), + ); + assert.match( + recoverySource, + /artifact\.copySchedule\?\.pause\?\.status === "passed" &&[\s\S]*pausedDeploymentId === retainedDeploymentId[\s\S]*await setCopySchedules/, + ); +}); + +test("candidate cleanup refuses a live transition before any deletion", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const cleanupSource = source.slice( + source.indexOf("const cleanup = async"), + source.indexOf("const verifyPromoted = async"), + ); + const targetGuard = cleanupSource.indexOf("if (target !== requestedTarget)"); + assert.ok(targetGuard >= 0); + assert.ok(targetGuard < cleanupSource.indexOf("cleanupPreviewDatabaseState")); + assert.ok(targetGuard < cleanupSource.indexOf("deleteProductEventRows")); + assert.match( + cleanupSource, + /Tinybird cleanup target changed before scoped cleanup/, + ); +}); + test("event state reaches terminal success before canonical rebuilding", () => { const source = fs.readFileSync( new URL("../staging-ci.js", import.meta.url), From e560be9dd100359f790ae0aaaa9b8a91401c3809 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:07:01 +0100 Subject: [PATCH 104/110] fix: seed analytics after staging promotion --- .github/workflows/analytics.yml | 60 ++- ...st-party-analytics-production-readiness.md | 8 +- scripts/analytics/staging-ci-lib.js | 16 + scripts/analytics/staging-ci.js | 346 ++++++++++++------ scripts/analytics/tests/staging-ci.test.js | 70 +++- 5 files changed, 336 insertions(+), 164 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 02bece1a394..b468f3ca1d4 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -296,43 +296,14 @@ jobs: TINYBIRD_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TB_HOST: ${{ secrets.TINYBIRD_STAGING_URL }} run: node scripts/analytics/analytics-cli.js test - - name: Seed bounded duplicate and conflict probes - id: seed - env: - VERCEL_DEPLOYMENT_ID: ${{ steps.vercel.outputs.deployment_id }} - VERCEL_PREVIEW_URL: ${{ steps.vercel.outputs.url }} - TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TINYBIRD_STAGING_INGEST_TOKEN: ${{ secrets.TINYBIRD_STAGING_INGEST_TOKEN }} - run: >- - node scripts/analytics/staging-ci.js seed - --run-id "$ANALYTICS_TEST_RUN_ID" - --deployment-id "${{ steps.tinybird.outputs.id }}" - --state "$RUNNER_TEMP/analytics-staging-state.json" - --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Rebuild no-op live decision and health copies - id: staging-copies - if: steps.tinybird.outputs.needs_promotion != 'true' - env: - TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} - TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} - TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} - TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} - TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} - run: >- - node scripts/analytics/staging-ci.js run-copies - --deployment-id "${{ steps.tinybird.outputs.id }}" - --phase staged - --target live - --state "$RUNNER_TEMP/analytics-staging-state.json" - --artifact "$RUNNER_TEMP/analytics-staging-report.json" - - name: Prove candidate isolation or no-op live decision quality + - name: Prove exact candidate endpoints before promotion id: verify env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} run: >- - node scripts/analytics/staging-ci.js verify + node scripts/analytics/staging-ci.js verify-preseed --deployment-id "${{ steps.tinybird.outputs.id }}" --target "${{ steps.tinybird.outputs.needs_promotion == 'true' && 'staging' || 'live' }}" --state "$RUNNER_TEMP/analytics-staging-state.json" @@ -406,6 +377,30 @@ jobs: - name: Refuse to proceed without an authoritative live deployment if: steps.deployment-state.outputs.promoted != 'true' || steps.promote.outcome == 'failure' run: exit 1 + - name: Seed bounded duplicate and conflict probes into exact live staging + id: seed + env: + VERCEL_DEPLOYMENT_ID: ${{ steps.vercel.outputs.deployment_id }} + VERCEL_PREVIEW_URL: ${{ steps.vercel.outputs.url }} + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_INGEST_TOKEN: ${{ secrets.TINYBIRD_STAGING_INGEST_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js seed + --run-id "$ANALYTICS_TEST_RUN_ID" + --deployment-id "${{ steps.tinybird.outputs.id }}" + --state "$RUNNER_TEMP/analytics-staging-state.json" + --artifact "$RUNNER_TEMP/analytics-staging-report.json" + - name: Upload the immutable post-ingestion recovery checkpoint + if: steps.seed.outcome == 'success' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: analytics-recovery-${{ github.run_id }}-${{ github.run_attempt }}-35-postseed + path: | + ${{ runner.temp }}/analytics-staging-state.json + ${{ runner.temp }}/analytics-staging-report.json + if-no-files-found: error + retention-days: 30 - name: Prove least-privilege staging token scopes id: token-scopes env: @@ -544,6 +539,7 @@ jobs: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_CLEANUP_TOKEN: ${{ secrets.TINYBIRD_STAGING_CLEANUP_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: | @@ -627,7 +623,7 @@ jobs: --artifact "$RUNNER_TEMP/analytics-staging-report.json" - name: Restore the previous staging deployment on failure id: rollback - if: always() && steps.promote.outputs.previous_live_id != '' && steps.finalize.outcome != 'success' && steps.rollback-drill.outputs.rollback_target_usable != 'false' + if: always() && steps.promote.outputs.previous_live_id != '' && steps.finalize.outcome != 'success' && steps.rollback-drill.outputs.rollback_target_usable != 'false' && (steps.seed.outcome == 'skipped' || steps.verify-cleanup.outcome == 'success') env: TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index 0cff92f748c..ca3b0f88e3e 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -40,11 +40,11 @@ Raw, canonical, and decision data share an 800-day TTL. This supports two comple ## Data quality and performance gates -The staging workflow is restricted to PR 2003 and `codex/first-party-analytics`, hard-codes the staging Tinybird workspace ID, checks the exact Git SHA, waits for the exact-SHA Vercel preview, and requires that preview to attest the reviewed staging database credential fingerprint and migration `0042_lying_sharon_ventura` before any synthetic mutation. It creates an isolated Tinybird staging deployment, runs fixture and synthetic tests, and promotes only a verified deployment inside that staging workspace. Candidate reads use the exact numeric deployment ID and prove raw delivery, isolation, endpoint execution, and absolute latency before promotion. Tinybird's Copy API does not reliably route an on-demand Copy mutation into a candidate, so Copy mutations are prohibited until the exact candidate is live in staging. The prior staging deployment remains available until promoted Copy results, public business values, retained-deployment and representative-volume performance, erasure, cleanup, durable MySQL outbox health, and a live rollback-and-restoration drill pass. The first rollout of a new endpoint compares only shared endpoints to the retained deployment, records the new endpoint as having no historical baseline, and still applies candidate and representative absolute budgets. A no-op deployment uses absolute budgets rather than presenting the same deployment as an independent regression baseline. The rollback drill probes candidate-only endpoints against the retained deployment and excludes only endpoints that return `404`; every available typed endpoint must remain readable. The exact-SHA admin client applies the same narrow rollback compatibility rule, then the drill restores the candidate and re-proves the full endpoint suite, health, and synthetic business responses. Ambiguous live-switch responses are reconciled against the exact deployment pair before recovery continues. Any earlier failure restores a data-plane-proven prior deployment and removes the rejected candidate. +The staging workflow is restricted to PR 2003 and `codex/first-party-analytics`, hard-codes the staging Tinybird workspace ID, checks the exact Git SHA, waits for the exact-SHA Vercel preview, and requires that preview to attest the reviewed staging database credential fingerprint and migration `0042_lying_sharon_ventura` before any synthetic mutation. It creates a Tinybird candidate inside that staging workspace, pins every candidate read to its exact numeric deployment ID, proves all typed endpoints execute within absolute latency budgets, and proves the unique run IDs are absent from the candidate, pinned live deployment, and default live route before promotion. Tinybird staging empirically exposed rows written with `__tb__min_deployment` to the live deployment even though its deployment documentation describes candidate-only visibility. The workflow therefore never writes synthetic rows or runs Copy mutations against a candidate: it promotes the exact candidate inside the dedicated staging workspace first, re-resolves its authoritative live state, and only then seeds and fully tests that exact deployment. Cleanup also deletes the bounded run IDs from the pinned live source before discarding any rejected candidate, so a provider isolation regression cannot leave synthetic rows behind. The prior staging deployment remains available until promoted Copy results, public business values, retained-deployment and representative-volume performance, erasure, cleanup, durable MySQL outbox health, and a live rollback-and-restoration drill pass. The first rollout of a new endpoint compares only shared endpoints to the retained deployment, records the new endpoint as having no historical baseline, and still applies exact-deployment and representative absolute budgets. A no-op deployment uses absolute budgets rather than presenting the same deployment as an independent regression baseline. The rollback drill probes candidate-only endpoints against the retained deployment and excludes only endpoints that return `404`; every available typed endpoint must remain readable. The exact-SHA admin client applies the same narrow rollback compatibility rule, then the drill restores the candidate and re-proves the full endpoint suite, health, and synthetic business responses. Ambiguous live-switch responses are reconciled against the exact deployment pair before recovery continues. Any earlier failure cleans the exact synthetic run, restores a data-plane-proven prior deployment, and removes only the rejected candidate. -The workflow persists immutable checkpoints before deployment creation, seeding, promotion, and finalization. Its same-run recovery job resolves uncertain creation from the exact deployment-ID set difference, restores the prior read plane, resumes paused copies, removes owned synthetic state, and discards only the owned candidate. GitHub force-cancellation or a GitHub control-plane outage can still prevent that same-workflow recovery job from starting; the recorded checkpoint is the manual recovery input in that case. The workflow has no push trigger, production environment, production token, or production deployment command. +The workflow persists immutable checkpoints before deployment creation, before ingestion, before promotion, after successful ingestion, and before finalization. Its same-run recovery job resolves uncertain creation from the exact deployment-ID set difference. If the candidate was promoted, recovery pauses its reviewed Copy schedules, removes the bounded synthetic run, rebuilds derived state, resumes the schedules, restores and probes the prior read plane, and only then discards the owned candidate. GitHub force-cancellation or a GitHub control-plane outage can still prevent that same-workflow recovery job from starting; the recorded checkpoint is the manual recovery input in that case. The workflow has no push trigger, production environment, production token, or production deployment command. -The redacted evidence artifact records the Git SHA, GitHub run, Vercel and Tinybird deployment IDs, hashed synthetic-run identity, delivery attempts, ingestion throughput, endpoint and full-dashboard p50/p95/p99, measured-baseline regressions, visibility time, health totals, decision-dedup assertions, conflict quarantine, least-privilege token checks, scoped identity erasure, and cleanup. Synthetic rows are excluded from normal metrics and cleanup is verified after every promoted run. +The redacted evidence artifact records the Git SHA, GitHub run, Vercel and Tinybird deployment IDs, pre-seed exact-deployment proof, hashed synthetic-run identity, delivery attempts, ingestion throughput, endpoint and full-dashboard p50/p95/p99, measured-baseline regressions, visibility time, health totals, decision-dedup assertions, conflict quarantine, least-privilege token checks, scoped identity erasure, and cleanup. Synthetic rows are excluded from normal metrics and cleanup is verified after every promoted run. Required live gates are: @@ -60,7 +60,7 @@ Required live gates are: - a populated-table performance pass measures every typed decision endpoint and full dashboard fanout after materialization; - wrong workspace, stale SHA, missing credentials, partial execution, failed promotion, and failed cleanup all fail closed. -Copy-backed decision tables rebuild through a durable single-owner Vercel Workflow every ten minutes to avoid competing for the same Tinybird worker pool. The routine refresh runs the eight decision models from the streaming exact event-ID state. Exact-SHA staging and erasure runs additionally rebuild global and day-partitioned event state, canonical state, and health, for twelve ordered copies in total. Both event-state copies must reach terminal success before canonical rebuilding begins. Candidate ingestion visibility is measured directly against the exact candidate and promoted Copy visibility is measured separately, so neither proof waits for the periodic refresh. Dashboard freshness exposes product, traffic, retention, identity-funnel, attribution, and experiment-outcome aggregate timestamps; scheduled production freshness is therefore bounded by the documented workflow cadence plus provider execution time. +Copy-backed decision tables rebuild through a durable single-owner Vercel Workflow every ten minutes to avoid competing for the same Tinybird worker pool. The routine refresh runs the eight decision models from the streaming exact event-ID state. Exact-SHA staging and erasure runs additionally rebuild global and day-partitioned event state, canonical state, and health, for twelve ordered copies in total. Both event-state copies must reach terminal success before canonical rebuilding begins. Exact live-staging ingestion visibility and promoted Copy visibility are measured separately, so neither proof waits for the periodic refresh. Dashboard freshness exposes product, traffic, retention, identity-funnel, attribution, and experiment-outcome aggregate timestamps; scheduled production freshness is therefore bounded by the documented workflow cadence plus provider execution time. The checked-in bounded v2 hot/cold models are deployment-validated side-by-side resources, not the current public read plane. Routine dashboard and agent endpoints continue to read the exact pre-aggregated decision tables and never scan raw events. A future cutover must backfill settled UTC dates, publish complete cross-model generations, prove parity against the current tables, add physical cold-state erasure rebuilding, and then switch endpoints. Until those gates exist, the v2 resources remain unscheduled and cannot affect decision metrics. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 303a1541191..c63eab0fbcf 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -3092,10 +3092,12 @@ export const assertWorkflowSafety = (workflow) => { "TINYBIRD_STAGING_INGEST_TOKEN", "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_CLEANUP_TOKEN", + "staging-ci.js verify-preseed", "staging-ci.js run-copies", "staging-ci.js set-copy-schedules", "steps.deployment-state.outputs.promoted == 'true'", "steps.deployment-state.outputs.target == 'staging' || steps.pause-copies.outcome == 'success'", + "steps.seed.outcome == 'skipped' || steps.verify-cleanup.outcome == 'success'", "attest-preview", "probe-preview", "verify-promoted", @@ -3116,4 +3118,18 @@ export const assertWorkflowSafety = (workflow) => { ) { throw new Error("Workflow contains a non-staging Tinybird workspace ID"); } + const candidateValidation = workflow.indexOf("staging-ci.js verify-preseed"); + const promotion = workflow.indexOf("staging-ci.js promote-deployment"); + const seed = workflow.indexOf("staging-ci.js seed"); + if ( + candidateValidation < 0 || + promotion < 0 || + seed < 0 || + candidateValidation > promotion || + promotion > seed + ) { + throw new Error( + "Workflow must validate, promote, and then seed the exact staging deployment", + ); + } }; diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index c7d014b69e3..1da0684c4e7 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -2248,8 +2248,9 @@ const seed = async () => { const state = readJson(statePath); const artifact = readJson(artifactPath); assertRecoveryIdentity(state.recoveryIdentity); + const expectedRecoveryPhase = state.needsPromotion ? "prepromote" : "preseed"; if ( - state.recoveryPhase !== "preseed" || + state.recoveryPhase !== expectedRecoveryPhase || state.runId !== runId || String(state.deploymentId) !== deploymentId || artifact.sha !== environment("EXPECTED_SHA") || @@ -2285,8 +2286,23 @@ const seed = async () => { now: startedAt, }); const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", "TINYBIRD_STAGING_INGEST_TOKEN", ]); + const assertExactLiveOwnership = async () => { + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== "live" + ) { + throw new Error( + "Synthetic seed requires the exact live staging deployment", + ); + } + }; + await assertExactLiveOwnership(); const deliver = async (row, fixtureRow = false) => { if (fixtureRow) { artifact.delivery.rowsAttempted += 1; @@ -2464,8 +2480,11 @@ const seed = async () => { controlDeliveryLatencyMs: erasureControlDelivery.latencyMs, controlRetryAttempts: erasureControlDelivery.attempts - 1, }; + await assertExactLiveOwnership(); artifact.assertions.seedAccepted = true; writeJson(artifactPath, artifact); + state.recoveryPhase = "postseed"; + writeJson(statePath, state, 0o600); }; const waitForCopyVisibility = async ({ label, read, assert }) => { @@ -3102,101 +3121,86 @@ const setCopySchedules = async (parameters = {}) => { writeJson(artifactPath, artifact); }; -const rawAssertionMetrics = (assertions) => ({ - receivedRows: assertions.receivedRows, - uniqueEvents: assertions.uniqueEvents, - uniquePayloads: assertions.uniquePayloads, - duplicateRows: assertions.duplicateRows, - payloadConflicts: assertions.payloadConflicts, -}); +const assertZeroCiAssertions = (assertions, label) => { + if (Object.values(assertions).some((value) => value !== 0)) { + throw new Error(`${label} contained synthetic rows before staging seed`); + } +}; -const verifyCandidate = async ({ state, artifact, artifactPath }) => { +const verifyPreSeedDeployment = async ({ + state, + artifact, + artifactPath, + target, +}) => { const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_DEPLOY_TOKEN", ]); - if ( - (await ownedMutationTarget({ - state, - origin, - token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, - })) !== "staging" - ) { - throw new Error("Candidate validation lost exact deployment ownership"); - } - const visibility = await waitForCopyVisibility({ - label: "Tinybird candidate raw delivery", - read: async () => - Promise.all( - [ - state.runId, - state.loadRunId, - state.largeLoadRunId, - state.decisionRunId, - state.erasureControlRunId, - ].map(async (syntheticRunId) => - normalizeCiAssertions( - ( - await ciAssertionsQuery({ - state, - deploymentId: state.deploymentId, - syntheticRunId, - }) - ).data, - ), - ), - ), - assert: ([main, load, largeLoad, decisions, control]) => { - assertSyntheticHealth(main); - assertSyntheticLoadHealth(load, state.loadEventCount); - assertSyntheticLoadHealth(largeLoad, state.largeLoadEventCount); - assertSyntheticLoadHealth(decisions, state.decisionEventCount); - assertSingleHealth(control); - }, - }); - const ingestionVisibilityMs = Date.now() - Date.parse(state.startedAt); - const [main, load, largeLoad, decisions, control] = visibility.value; - const liveAssertions = await Promise.all( - [ - state.runId, - state.loadRunId, - state.largeLoadRunId, - state.decisionRunId, - state.erasureControlRunId, - ].map(async (syntheticRunId) => - normalizeCiAssertions( - (await ciAssertionsQuery({ state, syntheticRunId })).data, - ), - ), + const deploymentId = String(state.deploymentId); + const liveDeploymentId = String( + target === "staging" ? state.liveBeforeDeploymentId : state.deploymentId, ); if ( - liveAssertions.some((assertions) => - Object.values(assertions).some((value) => value !== 0), - ) + !/^[0-9]+$/.test(deploymentId) || + !/^[0-9]+$/.test(liveDeploymentId) || + (target === "staging" && deploymentId === liveDeploymentId) ) { - throw new Error("Candidate-only synthetic events affected live analytics"); + throw new Error("Pre-seed validation has invalid Tinybird deployment IDs"); } - const queries = decisionEndpointQueries({ - startDate: state.startTime.slice(0, 10), - endDate: state.endTime.slice(0, 10), - deploymentId: state.deploymentId, + const resolvedTarget = await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, }); - const samples = Object.fromEntries(queries.map(({ name }) => [name, []])); + if (resolvedTarget !== target) { + throw new Error("Pre-seed validation lost exact deployment ownership"); + } + const runIds = [ + state.runId, + state.loadRunId, + state.largeLoadRunId, + state.decisionRunId, + state.erasureControlRunId, + ]; + for (const syntheticRunId of runIds) { + const [exact, pinnedLive, defaultLive] = await Promise.all([ + ciAssertionsQuery({ state, deploymentId, syntheticRunId }), + ciAssertionsQuery({ + state, + deploymentId: liveDeploymentId, + syntheticRunId, + }), + ciAssertionsQuery({ state, syntheticRunId }), + ]); + assertZeroCiAssertions( + normalizeCiAssertions(exact.data), + "Exact Tinybird deployment", + ); + assertZeroCiAssertions( + normalizeCiAssertions(pinnedLive.data), + "Pinned live Tinybird deployment", + ); + assertZeroCiAssertions( + normalizeCiAssertions(defaultLive.data), + "Default live Tinybird deployment", + ); + } + const samples = {}; const fanoutSamples = []; for (let round = 0; round < 5; round += 1) { const startedAt = performance.now(); - const results = await Promise.all( - queries.map((query) => - decisionEndpointQuery({ - origin, - token: tokens.TINYBIRD_STAGING_READ_TOKEN, - ...query, - }), - ), - ); + const suite = await queryDecisionEndpointSuite({ + deploymentId, + origin, + state, + syntheticRunId: state.decisionRunId, + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); + assertDecisionEndpointSuiteReadable(suite.payloads); fanoutSamples.push(Math.round(performance.now() - startedAt)); - for (let index = 0; index < results.length; index += 1) { - samples[queries[index].name].push(results[index].latencyMs); + for (const [name, latencyMs] of Object.entries(suite.latencyMs)) { + samples[name] = [...(samples[name] ?? []), latencyMs]; } } const endpointLatency = Object.fromEntries( @@ -3215,39 +3219,23 @@ const verifyCandidate = async ({ state, artifact, artifactPath }) => { const failedEndpoints = Object.entries(endpointLatency) .filter(([, latency]) => latency.p95Ms > endpointP95BudgetMs) .map(([name]) => name); - const ingestionSloMs = Number(process.env.INGESTION_SLO_MS ?? 180_000); artifact.candidateValidation = { - raw: { - main: rawAssertionMetrics(main), - load: rawAssertionMetrics(load), - largeLoad: rawAssertionMetrics(largeLoad), - decisions: rawAssertionMetrics(decisions), - control: rawAssertionMetrics(control), - }, - liveIsolated: true, - polls: visibility.polls, - ingestionVisibilityMs, + strategy: "promote_then_seed", + exactDeploymentId: deploymentId, + liveDeploymentId, + preSeedSyntheticRows: 0, endpointLatency, dashboardFanoutLatency, }; artifact.assertions = { ...artifact.assertions, - candidateRawDeliveryPassed: true, - candidateDuplicateVisible: true, - candidatePayloadConflictVisible: true, - candidateIsolationPassed: true, + candidateExactDeploymentPassed: true, + candidatePreSeedCleanPassed: true, candidateEndpointsPassed: failedEndpoints.length === 0 && dashboardFanoutLatency.p95Ms <= fanoutP95BudgetMs, - candidateIngestionSloPassed: ingestionVisibilityMs <= ingestionSloMs, }; - artifact.visibilityMs = ingestionVisibilityMs; writeJson(artifactPath, artifact); - if (ingestionVisibilityMs > ingestionSloMs) { - throw new Error( - `Candidate events became visible in ${ingestionVisibilityMs}ms, over ${ingestionSloMs}ms`, - ); - } if ( failedEndpoints.length > 0 || dashboardFanoutLatency.p95Ms > fanoutP95BudgetMs @@ -3263,6 +3251,19 @@ const verifyCandidate = async ({ state, artifact, artifactPath }) => { } }; +const verifyPreSeed = async () => { + const state = readJson(option("state")); + const artifactPath = option("artifact"); + const artifact = readJson(artifactPath); + const target = option("target"); + dataMutationDeploymentParameters({ + target, + deploymentId: option("deployment-id"), + expectedDeploymentId: String(state.deploymentId), + }); + await verifyPreSeedDeployment({ state, artifact, artifactPath, target }); +}; + const verify = async () => { const state = readJson(option("state")); const artifactPath = option("artifact"); @@ -3274,8 +3275,7 @@ const verify = async () => { expectedDeploymentId: String(state.deploymentId), }); if (target === "staging") { - await verifyCandidate({ state, artifact, artifactPath }); - return; + throw new Error("Synthetic verification requires a promoted deployment"); } const deadline = Date.now() + Number(process.env.INGESTION_SLO_MS ?? 180_000); let result; @@ -4009,6 +4009,7 @@ const cleanup = async (parameters = {}) => { const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_CLEANUP_TOKEN", "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_READ_TOKEN", ]); let target = await waitForOwnedMutationTarget({ state, @@ -4063,6 +4064,56 @@ const cleanup = async (parameters = {}) => { writeJson(artifactPath, databaseArtifact); } if (target === "staging") { + const assertStagingLeakCleanupOwnership = async () => { + if ( + (await ownedMutationTarget({ + state, + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + })) !== "staging" + ) { + throw new Error("The staged Tinybird cleanup candidate changed"); + } + const deployments = await deploymentList({ + origin, + token: tokens.TINYBIRD_STAGING_DEPLOY_TOKEN, + }); + if ( + createDeploymentBoundary(deployments.data).liveDeploymentId !== + String(state.liveBeforeDeploymentId) + ) { + throw new Error("The staged Tinybird cleanup live deployment changed"); + } + }; + const rowsAffected = await deleteProductEventRows({ + origin, + token: tokens.TINYBIRD_STAGING_CLEANUP_TOKEN, + condition: `synthetic_run_id IN (${runIds.map((runId) => `'${runId}'`).join(", ")})`, + beforeAttempt: assertStagingLeakCleanupOwnership, + }); + const liveDeploymentId = String(state.liveBeforeDeploymentId); + await waitForCopyVisibility({ + label: "Tinybird staged-write live cleanup", + read: () => + Promise.all( + runIds.flatMap((syntheticRunId) => [ + ciAssertionsQuery({ state, syntheticRunId }), + ciAssertionsQuery({ + state, + deploymentId: liveDeploymentId, + syntheticRunId, + }), + ]), + ), + assert: (results) => { + for (const result of results) { + assertZeroCiAssertions( + normalizeCiAssertions(result.data), + "Live Tinybird cleanup", + ); + } + }, + }); writeOutput("target", target); writeOutput("requires_copies", "false"); writeOutput("requires_discard", "true"); @@ -4071,6 +4122,8 @@ const cleanup = async (parameters = {}) => { ...artifact.cleanup, strategy: "deployment_discard", candidateDiscarded: false, + liveSyntheticRowsDeleted: true, + rowsAffected, }; writeJson(artifactPath, artifact); return; @@ -4700,7 +4753,7 @@ const markRecoveryReadyToFinalize = async () => { assertRecoveryIdentity(state.recoveryIdentity); if ( state.needsPromotion !== true || - state.recoveryPhase !== "prepromote" || + state.recoveryPhase !== "postseed" || artifact.assertions?.cleanupPassed !== true || artifact.rollbackDrill?.passed !== true || artifact.copySchedule?.pause?.status !== "passed" || @@ -4725,7 +4778,8 @@ const recoveryCheckpoint = (directory) => { const phaseRank = { preseed: 1, prepromote: 2, - ready_to_finalize: 3, + postseed: 3, + ready_to_finalize: 4, }[state.recoveryPhase]; if (!phaseRank) { throw new Error("Recovery checkpoint has an unsupported phase"); @@ -4814,6 +4868,8 @@ const recoverStaging = async () => { ); let retainedDeploymentId = candidateId; let strategy = "clean_noop_live"; + let databaseCleanup; + let syntheticCleanupCompleted = false; const candidateLifecycle = await settledDeploymentLifecycle({ origin, token: deployToken, @@ -4830,6 +4886,37 @@ const recoverStaging = async () => { deploymentId: previousLiveDeploymentId, }); if (previousLifecycle === "ready") { + await setCopySchedules({ + action: "pause", + artifactPath: checkpoint.artifactPath, + state, + }); + try { + await cleanup({ + artifactPath: checkpoint.artifactPath, + deploymentId: candidateId, + state, + target: "live", + }); + databaseCleanup = readJson(checkpoint.artifactPath).cleanup?.database; + if (!databaseCleanup?.cleaned) { + throw new Error("Recovery did not attest staging database cleanup"); + } + await runCopies({ + artifactPath: checkpoint.artifactPath, + deploymentId: candidateId, + phase: "cleanup", + state, + target: "live", + }); + syntheticCleanupCompleted = true; + } finally { + await setCopySchedules({ + action: "resume", + artifactPath: checkpoint.artifactPath, + state, + }); + } await switchLiveDeployment({ origin, token: deployToken, @@ -4879,6 +4966,19 @@ const recoverStaging = async () => { ); } } else if (["ready", "failed", "pending"].includes(candidateLifecycle)) { + if (candidateLifecycle === "ready") { + await cleanup({ + artifactPath: checkpoint.artifactPath, + deploymentId: candidateId, + state, + target: "staging", + }); + databaseCleanup = readJson(checkpoint.artifactPath).cleanup?.database; + if (!databaseCleanup?.cleaned) { + throw new Error("Recovery did not attest staging database cleanup"); + } + syntheticCleanupCompleted = true; + } retainedDeploymentId = previousLiveDeploymentId; strategy = "discard_rejected_candidate"; } else if (candidateLifecycle === "deleted") { @@ -4889,7 +4989,6 @@ const recoverStaging = async () => { } } const serverRunId = validateSyntheticRunId(`${state.runId}_server`); - let databaseCleanup; if ( state.needsPromotion !== true || strategy === "retain_finalized_candidate" @@ -4942,19 +5041,21 @@ const recoverStaging = async () => { if (candidateLifecycle !== "deleted") { await discardOwnedDeployment({ deploymentId: candidateId }); } - databaseCleanup = await cleanupPreviewDatabaseState({ - anonymousIdentityHashes: [ - state.browserAnonymousIdentityHash, - state.previewAnonymousIdentityHash, - ].filter( - (identityHash) => - typeof identityHash === "string" && - /^[0-9a-f]{64}$/.test(identityHash), - ), - artifact, - runIds: [state.runId, serverRunId], - secret: environment("CAP_ANALYTICS_STAGING_TEST_SECRET"), - }); + if (!syntheticCleanupCompleted) { + databaseCleanup = await cleanupPreviewDatabaseState({ + anonymousIdentityHashes: [ + state.browserAnonymousIdentityHash, + state.previewAnonymousIdentityHash, + ].filter( + (identityHash) => + typeof identityHash === "string" && + /^[0-9a-f]{64}$/.test(identityHash), + ), + artifact, + runIds: [state.runId, serverRunId], + secret: environment("CAP_ANALYTICS_STAGING_TEST_SECRET"), + }); + } } writeJson(recoveryArtifactPath, { databaseCleanup, @@ -5037,6 +5138,7 @@ const handlers = { seed, "run-copies": runCopies, "set-copy-schedules": setCopySchedules, + "verify-preseed": verifyPreSeed, verify, "probe-preview": probePreview, "probe-server": probeDurableServerPath, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 4289c0e3cb9..eed9fa0c726 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -2015,6 +2015,14 @@ test("the analytics workflow is statically restricted to staging", () => { "utf8", ); assert.doesNotThrow(() => assertWorkflowSafety(workflow)); + const reorderedWorkflow = workflow + .replace("staging-ci.js seed", "staging-ci.js __seed_placeholder__") + .replace("staging-ci.js promote-deployment", "staging-ci.js seed") + .replace( + "staging-ci.js __seed_placeholder__", + "staging-ci.js promote-deployment", + ); + assert.throws(() => assertWorkflowSafety(reorderedWorkflow)); assert.equal( workflow.match( /deployment create --allow-destructive-operations --(?:check|wait)/g, @@ -2024,13 +2032,13 @@ test("the analytics workflow is statically restricted to staging", () => { assert.equal( workflow.match(/node scripts\/analytics\/staging-ci\.js run-copies/g) ?.length, - 3, + 2, ); assert.equal( workflow.match( /--deployment-id "\$\{\{ steps\.tinybird\.outputs\.id \}\}"/g, )?.length, - 16, + 15, ); assert.doesNotMatch(workflow, /tinybird-cloud-cli --cloud copy run/); assert.ok( @@ -2121,7 +2129,7 @@ test("the analytics workflow is statically restricted to staging", () => { assert.match(workflow, /staging-ci\.js rollback-promotion/); assert.match( workflow, - /steps\.promote\.outputs\.previous_live_id != '' && steps\.finalize\.outcome != 'success' && steps\.rollback-drill\.outputs\.rollback_target_usable != 'false'/, + /steps\.promote\.outputs\.previous_live_id != '' && steps\.finalize\.outcome != 'success' && steps\.rollback-drill\.outputs\.rollback_target_usable != 'false' && \(steps\.seed\.outcome == 'skipped' \|\| steps\.verify-cleanup\.outcome == 'success'\)/, ); assert.doesNotMatch( workflow, @@ -2145,8 +2153,26 @@ test("the analytics workflow is statically restricted to staging", () => { ); assert.match( workflow, - /Rebuild no-op live decision and health copies\n {8}id: staging-copies\n {8}if: steps\.tinybird\.outputs\.needs_promotion != 'true'[\s\S]*--target live/, + /Prove exact candidate endpoints before promotion[\s\S]*staging-ci\.js verify-preseed/, + ); + assert.ok( + workflow.indexOf("Prove exact candidate endpoints before promotion") < + workflow.indexOf("Promote the verified staging deployment"), ); + assert.ok( + workflow.indexOf( + "Refuse to proceed without an authoritative live deployment", + ) < + workflow.indexOf( + "Seed bounded duplicate and conflict probes into exact live staging", + ), + ); + assert.ok( + workflow.indexOf( + "Seed bounded duplicate and conflict probes into exact live staging", + ) < workflow.indexOf("Prove least-privilege staging token scopes"), + ); + assert.match(workflow, /35-postseed/); assert.ok( workflow.indexOf("Prove the exact-SHA deployed browser tracker") < workflow.indexOf( @@ -2197,7 +2223,10 @@ test("the analytics workflow is statically restricted to staging", () => { /Resume reviewed Copy schedules\n {8}id: resume-copies\n {8}if: always\(\) && steps\.pause-copies\.outcome == 'success'/, ); assert.match(workflow, /steps\.cleanup\.outputs\.requires_copies == 'true'/); - assert.doesNotMatch(workflow, /steps\.seed\.outcome == 'success'/); + assert.match( + workflow, + /Upload the immutable post-ingestion recovery checkpoint\n {8}if: steps\.seed\.outcome == 'success'/, + ); assert.match(workflow, /echo "required=false" >> "\$GITHUB_OUTPUT"/); assert.match(workflow, /echo "required=true" >> "\$GITHUB_OUTPUT"/); assert.match( @@ -2352,9 +2381,11 @@ test("the seed checkpoint is persisted before ingestion", () => { assert.match(prepareSource, /rowsAttempted: 0/); assert.match( seedSource, - /tinybirdEnvironment\(\[\s*"TINYBIRD_STAGING_INGEST_TOKEN",?\s*\]\)/, + /tinybirdEnvironment\(\[\s*"TINYBIRD_STAGING_DEPLOY_TOKEN",\s*"TINYBIRD_STAGING_INGEST_TOKEN",?\s*\]\)/, ); assert.doesNotMatch(seedSource, /TINYBIRD_STAGING_COPY_TOKEN/); + assert.match(seedSource, /assertExactLiveOwnership/); + assert.match(seedSource, /state\.recoveryPhase = "postseed"/); assert.ok( seedSource.indexOf("artifact.delivery.rowsAttempted += 1") < seedSource.indexOf("const result = await request"), @@ -2365,6 +2396,21 @@ test("the seed checkpoint is persisted before ingestion", () => { ); }); +test("candidate validation performs no synthetic writes before promotion", () => { + const source = fs.readFileSync( + new URL("../staging-ci.js", import.meta.url), + "utf8", + ); + const preSeedSource = source.slice( + source.indexOf("const verifyPreSeedDeployment = async"), + source.indexOf("const verify = async"), + ); + assert.match(preSeedSource, /strategy: "promote_then_seed"/); + assert.match(preSeedSource, /candidatePreSeedCleanPassed: true/); + assert.doesNotMatch(preSeedSource, /\/v0\/events/); + assert.doesNotMatch(preSeedSource, /TINYBIRD_STAGING_INGEST_TOKEN/); +}); + test("recovery avoids unowned schedule changes and publishes failure evidence", () => { const source = fs.readFileSync( new URL("../staging-ci.js", import.meta.url), @@ -2382,6 +2428,14 @@ test("recovery avoids unowned schedule changes and publishes failure evidence", recoverySource, /artifact\.copySchedule\?\.pause\?\.status === "passed" &&[\s\S]*pausedDeploymentId === retainedDeploymentId[\s\S]*await setCopySchedules/, ); + assert.match( + recoverySource, + /previousLifecycle === "ready"[\s\S]*action: "pause"[\s\S]*await cleanup\([\s\S]*await runCopies\([\s\S]*action: "resume"[\s\S]*await switchLiveDeployment/, + ); + assert.match( + recoverySource, + /candidateLifecycle === "ready"[\s\S]*target: "staging"[\s\S]*syntheticCleanupCompleted = true/, + ); }); test("candidate cleanup refuses a live transition before any deletion", () => { @@ -2401,6 +2455,10 @@ test("candidate cleanup refuses a live transition before any deletion", () => { cleanupSource, /Tinybird cleanup target changed before scoped cleanup/, ); + assert.match( + cleanupSource, + /if \(target === "staging"\)[\s\S]*liveBeforeDeploymentId[\s\S]*deleteProductEventRows[\s\S]*liveSyntheticRowsDeleted: true/, + ); }); test("event state reaches terminal success before canonical rebuilding", () => { From 7a2b9e2ab60a7445224c54d5ef5abacedfe3c0d2 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:59:38 +0100 Subject: [PATCH 105/110] fix: attest analytics copy jobs before erasure --- .../unit/product-analytics-erasure.test.ts | 84 ++++++++++++++ .../unit/product-analytics-refresh.test.ts | 8 +- .../workflows/refresh-product-analytics.ts | 18 ++- packages/web-backend/src/Tinybird/index.ts | 104 ++++++++++++++++-- 4 files changed, 201 insertions(+), 13 deletions(-) diff --git a/apps/web/__tests__/unit/product-analytics-erasure.test.ts b/apps/web/__tests__/unit/product-analytics-erasure.test.ts index ce5fbe4c4ba..7f2f1703ae7 100644 --- a/apps/web/__tests__/unit/product-analytics-erasure.test.ts +++ b/apps/web/__tests__/unit/product-analytics-erasure.test.ts @@ -108,6 +108,12 @@ describe.sequential("product analytics erasure", () => { vi.fn(async (input: string | URL | Request, init: RequestInit = {}) => { const url = new URL(String(input)); requests.push({ url, init }); + if (/\/copy\/(cancel|resume)$/.test(url.pathname)) { + return Response.json( + { error: "The copy Pipe is not scheduled" }, + { status: 422 }, + ); + } const scheduleResponse = copyScheduleResponse(url, pausedPipes); if (scheduleResponse) return scheduleResponse; if (url.pathname === "/v0/sql") { @@ -140,6 +146,9 @@ describe.sequential("product analytics erasure", () => { ], }); } + if (url.pathname === "/v0/jobs") { + return Response.json({ jobs: [] }); + } if (url.pathname.startsWith("/v0/jobs/")) { return Response.json({ status: "done" }); } @@ -206,9 +215,39 @@ describe.sequential("product analytics erasure", () => { )) { expect(request.url.searchParams.get("_mode")).toBe("replace"); } + const jobListRequests = requests.filter( + ({ url }) => url.pathname === "/v0/jobs", + ); + expect(jobListRequests).toHaveLength(12); + expect( + new Set( + jobListRequests.map(({ url }) => url.searchParams.get("pipe_name")), + ).size, + ).toBe(12); + expect( + jobListRequests.every( + ({ init }) => + new Headers(init.headers).get("Authorization") === + "Bearer scheduler-token", + ), + ).toBe(true); + expect( + requests.filter(({ url }) => + /^\/v0\/pipes\/[A-Za-z0-9_]+$/.test(url.pathname), + ), + ).toHaveLength(0); expect( requests.filter(({ url }) => url.pathname.startsWith("/v0/jobs/")), ).toHaveLength(12); + expect( + requests + .filter(({ url }) => url.pathname.startsWith("/v0/jobs/")) + .every( + ({ init }) => + new Headers(init.headers).get("Authorization") === + "Bearer scheduler-token", + ), + ).toBe(true); expect( requests .filter(({ url }) => url.pathname === "/v0/sql") @@ -255,6 +294,39 @@ describe.sequential("product analytics erasure", () => { ); }); + it("fails closed before deletion when Copy job state is malformed", async () => { + const requests: URL[] = []; + const pausedPipes = new Set(); + vi.stubGlobal( + "fetch", + vi.fn(async (input: string | URL | Request) => { + const url = new URL(String(input)); + requests.push(url); + const scheduleResponse = copyScheduleResponse(url, pausedPipes); + if (scheduleResponse) return scheduleResponse; + if (url.pathname === "/v0/sql") { + return Response.json({ data: [{ matching_rows: 1 }] }); + } + if (url.pathname === "/v0/jobs") return Response.json({}); + return Response.json({}); + }), + ); + + const error = await Effect.runPromise( + Effect.gen(function* () { + const tinybird = yield* Tinybird; + yield* tinybird.eraseProductAnalytics({ organizationId: "org-1" }); + }).pipe(Effect.provide(tinybirdTestLayer()), Effect.flip), + ); + + expect(error.message).toBe( + "Product analytics Jobs API response was invalid", + ); + expect(requests.some((url) => url.pathname.includes("/delete"))).toBe( + false, + ); + }); + it("fails closed when Tinybird does not confirm deletion", async () => { const pausedPipes = new Set(); vi.stubGlobal( @@ -266,6 +338,9 @@ describe.sequential("product analytics erasure", () => { if (url.pathname === "/v0/sql") { return Response.json({ data: [{ matching_rows: 1 }] }); } + if (url.pathname === "/v0/jobs") { + return Response.json({ jobs: [] }); + } return Response.json({}); }), ); @@ -311,6 +386,9 @@ describe.sequential("product analytics erasure", () => { ], }); } + if (url.pathname === "/v0/jobs") { + return Response.json({ jobs: [] }); + } if (url.pathname.startsWith("/v0/jobs/")) { return Response.json({ status: "done" }); } @@ -351,6 +429,9 @@ describe.sequential("product analytics erasure", () => { if (url.pathname === "/v0/sql") { return Response.json({ data: [{ matching_rows: 0 }] }); } + if (url.pathname === "/v0/jobs") { + return Response.json({ jobs: [] }); + } if ( url.pathname === "/v0/pipes/snapshot_product_event_id_states_v2/copy" ) { @@ -485,6 +566,9 @@ describe.sequential("product analytics erasure", () => { ], }); } + if (url.pathname === "/v0/jobs") { + return Response.json({ jobs: [] }); + } if (url.pathname.startsWith("/v0/jobs/")) { return Response.json({ status: "done" }); } diff --git a/apps/web/__tests__/unit/product-analytics-refresh.test.ts b/apps/web/__tests__/unit/product-analytics-refresh.test.ts index 6e8f905506b..b593bdf9878 100644 --- a/apps/web/__tests__/unit/product-analytics-refresh.test.ts +++ b/apps/web/__tests__/unit/product-analytics-refresh.test.ts @@ -12,6 +12,7 @@ vi.mock("@cap/env", () => ({ PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN: "copy-token", PRODUCT_ANALYTICS_TINYBIRD_HOST: "https://staging.tinybird.test", PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN: "read-token", + PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN: "scheduler-token", }), })); @@ -84,13 +85,18 @@ describe("product analytics refresh", () => { expect(result.jobs).toHaveLength(8); expect(mocks.renew).toHaveBeenCalledTimes(8); expect(mocks.release).toHaveBeenCalledWith("refresh-owner-1", undefined); + for (const request of requestedUrls.filter((url) => + url.pathname.startsWith("/v0/jobs/"), + )) { + expect(request.pathname).toMatch(/^\/v0\/jobs\/[A-Za-z0-9_-]+$/); + } const copyUrls = requestedUrls.filter((url) => url.pathname.endsWith("/copy"), ); expect(copyUrls).toHaveLength(8); expect( copyUrls.map((url) => url.searchParams.get("source_cutoff")), - ).toEqual(Array.from({ length: 8 }, () => "2026-07-31T12:00:00.000Z")); + ).toEqual(Array.from({ length: 8 }, () => "2026-07-31 12:00:00.000")); expect( new Set(copyUrls.map((url) => url.searchParams.get("copy_run_id"))).size, ).toBe(1); diff --git a/apps/web/workflows/refresh-product-analytics.ts b/apps/web/workflows/refresh-product-analytics.ts index 1541e9d272e..2a655c3b423 100644 --- a/apps/web/workflows/refresh-product-analytics.ts +++ b/apps/web/workflows/refresh-product-analytics.ts @@ -60,6 +60,14 @@ const jobStatus = (response: TinybirdJobResponse) => "", ).toLowerCase(); +const formatTinybirdDateTime64 = (value: string) => { + const parsed = new Date(value); + if (!Number.isFinite(parsed.getTime())) { + throw new FatalError("Product analytics refresh cutoff is invalid"); + } + return parsed.toISOString().replace("T", " ").replace(/Z$/, ""); +}; + export async function acquireProductAnalyticsRefreshStep(sourceCutoff: string) { "use step"; const parsed = new Date(sourceCutoff); @@ -84,7 +92,8 @@ export async function runProductAnalyticsCopyStep( const host = env.PRODUCT_ANALYTICS_TINYBIRD_HOST; const copyToken = env.PRODUCT_ANALYTICS_TINYBIRD_COPY_TOKEN; const readToken = env.PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN; - if (!host || !copyToken || !readToken) { + const schedulerToken = env.PRODUCT_ANALYTICS_TINYBIRD_SCHEDULER_TOKEN; + if (!host || !copyToken || !readToken || !schedulerToken) { throw new FatalError("Product analytics refresh is not configured"); } const origin = new URL(host); @@ -95,7 +104,10 @@ export async function runProductAnalyticsCopyStep( copyUrl.searchParams.set("_mode", "replace"); copyUrl.searchParams.set("copy_max_threads", "2"); copyUrl.searchParams.set("copy_run_id", copyRunId); - copyUrl.searchParams.set("source_cutoff", sourceCutoff); + copyUrl.searchParams.set( + "source_cutoff", + formatTinybirdDateTime64(sourceCutoff), + ); const created = await request(copyUrl, copyToken, { method: "POST", }); @@ -106,7 +118,7 @@ export async function runProductAnalyticsCopyStep( while (Date.now() < deadline) { const job = await request( new URL(`/v0/jobs/${encodeURIComponent(id)}`, origin), - copyToken, + schedulerToken, ); const status = jobStatus(job); if (["done", "success", "finished", "completed"].includes(status)) break; diff --git a/packages/web-backend/src/Tinybird/index.ts b/packages/web-backend/src/Tinybird/index.ts index e32974eaef5..c6115219cb1 100644 --- a/packages/web-backend/src/Tinybird/index.ts +++ b/packages/web-backend/src/Tinybird/index.ts @@ -42,10 +42,44 @@ interface TinybirdJobResponse { }; } +interface TinybirdJobListResponse { + jobs?: Array<{ + pipe_name?: unknown; + status?: unknown; + }>; +} + interface TinybirdPipeResponse { schedule?: { status?: string }; } +class ProductAnalyticsRequestError extends Error { + constructor( + readonly status: number, + readonly responseBody: string, + ) { + super(`Product analytics erasure request failed (${status})`); + } +} + +const isUnscheduledProductAnalyticsCopyError = (error: Error) => { + if ( + !(error instanceof ProductAnalyticsRequestError) || + error.status !== 422 + ) { + return false; + } + try { + const payload = JSON.parse(error.responseBody) as { error?: unknown }; + return ( + typeof payload.error === "string" && + payload.error.startsWith("The copy Pipe is not scheduled") + ); + } catch { + return false; + } +}; + const tinybirdJobId = (response: TinybirdJobResponse) => { const id = response.job_id ?? response.job?.id ?? response.id; return typeof id === "string" && id ? id : undefined; @@ -390,9 +424,7 @@ export class Tinybird extends Effect.Service()("Tinybird", { }); const body = await response.text(); if (!response.ok) { - throw new Error( - `Product analytics erasure request failed (${response.status})`, - ); + throw new ProductAnalyticsRequestError(response.status, body); } return body ? (JSON.parse(body) as T) : ({} as T); }, @@ -446,7 +478,10 @@ export class Tinybird extends Effect.Service()("Tinybird", { for (let attempt = 0; attempt < 450; attempt += 1) { const job = yield* productAnalyticsRequest( `/v0/jobs/${encodeURIComponent(id)}`, - { token: productAnalyticsCopyToken, purpose: "Copy execution" }, + { + token: productAnalyticsSchedulerToken, + purpose: "Copy job attestation", + }, ); const status = tinybirdJobStatus(job); if (["done", "success", "finished", "completed"].includes(status)) { @@ -488,8 +523,15 @@ export class Tinybird extends Effect.Service()("Tinybird", { { method: "POST" }, ).pipe( Effect.either, - Effect.flatMap((mutation) => - productAnalyticsRequest( + Effect.flatMap((mutation) => { + if (mutation._tag === "Left") { + const error = mutation.left; + if (isUnscheduledProductAnalyticsCopyError(error)) { + return Effect.void; + } + return Effect.fail(error); + } + return productAnalyticsRequest( `/v0/pipes/${encodeURIComponent(name)}`, auth, ).pipe( @@ -500,15 +542,14 @@ export class Tinybird extends Effect.Service()("Tinybird", { ? status === "paused" : status === "scheduled" || status === "active"; if (matches) return Effect.void; - if (mutation._tag === "Left") return Effect.fail(mutation.left); return Effect.fail( new Error( `Product analytics schedule state did not become ${paused ? "paused" : "active"}`, ), ); }), - ), - ), + ); + }), ); return attempt.pipe(Effect.retry({ times: 3 })); }; @@ -558,6 +599,50 @@ export class Tinybird extends Effect.Service()("Tinybird", { return paused; }); + const waitForProductAnalyticsCopyQuiescence = Effect.gen(function* () { + for (let attempt = 0; attempt < 90; attempt += 1) { + const responses = yield* Effect.all( + PRODUCT_ANALYTICS_REBUILD_PIPES.map((name) => + productAnalyticsRequest( + `/v0/jobs?kind=copy&pipe_name=${encodeURIComponent(name)}`, + { + token: productAnalyticsSchedulerToken, + purpose: "Copy job attestation", + }, + ), + ), + ); + if (responses.some((response) => !Array.isArray(response.jobs))) { + return yield* Effect.fail( + new Error("Product analytics Jobs API response was invalid"), + ); + } + const active = responses + .flatMap((response) => response.jobs ?? []) + .filter((job) => { + const status = + typeof job.status === "string" + ? job.status.toLowerCase() + : "unknown"; + return ![ + "done", + "success", + "finished", + "completed", + "failed", + "error", + "cancelled", + "canceled", + ].includes(status); + }); + if (active.length === 0) return; + yield* Effect.sleep(2_000); + } + return yield* Effect.fail( + new Error("Product analytics Copy jobs did not quiesce"), + ); + }); + const waitForProductAnalytics = ( label: string, read: () => Effect.Effect, @@ -659,6 +744,7 @@ export class Tinybird extends Effect.Service()("Tinybird", { pausedPipes = [...paused]; return advance("pausing", paused); }); + yield* waitForProductAnalyticsCopyQuiescence; yield* advance("deleting"); const conditions: string[] = []; if (lease.scope.organizationId) { From 078f69e6d6a35739c86c5c8068d024fbaef2ab1d Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 18:59:57 +0100 Subject: [PATCH 106/110] fix: harden Tinybird staging token boundaries --- .github/workflows/analytics.yml | 2 + ...st-party-analytics-production-readiness.md | 8 +- scripts/analytics/README.md | 13 +- scripts/analytics/staging-ci-lib.js | 117 ++++++++++++++- scripts/analytics/staging-ci.js | 129 +++++++++++----- scripts/analytics/tests/datafiles.test.js | 6 +- scripts/analytics/tests/staging-ci.test.js | 140 +++++++++++++++++- scripts/analytics/tests/tooling.test.js | 12 +- .../tinybird/pipes/product_activation.pipe | 1 - .../product_analytics_ci_assertions.pipe | 1 - .../product_analytics_copy_assertions.pipe | 1 - .../pipes/product_analytics_freshness.pipe | 1 - .../tinybird/pipes/product_attribution.pipe | 1 - .../pipes/product_creator_activity.pipe | 1 - .../pipes/product_creator_retention.pipe | 1 - .../tinybird/pipes/product_events_daily.pipe | 1 - .../tinybird/pipes/product_events_health.pipe | 1 - .../pipes/product_experiment_outcomes.pipe | 1 - .../pipes/product_feature_adoption.pipe | 1 - .../pipes/product_identity_funnel.pipe | 1 - .../pipes/product_traffic_countries.pipe | 1 - .../pipes/product_traffic_overview.pipe | 1 - .../tinybird/pipes/product_traffic_pages.pipe | 1 - .../pipes/product_traffic_sources.pipe | 1 - .../pipes/product_traffic_technology.pipe | 1 - .../pipes/product_traffic_totals.pipe | 1 - scripts/analytics/tooling.js | 63 ++++---- 27 files changed, 394 insertions(+), 114 deletions(-) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index b468f3ca1d4..624b8ed23c7 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -457,6 +457,7 @@ jobs: TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" @@ -561,6 +562,7 @@ jobs: TINYBIRD_STAGING_COPY_TOKEN: ${{ secrets.TINYBIRD_STAGING_COPY_TOKEN }} TINYBIRD_STAGING_READ_TOKEN: ${{ secrets.TINYBIRD_STAGING_READ_TOKEN }} TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + TINYBIRD_STAGING_SCHEDULER_TOKEN: ${{ secrets.TINYBIRD_STAGING_SCHEDULER_TOKEN }} run: >- node scripts/analytics/staging-ci.js run-copies --deployment-id "${{ steps.tinybird.outputs.id }}" diff --git a/analysis/first-party-analytics-production-readiness.md b/analysis/first-party-analytics-production-readiness.md index ca3b0f88e3e..8999ff9b743 100644 --- a/analysis/first-party-analytics-production-readiness.md +++ b/analysis/first-party-analytics-production-readiness.md @@ -55,7 +55,7 @@ Required live gates are: - representative endpoint p95 is within the measured baseline budget; - raw and all derived synthetic state is removed successfully; - an erased identity disappears from raw and derived results while an out-of-scope control remains; -- aggregate read tokens cannot query raw identifiers or append, and append/cleanup tokens cannot read aggregate endpoints; +- the expiring aggregate-read JWT has exactly the reviewed endpoint scopes, cannot query raw identifiers, append, list jobs, or execute Copy Pipes, and append/cleanup tokens cannot read aggregate endpoints; - a bounded five-event fixture produces exact non-zero traffic, page, activation, and retention values while remaining absent from normal decision endpoints; - a populated-table performance pass measures every typed decision endpoint and full dashboard fanout after materialization; - wrong workspace, stale SHA, missing credentials, partial execution, failed promotion, and failed cleanup all fail closed. @@ -76,10 +76,10 @@ The preview-only `/api/analytics/staging-test` route must not receive a producti 2. Create an analytics-only production Tinybird workspace, then create its resources from the reviewed datafiles with a deploy credential scoped only to that workspace. Run `deployment create --check`, review destructive/schema changes, create an isolated deployment, run fixture tests, rebuild every copy, query all aggregate endpoints, then promote. Record the workspace and deployment IDs. Do not reuse the staging workspace or tokens. 3. Create least-privilege Tinybird tokens: - append-only token for `product_events_v1`; - - aggregate endpoint read token with no raw or canonical datasource access; - - resource-scoped copy token for the reviewed copy pipes enumerated by the staging runner, with access to read the status and recent Jobs API records for only its own Copy jobs and no raw identity datasource access; + - expiring JWT with exactly the reviewed 18 aggregate endpoint `PIPES:READ` scopes, no raw or canonical datasource access, and an expiry alert plus rotation procedure that replaces it before the six-hour runway; never create a static endpoint-read token because Tinybird's static `PIPES:READ` credential can start an on-demand Copy Pipe by name outside its declared resources; + - resource-scoped copy token for only the reviewed Copy Pipes and targets enumerated by the staging runner, with no endpoint, raw identity, or Jobs API access; submit each mutation once and use the separate schedule controller for job attestation; - erasure-lookup token limited to read access on `product_events_v1` and `product_events_canonical_v1`, protected from all agent and admin surfaces; - - schedule-controller token limited to cancelling, pausing, and resuming the reviewed Copy Pipes enumerated by the staging runner; + - schedule-controller token with only Tinybird's `PIPES:CREATE` and `DATASOURCES:CREATE` scopes in the target workspace; Tinybird does not resource-bind these operational scopes, so application code must enforce the reviewed Copy Pipe allowlist, use it only for schedule control and Copy job-list attestation, and prove the token fails aggregate-read probes; - dedicated erasure token with Tinybird's required `DATASOURCES:CREATE` scope, no read/deploy scopes, protected as a high-impact operational secret until Tinybird offers resource-scoped row deletion; - deployment token used only by the controlled production release path. 4. Set these Vercel production variables without copying values into logs or artifacts: diff --git a/scripts/analytics/README.md b/scripts/analytics/README.md index b980fc0e44d..9aa8466e000 100644 --- a/scripts/analytics/README.md +++ b/scripts/analytics/README.md @@ -34,16 +34,17 @@ Cloud deployment requires: - `TINYBIRD_URL`: the regional Tinybird API URL. - `TINYBIRD_WORKSPACE_ID`: the production Workspace UUID verified before every cloud check or deployment. -The deployed datafiles create four runtime tokens: +The deployed datafiles create three static runtime tokens: - `product_events_ingest`: append-only access to `product_events_v1`. -- `product_events_agent_read`: read-only access to privacy-safe product endpoints. It has no raw identity datasource or Copy Pipe scope. - `product_events_copy_runner`: execution-only access to the reviewed Copy Pipes enumerated by the staging runner. It cannot query endpoints or raw identity data. - `product_events_erasure_lookup`: read-only access to `product_events_v1` and `product_events_canonical_v1` for bounded identity erasure. It cannot query decision endpoints, append rows, or execute Copy Pipes. -Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents the read token, never the deployment or append token. +Tinybird's static `PIPES:READ` token can also start an on-demand Copy Pipe by name even when that Copy Pipe is not one of its declared resources. Decision endpoints therefore have no static token grants. Create an expiring JWT with exactly the 18 reviewed `PIPES:READ` endpoint scopes, set it as `PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN`, and rotate it before expiry. The staging runner rejects a static token, an extra or missing scope, or a JWT with less than six hours remaining. Tinybird rejects Copy mutations made with that JWT while allowing the scoped endpoint reads. -Staging also requires `TINYBIRD_STAGING_SCHEDULER_TOKEN`, a protected token with schedule-control access only to the Copy Pipes enumerated by the staging runner. The destructive erasure phase cancels and pauses those schedules before deleting rows, runs each replacement Copy exactly once, proves completion with scoped state transitions or copy-run markers, and resumes every schedule in an always-run recovery step. A failed or ambiguous Copy submission is never retried automatically. +Set the append token as `PRODUCT_ANALYTICS_TINYBIRD_TOKEN` in the application. Give agents only the expiring endpoint JWT, never a deployment, append, Copy, scheduler, or erasure token. + +Staging also requires `TINYBIRD_STAGING_SCHEDULER_TOKEN`, a protected token in the staging workspace with only Tinybird's `PIPES:CREATE` and `DATASOURCES:CREATE` operational scopes. Tinybird does not support resource-bound variants of those scopes, so the runner independently restricts operations to its enumerated Copy Pipes, uses the token only for schedule control and Copy job-list attestation, and proves it cannot read aggregate endpoints. The destructive erasure phase cancels scheduled work where present, treats Tinybird's exact unscheduled response as an on-demand no-op, waits for active Copy jobs to quiesce, runs each replacement Copy exactly once, and resumes every schedule in an always-run recovery step. A failed or ambiguous Copy submission is never retried automatically. The staging workflow validates candidate ingestion, duplicate/conflict visibility, live isolation, and every typed endpoint through the exact numeric deployment ID before promotion. Tinybird's Copy service does not reliably route an on-demand Copy mutation to a deployment candidate, so CI refuses that path. After the exact candidate is promoted inside the staging workspace, CI triggers the reviewed Copy Pipes through Tinybird's direct Copy API with the protected `product_events_copy_runner` token. `tb copy run` performs a workspace-level lookup that rejects otherwise sufficient per-pipe read scopes. CI polls both exact event-state jobs to terminal success before rebuilding canonical state, then proves every downstream Copy with canonical, daily, and health transitions plus unique zero-valued markers. A real browser drives the exact-SHA preview through reload, shared-tab, inactivity, SPA retry, unload, and matched-control main-thread measurements. A preview-only authenticated route starts durable server workflows for deduplicated activation and paid-purchase facts. Registry-validated synthetic fixtures separately prove exact anonymous-to-authenticated, guest checkout, cross-device checkout, lifecycle revenue, explicit experiment outcomes, and public endpoint values while remaining excluded from normal queries. Performance compares shared endpoints with the retained deployment, applies absolute budgets when no independent baseline exists, and validates mixed 1,000-row and 100,000-row traffic, activation, retention, identity, attribution, experimentation, adoption, and revenue corpora spread across 30 and 80 UTC days. Direct provider ingestion, the exact-SHA application collector, browser capture, desktop encrypted journal writes, every typed endpoint, and full-dashboard fanout have separate budgets and redacted measurements. Before the prior deployment can be deleted, CI switches it live, executes every endpoint available on that retained schema, proves the exact-SHA admin client degrades only for explicitly optional candidate-only endpoints that return `404`, restores the candidate, and repeats the full synthetic business and health assertions. @@ -53,9 +54,9 @@ Set `TINYBIRD_AGENT_TOKEN` and `TINYBIRD_URL` for the query command. Set a separ ## Agent access -Tinybird exposes published endpoints through its hosted MCP server. Copy `scripts/analytics/tinybird-mcp.example.json` into your agent configuration, replace the two placeholders with the resource-scoped `product_events_agent_read` token and regional API host, and keep the resulting file out of version control. The MCP setup is documented at . +Tinybird exposes published endpoints through its hosted MCP server. Copy `scripts/analytics/tinybird-mcp.example.json` into your agent configuration, replace the two placeholders with the expiring 18-endpoint JWT and regional API host, and keep the resulting file out of version control. The MCP setup is documented at . -Agents should use `product_events_daily` for funnels and trends, and `product_events_health` for delivery checks. The daily endpoint defaults to the latest 30 days, caps results at 1,000 groups, returns newest dates first, and exposes payment and subscription status so paid purchases are not conflated with trials. Health is hourly and rejects windows over 31 days. +Agents should use `product_events_daily` for funnels and trends, and `product_events_health` for delivery checks. The daily endpoint defaults to the latest 30 days, caps results at 1,000 groups, returns newest dates first, and exposes payment and subscription status so paid purchases are not conflated with trials. Health is hourly and rejects windows over 31 days. Alert before the endpoint JWT enters its six-hour rotation window; an expired token must fail reads rather than be replaced with a broader static credential. The Analytics GitHub workflow runs static tests, Docker Compose validation, a complete Tinybird Local build, and fixture tests on relevant pull requests. Merges to `main` deploy only after those gates pass. diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index c63eab0fbcf..38e611bf3a8 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -6,6 +6,28 @@ export const STAGING_DATABASE_FINGERPRINT = export const STAGING_DATABASE_SCHEMA = "0042_lying_sharon_ventura"; export const FEATURE_BRANCH = "codex/first-party-analytics"; export const FEATURE_PULL_REQUEST = 2003; +export const STAGING_READ_ENDPOINTS = [ + "product_activation", + "product_analytics_ci_assertions", + "product_analytics_copy_assertions", + "product_analytics_freshness", + "product_attribution", + "product_creator_activity", + "product_creator_retention", + "product_events_daily", + "product_events_health", + "product_experiment_outcomes", + "product_feature_adoption", + "product_identity_funnel", + "product_traffic_countries", + "product_traffic_overview", + "product_traffic_pages", + "product_traffic_sources", + "product_traffic_technology", + "product_traffic_totals", +]; +export const STAGING_READ_TOKEN_MINIMUM_LIFETIME_MS = 6 * 60 * 60 * 1_000; +export const STAGING_READ_TOKEN_MAXIMUM_LIFETIME_MS = 45 * 24 * 60 * 60 * 1_000; export const PREVIEW_TINYBIRD_TOKEN_NAMES = [ "PRODUCT_ANALYTICS_TINYBIRD_TOKEN", "PRODUCT_ANALYTICS_TINYBIRD_READ_TOKEN", @@ -114,6 +136,39 @@ export const copyScheduleMatchesAction = (value, action) => { : status === "scheduled" || status === "active"; }; +export const isUnscheduledCopyMutation = (status, payload) => + status === 422 && + typeof payload?.error === "string" && + payload.error.startsWith("The copy Pipe is not scheduled"); + +export const formatTinybirdDateTime64 = (value) => { + if (/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}$/.test(value)) { + return value; + } + if (!/(?:Z|[+-]\d{2}:\d{2})$/.test(value)) { + throw new Error("Tinybird DateTime64 value must include a timezone"); + } + const parsed = new Date(value); + if (!Number.isFinite(parsed.getTime())) { + throw new Error("Tinybird DateTime64 value is invalid"); + } + return parsed.toISOString().replace("T", " ").replace(/Z$/, ""); +}; + +export const tokenScopeProbeWindow = (startTime, endTime) => { + const startMs = Date.parse(startTime); + const endMs = Date.parse(endTime); + if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) { + throw new Error("Token scope probe window is invalid"); + } + return { + start_time: formatTinybirdDateTime64(new Date(startMs).toISOString()), + end_time: formatTinybirdDateTime64( + new Date(Math.min(endMs, startMs + 86_400_000)).toISOString(), + ), + }; +}; + export const waitForTinybirdCopyPipesQuiescent = async ({ origin, token, @@ -213,19 +268,20 @@ export const assertExecutionScope = ({ throw new Error("This event cannot deploy analytics staging"); }; -export const tokenWorkspaceId = (token) => { +const tinybirdTokenPayload = (token) => { const segments = token.split("."); - if (segments.length < 3 || !segments[1]) { + if (segments.length !== 3 || !segments[1]) { throw new Error("A Tinybird staging token has an unsupported format"); } - let payload; try { - payload = JSON.parse( - Buffer.from(segments[1], "base64url").toString("utf8"), - ); + return JSON.parse(Buffer.from(segments[1], "base64url").toString("utf8")); } catch { throw new Error("A Tinybird staging token cannot be decoded"); } +}; + +export const tokenWorkspaceId = (token) => { + const payload = tinybirdTokenPayload(token); const workspaceId = payload.u ?? payload.workspace_id ?? payload.workspaceId; if (typeof workspaceId !== "string") { throw new Error("A Tinybird staging token does not identify its workspace"); @@ -233,7 +289,51 @@ export const tokenWorkspaceId = (token) => { return workspaceId; }; -export const validateTinybirdCredentials = ({ url, tokens }) => { +const validateStagingReadJwt = (token, now) => { + const payload = tinybirdTokenPayload(token); + if ( + !Number.isInteger(payload.exp) || + payload.exp * 1_000 < now + STAGING_READ_TOKEN_MINIMUM_LIFETIME_MS || + payload.exp * 1_000 > now + STAGING_READ_TOKEN_MAXIMUM_LIFETIME_MS + ) { + throw new Error( + "TINYBIRD_STAGING_READ_TOKEN must expire between six hours and 45 days from now", + ); + } + if (!Array.isArray(payload.scopes)) { + throw new Error( + "TINYBIRD_STAGING_READ_TOKEN must be an expiring resource-scoped JWT", + ); + } + const scopes = payload.scopes.map((scope) => { + if ( + scope === null || + typeof scope !== "object" || + scope.type !== "PIPES:READ" || + typeof scope.resource !== "string" || + JSON.stringify(Object.keys(scope).sort()) !== + JSON.stringify(["resource", "type"]) + ) { + throw new Error("TINYBIRD_STAGING_READ_TOKEN has an unauthorized scope"); + } + return scope.resource; + }); + const actual = [...new Set(scopes)].sort(); + if ( + actual.length !== scopes.length || + JSON.stringify(actual) !== JSON.stringify(STAGING_READ_ENDPOINTS) + ) { + throw new Error( + "TINYBIRD_STAGING_READ_TOKEN must grant only the reviewed decision endpoints", + ); + } +}; + +export const validateTinybirdCredentials = ({ + url, + tokens, + now = Date.now(), +}) => { let parsedUrl; try { parsedUrl = new URL(url); @@ -263,6 +363,9 @@ export const validateTinybirdCredentials = ({ url, tokens }) => { `${name} is not scoped to the analytics staging workspace`, ); } + if (name === "TINYBIRD_STAGING_READ_TOKEN") { + validateStagingReadJwt(token, now); + } } return parsedUrl.origin; }; diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 1da0684c4e7..39f28b36e30 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -28,7 +28,9 @@ import { evaluateCopyPerformanceBudget, evaluateLatencyBudget, extractSameOriginNextScriptUrls, + formatTinybirdDateTime64, hashIdentifier, + isUnscheduledCopyMutation, latencySummary, normalizeCiAssertions, normalizeCopyAssertions, @@ -45,8 +47,10 @@ import { submitTinybirdCopyJobs, syntheticIdentityFilterQueries, syntheticMonetizationFilterQueries, + tokenScopeProbeWindow, validateSyntheticRunId, validateTinybirdCredentials, + waitForTinybirdCopyJob, waitForTinybirdCopyPipesQuiescent, } from "./staging-ci-lib.js"; @@ -242,7 +246,12 @@ const tinybirdUrl = (origin, pathname, parameters = {}) => { const url = new URL(pathname, origin); for (const [name, value] of Object.entries(parameters)) { if (value !== undefined && value !== "") { - url.searchParams.set(name, value); + url.searchParams.set( + name, + ["end_time", "source_cutoff", "start_time"].includes(name) + ? formatTinybirdDateTime64(String(value)) + : value, + ); } } return url; @@ -2727,6 +2736,7 @@ const runCopies = async (parameters = {}) => { const { origin, tokens } = tinybirdEnvironment([ "TINYBIRD_STAGING_COPY_TOKEN", "TINYBIRD_STAGING_DEPLOY_TOKEN", + "TINYBIRD_STAGING_SCHEDULER_TOKEN", ]); const copyRunId = validateSyntheticRunId(`${state.runId}_${phase}`); const expectations = phaseRunExpectations({ state, phase }); @@ -2770,7 +2780,7 @@ const runCopies = async (parameters = {}) => { const stateJobCompletions = await waitForCopyJobs({ jobs: stateJobs, origin, - token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, assertMutationOwnership, }); const canonicalJobs = await submitTinybirdCopyJobs({ @@ -2794,6 +2804,12 @@ const runCopies = async (parameters = {}) => { }, }; writeJson(artifactPath, artifact); + const canonicalJobCompletions = await waitForCopyJobs({ + jobs: canonicalJobs, + origin, + token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, + assertMutationOwnership, + }); const canonicalVisibility = await waitForCopyVisibility({ label: "Tinybird canonical copy", read: () => @@ -2806,6 +2822,7 @@ const runCopies = async (parameters = {}) => { assertPhaseCiAssertions(results, ["canonicalEvents"]), }); const downstreamJobs = []; + const downstreamJobCompletions = []; const downstreamVisibility = {}; const copySteps = [ { @@ -2846,15 +2863,22 @@ const runCopies = async (parameters = {}) => { }, ]; for (const copyStep of copySteps) { - downstreamJobs.push( - ...(await submitTinybirdCopyJobs({ + const copyJobs = await submitTinybirdCopyJobs({ + origin, + token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + deploymentId: state.deploymentId, + request, + pipes: [copyStep.pipe], + copyRunId, + sourceCutoff, + assertMutationOwnership, + }); + downstreamJobs.push(...copyJobs); + downstreamJobCompletions.push( + ...(await waitForCopyJobs({ + jobs: copyJobs, origin, - token: tokens.TINYBIRD_STAGING_COPY_TOKEN, - deploymentId: state.deploymentId, - request, - pipes: [copyStep.pipe], - copyRunId, - sourceCutoff, + token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, assertMutationOwnership, })), ); @@ -2890,6 +2914,8 @@ const runCopies = async (parameters = {}) => { await assertMutationOwnership(); const visibility = latencySummary([ ...stateJobCompletions.map((completion) => completion.completionMs), + ...canonicalJobCompletions.map((completion) => completion.completionMs), + ...downstreamJobCompletions.map((completion) => completion.completionMs), canonicalVisibility.visibilityMs, ...Object.values(downstreamVisibility).map( (copyVisibility) => copyVisibility.visibilityMs, @@ -2901,6 +2927,8 @@ const runCopies = async (parameters = {}) => { copyRunHash: hashIdentifier(copyRunId), jobs: [...stateJobs, ...canonicalJobs, ...downstreamJobs], stateJobCompletions, + canonicalJobCompletions, + downstreamJobCompletions, canonicalVisibility: { polls: canonicalVisibility.polls, visibilityMs: canonicalVisibility.visibilityMs, @@ -2943,7 +2971,7 @@ const runCopies = async (parameters = {}) => { providerResourceMetrics: { available: false, limitation: - "Tinybird Copy job completion is polled for the prerequisite state rebuild, but provider resource metrics remain unavailable; CI measures job completion, marker visibility, and end-to-end pipeline wall-clock.", + "Tinybird Copy job completion is polled for every rebuild, but provider resource metrics remain unavailable; CI measures job completion, marker visibility, and end-to-end pipeline wall-clock.", }, baseline: baseline ?? { phase, @@ -3038,26 +3066,42 @@ const setCopySchedules = async (parameters = {}) => { ); } }; + const unscheduledPipes = new Set(); await applyCopyScheduleAction({ pipes: COPY_PIPES, action, setSchedule: async (pipe, scheduleAction) => { let mutationError; + let unscheduled = false; try { - await request( + const response = await tokenScopeProbe( tinybirdUrl( origin, `/v0/pipes/${encodeURIComponent(pipe)}/copy/${scheduleAction === "pause" ? "cancel" : "resume"}`, ), - { - token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, - method: "POST", - attempts: 1, - }, + tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, + { method: "POST" }, ); + if (!response.ok) { + const payload = + response.status === 422 + ? await response.json().catch(() => ({})) + : {}; + if (isUnscheduledCopyMutation(response.status, payload)) { + unscheduled = true; + unscheduledPipes.add(pipe); + } else { + const error = new Error( + `Tinybird request was rejected with HTTP ${response.status}`, + ); + error.status = response.status; + throw error; + } + } } catch (error) { mutationError = error; } + if (unscheduled) return; if (scheduleAction === "pause") { if (!mutationError) return; if ( @@ -3066,7 +3110,7 @@ const setCopySchedules = async (parameters = {}) => { ) { await waitForTinybirdCopyPipesQuiescent({ origin, - token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, pipes: [pipe], request, assertMutationOwnership: assertLiveOwnership, @@ -3100,7 +3144,7 @@ const setCopySchedules = async (parameters = {}) => { action === "pause" ? await waitForTinybirdCopyPipesQuiescent({ origin, - token: tokens.TINYBIRD_STAGING_COPY_TOKEN, + token: tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, request, requiredVisibleJobIds: Object.values(artifact.copyJobs ?? {}) .flatMap((phase) => (Array.isArray(phase?.jobs) ? phase.jobs : [])) @@ -3115,6 +3159,7 @@ const setCopySchedules = async (parameters = {}) => { status: "passed", deploymentId: String(state.deploymentId), pipeCount: COPY_PIPES.length, + unscheduledPipeCount: unscheduledPipes.size, ...(quiescence ? { quiescence } : {}), }, }; @@ -4591,28 +4636,37 @@ const verifyTokenScopes = async () => { "TINYBIRD_STAGING_READ_TOKEN", "TINYBIRD_STAGING_CLEANUP_TOKEN", ]); - await request( - tinybirdUrl(origin, "/v0/pipes/product_events_health.json", { - start_time: state.startTime, - end_time: state.endTime, - }), - { token: tokens.TINYBIRD_STAGING_READ_TOKEN }, + const scopeWindow = tokenScopeProbeWindow(state.startTime, state.endTime); + const aggregateReadProbeUrl = tinybirdUrl( + origin, + "/v0/pipes/product_events_health.json", + scopeWindow, ); + await request(aggregateReadProbeUrl, { + token: tokens.TINYBIRD_STAGING_READ_TOKEN, + }); await request( tinybirdUrl(origin, "/v0/sql", { - q: "SELECT countIf(user_id != '') AS rows FROM product_events_v1 UNION ALL SELECT countIf(user_id != '') AS rows FROM product_events_canonical_v1", + q: "SELECT countIf(user_id != '') AS rows FROM product_events_v1 UNION ALL SELECT countIf(user_id != '') AS rows FROM product_events_canonical_v1 FORMAT JSON", }), { token: tokens.TINYBIRD_STAGING_ERASURE_LOOKUP_TOKEN }, ); const copyJobsProbe = await tokenScopeProbe( tinybirdUrl(origin, "/v0/jobs", { kind: "copy" }), - tokens.TINYBIRD_STAGING_COPY_TOKEN, + tokens.TINYBIRD_STAGING_SCHEDULER_TOKEN, ); if (!copyJobsProbe.ok) { throw new Error( - `The copy-runner token cannot attest Copy job quiescence: HTTP ${copyJobsProbe.status}`, + `The schedule-controller token cannot attest Copy job quiescence: HTTP ${copyJobsProbe.status}`, ); } + await assertScopeDenied( + "The copy-runner token job-list probe", + tokenScopeProbe( + tinybirdUrl(origin, "/v0/jobs", { kind: "copy" }), + tokens.TINYBIRD_STAGING_COPY_TOKEN, + ), + ); await assertScopeDenied( "The aggregate read token raw identity query", tokenScopeProbe( @@ -4637,6 +4691,13 @@ const verifyTokenScopes = async () => { }, ), ); + await assertScopeDenied( + "The aggregate read token job-list probe", + tokenScopeProbe( + tinybirdUrl(origin, "/v0/jobs", { kind: "copy" }), + tokens.TINYBIRD_STAGING_READ_TOKEN, + ), + ); await assertScopeDenied( "The aggregate read token Copy mutation probe", tokenScopeProbe( @@ -4702,13 +4763,7 @@ const verifyTokenScopes = async () => { ]) { await assertScopeDenied( `The ${name} aggregate read probe`, - tokenScopeProbe( - tinybirdUrl(origin, "/v0/pipes/product_events_health.json", { - start_time: state.startTime, - end_time: state.endTime, - }), - token, - ), + tokenScopeProbe(aggregateReadProbeUrl, token), ); } artifact.tokenScopes = { @@ -4716,15 +4771,17 @@ const verifyTokenScopes = async () => { rawIdentityReadDenied: true, readTokenAppendDenied: true, readTokenCopyMutationDenied: true, + readTokenJobListDenied: true, ingestTokenAppendAuthorized: true, ingestTokenAggregateReadDenied: true, cleanupTokenAggregateReadDenied: true, copyTokenAggregateReadDenied: true, - copyTokenJobsReadPassed: true, + copyTokenJobListDenied: true, erasureLookupRawReadPassed: true, erasureLookupAppendDenied: true, erasureLookupCopyMutationDenied: true, erasureLookupAggregateReadDenied: true, + schedulerTokenJobsReadPassed: true, }; artifact.assertions = { ...artifact.assertions, diff --git a/scripts/analytics/tests/datafiles.test.js b/scripts/analytics/tests/datafiles.test.js index d572601f1c3..5aa20149877 100644 --- a/scripts/analytics/tests/datafiles.test.js +++ b/scripts/analytics/tests/datafiles.test.js @@ -584,15 +584,13 @@ test("copy rebuilds are on-demand and require the sequential controller", () => } }); -test("decision endpoints cannot be executed with the Copy runner token", () => { +test("decision endpoints do not create reusable static tokens", () => { const project = loadTinybirdProject(TINYBIRD_PROJECT_DIR); for (const pipe of project.pipes.filter( (candidate) => candidate.name.startsWith("product_") && candidate.type === "endpoint", )) { - assert.deepEqual(pipe.tokens, [ - { name: "product_events_agent_read", scope: "READ" }, - ]); + assert.deepEqual(pipe.tokens, []); } }); diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index eed9fa0c726..773a32a0a0c 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -30,6 +30,8 @@ import { extractSameOriginNextScriptUrls, FEATURE_BRANCH, FEATURE_PULL_REQUEST, + formatTinybirdDateTime64, + isUnscheduledCopyMutation, latencySummary, normalizeCiAssertions, normalizeCopyAssertions, @@ -44,11 +46,15 @@ import { resolveOwnedMutationTarget, STAGING_DATABASE_FINGERPRINT, STAGING_DATABASE_SCHEMA, + STAGING_READ_ENDPOINTS, + STAGING_READ_TOKEN_MAXIMUM_LIFETIME_MS, + STAGING_READ_TOKEN_MINIMUM_LIFETIME_MS, STAGING_WORKSPACE_ID, selectStagingDeployment, submitTinybirdCopyJobs, syntheticIdentityFilterQueries, syntheticMonetizationFilterQueries, + tokenScopeProbeWindow, tokenWorkspaceId, validateSyntheticRunId, validateTinybirdCredentials, @@ -59,6 +65,21 @@ import { const SHA = "1234567890abcdef1234567890abcdef12345678"; const token = (workspaceId = STAGING_WORKSPACE_ID) => `p.${Buffer.from(JSON.stringify({ u: workspaceId })).toString("base64url")}.signature`; +const readJwt = ({ + workspaceId = STAGING_WORKSPACE_ID, + expiresAt = Date.now() + STAGING_READ_TOKEN_MINIMUM_LIFETIME_MS + 1_000, + scopes = STAGING_READ_ENDPOINTS.map((resource) => ({ + type: "PIPES:READ", + resource, + })), +} = {}) => + `ey.${Buffer.from( + JSON.stringify({ + exp: Math.ceil(expiresAt / 1_000), + scopes, + workspace_id: workspaceId, + }), + ).toString("base64url")}.signature`; test("schedule pause compensates every schedule already paused", async () => { const calls = []; @@ -122,6 +143,75 @@ test("schedule state attestation distinguishes paused from active copies", () => assert.equal(copyScheduleMatchesAction({}, "resume"), true); }); +test("on-demand Copy responses are distinguished from schedule failures", () => { + assert.equal( + isUnscheduledCopyMutation(422, { + error: "The copy Pipe is not scheduled", + }), + true, + ); + assert.equal( + isUnscheduledCopyMutation(422, { error: "Another provider failure" }), + false, + ); + assert.equal( + isUnscheduledCopyMutation(403, { + error: "The copy Pipe is not scheduled", + }), + false, + ); +}); + +test("token scope probes stay inside the health endpoint window", () => { + assert.deepEqual( + tokenScopeProbeWindow( + "2026-05-14T05:00:00.000Z", + "2026-08-01T17:20:36.520Z", + ), + { + start_time: "2026-05-14 05:00:00.000", + end_time: "2026-05-15 05:00:00.000", + }, + ); + assert.deepEqual( + tokenScopeProbeWindow( + "2026-08-01T17:00:00.000Z", + "2026-08-01T17:20:36.520Z", + ), + { + start_time: "2026-08-01 17:00:00.000", + end_time: "2026-08-01 17:20:36.520", + }, + ); + assert.throws( + () => + tokenScopeProbeWindow( + "2026-08-02T00:00:00.000Z", + "2026-08-01T00:00:00.000Z", + ), + /Token scope probe window is invalid/, + ); +}); + +test("Tinybird DateTime64 parameters are normalized to UTC without a suffix", () => { + assert.equal( + formatTinybirdDateTime64("2026-08-01T17:20:36.520Z"), + "2026-08-01 17:20:36.520", + ); + assert.equal( + formatTinybirdDateTime64("2026-08-01T18:20:36.520+01:00"), + "2026-08-01 17:20:36.520", + ); + assert.equal( + formatTinybirdDateTime64("2026-08-01 17:20:36.520"), + "2026-08-01 17:20:36.520", + ); + assert.throws( + () => formatTinybirdDateTime64("2026-08-01 17:20:36"), + /must include a timezone/, + ); +}); + test("Copy quiescence waits until every approved pipe has no active jobs", async () => { let now = 1_000; let round = 0; @@ -246,10 +336,10 @@ test("Tinybird credentials must all decode to the hard-coded staging workspace", validateTinybirdCredentials({ url: "https://api.us-east.aws.tinybird.co", tokens: { - deploy: token(), - ingest: token(), - read: token(), - cleanup: token(), + TINYBIRD_STAGING_DEPLOY_TOKEN: token(), + TINYBIRD_STAGING_INGEST_TOKEN: token(), + TINYBIRD_STAGING_READ_TOKEN: readJwt(), + TINYBIRD_STAGING_CLEANUP_TOKEN: token(), }, }), "https://api.us-east.aws.tinybird.co", @@ -268,6 +358,40 @@ test("Tinybird credentials must all decode to the hard-coded staging workspace", ); }); +test("staging read credentials require an exact, sufficiently-lived endpoint JWT", () => { + for (const invalidToken of [ + token(), + readJwt({ expiresAt: Date.now() + 60_000 }), + readJwt({ + expiresAt: Date.now() + STAGING_READ_TOKEN_MAXIMUM_LIFETIME_MS + 60_000, + }), + readJwt({ + scopes: [ + ...STAGING_READ_ENDPOINTS.map((resource) => ({ + type: "PIPES:READ", + resource, + })), + { type: "DATASOURCES:READ", resource: "product_events_v1" }, + ], + }), + readJwt({ scopes: [] }), + readJwt({ + scopes: STAGING_READ_ENDPOINTS.map((resource) => ({ + fixed_params: { hidden: "true" }, + type: "PIPES:READ", + resource, + })), + }), + ]) { + assert.throws(() => + validateTinybirdCredentials({ + url: "https://api.us-east.aws.tinybird.co", + tokens: { TINYBIRD_STAGING_READ_TOKEN: invalidToken }, + }), + ); + } +}); + test("preview Tinybird attestation requires the exact SHA, host, and staging workspace", () => { const expectedTokenHashes = Object.fromEntries( PREVIEW_TINYBIRD_TOKEN_NAMES.map((name, index) => [ @@ -2196,6 +2320,14 @@ test("the analytics workflow is statically restricted to staging", () => { workflow.indexOf("Rebuild promoted decision and health copies") < workflow.indexOf("Measure populated decision endpoint performance"), ); + assert.match( + workflow, + /Rebuild promoted decision and health copies[\s\S]{0,600}TINYBIRD_STAGING_SCHEDULER_TOKEN:[\s\S]{0,400}staging-ci\.js run-copies/, + ); + assert.match( + workflow, + /Retract synthetic rows from every derived copy[\s\S]{0,600}TINYBIRD_STAGING_SCHEDULER_TOKEN:[\s\S]{0,400}staging-ci\.js run-copies/, + ); assert.ok( workflow.indexOf("Measure populated decision endpoint performance") < workflow.indexOf( diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index 72d68cb98c4..4e510ca5c9a 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -497,14 +497,16 @@ test("project validation rejects the Copy runner token on decision endpoints", ( fs.writeFileSync( pipePath, contents.replace( - "TOKEN product_events_agent_read READ", - "TOKEN product_events_copy_runner READ", + "\n\nNODE product_events_health_node", + "\nTOKEN product_events_copy_runner READ\n\nNODE product_events_health_node", ), ); const issues = validateAnalyticsProject(projectDir); assert.ok( issues.some((issue) => - issue.includes("missing its read-only agent token"), + issue.includes( + "must use an expiring resource-scoped JWT instead of static token grants", + ), ), ); assert.ok( @@ -544,8 +546,8 @@ test("project validation rejects extra Copy and erasure lookup grants", () => { fs.writeFileSync( endpointPath, endpoint.replace( - "TOKEN product_events_agent_read READ", - "TOKEN product_events_agent_read READ\nTOKEN product_events_erasure_lookup READ", + "\n\nNODE product_events_health_node", + "\nTOKEN product_events_erasure_lookup READ\n\nNODE product_events_health_node", ), ); const issues = validateAnalyticsProject(projectDir); diff --git a/scripts/analytics/tinybird/pipes/product_activation.pipe b/scripts/analytics/tinybird/pipes/product_activation.pipe index 9b37b82c9fb..bdb1647c077 100644 --- a/scripts/analytics/tinybird/pipes/product_activation.pipe +++ b/scripts/analytics/tinybird/pipes/product_activation.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query signup cohorts and first-share activation within seven UTC days. -TOKEN product_events_agent_read READ NODE product_activation_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe b/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe index acfcd58f294..393d1e2983f 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_ci_assertions.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Prove synthetic retry deduplication and payload-conflict quarantine without exposing raw event identifiers. -TOKEN product_events_agent_read READ NODE product_analytics_ci_assertions_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe index da8b484ef40..c34608e471c 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_copy_assertions.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Prove each privacy-safe aggregate copy completed for one bounded staging phase. -TOKEN product_events_agent_read READ NODE product_analytics_copy_assertions_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe index 74892360b67..f33ebbaf704 100644 --- a/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe +++ b/scripts/analytics/tinybird/pipes/product_analytics_freshness.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Expose aggregate freshness and bounded raw ingestion lag without identifiers. -TOKEN product_events_agent_read READ NODE product_analytics_freshness_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_attribution.pipe b/scripts/analytics/tinybird/pipes/product_attribution.pipe index 387b3516e02..c786b422694 100644 --- a/scripts/analytics/tinybird/pipes/product_attribution.pipe +++ b/scripts/analytics/tinybird/pipes/product_attribution.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query aggregate first, session, and last campaign attribution without returning visitor or session identifiers. -TOKEN product_events_agent_read READ NODE product_attribution_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_creator_activity.pipe b/scripts/analytics/tinybird/pipes/product_creator_activity.pipe index e6aed7458d9..46a9b15b152 100644 --- a/scripts/analytics/tinybird/pipes/product_creator_activity.pipe +++ b/scripts/analytics/tinybird/pipes/product_creator_activity.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query active creators, organizations, new-versus-returning activity, and DAU-WAU-MAU stickiness. -TOKEN product_events_agent_read READ NODE product_creator_activity_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_creator_retention.pipe b/scripts/analytics/tinybird/pipes/product_creator_retention.pipe index 6dea19f4d76..3b8a61f4bde 100644 --- a/scripts/analytics/tinybird/pipes/product_creator_retention.pipe +++ b/scripts/analytics/tinybird/pipes/product_creator_retention.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query exact creator and organization retention cohorts with an optional activity-platform filter and an all-platform cohort baseline. -TOKEN product_events_agent_read READ NODE product_creator_retention_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_events_daily.pipe b/scripts/analytics/tinybird/pipes/product_events_daily.pipe index e28c61f70ea..58808091808 100644 --- a/scripts/analytics/tinybird/pipes/product_events_daily.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_daily.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query daily product events using typed filters and pre-aggregated data. -TOKEN product_events_agent_read READ NODE product_events_daily_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_events_health.pipe b/scripts/analytics/tinybird/pipes/product_events_health.pipe index ee3edb4e628..bd2daa73615 100644 --- a/scripts/analytics/tinybird/pipes/product_events_health.pipe +++ b/scripts/analytics/tinybird/pipes/product_events_health.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Inspect hourly product event delivery quality over a maximum 31-day window. -TOKEN product_events_agent_read READ NODE product_events_health_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe b/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe index 843120aeb9a..1501a8256d7 100644 --- a/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe +++ b/scripts/analytics/tinybird/pipes/product_experiment_outcomes.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query aggregate 30-day first authoritative outcomes for actors with an explicit stable experiment exposure. -TOKEN product_events_agent_read READ NODE product_experiment_outcomes_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe b/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe index 81bf7610554..9962d31e55c 100644 --- a/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe +++ b/scripts/analytics/tinybird/pipes/product_feature_adoption.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query feature event volume and daily unique adopters after applying decision filters. -TOKEN product_events_agent_read READ NODE product_feature_adoption_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_identity_funnel.pipe b/scripts/analytics/tinybird/pipes/product_identity_funnel.pipe index 2ad1bede8e1..2eaf05f4610 100644 --- a/scripts/analytics/tinybird/pipes/product_identity_funnel.pipe +++ b/scripts/analytics/tinybird/pipes/product_identity_funnel.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query complete privacy-safe anonymous acquisition to authenticated conversion totals without raw identifiers. -TOKEN product_events_agent_read READ NODE product_identity_funnel_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe index 905097520a2..01f9677559c 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_countries.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query country traffic without raw identifiers or raw network addresses. -TOKEN product_events_agent_read READ NODE product_traffic_countries_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe index 0d204b857c5..5697b154205 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_overview.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query exact daily Plausible-style traffic metrics from privacy-safe aggregates. -TOKEN product_events_agent_read READ NODE product_traffic_overview_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe index a8aa92b0740..9abbfb154cb 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_pages.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query top, landing, exit, and engagement page metrics without raw identifiers. -TOKEN product_events_agent_read READ NODE product_traffic_pages_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe index 003efb59144..b7fb0b81abf 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_sources.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query source, channel, medium, and campaign traffic without raw identifiers. -TOKEN product_events_agent_read READ NODE product_traffic_sources_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe index 11cd3966595..7b209ee822b 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_technology.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query device, browser, and operating-system traffic without raw user agents. -TOKEN product_events_agent_read READ NODE product_traffic_technology_node SQL > diff --git a/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe b/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe index c1021bd0505..d961e16cfec 100644 --- a/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe +++ b/scripts/analytics/tinybird/pipes/product_traffic_totals.pipe @@ -1,7 +1,6 @@ DESCRIPTION > Query selected-range unique visitors and exact traffic totals by merging privacy-safe aggregate states across UTC dates. -TOKEN product_events_agent_read READ NODE product_traffic_totals_node SQL > diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index d1188d81397..ee28c05067e 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -98,6 +98,26 @@ const PRODUCT_COPY_SOURCES = [ "product_traffic_daily_bounded_v2", "product_traffic_pages_daily_bounded_v2", ]; +const PRODUCT_DECISION_ENDPOINTS = [ + "product_activation", + "product_analytics_ci_assertions", + "product_analytics_copy_assertions", + "product_analytics_freshness", + "product_attribution", + "product_creator_activity", + "product_creator_retention", + "product_events_daily", + "product_events_health", + "product_experiment_outcomes", + "product_feature_adoption", + "product_identity_funnel", + "product_traffic_countries", + "product_traffic_overview", + "product_traffic_pages", + "product_traffic_sources", + "product_traffic_technology", + "product_traffic_totals", +]; const WORKSPACE_ID_SOURCE = "[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}"; const WORKSPACE_ID_PATTERN = new RegExp(`^${WORKSPACE_ID_SOURCE}$`, "i"); @@ -441,6 +461,12 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { } const project = loadTinybirdProject(projectDir); + validateExactTokenGrants( + project, + "product_events_agent_read", + new Set(), + issues, + ); validateExactTokenGrants( project, "product_events_copy_runner", @@ -749,36 +775,16 @@ const validateAnalyticsProject = (projectDir = TINYBIRD_PROJECT_DIR) => { issues.push(`${name} must not grant Copy execution to the agent token`); } } - for (const name of [ - "product_events_daily", - "product_events_health", - "product_traffic_overview", - "product_traffic_totals", - "product_traffic_pages", - "product_traffic_sources", - "product_attribution", - "product_traffic_countries", - "product_traffic_technology", - "product_activation", - "product_creator_retention", - "product_creator_activity", - "product_feature_adoption", - "product_identity_funnel", - "product_experiment_outcomes", - "product_analytics_freshness", - "product_analytics_copy_assertions", - ]) { + for (const name of PRODUCT_DECISION_ENDPOINTS) { const pipe = project.pipes.find((candidate) => candidate.name === name); if (!pipe) { issues.push(`Missing product analytics pipe ${name}`); continue; } - if ( - pipe.type !== "materialized" && - pipe.type !== "copy" && - !hasToken(pipe, "product_events_agent_read", "READ") - ) { - issues.push(`${name} is missing its read-only agent token`); + if (pipe.tokens.length > 0) { + issues.push( + `${name} must use an expiring resource-scoped JWT instead of static token grants`, + ); } if (hasToken(pipe, "product_events_copy_runner", "READ")) { issues.push(`${name} must not be queryable by the Copy runner token`); @@ -1171,14 +1177,11 @@ const runAnalyticsCommand = async (operation) => { } if (step.type === "verify-local") { const environment = localEnvironment(); - const readToken = await localResourceToken( - environment, - "product_events_agent_read", - ); await runProcess(process.execPath, [LOCAL_VERIFY_SCRIPT], { env: { ...environment, - PRODUCT_ANALYTICS_TINYBIRD_TOKEN: readToken, + PRODUCT_ANALYTICS_TINYBIRD_TOKEN: + environment.TB_LOCAL_WORKSPACE_TOKEN, }, }); continue; From 8001bc9cffc16896c4bbd42d33262c9e89259386 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:07:56 +0100 Subject: [PATCH 107/110] fix: mint scoped JWT for local analytics checks --- scripts/analytics/tests/tooling.test.js | 39 +++++++++++++++++++++ scripts/analytics/tooling.js | 45 +++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index 4e510ca5c9a..186cdd4094f 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -9,6 +9,7 @@ import { COMPOSE_FILE, cloudEnvironment, LOCAL_ENV_FILE, + localDecisionJwt, localEnvironment, localResourceToken, operationPlan, @@ -205,6 +206,44 @@ test("local resource discovery returns only the named scoped token", async () => assert.equal(token, "p.resource-token-value"); }); +test("local decision verification uses an expiring endpoint-scoped JWT", async () => { + const requests = []; + const token = await localDecisionJwt( + { + PRODUCT_ANALYTICS_TINYBIRD_HOST: "http://127.0.0.1:7181", + TB_LOCAL_WORKSPACE_TOKEN: "local-workspace-token", + }, + async (url, init) => { + requests.push({ url, init }); + return Response.json({ token: "local-decision-jwt-value" }); + }, + 1_785_607_200_000, + ); + + assert.equal(token, "local-decision-jwt-value"); + assert.equal(requests.length, 1); + assert.equal(requests[0].url.pathname, "/v0/tokens"); + assert.equal( + requests[0].url.searchParams.get("expiration_time"), + "1785610800", + ); + assert.equal(requests[0].init.method, "POST"); + assert.equal( + requests[0].init.headers.Authorization, + "Bearer local-workspace-token", + ); + const payload = JSON.parse(requests[0].init.body); + assert.equal(payload.scopes.length, 18); + assert.ok( + payload.scopes.every( + (scope) => + scope.type === "PIPES:READ" && + typeof scope.resource === "string" && + Object.keys(scope).length === 2, + ), + ); +}); + test("local resource discovery falls back to the local user token", async () => { const authorizations = []; const token = await localResourceToken( diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index ee28c05067e..d0e3e102559 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -945,6 +945,46 @@ const localResourceToken = async ( ); }; +const localDecisionJwt = async ( + environment, + fetcher = fetch, + now = Date.now(), +) => { + const url = new URL( + "/v0/tokens", + environment.PRODUCT_ANALYTICS_TINYBIRD_HOST, + ); + url.searchParams.set("name", "product_events_local_verification"); + url.searchParams.set( + "expiration_time", + String(Math.floor(now / 1_000) + 3_600), + ); + const response = await fetcher(url, { + method: "POST", + headers: { + Authorization: `Bearer ${environment.TB_LOCAL_WORKSPACE_TOKEN}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + scopes: PRODUCT_DECISION_ENDPOINTS.map((resource) => ({ + type: "PIPES:READ", + resource, + })), + }), + signal: AbortSignal.timeout(15_000), + }); + if (!response.ok) { + throw new Error( + `Tinybird Local could not create its decision-endpoint JWT: HTTP ${response.status}`, + ); + } + const payload = await response.json(); + if (typeof payload.token !== "string" || payload.token.length < 16) { + throw new Error("Tinybird Local returned an invalid decision-endpoint JWT"); + } + return payload.token; +}; + const assertSafeStep = (step) => { const command = [step.command, ...(step.args ?? [])].join(" "); if ( @@ -1177,11 +1217,11 @@ const runAnalyticsCommand = async (operation) => { } if (step.type === "verify-local") { const environment = localEnvironment(); + const readToken = await localDecisionJwt(environment); await runProcess(process.execPath, [LOCAL_VERIFY_SCRIPT], { env: { ...environment, - PRODUCT_ANALYTICS_TINYBIRD_TOKEN: - environment.TB_LOCAL_WORKSPACE_TOKEN, + PRODUCT_ANALYTICS_TINYBIRD_TOKEN: readToken, }, }); continue; @@ -1207,6 +1247,7 @@ export { assertSafeStep, cloudEnvironment, composeArgs, + localDecisionJwt, localEnvironment, localResourceToken, operationPlan, From 3c92e69b0a4a93387d5ff3d27e914d263de82b29 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:16:47 +0100 Subject: [PATCH 108/110] fix: isolate local analytics read token --- scripts/analytics/tests/tooling.test.js | 57 +++++++++------------- scripts/analytics/tooling.js | 65 ++++++++++++------------- 2 files changed, 55 insertions(+), 67 deletions(-) diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index 186cdd4094f..d4f4643de86 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -9,7 +9,7 @@ import { COMPOSE_FILE, cloudEnvironment, LOCAL_ENV_FILE, - localDecisionJwt, + localDecisionToken, localEnvironment, localResourceToken, operationPlan, @@ -206,42 +206,33 @@ test("local resource discovery returns only the named scoped token", async () => assert.equal(token, "p.resource-token-value"); }); -test("local decision verification uses an expiring endpoint-scoped JWT", async () => { - const requests = []; - const token = await localDecisionJwt( - { - PRODUCT_ANALYTICS_TINYBIRD_HOST: "http://127.0.0.1:7181", - TB_LOCAL_WORKSPACE_TOKEN: "local-workspace-token", +test("local decision verification creates an isolated exact-scope token", () => { + const calls = []; + const environment = localEnvironment({}); + const token = localDecisionToken( + environment, + (command, args, options) => { + calls.push({ command, args, options }); + return { status: 0 }; }, - async (url, init) => { - requests.push({ url, init }); - return Response.json({ token: "local-decision-jwt-value" }); + (actualEnvironment, tokenName) => { + assert.equal(actualEnvironment, environment); + assert.equal(tokenName, "product_events_local_verification"); + return "local-decision-token-value"; }, - 1_785_607_200_000, ); - assert.equal(token, "local-decision-jwt-value"); - assert.equal(requests.length, 1); - assert.equal(requests[0].url.pathname, "/v0/tokens"); - assert.equal( - requests[0].url.searchParams.get("expiration_time"), - "1785610800", - ); - assert.equal(requests[0].init.method, "POST"); - assert.equal( - requests[0].init.headers.Authorization, - "Bearer local-workspace-token", - ); - const payload = JSON.parse(requests[0].init.body); - assert.equal(payload.scopes.length, 18); - assert.ok( - payload.scopes.every( - (scope) => - scope.type === "PIPES:READ" && - typeof scope.resource === "string" && - Object.keys(scope).length === 2, - ), - ); + assert.equal(token, "local-decision-token-value"); + assert.equal(calls.length, 1); + assert.equal(calls[0].command, "docker"); + assert.ok(calls[0].args.includes("create")); + assert.ok(calls[0].args.includes("static")); + const scopes = calls[0].args.filter((value) => + value.startsWith("PIPES:READ:"), + ); + assert.equal(scopes.length, 18); + assert.equal(new Set(scopes).size, 18); + assert.ok(scopes.every((scope) => !scope.includes("product_events_v1"))); }); test("local resource discovery falls back to the local user token", async () => { diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index d0e3e102559..8326e1151cf 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -945,44 +945,41 @@ const localResourceToken = async ( ); }; -const localDecisionJwt = async ( +const localDecisionToken = ( environment, - fetcher = fetch, - now = Date.now(), + runner = spawnSync, + tokenLister = listLocalStaticToken, ) => { - const url = new URL( - "/v0/tokens", - environment.PRODUCT_ANALYTICS_TINYBIRD_HOST, - ); - url.searchParams.set("name", "product_events_local_verification"); - url.searchParams.set( - "expiration_time", - String(Math.floor(now / 1_000) + 3_600), - ); - const response = await fetcher(url, { - method: "POST", - headers: { - Authorization: `Bearer ${environment.TB_LOCAL_WORKSPACE_TOKEN}`, - "Content-Type": "application/json", + const tokenName = "product_events_local_verification"; + const result = runner( + "docker", + composeArgs( + "run", + "--rm", + "tinybird-cli", + "--local", + "token", + "create", + "static", + tokenName, + ...PRODUCT_DECISION_ENDPOINTS.flatMap((resource) => [ + "--scope", + `PIPES:READ:${resource}`, + ]), + ), + { + cwd: PROJECT_ROOT, + encoding: "utf8", + env: environment, + maxBuffer: 1024 * 1024, }, - body: JSON.stringify({ - scopes: PRODUCT_DECISION_ENDPOINTS.map((resource) => ({ - type: "PIPES:READ", - resource, - })), - }), - signal: AbortSignal.timeout(15_000), - }); - if (!response.ok) { + ); + if (result.error || result.status !== 0) { throw new Error( - `Tinybird Local could not create its decision-endpoint JWT: HTTP ${response.status}`, + "Tinybird Local could not create its decision-endpoint token", ); } - const payload = await response.json(); - if (typeof payload.token !== "string" || payload.token.length < 16) { - throw new Error("Tinybird Local returned an invalid decision-endpoint JWT"); - } - return payload.token; + return tokenLister(environment, tokenName); }; const assertSafeStep = (step) => { @@ -1217,7 +1214,7 @@ const runAnalyticsCommand = async (operation) => { } if (step.type === "verify-local") { const environment = localEnvironment(); - const readToken = await localDecisionJwt(environment); + const readToken = localDecisionToken(environment); await runProcess(process.execPath, [LOCAL_VERIFY_SCRIPT], { env: { ...environment, @@ -1247,7 +1244,7 @@ export { assertSafeStep, cloudEnvironment, composeArgs, - localDecisionJwt, + localDecisionToken, localEnvironment, localResourceToken, operationPlan, From ebdb814a22812eb377705f2cc9ef6771210610eb Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:24:28 +0100 Subject: [PATCH 109/110] fix: scope Tinybird Local verification at build --- scripts/analytics/tests/tooling.test.js | 50 ++++++++---------- scripts/analytics/tooling.js | 70 ++++++++++++------------- 2 files changed, 57 insertions(+), 63 deletions(-) diff --git a/scripts/analytics/tests/tooling.test.js b/scripts/analytics/tests/tooling.test.js index d4f4643de86..3e2df10c3a2 100644 --- a/scripts/analytics/tests/tooling.test.js +++ b/scripts/analytics/tests/tooling.test.js @@ -9,7 +9,6 @@ import { COMPOSE_FILE, cloudEnvironment, LOCAL_ENV_FILE, - localDecisionToken, localEnvironment, localResourceToken, operationPlan, @@ -19,6 +18,7 @@ import { TINYBIRD_PROJECT_DIR, validateAnalyticsProject, verifyCloudWorkspace, + withLocalDecisionTokenDatafiles, writeLocalEnvironmentFile, } from "../tooling.js"; @@ -64,7 +64,9 @@ test("local setup builds, verifies copied endpoints and writes its deterministic const commands = steps .filter((step) => step.command) .map((step) => step.args.join(" ")); - assert.ok(commands.some((command) => command.endsWith("--local build"))); + assert.ok( + steps.some((step) => step.type === "build-local-with-decision-token"), + ); assert.ok( commands.some((command) => command.endsWith( @@ -206,33 +208,25 @@ test("local resource discovery returns only the named scoped token", async () => assert.equal(token, "p.resource-token-value"); }); -test("local decision verification creates an isolated exact-scope token", () => { - const calls = []; - const environment = localEnvironment({}); - const token = localDecisionToken( - environment, - (command, args, options) => { - calls.push({ command, args, options }); - return { status: 0 }; - }, - (actualEnvironment, tokenName) => { - assert.equal(actualEnvironment, environment); - assert.equal(tokenName, "product_events_local_verification"); - return "local-decision-token-value"; - }, - ); +test("local decision verification restores its temporary token datafiles", async () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "analytics-token-")); + const pipesDir = path.join(tempDir, "pipes"); + fs.mkdirSync(pipesDir); + const pipeFile = path.join(pipesDir, "product_events_daily.pipe"); + const original = "NODE endpoint\nSQL >\n\tSELECT 1\n"; + fs.writeFileSync(pipeFile, original); - assert.equal(token, "local-decision-token-value"); - assert.equal(calls.length, 1); - assert.equal(calls[0].command, "docker"); - assert.ok(calls[0].args.includes("create")); - assert.ok(calls[0].args.includes("static")); - const scopes = calls[0].args.filter((value) => - value.startsWith("PIPES:READ:"), - ); - assert.equal(scopes.length, 18); - assert.equal(new Set(scopes).size, 18); - assert.ok(scopes.every((scope) => !scope.includes("product_events_v1"))); + await assert.rejects( + withLocalDecisionTokenDatafiles(tempDir, async () => { + assert.equal( + fs.readFileSync(pipeFile, "utf8"), + `TOKEN product_events_local_verification READ\n${original}`, + ); + throw new Error("synthetic build failure"); + }, ["product_events_daily"]), + /synthetic build failure/, + ); + assert.equal(fs.readFileSync(pipeFile, "utf8"), original); }); test("local resource discovery falls back to the local user token", async () => { diff --git a/scripts/analytics/tooling.js b/scripts/analytics/tooling.js index 8326e1151cf..838b577095a 100644 --- a/scripts/analytics/tooling.js +++ b/scripts/analytics/tooling.js @@ -264,7 +264,7 @@ const operationPlan = (operation) => { ), localAuth: true, }, - localCliStep("--local", "build"), + { type: "build-local-with-decision-token" }, ...PRODUCT_COPY_PIPES.map((name) => localCliStep("--local", "copy", "pause", name), ), @@ -945,41 +945,28 @@ const localResourceToken = async ( ); }; -const localDecisionToken = ( - environment, - runner = spawnSync, - tokenLister = listLocalStaticToken, +const withLocalDecisionTokenDatafiles = async ( + projectDir, + operation, + endpoints = PRODUCT_DECISION_ENDPOINTS, ) => { - const tokenName = "product_events_local_verification"; - const result = runner( - "docker", - composeArgs( - "run", - "--rm", - "tinybird-cli", - "--local", - "token", - "create", - "static", - tokenName, - ...PRODUCT_DECISION_ENDPOINTS.flatMap((resource) => [ - "--scope", - `PIPES:READ:${resource}`, - ]), - ), - { - cwd: PROJECT_ROOT, - encoding: "utf8", - env: environment, - maxBuffer: 1024 * 1024, - }, - ); - if (result.error || result.status !== 0) { - throw new Error( - "Tinybird Local could not create its decision-endpoint token", + const originals = new Map(); + for (const endpoint of endpoints) { + const file = path.join(projectDir, "pipes", `${endpoint}.pipe`); + const contents = fs.readFileSync(file, "utf8"); + originals.set(file, contents); + fs.writeFileSync( + file, + `TOKEN product_events_local_verification READ\n${contents}`, ); } - return tokenLister(environment, tokenName); + try { + return await operation(); + } finally { + for (const [file, contents] of originals) { + fs.writeFileSync(file, contents); + } + } }; const assertSafeStep = (step) => { @@ -1176,6 +1163,16 @@ const runAnalyticsCommand = async (operation) => { prepareLocalFixture(); continue; } + if (step.type === "build-local-with-decision-token") { + await withLocalDecisionTokenDatafiles(TINYBIRD_PROJECT_DIR, () => + runProcess( + "docker", + composeArgs("run", "--rm", "tinybird-cli", "--local", "build"), + { env: localEnvironment() }, + ), + ); + continue; + } if (step.type === "reset-local-fixture") { await runProcess( "docker", @@ -1214,7 +1211,10 @@ const runAnalyticsCommand = async (operation) => { } if (step.type === "verify-local") { const environment = localEnvironment(); - const readToken = localDecisionToken(environment); + const readToken = await localResourceToken( + environment, + "product_events_local_verification", + ); await runProcess(process.execPath, [LOCAL_VERIFY_SCRIPT], { env: { ...environment, @@ -1244,7 +1244,6 @@ export { assertSafeStep, cloudEnvironment, composeArgs, - localDecisionToken, localEnvironment, localResourceToken, operationPlan, @@ -1254,5 +1253,6 @@ export { runAnalyticsCommand, validateAnalyticsProject, verifyCloudWorkspace, + withLocalDecisionTokenDatafiles, writeLocalEnvironmentFile, }; From 53890769e3db23887535579f2c16e46de807ec93 Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Sat, 1 Aug 2026 19:37:34 +0100 Subject: [PATCH 110/110] fix: retire superseded Tinybird staging deployments --- .github/workflows/analytics.yml | 15 +++++ scripts/analytics/staging-ci-lib.js | 53 +++++++++++++++++ scripts/analytics/staging-ci.js | 37 ++++++++++++ scripts/analytics/tests/staging-ci.test.js | 69 ++++++++++++++++++++++ 4 files changed, 174 insertions(+) diff --git a/.github/workflows/analytics.yml b/.github/workflows/analytics.yml index 624b8ed23c7..ea3d3b35682 100644 --- a/.github/workflows/analytics.yml +++ b/.github/workflows/analytics.yml @@ -210,6 +210,21 @@ jobs: VERCEL_AUTOMATION_BYPASS_SECRET: ${{ secrets.VERCEL_AUTOMATION_BYPASS_SECRET }} VERCEL_PREVIEW_SHARE_SECRET: ${{ secrets.VERCEL_PREVIEW_SHARE_SECRET }} run: node scripts/analytics/staging-ci.js attest-preview + - name: Retire only a superseded Tinybird staging predecessor + id: retired-deployment + env: + TINYBIRD_STAGING_URL: ${{ secrets.TINYBIRD_STAGING_URL }} + TINYBIRD_STAGING_DEPLOY_TOKEN: ${{ secrets.TINYBIRD_STAGING_DEPLOY_TOKEN }} + run: >- + node scripts/analytics/staging-ci.js discard-retired-deployment + --artifact "$RUNNER_TEMP/analytics-retired-deployment.json" + - name: Upload retired Tinybird deployment evidence + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: analytics-retired-deployment-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/analytics-retired-deployment.json + if-no-files-found: error + retention-days: 30 - name: Persist the pre-create Tinybird recovery boundary env: VERCEL_DEPLOYMENT_ID: ${{ steps.vercel.outputs.deployment_id }} diff --git a/scripts/analytics/staging-ci-lib.js b/scripts/analytics/staging-ci-lib.js index 38e611bf3a8..65cfac54db2 100644 --- a/scripts/analytics/staging-ci-lib.js +++ b/scripts/analytics/staging-ci-lib.js @@ -466,6 +466,59 @@ export const createDeploymentBoundary = (value) => { }; }; +export const selectRetiredStagingDeployment = (value) => { + const deployments = deploymentsFromResponse(value); + const liveDeployments = deployments.filter(isLiveDeployment); + if (liveDeployments.length !== 1) { + throw new Error( + "Tinybird retired-deployment cleanup requires exactly one live deployment", + ); + } + const pendingDeployments = deployments.filter(isPendingDeployment); + if (pendingDeployments.length > 0) { + throw new Error( + "Tinybird has an active deployment that cannot be retired automatically", + ); + } + const stagingDeployments = deployments.filter(isStagingDeployment); + const liveDeployment = liveDeployments[0]; + const liveDeploymentId = String(deploymentId(liveDeployment)); + if (!DEPLOYMENT_ID_PATTERN.test(liveDeploymentId)) { + throw new Error("Tinybird returned an invalid live deployment ID"); + } + if (stagingDeployments.length === 0) { + return { liveDeploymentId, retiredDeploymentId: undefined }; + } + if (stagingDeployments.length !== 1) { + throw new Error( + "Tinybird has multiple staging deployments that cannot be retired automatically", + ); + } + const retiredDeployment = stagingDeployments[0]; + const retiredDeploymentId = String(deploymentId(retiredDeployment)); + const createdAt = (deployment) => + Date.parse( + deployment.created_at ?? + deployment.createdAt ?? + deployment["Created at"] ?? + deployment.created ?? + "", + ); + const liveCreatedAt = createdAt(liveDeployment); + const retiredCreatedAt = createdAt(retiredDeployment); + if ( + !DEPLOYMENT_ID_PATTERN.test(retiredDeploymentId) || + !Number.isFinite(liveCreatedAt) || + !Number.isFinite(retiredCreatedAt) || + retiredCreatedAt >= liveCreatedAt + ) { + throw new Error( + "Tinybird staging deployment is not a proven retired predecessor", + ); + } + return { liveDeploymentId, retiredDeploymentId }; +}; + export const resolveDeploymentCreatedAfterBoundary = ( value, boundary, diff --git a/scripts/analytics/staging-ci.js b/scripts/analytics/staging-ci.js index 39f28b36e30..7eff8ca9da0 100644 --- a/scripts/analytics/staging-ci.js +++ b/scripts/analytics/staging-ci.js @@ -43,6 +43,7 @@ import { resolveOwnedDiscardTarget, resolveOwnedMutationTarget, STAGING_WORKSPACE_ID, + selectRetiredStagingDeployment, selectStagingDeployment, submitTinybirdCopyJobs, syntheticIdentityFilterQueries, @@ -596,6 +597,41 @@ const deleteRetiredDeployment = async ({ }); }; +const discardRetiredStagingDeployment = async () => { + const artifactPath = option("artifact"); + const { origin, tokens } = tinybirdEnvironment([ + "TINYBIRD_STAGING_DEPLOY_TOKEN", + ]); + const token = tokens.TINYBIRD_STAGING_DEPLOY_TOKEN; + const before = await deploymentList({ origin, token }); + const pair = selectRetiredStagingDeployment(before.data); + if (pair.retiredDeploymentId) { + await deleteRetiredDeployment({ + origin, + token, + liveDeploymentId: pair.liveDeploymentId, + retiredDeploymentId: pair.retiredDeploymentId, + }); + } + const after = selectRetiredStagingDeployment( + (await deploymentList({ origin, token })).data, + ); + if ( + after.liveDeploymentId !== pair.liveDeploymentId || + after.retiredDeploymentId !== undefined + ) { + throw new Error("Tinybird retired deployment cleanup did not settle"); + } + writeJson(artifactPath, { + liveDeploymentId: pair.liveDeploymentId, + retiredDeploymentId: pair.retiredDeploymentId ?? null, + retired: pair.retiredDeploymentId !== undefined, + verifiedAt: new Date().toISOString(), + workspaceId: STAGING_WORKSPACE_ID, + }); + writeOutput("retired", pair.retiredDeploymentId ? "true" : "false"); +}; + const switchLiveDeployment = async ({ origin, token, @@ -5141,6 +5177,7 @@ const handlers = { "prepare-seed": prepareSeed, "promote-deployment": promoteOwnedDeployment, "drill-rollback": drillOwnedRollback, + "discard-retired-deployment": discardRetiredStagingDeployment, "finalize-promotion": finalizeOwnedPromotion, "rollback-promotion": rollbackOwnedPromotion, "discard-deployment": discardOwnedDeployment, diff --git a/scripts/analytics/tests/staging-ci.test.js b/scripts/analytics/tests/staging-ci.test.js index 773a32a0a0c..68c3dc7f404 100644 --- a/scripts/analytics/tests/staging-ci.test.js +++ b/scripts/analytics/tests/staging-ci.test.js @@ -50,6 +50,7 @@ import { STAGING_READ_TOKEN_MAXIMUM_LIFETIME_MS, STAGING_READ_TOKEN_MINIMUM_LIFETIME_MS, STAGING_WORKSPACE_ID, + selectRetiredStagingDeployment, selectStagingDeployment, submitTinybirdCopyJobs, syntheticIdentityFilterQueries, @@ -621,6 +622,66 @@ test("deployment recovery resolves only one candidate created after its boundary ); }); +test("retired deployment cleanup selects only an older staging predecessor", () => { + assert.deepEqual( + selectRetiredStagingDeployment({ + deployments: [ + { + id: "13", + status: "Live", + created_at: "2026-08-01T17:00:00.000Z", + }, + { + id: "10", + status: "Staging", + created_at: "2026-08-01T07:00:00.000Z", + }, + ], + }), + { liveDeploymentId: "13", retiredDeploymentId: "10" }, + ); + assert.deepEqual( + selectRetiredStagingDeployment({ + deployments: [ + { + id: "13", + status: "Live", + created_at: "2026-08-01T17:00:00.000Z", + }, + ], + }), + { liveDeploymentId: "13", retiredDeploymentId: undefined }, + ); + for (const deployments of [ + [ + { + id: "13", + status: "Live", + created_at: "2026-08-01T17:00:00.000Z", + }, + { + id: "14", + status: "Staging", + created_at: "2026-08-01T18:00:00.000Z", + }, + ], + [ + { + id: "13", + status: "Live", + created_at: "2026-08-01T17:00:00.000Z", + }, + { + id: "14", + status: "creating_schema", + created_at: "2026-08-01T18:00:00.000Z", + }, + ], + ]) { + assert.throws(() => selectRetiredStagingDeployment({ deployments })); + } +}); + test("data mutations validate the exact deployment before using the staging selector", () => { assert.deepEqual( dataMutationDeploymentParameters({ @@ -2365,10 +2426,18 @@ test("the analytics workflow is statically restricted to staging", () => { workflow, /Upload redacted staging evidence\n {8}if: always\(\) && steps\.cleanup\.outputs\.required == 'true'/, ); + assert.ok( + workflow.indexOf("Retire only a superseded Tinybird staging predecessor") < + workflow.indexOf("Persist the pre-create Tinybird recovery boundary"), + ); assert.ok( workflow.indexOf("Upload the immutable pre-create recovery boundary") < workflow.indexOf("Create isolated Tinybird staging deployment"), ); + assert.match( + workflow, + /staging-ci\.js discard-retired-deployment[\s\S]*analytics-retired-deployment\.json/, + ); assert.ok( workflow.indexOf("Upload the immutable pre-ingestion recovery checkpoint") < workflow.indexOf("Seed bounded duplicate and conflict probes"),