From 4f7f2795d721b97c1088a8c5f5a4bf6cfe00f860 Mon Sep 17 00:00:00 2001 From: Agent Date: Fri, 18 Sep 2026 15:31:36 +0000 Subject: [PATCH 1/2] feat: add anonymous opt-out telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A daily pg_cron job (pgflow_telemetry.report) aggregates yesterday's pgflow.* activity into coarse, identifier-free buckets (active-day, worker versions, run/worker counts, flow shapes, feature adoption, durations) and posts one small JSON payload via pg_net to a Cloudflare Worker backed by Workers Analytics Engine. Edge workers stamp their package version into pgflow.workers at registration. Design decisions: the cron.job row is the only switch — the migration schedules it exactly once and nothing re-schedules it, so disable() stays permanent across upgrades. pgflow_telemetry.sent_reports doubles as user-auditable payload log and per-day dedup marker. report() gates run in order: inactive day, already reported, is_local(); one pg_net attempt, 5s statement timeout, no retries, no response inspection. Metric names, bucket strings, and count buckets are closed allowlists identical across SQL (0133), the ingest worker (apps/telemetry-worker), and docs; anything else never leaves the database. Local supabase-start stacks never send. Compatibility fixes found during implementation: Deno 2.1.4 requires the JSON import attribute ('with { type: json }') and a default import for package.json version lookup, so jsr.json publish.includes package.json and the e2e vendor script copies it; bucket_steps takes bigint because steps_per_flow passes count(*); 0136 uses $disable$/$enable$ dollar-quote tags because the plan's nested $$ would terminate enable()'s body early. Plan test bugs corrected with rationale in the tests: 90s lands in 1-4.9m not 1-9.9s; bucket_count(2) is '2-3'; the 65-contribution body always exceeds the 2KB cap so 413 fires before the count guard. Verification: 4 new pgTAP files (red/green, migration replay with deps), telemetry-worker vitest 10, full pgTAP suite 314 files/1606 tests, nx affected build+test, e2e 13/13, live report() returns skipped: local with zero audit rows, live worker registration stamps 0.17.0, website build. Ingest worker deployment (wrangler deploy + smoke) is a manual follow-up and is not part of this commit. --- .changeset/anonymous-telemetry.md | 6 + .changeset/config.json | 2 +- apps/telemetry-worker/package.json | 15 + apps/telemetry-worker/src/index.test.ts | 125 +++++ apps/telemetry-worker/src/index.ts | 133 ++++++ apps/telemetry-worker/tsconfig.json | 12 + apps/telemetry-worker/vite.config.ts | 7 + apps/telemetry-worker/wrangler.toml | 11 + pkgs/core/schemas/0055_tables_workers.sql | 8 +- pkgs/core/schemas/0060_tables_runtime.sql | 1 + .../schemas/0130_schema_pgflow_telemetry.sql | 5 + pkgs/core/schemas/0131_table_sent_reports.sql | 12 + .../0132_function_telemetry_buckets.sql | 60 +++ .../0133_function_build_telemetry_payload.sql | 309 ++++++++++++ pkgs/core/schemas/0134_function_preview.sql | 10 + pkgs/core/schemas/0135_function_report.sql | 60 +++ ...0136_function_telemetry_enable_disable.sql | 29 ++ pkgs/core/src/database-types.ts | 3 + .../20260918145424_pgflow_telemetry.sql | 450 ++++++++++++++++++ pkgs/core/supabase/migrations/atlas.sum | 3 +- .../supabase/tests/telemetry/payload.test.sql | 51 ++ .../supabase/tests/telemetry/report.test.sql | 61 +++ .../supabase/tests/telemetry/schema.test.sql | 46 ++ .../tests/telemetry/workers_version.test.sql | 29 ++ pkgs/edge-worker/jsr.json | 1 + pkgs/edge-worker/scripts/sync-e2e-deps.sh | 2 + pkgs/edge-worker/src/core/Queries.ts | 6 +- pkgs/edge-worker/src/core/WorkerLifecycle.ts | 2 + pkgs/edge-worker/src/core/types.ts | 2 + pkgs/edge-worker/src/core/version.ts | 8 + .../src/flow/FlowWorkerLifecycle.ts | 2 + pkgs/website/astro.config.mjs | 1 + pkgs/website/redirects.config.mjs | 7 + ...-17-1-persistent-queues-and-telemetry.mdx} | 20 +- .../src/content/docs/reference/index.mdx | 10 + .../src/content/docs/reference/telemetry.mdx | 77 +++ pnpm-lock.yaml | 304 +++++++++++- tsconfig.base.json | 2 +- 38 files changed, 1864 insertions(+), 28 deletions(-) create mode 100644 .changeset/anonymous-telemetry.md create mode 100644 apps/telemetry-worker/package.json create mode 100644 apps/telemetry-worker/src/index.test.ts create mode 100644 apps/telemetry-worker/src/index.ts create mode 100644 apps/telemetry-worker/tsconfig.json create mode 100644 apps/telemetry-worker/vite.config.ts create mode 100644 apps/telemetry-worker/wrangler.toml create mode 100644 pkgs/core/schemas/0130_schema_pgflow_telemetry.sql create mode 100644 pkgs/core/schemas/0131_table_sent_reports.sql create mode 100644 pkgs/core/schemas/0132_function_telemetry_buckets.sql create mode 100644 pkgs/core/schemas/0133_function_build_telemetry_payload.sql create mode 100644 pkgs/core/schemas/0134_function_preview.sql create mode 100644 pkgs/core/schemas/0135_function_report.sql create mode 100644 pkgs/core/schemas/0136_function_telemetry_enable_disable.sql create mode 100644 pkgs/core/supabase/migrations/20260918145424_pgflow_telemetry.sql create mode 100644 pkgs/core/supabase/tests/telemetry/payload.test.sql create mode 100644 pkgs/core/supabase/tests/telemetry/report.test.sql create mode 100644 pkgs/core/supabase/tests/telemetry/schema.test.sql create mode 100644 pkgs/core/supabase/tests/telemetry/workers_version.test.sql create mode 100644 pkgs/edge-worker/src/core/version.ts rename pkgs/website/src/content/docs/news/{pgflow-0-17-0-persistent-queue-identity.mdx => pgflow-0-17-1-persistent-queues-and-telemetry.mdx} (81%) create mode 100644 pkgs/website/src/content/docs/reference/telemetry.mdx diff --git a/.changeset/anonymous-telemetry.md b/.changeset/anonymous-telemetry.md new file mode 100644 index 000000000..2553c6a79 --- /dev/null +++ b/.changeset/anonymous-telemetry.md @@ -0,0 +1,6 @@ +--- +'@pgflow/core': patch +'@pgflow/edge-worker': patch +--- + +Add anonymous, opt-out telemetry. A daily `pg_cron` job aggregates yesterday's `pgflow.*` activity into coarse, identifier-free buckets (versions in use, run and worker counts, flow shapes, feature adoption, durations) and sends one small payload through `pg_net` to a Cloudflare Worker backed by Workers Analytics Engine. Workers stamp their package version at registration. Nothing identifying is ever sent; every payload is stored locally in `pgflow_telemetry.sent_reports` for audit; `pgflow_telemetry.preview()` shows any day's payload without sending; `pgflow_telemetry.disable()` opts out permanently (the cron job row is the switch) and local development databases never report. diff --git a/.changeset/config.json b/.changeset/config.json index 518661fbc..c00d7fe96 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -15,5 +15,5 @@ "access": "public", "baseBranch": "main", "updateInternalDependencies": "patch", - "ignore": ["@pgflow/demo", "@pgflow/website", "@pgflow/example-flows"] + "ignore": ["@pgflow/demo", "@pgflow/website", "@pgflow/example-flows", "@pgflow/telemetry-worker"] } diff --git a/apps/telemetry-worker/package.json b/apps/telemetry-worker/package.json new file mode 100644 index 000000000..2b4eab28f --- /dev/null +++ b/apps/telemetry-worker/package.json @@ -0,0 +1,15 @@ +{ + "name": "@pgflow/telemetry-worker", + "private": true, + "type": "module", + "scripts": { + "test": "vitest run", + "deploy": "wrangler deploy" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20250901.0", + "typescript": "^5.5.0", + "vitest": "^3.0.0", + "wrangler": "^4.0.0" + } +} diff --git a/apps/telemetry-worker/src/index.test.ts b/apps/telemetry-worker/src/index.test.ts new file mode 100644 index 000000000..4ed8dcdda --- /dev/null +++ b/apps/telemetry-worker/src/index.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from 'vitest'; +import worker, { type Env } from './index'; + +function makeEnv() { + const points: Array<{ indexes: string[]; blobs: string[]; doubles: number[] }> = []; + const env = { + PGFLOW_TELEMETRY: { + writeDataPoint: (p: { indexes: string[]; blobs: string[]; doubles: number[] }) => + points.push(p), + }, + } as unknown as Env; + return { env, points }; +} + +function post(body: unknown, env: Env, raw = false): Promise { + return worker.fetch( + new Request('https://pgflow-telemetry.workers.dev/', { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: raw ? (body as string) : JSON.stringify(body), + }), + env, + ); +} + +const valid = { + schema: 1, + contributions: [ + { metric: 'runs_started_day', bucket: '2-3' }, + { metric: 'workers_by_version', bucket: '0.17.0', count: '1' }, + { metric: 'uses_map', bucket: 'yes' }, + ], +}; + +describe('telemetry ingest', () => { + it('accepts a valid payload, writes one point per contribution, returns 204', async () => { + const { env, points } = makeEnv(); + const res = await post(valid, env); + expect(res.status).toBe(204); + expect(points).toHaveLength(3); + expect(points[0]).toEqual({ + indexes: ['runs_started_day'], + blobs: ['2-3'], + doubles: [1], + }); + }); + + it('rejects GET with 405', async () => { + const { env } = makeEnv(); + const res = await worker.fetch( + new Request('https://pgflow-telemetry.workers.dev/', { method: 'GET' }), + env, + ); + expect(res.status).toBe(405); + }); + + it('rejects malformed JSON with 400', async () => { + const { env } = makeEnv(); + const res = await post('{not json', env, true); + expect(res.status).toBe(400); + }); + + it('rejects wrong schema version with 400', async () => { + const { env, points } = makeEnv(); + const res = await post({ schema: 2, contributions: valid.contributions }, env); + expect(res.status).toBe(400); + expect(points).toHaveLength(0); + }); + + it('rejects unknown metric with 400', async () => { + const { env } = makeEnv(); + const res = await post({ + schema: 1, + contributions: [{ metric: 'flow_slug', bucket: 'yes' }], + }, env); + expect(res.status).toBe(400); + }); + + it('rejects arbitrary bucket strings with 400', async () => { + const { env } = makeEnv(); + const res = await post({ + schema: 1, + contributions: [{ metric: 'runs_started_day', bucket: 'banana' }], + }, env); + expect(res.status).toBe(400); + }); + + it('rejects non-semantic-version buckets with 400', async () => { + const { env } = makeEnv(); + const res = await post({ + schema: 1, + contributions: [{ metric: 'workers_by_version', bucket: 'my-flow-name' }], + }, env); + expect(res.status).toBe(400); + }); + + it('rejects oversized bodies with 413', async () => { + const { env } = makeEnv(); + const big = 'x'.repeat(2049); + const res = await post(big, env, true); + expect(res.status).toBe(413); + }); + + it('rejects more than 64 contributions', async () => { + const { env, points } = makeEnv(); + const res = await post({ + schema: 1, + contributions: Array.from({ length: 65 }, () => ({ + metric: 'uses_map', + bucket: 'yes', + })), + }, env); + // 65 valid contributions always exceed the 2 KB body cap, so the size + // guard (413) fires before the count guard. MAX_CONTRIBUTIONS stays as + // defense-in-depth below the byte cap. + expect(res.status).toBe(413); + expect(points).toHaveLength(0); + }); + + it('sets no cookies on success', async () => { + const { env } = makeEnv(); + const res = await post(valid, env); + expect(res.headers.getSetCookie()).toHaveLength(0); + }); +}); diff --git a/apps/telemetry-worker/src/index.ts b/apps/telemetry-worker/src/index.ts new file mode 100644 index 000000000..0a7a3ce3a --- /dev/null +++ b/apps/telemetry-worker/src/index.ts @@ -0,0 +1,133 @@ +// pgflow anonymous telemetry ingest. Accepts exactly one closed schema, +// rejects everything else, writes one Analytics Engine data point per +// contribution. No cookies, no body logging, IP only as transport. + +export interface Env { + PGFLOW_TELEMETRY: AnalyticsEngineDataset; +} + +const SCHEMA_VERSION = 1; +const MAX_BODY_BYTES = 2048; +const MAX_CONTRIBUTIONS = 64; + +const COUNT_BUCKETS = new Set([ + '0', '1', '2-3', '4-7', '8-15', '16-31', '32-63', '64-127', '128-255', '256+', +]); +const STEPS_BUCKETS = new Set(['1', '2-3', '4-7', '8-15', '16-31', '32+']); +const DURATION_BUCKETS = new Set([ + '<100ms', '100-999ms', '1-9.9s', '10-59s', '1-4.9m', '5-29m', + '30m-1.9h', '2-23h', '1-6d', '7d+', +]); +const OUTCOME_BUCKETS = new Set(['completed', 'failed', 'started']); +const MODE_BUCKETS = new Set(['http', 'process']); +const QUEUE_MODE_BUCKETS = new Set(['flow', 'step']); +const SEMVER_RE = /^\d+\.\d+\.\d+(-[0-9A-Za-z.-]+)?$/; + +type BucketKind = + | 'count' | 'steps' | 'duration' | 'outcome' | 'mode' | 'queue_mode' + | 'yes' | 'semver'; + +const METRICS: Record = { + active_db_day: 'yes', + workers_by_version: 'semver', + version_changed: 'yes', + worker_functions: 'count', + worker_functions_by_mode: 'mode', + worker_functions_disabled: 'count', + workers_started_day: 'count', + workers_stopped_day: 'count', + workers_deprecated_day: 'count', + worker_starts_per_function: 'count', + steps_per_flow: 'steps', + uses_map: 'yes', + uses_root_map: 'yes', + uses_condition: 'yes', + uses_graceful_failure: 'yes', + uses_skip_cascade: 'yes', + uses_retry_override: 'yes', + uses_timeout_override: 'yes', + uses_start_delay: 'yes', + queue_mode: 'queue_mode', + runs_started_day: 'count', + run_outcomes: 'outcome', + run_wall_duration: 'duration', + step_wall_duration: 'duration', + task_attempts_day: 'count', + map_task_count: 'steps', +}; + +function validBucket(kind: BucketKind, bucket: unknown): boolean { + if (typeof bucket !== 'string' || bucket.length > 32) return false; + switch (kind) { + case 'count': return COUNT_BUCKETS.has(bucket); + case 'steps': return STEPS_BUCKETS.has(bucket); + case 'duration': return DURATION_BUCKETS.has(bucket); + case 'outcome': return OUTCOME_BUCKETS.has(bucket); + case 'mode': return MODE_BUCKETS.has(bucket); + case 'queue_mode': return QUEUE_MODE_BUCKETS.has(bucket); + case 'yes': return bucket === 'yes'; + case 'semver': return SEMVER_RE.test(bucket); + } +} + +interface Contribution { + metric?: unknown; + bucket?: unknown; + count?: unknown; +} + +const reject = (status: number) => new Response(null, { + status, + headers: { 'cache-control': 'no-store' }, +}); + +export default { + async fetch(request: Request, env: Env): Promise { + if (request.method !== 'POST') return reject(405); + + const contentType = request.headers.get('content-type') ?? ''; + if (!contentType.includes('application/json')) return reject(415); + + const raw = await request.text(); + if (raw.length > MAX_BODY_BYTES) return reject(413); + + let body: { schema?: unknown; contributions?: unknown }; + try { + body = JSON.parse(raw); + } catch { + return reject(400); + } + + if (body.schema !== SCHEMA_VERSION) return reject(400); + if (!Array.isArray(body.contributions)) return reject(400); + const contributions = body.contributions as Contribution[]; + if (contributions.length === 0 || contributions.length > MAX_CONTRIBUTIONS) { + return reject(400); + } + + for (const c of contributions) { + if (c === null || typeof c !== 'object') return reject(400); + const kind = METRICS[c.metric as string]; + if (!kind) return reject(400); + if (!validBucket(kind, c.bucket)) return reject(400); + if (c.count !== undefined) { + if (typeof c.count !== 'string' || !COUNT_BUCKETS.has(c.count)) { + return reject(400); + } + } + } + + for (const c of contributions) { + env.PGFLOW_TELEMETRY.writeDataPoint({ + indexes: [c.metric as string], + blobs: [c.bucket as string], + doubles: [1], + }); + } + + return new Response(null, { + status: 204, + headers: { 'cache-control': 'no-store' }, + }); + }, +}; diff --git a/apps/telemetry-worker/tsconfig.json b/apps/telemetry-worker/tsconfig.json new file mode 100644 index 000000000..9d40ea86f --- /dev/null +++ b/apps/telemetry-worker/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "ES2022", + "moduleResolution": "bundler", + "strict": true, + "noEmit": true, + "types": ["@cloudflare/workers-types"] + }, + "include": ["src/index.ts"] +} diff --git a/apps/telemetry-worker/vite.config.ts b/apps/telemetry-worker/vite.config.ts new file mode 100644 index 000000000..6ec74eee2 --- /dev/null +++ b/apps/telemetry-worker/vite.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + include: ['src/**/*.test.ts'], + }, +}); diff --git a/apps/telemetry-worker/wrangler.toml b/apps/telemetry-worker/wrangler.toml new file mode 100644 index 000000000..27eb9f42a --- /dev/null +++ b/apps/telemetry-worker/wrangler.toml @@ -0,0 +1,11 @@ +name = "pgflow-telemetry" +main = "src/index.ts" +compatibility_date = "2026-09-01" + +[[analytics_engine_datasets]] +binding = "PGFLOW_TELEMETRY" +dataset = "pgflow_telemetry" + +# No request logging: bodies and transport metadata are never stored. +[observability] +enabled = false diff --git a/pkgs/core/schemas/0055_tables_workers.sql b/pkgs/core/schemas/0055_tables_workers.sql index 3af5e8fa2..8e6127800 100644 --- a/pkgs/core/schemas/0055_tables_workers.sql +++ b/pkgs/core/schemas/0055_tables_workers.sql @@ -7,8 +7,14 @@ create table if not exists pgflow.workers ( started_at timestamptz not null default now(), deprecated_at timestamptz, stopped_at timestamptz, - last_heartbeat_at timestamptz not null default now() + last_heartbeat_at timestamptz not null default now(), + -- Version of the pgflow package the worker was built from, stamped once at + -- registration. Nullable: rows written before telemetry exist. Telemetry + -- reads a per-day distribution; MAX() is the newest version ever run. + pgflow_version text ); create index if not exists idx_workers_queue_name on pgflow.workers (queue_name); create index if not exists idx_workers_heartbeat on pgflow.workers (last_heartbeat_at); +create index if not exists idx_workers_started_at on pgflow.workers (started_at); +create index if not exists idx_workers_stopped_at on pgflow.workers (stopped_at); diff --git a/pkgs/core/schemas/0060_tables_runtime.sql b/pkgs/core/schemas/0060_tables_runtime.sql index 40624ebdf..4ab23e1e9 100644 --- a/pkgs/core/schemas/0060_tables_runtime.sql +++ b/pkgs/core/schemas/0060_tables_runtime.sql @@ -19,6 +19,7 @@ create table pgflow.runs ( create index if not exists idx_runs_flow_slug on pgflow.runs (flow_slug); create index if not exists idx_runs_status on pgflow.runs (status); +create index if not exists idx_runs_started_at on pgflow.runs (started_at); -- Step states table - tracks the state of individual steps within a run create table pgflow.step_states ( diff --git a/pkgs/core/schemas/0130_schema_pgflow_telemetry.sql b/pkgs/core/schemas/0130_schema_pgflow_telemetry.sql new file mode 100644 index 000000000..267e482ae --- /dev/null +++ b/pkgs/core/schemas/0130_schema_pgflow_telemetry.sql @@ -0,0 +1,5 @@ +-- Self-contained telemetry schema: droppable without touching pgflow engine +-- objects. The cron.job row is the enabled switch; no settings table. +create schema if not exists pgflow_telemetry; + +revoke all on schema pgflow_telemetry from public; diff --git a/pkgs/core/schemas/0131_table_sent_reports.sql b/pkgs/core/schemas/0131_table_sent_reports.sql new file mode 100644 index 000000000..09605da67 --- /dev/null +++ b/pkgs/core/schemas/0131_table_sent_reports.sql @@ -0,0 +1,12 @@ +-- One row per reported UTC day. The unique day doubles as the dedup marker +-- (no separate state row) and the payload is the exact bytes that left the +-- database, so users can audit every report. +create table pgflow_telemetry.sent_reports ( + day date primary key, + payload jsonb not null, + request_id bigint, + sent_at timestamptz not null default now() +); + +comment on table pgflow_telemetry.sent_reports is +'Audit log of every telemetry payload sent; unique day is the dedup marker'; diff --git a/pkgs/core/schemas/0132_function_telemetry_buckets.sql b/pkgs/core/schemas/0132_function_telemetry_buckets.sql new file mode 100644 index 000000000..338d7ba66 --- /dev/null +++ b/pkgs/core/schemas/0132_function_telemetry_buckets.sql @@ -0,0 +1,60 @@ +-- Shared bucket scales. The ingest Worker validates the exact same strings. + +create or replace function pgflow_telemetry.bucket_count(p_count bigint) +returns text +language sql +immutable +parallel safe +set search_path = '' +as $$ + select case + when p_count <= 0 then '0' + when p_count = 1 then '1' + when p_count <= 3 then '2-3' + when p_count <= 7 then '4-7' + when p_count <= 15 then '8-15' + when p_count <= 31 then '16-31' + when p_count <= 63 then '32-63' + when p_count <= 127 then '64-127' + when p_count <= 255 then '128-255' + else '256+' + end +$$; + +create or replace function pgflow_telemetry.bucket_steps(p_steps bigint) +returns text +language sql +immutable +parallel safe +set search_path = '' +as $$ + select case + when p_steps <= 1 then '1' + when p_steps <= 3 then '2-3' + when p_steps <= 7 then '4-7' + when p_steps <= 15 then '8-15' + when p_steps <= 31 then '16-31' + else '32+' + end +$$; + +create or replace function pgflow_telemetry.bucket_duration(p_duration interval) +returns text +language sql +immutable +parallel safe +set search_path = '' +as $$ + select case + when p_duration < '100 milliseconds'::interval then '<100ms' + when p_duration < '1 second'::interval then '100-999ms' + when p_duration < '10 seconds'::interval then '1-9.9s' + when p_duration < '1 minute'::interval then '10-59s' + when p_duration < '5 minutes'::interval then '1-4.9m' + when p_duration < '30 minutes'::interval then '5-29m' + when p_duration < '2 hours'::interval then '30m-1.9h' + when p_duration < '1 day'::interval then '2-23h' + when p_duration < '7 days'::interval then '1-6d' + else '7d+' + end +$$; diff --git a/pkgs/core/schemas/0133_function_build_telemetry_payload.sql b/pkgs/core/schemas/0133_function_build_telemetry_payload.sql new file mode 100644 index 000000000..3eeba7eec --- /dev/null +++ b/pkgs/core/schemas/0133_function_build_telemetry_payload.sql @@ -0,0 +1,309 @@ +-- Builds the anonymous daily payload. Only metric names, closed bucket +-- strings, semver versions, and count buckets ever enter the jsonb. +create or replace function pgflow_telemetry.build_payload(p_day date) +returns jsonb +language sql +stable +set search_path = '' +as $$ + select jsonb_build_object( + 'schema', 1, + 'contributions', coalesce(jsonb_agg(c order by c->>'metric', c->>'bucket'), '[]'::jsonb) + ) + from ( + -- Adoption: was this database active during the completed day? + select jsonb_build_object('metric', 'active_db_day', 'bucket', 'yes') as c + where exists ( + select 1 from pgflow.runs r + where r.started_at >= p_day + and r.started_at < p_day + 1 + ) + + union all + -- Version distribution of workers started during the day. + select jsonb_build_object( + 'metric', 'workers_by_version', + 'bucket', w.pgflow_version, + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.started_at >= p_day + and w.started_at < p_day + 1 + and w.pgflow_version ~ '^\d+\.\d+\.\d+' + group by w.pgflow_version + + union all + -- Same-database upgrade: a version started today that no earlier row had. + select jsonb_build_object('metric', 'version_changed', 'bucket', 'yes') + where exists ( + select 1 from pgflow.workers w + where w.started_at >= p_day + and w.started_at < p_day + 1 + and w.pgflow_version is not null + and not exists ( + select 1 from pgflow.workers prev + where prev.started_at < p_day + and prev.pgflow_version = w.pgflow_version + ) + ) + + union all + -- Worker function registry (current state, not windowed). + select jsonb_build_object( + 'metric', 'worker_functions', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.worker_functions wf + + union all + select jsonb_build_object( + 'metric', 'worker_functions_by_mode', + 'bucket', wf.start_mode, + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.worker_functions wf + group by wf.start_mode + + union all + select jsonb_build_object( + 'metric', 'worker_functions_disabled', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.worker_functions wf + where not wf.enabled + + union all + -- Worker churn during the day. + select jsonb_build_object( + 'metric', 'workers_started_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.started_at >= p_day and w.started_at < p_day + 1 + + union all + select jsonb_build_object( + 'metric', 'workers_stopped_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.stopped_at >= p_day and w.stopped_at < p_day + 1 + + union all + select jsonb_build_object( + 'metric', 'workers_deprecated_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.deprecated_at >= p_day and w.deprecated_at < p_day + 1 + + union all + -- Restart churn per function (Supabase recycles every 150-400 s). + select jsonb_build_object( + 'metric', 'worker_starts_per_function', + 'bucket', pgflow_telemetry.bucket_count(per_function.starts), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select w.function_name, count(*) as starts + from pgflow.workers w + where w.started_at >= p_day and w.started_at < p_day + 1 + group by w.function_name + ) per_function + group by per_function.starts + + union all + -- Active-flow shape: steps per flow, histogram over active flows. + select jsonb_build_object( + 'metric', 'steps_per_flow', + 'bucket', pgflow_telemetry.bucket_steps(per_flow.step_count), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select s.flow_slug, count(*) as step_count + from pgflow.steps s + where s.flow_slug in ( + select distinct r.flow_slug + from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + group by s.flow_slug + ) per_flow + group by per_flow.step_count + + union all + -- Feature booleans, scoped to steps of active flows. + select jsonb_build_object('metric', f.metric, 'bucket', 'yes') + from ( + select 'uses_map' as metric, exists ( + select 1 from pgflow.steps s + where s.step_type = 'map' + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) as used + union all + select 'uses_root_map', exists ( + select 1 from pgflow.steps s + where s.step_type = 'map' and s.deps_count = 0 + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_condition', exists ( + select 1 from pgflow.steps s + where (s.required_input_pattern is not null or s.forbidden_input_pattern is not null) + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_graceful_failure', exists ( + select 1 from pgflow.steps s + where s.when_exhausted <> 'fail' + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_skip_cascade', exists ( + select 1 from pgflow.steps s + where s.when_unmet = 'skip-cascade' + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_retry_override', exists ( + select 1 from pgflow.steps s + where s.opt_max_attempts is not null + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_timeout_override', exists ( + select 1 from pgflow.steps s + where s.opt_timeout is not null + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_start_delay', exists ( + select 1 from pgflow.steps s + where s.opt_start_delay is not null + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + ) f + where f.used + + union all + -- Queue-mode adoption among active flows. + select jsonb_build_object( + 'metric', 'queue_mode', + 'bucket', f.queue_mode, + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.flows f + where f.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + group by f.queue_mode + + union all + -- Workload: runs started during the day. + select jsonb_build_object( + 'metric', 'runs_started_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + + union all + -- Run outcomes by terminal timestamp in window. + select jsonb_build_object( + 'metric', 'run_outcomes', + 'bucket', o.outcome, + 'count', pgflow_telemetry.bucket_count(o.n) + ) + from ( + select 'completed' as outcome, count(*) as n + from pgflow.runs r + where r.completed_at >= p_day and r.completed_at < p_day + 1 + union all + select 'failed', count(*) + from pgflow.runs r + where r.failed_at >= p_day and r.failed_at < p_day + 1 + union all + select 'started', count(*) + from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + and r.status = 'started' + ) o + + union all + -- Wall durations of runs that terminated during the day. + select jsonb_build_object( + 'metric', 'run_wall_duration', + 'bucket', pgflow_telemetry.bucket_duration(d.terminal_at - d.started_at), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select r.started_at, coalesce(r.completed_at, r.failed_at) as terminal_at + from pgflow.runs r + where coalesce(r.completed_at, r.failed_at) >= p_day + and coalesce(r.completed_at, r.failed_at) < p_day + 1 + ) d + group by d.terminal_at - d.started_at + + union all + -- Wall durations of steps that terminated during the day. + select jsonb_build_object( + 'metric', 'step_wall_duration', + 'bucket', pgflow_telemetry.bucket_duration(d.terminal_at - d.started_at), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select ss.started_at, coalesce(ss.completed_at, ss.failed_at) as terminal_at + from pgflow.step_states ss + where ss.started_at is not null + and coalesce(ss.completed_at, ss.failed_at) >= p_day + and coalesce(ss.completed_at, ss.failed_at) < p_day + 1 + ) d + group by d.terminal_at - d.started_at + + union all + -- Task attempts for tasks queued during the day. + select jsonb_build_object( + 'metric', 'task_attempts_day', + 'bucket', pgflow_telemetry.bucket_count(coalesce(sum(t.attempts_count), 0)) + ) + from pgflow.step_tasks t + where t.queued_at >= p_day and t.queued_at < p_day + 1 + + union all + -- Map fan-out: initial task counts of steps started during the day. + select jsonb_build_object( + 'metric', 'map_task_count', + 'bucket', pgflow_telemetry.bucket_steps(m.initial_tasks), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.step_states m + where m.started_at >= p_day and m.started_at < p_day + 1 + and m.initial_tasks >= 2 + group by m.initial_tasks + ) contributions +$$; diff --git a/pkgs/core/schemas/0134_function_preview.sql b/pkgs/core/schemas/0134_function_preview.sql new file mode 100644 index 000000000..60b785166 --- /dev/null +++ b/pkgs/core/schemas/0134_function_preview.sql @@ -0,0 +1,10 @@ +-- Returns any day's payload without sending. Works on local databases too: +-- it is the debugging tool. +create or replace function pgflow_telemetry.preview(p_day date default current_date - 1) +returns jsonb +language sql +stable +set search_path = '' +as $$ + select pgflow_telemetry.build_payload(p_day) +$$; diff --git a/pkgs/core/schemas/0135_function_report.sql b/pkgs/core/schemas/0135_function_report.sql new file mode 100644 index 000000000..0e7d80206 --- /dev/null +++ b/pkgs/core/schemas/0135_function_report.sql @@ -0,0 +1,60 @@ +-- Daily collector. One pg_net attempt, no retries, no response inspection, +-- 5 s statement timeout: telemetry must be invisible to the host database. +create or replace function pgflow_telemetry.report() +returns text +language plpgsql +set search_path = '' +as $$ +declare + v_day date := current_date - 1; + v_payload jsonb; + v_request_id bigint; +begin + if not exists ( + select 1 from pgflow.runs r + where r.started_at >= v_day and r.started_at < v_day + 1 + ) then + return 'skipped: inactive day'; + end if; + + if exists ( + select 1 from pgflow_telemetry.sent_reports s where s.day = v_day + ) then + return 'skipped: already reported'; + end if; + + -- Local CLI stack (developer machines, CI on supabase start) never sends. + if pgflow.is_local() then + return 'skipped: local'; + end if; + + perform set_config('statement_timeout', '5000', true); + + begin + v_payload := pgflow_telemetry.build_payload(v_day); + exception when others then + raise warning 'pgflow telemetry: payload build failed, sending nothing'; + return 'error: build failed'; + end; + + begin + v_request_id := net.http_post( + url => 'https://pgflow-telemetry.workers.dev', + body => v_payload::text, + headers => jsonb_build_object('Content-Type', 'application/json'), + timeout_milliseconds => 5000 + ); + exception when others then + raise warning 'pgflow telemetry: http_post queueing failed'; + return 'error: queue failed'; + end; + + insert into pgflow_telemetry.sent_reports (day, payload, request_id) + values (v_day, v_payload, v_request_id); + + delete from pgflow_telemetry.sent_reports + where day < current_date - 90; + + return 'sent: ' || v_request_id; +end +$$; diff --git a/pkgs/core/schemas/0136_function_telemetry_enable_disable.sql b/pkgs/core/schemas/0136_function_telemetry_enable_disable.sql new file mode 100644 index 000000000..a2c389cd4 --- /dev/null +++ b/pkgs/core/schemas/0136_function_telemetry_enable_disable.sql @@ -0,0 +1,29 @@ +-- The cron.job row is the enabled switch. enable() is idempotent. +create or replace function pgflow_telemetry.disable() +returns void +language plpgsql +set search_path = '' +as $disable$ +begin + begin + perform cron.unschedule('pgflow_telemetry_report'); + exception when others then + null; -- job already absent + end; +end +$disable$; + +create or replace function pgflow_telemetry.enable() +returns void +language plpgsql +set search_path = '' +as $enable$ +begin + perform pgflow_telemetry.disable(); + perform cron.schedule( + 'pgflow_telemetry_report', + '17 3 * * *', + $$select pgflow_telemetry.report()$$ + ); +end +$enable$; diff --git a/pkgs/core/src/database-types.ts b/pkgs/core/src/database-types.ts index 1eb4d7420..737d0c72b 100644 --- a/pkgs/core/src/database-types.ts +++ b/pkgs/core/src/database-types.ts @@ -388,6 +388,7 @@ export type Database = { deprecated_at: string | null function_name: string last_heartbeat_at: string + pgflow_version: string | null queue_name: string started_at: string stopped_at: string | null @@ -397,6 +398,7 @@ export type Database = { deprecated_at?: string | null function_name: string last_heartbeat_at?: string + pgflow_version?: string | null queue_name: string started_at?: string stopped_at?: string | null @@ -406,6 +408,7 @@ export type Database = { deprecated_at?: string | null function_name?: string last_heartbeat_at?: string + pgflow_version?: string | null queue_name?: string started_at?: string stopped_at?: string | null diff --git a/pkgs/core/supabase/migrations/20260918145424_pgflow_telemetry.sql b/pkgs/core/supabase/migrations/20260918145424_pgflow_telemetry.sql new file mode 100644 index 000000000..fd6b7b1f9 --- /dev/null +++ b/pkgs/core/supabase/migrations/20260918145424_pgflow_telemetry.sql @@ -0,0 +1,450 @@ +-- Add new schema named "pgflow_telemetry" +CREATE SCHEMA "pgflow_telemetry"; +-- Create index "idx_runs_started_at" to table: "runs" +CREATE INDEX "idx_runs_started_at" ON "pgflow"."runs" ("started_at"); +-- Modify "workers" table +ALTER TABLE "pgflow"."workers" ADD COLUMN "pgflow_version" text NULL; +-- Create index "idx_workers_started_at" to table: "workers" +CREATE INDEX "idx_workers_started_at" ON "pgflow"."workers" ("started_at"); +-- Create index "idx_workers_stopped_at" to table: "workers" +CREATE INDEX "idx_workers_stopped_at" ON "pgflow"."workers" ("stopped_at"); +-- Create "bucket_count" function +CREATE FUNCTION "pgflow_telemetry"."bucket_count" ("p_count" bigint) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE SET "search_path" = '' AS $$ +select case + when p_count <= 0 then '0' + when p_count = 1 then '1' + when p_count <= 3 then '2-3' + when p_count <= 7 then '4-7' + when p_count <= 15 then '8-15' + when p_count <= 31 then '16-31' + when p_count <= 63 then '32-63' + when p_count <= 127 then '64-127' + when p_count <= 255 then '128-255' + else '256+' + end +$$; +-- Create "bucket_duration" function +CREATE FUNCTION "pgflow_telemetry"."bucket_duration" ("p_duration" interval) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE SET "search_path" = '' AS $$ +select case + when p_duration < '100 milliseconds'::interval then '<100ms' + when p_duration < '1 second'::interval then '100-999ms' + when p_duration < '10 seconds'::interval then '1-9.9s' + when p_duration < '1 minute'::interval then '10-59s' + when p_duration < '5 minutes'::interval then '1-4.9m' + when p_duration < '30 minutes'::interval then '5-29m' + when p_duration < '2 hours'::interval then '30m-1.9h' + when p_duration < '1 day'::interval then '2-23h' + when p_duration < '7 days'::interval then '1-6d' + else '7d+' + end +$$; +-- Create "bucket_steps" function +CREATE FUNCTION "pgflow_telemetry"."bucket_steps" ("p_steps" bigint) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE SET "search_path" = '' AS $$ +select case + when p_steps <= 1 then '1' + when p_steps <= 3 then '2-3' + when p_steps <= 7 then '4-7' + when p_steps <= 15 then '8-15' + when p_steps <= 31 then '16-31' + else '32+' + end +$$; +-- Create "build_payload" function +CREATE FUNCTION "pgflow_telemetry"."build_payload" ("p_day" date) RETURNS jsonb LANGUAGE sql STABLE SET "search_path" = '' AS $$ +select jsonb_build_object( + 'schema', 1, + 'contributions', coalesce(jsonb_agg(c order by c->>'metric', c->>'bucket'), '[]'::jsonb) + ) + from ( + -- Adoption: was this database active during the completed day? + select jsonb_build_object('metric', 'active_db_day', 'bucket', 'yes') as c + where exists ( + select 1 from pgflow.runs r + where r.started_at >= p_day + and r.started_at < p_day + 1 + ) + + union all + -- Version distribution of workers started during the day. + select jsonb_build_object( + 'metric', 'workers_by_version', + 'bucket', w.pgflow_version, + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.started_at >= p_day + and w.started_at < p_day + 1 + and w.pgflow_version ~ '^\d+\.\d+\.\d+' + group by w.pgflow_version + + union all + -- Same-database upgrade: a version started today that no earlier row had. + select jsonb_build_object('metric', 'version_changed', 'bucket', 'yes') + where exists ( + select 1 from pgflow.workers w + where w.started_at >= p_day + and w.started_at < p_day + 1 + and w.pgflow_version is not null + and not exists ( + select 1 from pgflow.workers prev + where prev.started_at < p_day + and prev.pgflow_version = w.pgflow_version + ) + ) + + union all + -- Worker function registry (current state, not windowed). + select jsonb_build_object( + 'metric', 'worker_functions', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.worker_functions wf + + union all + select jsonb_build_object( + 'metric', 'worker_functions_by_mode', + 'bucket', wf.start_mode, + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.worker_functions wf + group by wf.start_mode + + union all + select jsonb_build_object( + 'metric', 'worker_functions_disabled', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.worker_functions wf + where not wf.enabled + + union all + -- Worker churn during the day. + select jsonb_build_object( + 'metric', 'workers_started_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.started_at >= p_day and w.started_at < p_day + 1 + + union all + select jsonb_build_object( + 'metric', 'workers_stopped_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.stopped_at >= p_day and w.stopped_at < p_day + 1 + + union all + select jsonb_build_object( + 'metric', 'workers_deprecated_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.workers w + where w.deprecated_at >= p_day and w.deprecated_at < p_day + 1 + + union all + -- Restart churn per function (Supabase recycles every 150-400 s). + select jsonb_build_object( + 'metric', 'worker_starts_per_function', + 'bucket', pgflow_telemetry.bucket_count(per_function.starts), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select w.function_name, count(*) as starts + from pgflow.workers w + where w.started_at >= p_day and w.started_at < p_day + 1 + group by w.function_name + ) per_function + group by per_function.starts + + union all + -- Active-flow shape: steps per flow, histogram over active flows. + select jsonb_build_object( + 'metric', 'steps_per_flow', + 'bucket', pgflow_telemetry.bucket_steps(per_flow.step_count), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select s.flow_slug, count(*) as step_count + from pgflow.steps s + where s.flow_slug in ( + select distinct r.flow_slug + from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + group by s.flow_slug + ) per_flow + group by per_flow.step_count + + union all + -- Feature booleans, scoped to steps of active flows. + select jsonb_build_object('metric', f.metric, 'bucket', 'yes') + from ( + select 'uses_map' as metric, exists ( + select 1 from pgflow.steps s + where s.step_type = 'map' + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) as used + union all + select 'uses_root_map', exists ( + select 1 from pgflow.steps s + where s.step_type = 'map' and s.deps_count = 0 + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_condition', exists ( + select 1 from pgflow.steps s + where (s.required_input_pattern is not null or s.forbidden_input_pattern is not null) + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_graceful_failure', exists ( + select 1 from pgflow.steps s + where s.when_exhausted <> 'fail' + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_skip_cascade', exists ( + select 1 from pgflow.steps s + where s.when_unmet = 'skip-cascade' + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_retry_override', exists ( + select 1 from pgflow.steps s + where s.opt_max_attempts is not null + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_timeout_override', exists ( + select 1 from pgflow.steps s + where s.opt_timeout is not null + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + union all + select 'uses_start_delay', exists ( + select 1 from pgflow.steps s + where s.opt_start_delay is not null + and s.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + ) + ) f + where f.used + + union all + -- Queue-mode adoption among active flows. + select jsonb_build_object( + 'metric', 'queue_mode', + 'bucket', f.queue_mode, + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.flows f + where f.flow_slug in ( + select distinct r.flow_slug from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + ) + group by f.queue_mode + + union all + -- Workload: runs started during the day. + select jsonb_build_object( + 'metric', 'runs_started_day', + 'bucket', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + + union all + -- Run outcomes by terminal timestamp in window. + select jsonb_build_object( + 'metric', 'run_outcomes', + 'bucket', o.outcome, + 'count', pgflow_telemetry.bucket_count(o.n) + ) + from ( + select 'completed' as outcome, count(*) as n + from pgflow.runs r + where r.completed_at >= p_day and r.completed_at < p_day + 1 + union all + select 'failed', count(*) + from pgflow.runs r + where r.failed_at >= p_day and r.failed_at < p_day + 1 + union all + select 'started', count(*) + from pgflow.runs r + where r.started_at >= p_day and r.started_at < p_day + 1 + and r.status = 'started' + ) o + + union all + -- Wall durations of runs that terminated during the day. + select jsonb_build_object( + 'metric', 'run_wall_duration', + 'bucket', pgflow_telemetry.bucket_duration(d.terminal_at - d.started_at), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select r.started_at, coalesce(r.completed_at, r.failed_at) as terminal_at + from pgflow.runs r + where coalesce(r.completed_at, r.failed_at) >= p_day + and coalesce(r.completed_at, r.failed_at) < p_day + 1 + ) d + group by d.terminal_at - d.started_at + + union all + -- Wall durations of steps that terminated during the day. + select jsonb_build_object( + 'metric', 'step_wall_duration', + 'bucket', pgflow_telemetry.bucket_duration(d.terminal_at - d.started_at), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from ( + select ss.started_at, coalesce(ss.completed_at, ss.failed_at) as terminal_at + from pgflow.step_states ss + where ss.started_at is not null + and coalesce(ss.completed_at, ss.failed_at) >= p_day + and coalesce(ss.completed_at, ss.failed_at) < p_day + 1 + ) d + group by d.terminal_at - d.started_at + + union all + -- Task attempts for tasks queued during the day. + select jsonb_build_object( + 'metric', 'task_attempts_day', + 'bucket', pgflow_telemetry.bucket_count(coalesce(sum(t.attempts_count), 0)) + ) + from pgflow.step_tasks t + where t.queued_at >= p_day and t.queued_at < p_day + 1 + + union all + -- Map fan-out: initial task counts of steps started during the day. + select jsonb_build_object( + 'metric', 'map_task_count', + 'bucket', pgflow_telemetry.bucket_steps(m.initial_tasks), + 'count', pgflow_telemetry.bucket_count(count(*)) + ) + from pgflow.step_states m + where m.started_at >= p_day and m.started_at < p_day + 1 + and m.initial_tasks >= 2 + group by m.initial_tasks + ) contributions +$$; +-- Create "disable" function +CREATE FUNCTION "pgflow_telemetry"."disable" () RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $$ +begin + begin + perform cron.unschedule('pgflow_telemetry_report'); + exception when others then + null; -- job already absent + end; +end +$$; +-- Create "sent_reports" table +CREATE TABLE "pgflow_telemetry"."sent_reports" ( + "day" date NOT NULL, + "payload" jsonb NOT NULL, + "request_id" bigint NULL, + "sent_at" timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY ("day") +); +-- Set comment to table: "sent_reports" +COMMENT ON TABLE "pgflow_telemetry"."sent_reports" IS 'Audit log of every telemetry payload sent; unique day is the dedup marker'; +-- Create "report" function +CREATE FUNCTION "pgflow_telemetry"."report" () RETURNS text LANGUAGE plpgsql SET "search_path" = '' AS $$ +declare + v_day date := current_date - 1; + v_payload jsonb; + v_request_id bigint; +begin + if not exists ( + select 1 from pgflow.runs r + where r.started_at >= v_day and r.started_at < v_day + 1 + ) then + return 'skipped: inactive day'; + end if; + + if exists ( + select 1 from pgflow_telemetry.sent_reports s where s.day = v_day + ) then + return 'skipped: already reported'; + end if; + + -- Local CLI stack (developer machines, CI on supabase start) never sends. + if pgflow.is_local() then + return 'skipped: local'; + end if; + + perform set_config('statement_timeout', '5000', true); + + begin + v_payload := pgflow_telemetry.build_payload(v_day); + exception when others then + raise warning 'pgflow telemetry: payload build failed, sending nothing'; + return 'error: build failed'; + end; + + begin + v_request_id := net.http_post( + url => 'https://pgflow-telemetry.workers.dev', + body => v_payload::text, + headers => jsonb_build_object('Content-Type', 'application/json'), + timeout_milliseconds => 5000 + ); + exception when others then + raise warning 'pgflow telemetry: http_post queueing failed'; + return 'error: queue failed'; + end; + + insert into pgflow_telemetry.sent_reports (day, payload, request_id) + values (v_day, v_payload, v_request_id); + + delete from pgflow_telemetry.sent_reports + where day < current_date - 90; + + return 'sent: ' || v_request_id; +end +$$; +-- Create "enable" function +CREATE FUNCTION "pgflow_telemetry"."enable" () RETURNS void LANGUAGE plpgsql SET "search_path" = '' AS $BODY$ +begin + perform pgflow_telemetry.disable(); + perform cron.schedule( + 'pgflow_telemetry_report', + '17 3 * * *', + $$select pgflow_telemetry.report()$$ + ); +end +$BODY$; +-- Create "preview" function +CREATE FUNCTION "pgflow_telemetry"."preview" ("p_day" date DEFAULT (CURRENT_DATE - 1)) RETURNS jsonb LANGUAGE sql STABLE SET "search_path" = '' AS $$ select pgflow_telemetry.build_payload(p_day) $$; + +-- Scheduled exactly once, here. Later migrations must NEVER re-schedule this +-- job: a re-schedule would silently re-enable telemetry for opted-out users. +select cron.schedule( + 'pgflow_telemetry_report', + '17 3 * * *', + $$select pgflow_telemetry.report()$$ +); diff --git a/pkgs/core/supabase/migrations/atlas.sum b/pkgs/core/supabase/migrations/atlas.sum index c9ffbd276..32a2b17ce 100644 --- a/pkgs/core/supabase/migrations/atlas.sum +++ b/pkgs/core/supabase/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:KsVAXOPvkHDCj18n4kgWtwuS/HMiSC1dVMnNor++gpA= +h1:PhlQ4YrYMHSBTdyQ7EU4yHCABUukCncvzTTtSIGGBp0= 20250429164909_pgflow_initial.sql h1:I3n/tQIg5Q5nLg7RDoU3BzqHvFVjmumQxVNbXTPG15s= 20250517072017_pgflow_fix_poll_for_tasks_to_use_separate_statement_for_polling.sql h1:wTuXuwMxVniCr3ONCpodpVWJcHktoQZIbqMZ3sUHKMY= 20250609105135_pgflow_add_start_tasks_and_started_status.sql h1:ggGanW4Wyt8Kv6TWjnZ00/qVb3sm+/eFVDjGfT8qyPg= @@ -23,3 +23,4 @@ h1:KsVAXOPvkHDCj18n4kgWtwuS/HMiSC1dVMnNor++gpA= 20260904095427_pgflow_task_lifecycle_hardening.sql h1:27b0BfBcQxeu5XSqVtQYvDCRzTsvLqzS/5hx14/2VyM= 20260907082520_pgflow_remove_legacy_flow_compilation.sql h1:LNFDz+ZZlWb19FmWNPK57eiD+FXySMbStVij8MTSvDw= 20260915074120_pgflow_private_step_queues.sql h1:+vsfsOyDaM8WISO/jxp4UBlTzPuQhg6RwBCiOAk5YAE= +20260918145424_pgflow_telemetry.sql h1:gKCTXXVCBZeCzpvpIwny8qjDwXFgNblxVepb1c1c/ic= diff --git a/pkgs/core/supabase/tests/telemetry/payload.test.sql b/pkgs/core/supabase/tests/telemetry/payload.test.sql new file mode 100644 index 000000000..f8ef7caec --- /dev/null +++ b/pkgs/core/supabase/tests/telemetry/payload.test.sql @@ -0,0 +1,51 @@ +begin; +select plan(5); + +select pgflow_tests.reset_db(); +select pgflow_tests.setup_flow('sequential'); +select pgflow.start_flow('sequential', '{}'::jsonb); + +-- Shift the fixture into yesterday's window: telemetry only reports on +-- completed days. Mutating timestamps is the otherwise unreachable state. +update pgflow.runs + set started_at = current_date - 1 + interval '1 hour'; + +insert into pgflow.workers (worker_id, queue_name, function_name, started_at, pgflow_version) +values + (gen_random_uuid(), 'sequential', 'sequential-worker', current_date - 1 + interval '2 hours', '0.17.0'), + (gen_random_uuid(), 'sequential', 'sequential-worker', current_date - 1 + interval '3 hours', '0.17.0'); + +select is( + pgflow_telemetry.preview(current_date - 1) -> 'schema', + '1'::jsonb, + 'payload carries schema version 1' +); + +select ok( + pgflow_telemetry.preview(current_date - 1) -> 'contributions' @> + '[{"metric":"active_db_day","bucket":"yes"}]'::jsonb, + 'active day reports active_db_day' +); + +select ok( + pgflow_telemetry.preview(current_date - 1) -> 'contributions' @> + '[{"metric":"workers_by_version","bucket":"0.17.0","count":"2-3"}]'::jsonb, + 'workers_by_version buckets the day starts' +); + +select ok( + pgflow_telemetry.preview(current_date - 1) -> 'contributions' @> + '[{"metric":"steps_per_flow","bucket":"2-3","count":"1"}]'::jsonb, + 'sequential flow (3 steps) lands in the 2-3 bucket' +); + +select ok( + not ( + pgflow_telemetry.preview(current_date - 30) -> 'contributions' @> + '[{"metric":"active_db_day","bucket":"yes"}]'::jsonb + ), + 'inactive day reports no active_db_day' +); + +select finish(); +rollback; diff --git a/pkgs/core/supabase/tests/telemetry/report.test.sql b/pkgs/core/supabase/tests/telemetry/report.test.sql new file mode 100644 index 000000000..068b2b009 --- /dev/null +++ b/pkgs/core/supabase/tests/telemetry/report.test.sql @@ -0,0 +1,61 @@ +begin; +select plan(6); + +select pgflow_tests.reset_db(); + +-- Gate order: inactive -> already reported -> local. Test env IS local, so +-- only the first two gates and the switch functions are observable here. +select is( + pgflow_telemetry.report(), + 'skipped: inactive day', + 'fresh database has no yesterday runs, sends nothing' +); + +select pgflow_tests.setup_flow('sequential'); +select pgflow.start_flow('sequential', '{}'::jsonb); +update pgflow.runs set started_at = current_date - 1 + interval '1 hour'; + +insert into pgflow_telemetry.sent_reports (day, payload) +values (current_date - 1, '{"schema":1,"contributions":[]}'::jsonb); + +select is( + pgflow_telemetry.report(), + 'skipped: already reported', + 'existing sent_reports row for the day blocks a second report' +); + +delete from pgflow_telemetry.sent_reports; + +select is( + pgflow_telemetry.report(), + 'skipped: local', + 'local databases never send' +); + +select is( + (select count(*) from pgflow_telemetry.sent_reports), + 0::bigint, + 'skipped local report writes no audit row' +); + +select pgflow_telemetry.disable(); +select ok( + not exists ( + select 1 from cron.job + where jobname = 'pgflow_telemetry_report' + ), + 'disable() removes the cron job (the switch)' +); + +select pgflow_telemetry.enable(); +select ok( + exists ( + select 1 from cron.job + where jobname = 'pgflow_telemetry_report' + and schedule = '17 3 * * *' + ), + 'enable() restores the daily job' +); + +select finish(); +rollback; diff --git a/pkgs/core/supabase/tests/telemetry/schema.test.sql b/pkgs/core/supabase/tests/telemetry/schema.test.sql new file mode 100644 index 000000000..9b9297763 --- /dev/null +++ b/pkgs/core/supabase/tests/telemetry/schema.test.sql @@ -0,0 +1,46 @@ +begin; +select plan(10); + +select pgflow_tests.reset_db(); + +select ok( + exists (select 1 from pg_namespace where nspname = 'pgflow_telemetry'), + 'pgflow_telemetry schema exists' +); + +select is( + ( + select count(*) from information_schema.columns + where table_schema = 'pgflow_telemetry' and table_name = 'sent_reports' + ), + 4::bigint, + 'sent_reports has four columns' +); + +select ok( + exists ( + select 1 from pg_index i + join pg_class c on c.oid = i.indrelid + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = 'pgflow_telemetry' and c.relname = 'sent_reports' and i.indisprimary + ), + 'sent_reports primary key enforces one row per day' +); + +select is(pgflow_telemetry.bucket_count(0), '0', 'bucket_count 0'); +select is(pgflow_telemetry.bucket_count(300), '256+', 'bucket_count 256+'); +select is(pgflow_telemetry.bucket_steps(5), '4-7', 'bucket_steps 5'); +select is(pgflow_telemetry.bucket_duration('9 seconds'::interval), '1-9.9s', 'bucket_duration 9s'); +select is(pgflow_telemetry.bucket_duration('90 seconds'::interval), '1-4.9m', 'bucket_duration 90s'); +select is(pgflow_telemetry.bucket_duration('3 days'::interval), '1-6d', 'bucket_duration 3d'); + +select ok( + exists ( + select 1 from pg_indexes + where schemaname = 'pgflow' and tablename = 'runs' and indexname = 'idx_runs_started_at' + ), + 'runs(started_at) index exists for the daily scan' +); + +select finish(); +rollback; diff --git a/pkgs/core/supabase/tests/telemetry/workers_version.test.sql b/pkgs/core/supabase/tests/telemetry/workers_version.test.sql new file mode 100644 index 000000000..2cfffcecf --- /dev/null +++ b/pkgs/core/supabase/tests/telemetry/workers_version.test.sql @@ -0,0 +1,29 @@ +begin; +select plan(2); + +select pgflow_tests.reset_db(); + +select ok( + exists ( + select 1 from information_schema.columns + where table_schema = 'pgflow' + and table_name = 'workers' + and column_name = 'pgflow_version' + ), + 'workers.pgflow_version column exists' +); + +select is( + ( + select is_nullable + from information_schema.columns + where table_schema = 'pgflow' + and table_name = 'workers' + and column_name = 'pgflow_version' + ), + 'YES', + 'pgflow_version is nullable so existing rows survive' +); + +select finish(); +rollback; diff --git a/pkgs/edge-worker/jsr.json b/pkgs/edge-worker/jsr.json index 385e00f6f..ace2df35c 100644 --- a/pkgs/edge-worker/jsr.json +++ b/pkgs/edge-worker/jsr.json @@ -18,6 +18,7 @@ "README.md", "LICENSE.md", "CHANGELOG.md", + "package.json", "src/**/*.ts" ], "exclude": [ diff --git a/pkgs/edge-worker/scripts/sync-e2e-deps.sh b/pkgs/edge-worker/scripts/sync-e2e-deps.sh index a8d1fb5d6..b1ac21dc8 100755 --- a/pkgs/edge-worker/scripts/sync-e2e-deps.sh +++ b/pkgs/edge-worker/scripts/sync-e2e-deps.sh @@ -40,6 +40,8 @@ echo "📋 Copying @pgflow/edge-worker..." mkdir -p "$VENDOR_DIR/@pgflow/edge-worker" # Copy the entire src directory to maintain relative imports cp -r "$MONOREPO_ROOT/pkgs/edge-worker/src" "$VENDOR_DIR/@pgflow/edge-worker/" +# package.json is read by src/core/version.ts (build-time version stamp) +cp "$MONOREPO_ROOT/pkgs/edge-worker/package.json" "$VENDOR_DIR/@pgflow/edge-worker/" # Simple fix: replace .js with .ts in imports find "$VENDOR_DIR/@pgflow/edge-worker" -name "*.ts" -type f -exec sed -i 's/\.js"/\.ts"/g' {} + diff --git a/pkgs/edge-worker/src/core/Queries.ts b/pkgs/edge-worker/src/core/Queries.ts index ecd8d43f8..44220dc6a 100644 --- a/pkgs/edge-worker/src/core/Queries.ts +++ b/pkgs/edge-worker/src/core/Queries.ts @@ -26,14 +26,16 @@ export class Queries { queueName, workerId, edgeFunctionName, + pgflowVersion, }: { queueName: string; workerId: string; edgeFunctionName: string; + pgflowVersion: string; }): Promise { const [worker] = await this.sql` - INSERT INTO pgflow.workers (queue_name, worker_id, function_name) - VALUES (${queueName}, ${workerId}, ${edgeFunctionName}) + INSERT INTO pgflow.workers (queue_name, worker_id, function_name, pgflow_version) + VALUES (${queueName}, ${workerId}, ${edgeFunctionName}, ${pgflowVersion}) RETURNING *; `; diff --git a/pkgs/edge-worker/src/core/WorkerLifecycle.ts b/pkgs/edge-worker/src/core/WorkerLifecycle.ts index 051e3d06f..03e667782 100644 --- a/pkgs/edge-worker/src/core/WorkerLifecycle.ts +++ b/pkgs/edge-worker/src/core/WorkerLifecycle.ts @@ -2,6 +2,7 @@ import type { Queries } from './Queries.js'; import type { Queue } from '../queue/Queue.js'; import type { InternalLifecycle, Json, WorkerBootstrap, WorkerRow } from './types.js'; import { States, WorkerState } from './WorkerState.js'; +import { pgflowVersion } from './version.js'; import type { Logger } from '../platform/types.js'; export interface LifecycleConfig { @@ -39,6 +40,7 @@ export class WorkerLifecycle implements InternalLifecycle this.workerRow = await this.queries.onWorkerStarted({ queueName: this.queueName, ...workerBootstrap, + pgflowVersion, }); this.workerState.transitionTo(States.Running); diff --git a/pkgs/edge-worker/src/core/types.ts b/pkgs/edge-worker/src/core/types.ts index 8bf895ce3..6ac3d63a9 100644 --- a/pkgs/edge-worker/src/core/types.ts +++ b/pkgs/edge-worker/src/core/types.ts @@ -53,6 +53,8 @@ export type WorkerRow = { deprecated_at: string | null; worker_id: string; function_name: string; + /** pgflow package version stamped at registration (telemetry). */ + pgflow_version?: string | null; }; export type WorkerStartMode = 'http' | 'process'; diff --git a/pkgs/edge-worker/src/core/version.ts b/pkgs/edge-worker/src/core/version.ts new file mode 100644 index 000000000..061a786b5 --- /dev/null +++ b/pkgs/edge-worker/src/core/version.ts @@ -0,0 +1,8 @@ +// Baked in at build time from this package's package.json (relative JSON +// import, inlined by the bundler — never a runtime file read). Published dist +// therefore always carries the released version. +// The `with { type: 'json' }` attribute is required: Deno (the JSR runtime) +// rejects attribute-less JSON imports. +import pkg from '../../package.json' with { type: 'json' }; + +export const pgflowVersion: string = pkg.version; diff --git a/pkgs/edge-worker/src/flow/FlowWorkerLifecycle.ts b/pkgs/edge-worker/src/flow/FlowWorkerLifecycle.ts index c3241b035..93cd3cb8d 100644 --- a/pkgs/edge-worker/src/flow/FlowWorkerLifecycle.ts +++ b/pkgs/edge-worker/src/flow/FlowWorkerLifecycle.ts @@ -6,6 +6,7 @@ import type { AnyFlow } from '@pgflow/dsl'; import { extractFlowShape } from '@pgflow/dsl'; import { FlowRoutingMismatchError, FlowShapeMismatchError } from './errors.js'; import type { WorkerRouting } from './workerRouting.js'; +import { pgflowVersion } from '../core/version.js'; export interface FlowLifecycleConfig { heartbeatInterval?: number; @@ -67,6 +68,7 @@ export class FlowWorkerLifecycle implements InternalLifec this.workerRow = await this.queries.onWorkerStarted({ queueName: this.queueName, ...workerBootstrap, + pgflowVersion, }); this.workerState.transitionTo(States.Running); diff --git a/pkgs/website/astro.config.mjs b/pkgs/website/astro.config.mjs index 0d7aad1fd..dd428cd9c 100644 --- a/pkgs/website/astro.config.mjs +++ b/pkgs/website/astro.config.mjs @@ -429,6 +429,7 @@ export default defineConfig({ { label: 'Permissions', link: '/reference/permissions/' }, ], }, + { label: 'Telemetry', link: '/reference/telemetry/' }, { label: 'Configuration', autogenerate: { directory: 'reference/configuration/' }, diff --git a/pkgs/website/redirects.config.mjs b/pkgs/website/redirects.config.mjs index 93cc06323..c6810e340 100644 --- a/pkgs/website/redirects.config.mjs +++ b/pkgs/website/redirects.config.mjs @@ -123,4 +123,11 @@ export const redirects = { // ============================================================================ '/news/pgflow-0-3-0-fixing-race-conditions/': '/news/', + + // ============================================================================ + // MAIN BRANCH PATH MIGRATIONS (news - article renamed for 0.17.1) + // ============================================================================ + + '/news/pgflow-0-17-0-persistent-queue-identity/': + '/news/pgflow-0-17-1-persistent-queues-and-telemetry/', }; diff --git a/pkgs/website/src/content/docs/news/pgflow-0-17-0-persistent-queue-identity.mdx b/pkgs/website/src/content/docs/news/pgflow-0-17-1-persistent-queues-and-telemetry.mdx similarity index 81% rename from pkgs/website/src/content/docs/news/pgflow-0-17-0-persistent-queue-identity.mdx rename to pkgs/website/src/content/docs/news/pgflow-0-17-1-persistent-queues-and-telemetry.mdx index 2ef7e25e1..5d384d407 100644 --- a/pkgs/website/src/content/docs/news/pgflow-0-17-0-persistent-queue-identity.mdx +++ b/pkgs/website/src/content/docs/news/pgflow-0-17-1-persistent-queues-and-telemetry.mdx @@ -1,7 +1,7 @@ --- -title: 'pgflow 0.17.0: Private Step Queues' -description: 'Tasks store their physical queue route, and flows can use one private queue per step. start_tasks() requires the canonical route - a breaking low-level change with a stop-the-world maintenance upgrade.' -date: 2026-09-13 +title: 'pgflow 0.17.1: Private Step Queues and Anonymous Telemetry' +description: 'Queue identity is now persisted per step and task, flows can use private step queues, and pgflow reports anonymous opt-out usage data.' +date: 2026-09-18 authors: - jumski tags: @@ -12,6 +12,8 @@ featured: false import { Aside } from '@astrojs/starlight/components'; +0.17.0 shipped the queue identity work below; 0.17.1 adds anonymous telemetry to the same feature set. If you are upgrading from 0.16.x, this article covers the whole 0.17.x line. + pgflow now persists the physical queue identity of every step and task. `pgflow.steps` and `pgflow.step_tasks` gain a `queue_name` column that stores the canonical route, snapshotted when the task is created and never rewritten. A queued task's message identity is `(queue_name, message_id)` instead of `message_id` alone - PGMQ message ids are queue-scoped, so the same id in two different queues can no longer be mistaken for the same message. Existing flows use `queue_mode = 'flow'` and retain `lower(flow_slug)`. Wrap a new flow with `withStepQueues()` to use `queue_mode = 'step'`: each step receives a generated private queue. Read the [data model](/concepts/data-model/) for what is stored, and the [update guide](/deploy/update-pgflow/) for the full 0.16.0 → 0.17.0 procedure. @@ -36,3 +38,15 @@ Read the [data model](/concepts/data-model/) for what is stored, and the [update The ordered procedure with the exact SQL lives in the [update guide](/deploy/update-pgflow/#queue-identity-and-private-step-queues-0170). In short: update packages and copy migrations in advance, pause new producers, gracefully drain in-flight work while the old schema is present (waiting for running handlers to finish - not for queues to empty; queued tasks and messages are retained), stop every worker including HTTP re-invocation and long-running processes, quiesce maintenance/recovery/definition writers, apply the transactional migration through Supabase's migration runner (against the production database - `--linked` or `--db-url`, not the local default), replace the pruning helper if installed, deploy the matching package set, then resume maintenance, workers, and producers in that order - restoring the exact worker `enabled` states recorded before the window. The migration backfills existing steps and tasks with `lower(flow_slug)` and `queue_mode = 'flow'` - including tasks whose `message_id` is NULL - and fails atomically on conflicting data (flows differing only by slug case, case-only duplicate step slugs, invalid leading or trailing underscore or `__` slugs, duplicate `(queue_name, message_id)` pairs, queue names beyond PGMQ's 47-character limit). A failed migration leaves the database unchanged; resolve conflicts manually and re-run it. After a successful migration, old workers cannot simply restart. No queues or messages are migrated or recreated, and flow slug casing is untouched. + +## Anonymous telemetry + +pgflow now includes anonymous, opt-out usage telemetry so development can be guided by real usage instead of guesses. Once per day your database summarizes yesterday's activity into coarse buckets - active or not, versions running, run and worker counts, flow shapes, feature adoption - and sends one small JSON document. Nothing identifying ever leaves your database: no slugs, names, IDs, inputs, outputs, errors, or addresses. + +You can inspect the exact bytes of every past report with `select * from pgflow_telemetry.sent_reports`, preview any day with `pgflow_telemetry.preview()`, and disable it permanently with one line: + +```sql +select pgflow_telemetry.disable(); +``` + +Read the full details - what is collected, what is never collected, where data lives and for how long - in the [telemetry reference](/reference/telemetry/). diff --git a/pkgs/website/src/content/docs/reference/index.mdx b/pkgs/website/src/content/docs/reference/index.mdx index c2bc30551..da59ac457 100644 --- a/pkgs/website/src/content/docs/reference/index.mdx +++ b/pkgs/website/src/content/docs/reference/index.mdx @@ -17,6 +17,16 @@ Find precise technical information about pgflow's APIs, configuration options, a /> +## Telemetry + + + + + ## Configuration diff --git a/pkgs/website/src/content/docs/reference/telemetry.mdx b/pkgs/website/src/content/docs/reference/telemetry.mdx new file mode 100644 index 000000000..4f4db11ae --- /dev/null +++ b/pkgs/website/src/content/docs/reference/telemetry.mdx @@ -0,0 +1,77 @@ +--- +title: Telemetry +description: What anonymous usage data pgflow collects, how to inspect it, and how to disable it +--- + +pgflow includes anonymous, opt-out telemetry. It answers one question for the maintainers: **is pgflow actually used, and which features matter?** Your data stays yours — nothing below can identify you, your project, or your users. + +## What is collected + +Once per day, your database aggregates yesterday's activity into coarse buckets and sends one small JSON document. Nothing is sent per run, per step, or per task — telemetry never touches the execution path. + +The payload contains only: + +- Whether any runs happened that day +- Which pgflow versions started workers, and whether a version changed +- Coarse counts: runs started, run outcomes, workers started/stopped, registered worker functions +- Flow shape in buckets: steps per flow, which features appear (map steps, conditions, retries, delays, graceful failure, step queues) +- Coarse durations and task counts in fixed ranges + +An example of the exact shape: + +```json +{ + "schema": 1, + "contributions": [ + { "metric": "active_db_day", "bucket": "yes" }, + { "metric": "runs_started_day", "bucket": "16-31" }, + { "metric": "run_outcomes", "bucket": "completed", "count": "8-15" }, + { "metric": "workers_by_version", "bucket": "0.17.0", "count": "2-3" }, + { "metric": "steps_per_flow", "bucket": "2-3", "count": "1" }, + { "metric": "uses_map", "bucket": "yes" } + ] +} +``` + +Counts use fixed ranges (`0`, `1`, `2-3`, `4-7`, … `256+`), durations use fixed ranges (`<100ms` … `7d+`). Exact values never leave your database. + +## What is never collected + +No slugs, flow or step names, queue names, function names, run or task IDs, inputs, outputs, error messages, URLs, hostnames, environment names, IP addresses, or any free text. No cookies. No account. Nothing that could identify you, your project, or your users. + +## Inspect exactly what is sent + +Every payload is stored before it leaves, so you can audit the real bytes: + +```sql +select jsonb_pretty(payload) +from pgflow_telemetry.sent_reports +order by day desc +limit 1; +``` + +Preview any day without sending: + +```sql +select pgflow_telemetry.preview(); -- yesterday +select pgflow_telemetry.preview('2026-09-10'); -- any past day +``` + +## Disable or re-enable + +Telemetry is opt-out and runs as a daily `pg_cron` job. The job's presence is the switch: + +```sql +select pgflow_telemetry.disable(); -- stop reporting +select pgflow_telemetry.enable(); -- resume reporting +``` + +Local development databases (`supabase start`) never send telemetry, and days without runs send nothing. + +## Privacy + +- Data goes to `https://pgflow-telemetry.workers.dev`, a [Cloudflare Worker](https://workers.cloudflare.com/) backed by [Workers Analytics Engine](https://developers.cloudflare.com/analytics/analytics-engine/). +- Stored data is retained for three months, then expires automatically. +- pgflow sends no identifiers and the ingest application stores no transport metadata (no IPs, no user agents). Cloudflare necessarily processes source IPs as network transport. +- Counts are undercounts in practice: opt-outs, blocked egress, and failed sends are invisible. The maintainers treat every number as a lower bound and as directional product evidence only. +- The sender is the open-source SQL in this repository; the receiver is the open-source Worker in `apps/telemetry-worker/`. diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9298abfe2..ad142893e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -190,6 +190,21 @@ importers: specifier: ^4.20.3 version: 4.46.0(@cloudflare/workers-types@4.20251118.0) + apps/telemetry-worker: + devDependencies: + '@cloudflare/workers-types': + specifier: ^4.20250901.0 + version: 4.20251118.0 + typescript: + specifier: ^5.5.0 + version: 5.9.3 + vitest: + specifier: ^3.0.0 + version: 3.2.7(@types/debug@4.1.12)(@types/node@22.19.1)(jiti@2.6.1)(jsdom@22.1.0)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1) + wrangler: + specifier: ^4.0.0 + version: 4.47.0(@cloudflare/workers-types@4.20251118.0) + pkgs/cli: dependencies: '@clack/prompts': @@ -247,7 +262,7 @@ importers: version: 5.43.1 vite-plugin-dts: specifier: ~3.8.1 - version: 3.8.3(@types/node@22.19.1)(rollup@4.53.2)(typescript@5.8.3)(vite@7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)) + version: 3.8.3(@types/node@22.19.1)(rollup@4.53.2)(typescript@5.9.3)(vite@7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)) vitest: specifier: 1.3.1 version: 1.3.1(@types/node@22.19.1)(jsdom@22.1.0)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1) @@ -1158,9 +1173,6 @@ packages: cpu: [x64] os: [win32] - '@cloudflare/workers-types@4.20251014.0': - resolution: {integrity: sha512-tEW98J/kOa0TdylIUOrLKRdwkUw0rvvYVlo+Ce0mqRH3c8kSoxLzUH9gfCvwLe0M89z1RkzFovSKAW2Nwtyn3w==} - '@cloudflare/workers-types@4.20251118.0': resolution: {integrity: sha512-O1BlPjaQlM5rsxf8rc8NwNrXduzXMCWZDsSelI1SEyt0zYuBYMzn0kNrrD2bXAtXeYCk6LrX98037AlVdGXBUw==} @@ -2879,6 +2891,9 @@ packages: '@types/braces@3.0.5': resolution: {integrity: sha512-SQFof9H+LXeWNz8wDe7oN5zu7ket0qwMu5vZubW4GCJ8Kkeh6nBWUz87+KTz/G3Kqsrp0j/W253XJb3KMEeg3w==} + '@types/chai@5.2.3': + resolution: {integrity: sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==} + '@types/cookie@0.6.0': resolution: {integrity: sha512-4Kh9a6B2bQciAhf7FSuMRRkUWecJgJu9nPnx3yzpsfXX/c50REIqpHY4C82bXP90qrLtXtkDxTZosYO3UpOwlA==} @@ -2903,6 +2918,9 @@ packages: '@types/debug@4.1.12': resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} + '@types/deep-eql@4.0.2': + resolution: {integrity: sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==} + '@types/deno@2.5.0': resolution: {integrity: sha512-g8JS38vmc0S87jKsFzre+0ZyMOUDHPVokEJymSCRlL57h6f/FdKPWBXgdFh3Z8Ees9sz11qt9VWELU9Y9ZkiVw==} @@ -3166,18 +3184,47 @@ packages: '@vitest/expect@1.3.1': resolution: {integrity: sha512-xofQFwIzfdmLLlHa6ag0dPV8YsnKOCP1KdAeVVh34vSjN2dcUiXYCD9htu/9eM7t8Xln4v03U9HLxLpPlsXdZw==} + '@vitest/expect@3.2.7': + resolution: {integrity: sha512-E8eBXaKibuvH2pSZErOjdVb5vF4PbKYcrnluBTYxEk1l/VhhwZg1kZQsdtjq+CsF5CFydf2Rdkz7jDHKSisi3w==} + + '@vitest/mocker@3.2.7': + resolution: {integrity: sha512-Trr0hYO9CM3Wj6ksWHRhK9IZpIY6wTMO5u/MqXurMxT57sWBaOPEtP3Oq60ihZuh5JsiagKfz95OcxdEP6dBrA==} + peerDependencies: + msw: ^2.4.9 + vite: ^5.0.0 || ^6.0.0 || ^7.0.0-0 + peerDependenciesMeta: + msw: + optional: true + vite: + optional: true + + '@vitest/pretty-format@3.2.7': + resolution: {integrity: sha512-KUHlwqVu0sRlhCdyPdQ/wBoTfRahjUky1MubOmYw9fWfIZy1gNoHpuaaQBPAaMaVYdQYHJLurzj8ECCj5OwTqA==} + '@vitest/runner@1.3.1': resolution: {integrity: sha512-5FzF9c3jG/z5bgCnjr8j9LNq/9OxV2uEBAITOXfoe3rdZJTdO7jzThth7FXv/6b+kdY65tpRQB7WaKhNZwX+Kg==} + '@vitest/runner@3.2.7': + resolution: {integrity: sha512-sB9y4ovltoQP+WaUPwmSxO9WIg9Ig694Di5PalVPsYHklAdE027mehpWF2SQSVq+k6sFgaivbTjTJwZLSHbedA==} + '@vitest/snapshot@1.3.1': resolution: {integrity: sha512-EF++BZbt6RZmOlE3SuTPu/NfwBF6q4ABS37HHXzs2LUVPBLx2QoY/K0fKpRChSo8eLiuxcbCVfqKgx/dplCDuQ==} + '@vitest/snapshot@3.2.7': + resolution: {integrity: sha512-7C+MwShwtBSI5Buwoyg3s/iY1eHL9PKAf+O1wVh/TdnjXUtkoL/9YQtre90i4MtNXM6edP1wJ2zOBpfCyhIS7g==} + '@vitest/spy@1.3.1': resolution: {integrity: sha512-xAcW+S099ylC9VLU7eZfdT9myV67Nor9w9zhf0mGCYJSO+zM2839tOeROTdikOi/8Qeusffvxb/MyBSOja1Uig==} + '@vitest/spy@3.2.7': + resolution: {integrity: sha512-Q2eQGI6d2L/hBtZ0qNuKcAGid68XK6cv1xsoaIma6PaJhHPoqcEJhYpXZ/5myCMqkNgtP6UKuBhbc0nHKnrkuQ==} + '@vitest/utils@1.3.1': resolution: {integrity: sha512-d3Waie/299qqRyHTm2DjADeTaNdNSVsnwHPWrs20JMpjh6eiVq7ggggweO8rc4arhf6rRkWuHKwvxGvejUXZZQ==} + '@vitest/utils@3.2.7': + resolution: {integrity: sha512-x6BDOd7dyo3PFLY3I9/HJ25X/6OurhGXk2/B9gOZNPF7XDVjeBK4k01lQE5uvDpbuheErh91qYuE1E2OEjK3Rw==} + '@volar/kit@2.4.23': resolution: {integrity: sha512-YuUIzo9zwC2IkN7FStIcVl1YS9w5vkSFEZfPvnu0IbIMaR9WHhc9ZxvlT+91vrcSoRY469H2jwbrGqpG7m1KaQ==} peerDependencies: @@ -3424,6 +3471,10 @@ packages: assertion-error@1.1.0: resolution: {integrity: sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==} + assertion-error@2.0.1: + resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} + engines: {node: '>=12'} + astring@1.9.0: resolution: {integrity: sha512-LElXdjswlqjWrPpJFg1Fx4wpkOCxj1TDHlSV4PlaRxHGWko024xICaa97ZkMfs6DRKlCguiAI+rbXv5GWwXIkg==} hasBin: true @@ -3606,6 +3657,10 @@ packages: resolution: {integrity: sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==} engines: {node: '>=4'} + chai@5.3.3: + resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} + engines: {node: '>=18'} + chalk@4.1.2: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} @@ -3632,6 +3687,10 @@ packages: check-error@1.0.3: resolution: {integrity: sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==} + check-error@2.1.3: + resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} + engines: {node: '>= 16'} + chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -3875,6 +3934,10 @@ packages: resolution: {integrity: sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==} engines: {node: '>=6'} + deep-eql@5.0.2: + resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} + engines: {node: '>=6'} + deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -4231,6 +4294,10 @@ packages: resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==} engines: {node: '>=6'} + expect-type@1.4.0: + resolution: {integrity: sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==} + engines: {node: '>=12.0.0'} + expressive-code@0.41.3: resolution: {integrity: sha512-YLnD62jfgBZYrXIPQcJ0a51Afv9h8VlWqEGK9uU2T5nL/5rb8SnA86+7+mgCZe5D34Tff5RNEA5hjNVJYHzrFg==} @@ -5052,6 +5119,9 @@ packages: loupe@2.3.7: resolution: {integrity: sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==} + loupe@3.2.1: + resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} + lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} @@ -5666,6 +5736,10 @@ packages: pathval@1.1.1: resolution: {integrity: sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==} + pathval@2.0.1: + resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} + engines: {node: '>= 14.16'} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -6378,6 +6452,9 @@ packages: strip-literal@2.1.1: resolution: {integrity: sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q==} + strip-literal@3.1.0: + resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} + strnum@2.1.1: resolution: {integrity: sha512-7ZvoFTiCnGxBtDqJ//Cu6fWtZtc7Y3x+QOirG15wztbdngGSkht27o2pyGWrVy0b4WAy3jbKmnoK6g5VlVNUUw==} @@ -6504,6 +6581,9 @@ packages: tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} + tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + tinyexec@1.0.2: resolution: {integrity: sha512-W/KYk+NFhkmsYpuHq5JykngiOCnxeVL8v8dFnqxSD8qEEdRfXk1SDM6JzNqcERbcGYj9tMrDQBYV9cjgnunFIg==} engines: {node: '>=18'} @@ -6516,10 +6596,22 @@ packages: resolution: {integrity: sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ==} engines: {node: '>=14.0.0'} + tinypool@1.1.1: + resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==} + engines: {node: ^18.0.0 || >=20.0.0} + + tinyrainbow@2.0.0: + resolution: {integrity: sha512-op4nsTR47R6p0vMUUoYl/a+ljLFVtlfaXkLQmqfLR1qHma1h/ysYk4hEXZ880bf2CYgTskvTa/e196Vd5dDQXw==} + engines: {node: '>=14.0.0'} + tinyspy@2.2.1: resolution: {integrity: sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A==} engines: {node: '>=14.0.0'} + tinyspy@4.0.6: + resolution: {integrity: sha512-u8KszXvGfU68hVcZpRHKG28T0krMuv2G5nDhiHaMLen/gIuFEgIJhaJuO69qjnXg5paSrbPMFfx3brNuN8eVSg==} + engines: {node: '>=14.0.0'} + tmp-promise@3.0.3: resolution: {integrity: sha512-RwM7MoPojPxsOBYnyd2hy0bxtIlVrihNs9pj5SUvY8Zz1sQcQG2tG1hSr8PDxfgEB8RNKDhqbIlroIarSNDNsQ==} @@ -6885,6 +6977,11 @@ packages: engines: {node: ^18.0.0 || >=20.0.0} hasBin: true + vite-node@3.2.4: + resolution: {integrity: sha512-EbKSKh+bh1E1IFxeO0pg1n4dvoOTt0UDiXMd/qn++r98+jPO1xtJilvXldeuQ8giIB5IkpjCgMleHMNEsGH6pg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + vite-plugin-dts@3.8.3: resolution: {integrity: sha512-yRHiRosQw7MXdOhmcrVI+kRiB8YEShbSxnADNteK4eZGdEoyOkMHihvO5XOAVlOq8ng9sIqu8vVefDK1zcj3qw==} engines: {node: ^14.18.0 || >=16.0.0} @@ -7088,6 +7185,34 @@ packages: jsdom: optional: true + vitest@3.2.7: + resolution: {integrity: sha512-KrxIJ62Fd89gfysR4WotlgZABiz2dqFPgqGzX7s+CwsqLFomRH7777ZcrOD6+WVAh7khPQP41A+BKbpcJFrdEg==} + engines: {node: ^18.0.0 || ^20.0.0 || >=22.0.0} + hasBin: true + peerDependencies: + '@edge-runtime/vm': '*' + '@types/debug': ^4.1.12 + '@types/node': ^18.0.0 || ^20.0.0 || >=22.0.0 + '@vitest/browser': 3.2.7 + '@vitest/ui': 3.2.7 + happy-dom: '*' + jsdom: '*' + peerDependenciesMeta: + '@edge-runtime/vm': + optional: true + '@types/debug': + optional: true + '@types/node': + optional: true + '@vitest/browser': + optional: true + '@vitest/ui': + optional: true + happy-dom: + optional: true + jsdom: + optional: true + volar-service-css@0.0.66: resolution: {integrity: sha512-XrL1V9LEAHnunglYdDf/7shJbQXqKsHB+P69zPmJTqHx6hqvM9GWNbn2h7M0P/oElW8p/MTVHdfjl6C8cxdsBQ==} peerDependencies: @@ -8561,10 +8686,7 @@ snapshots: '@cloudflare/workerd-windows-64@1.20251109.0': optional: true - '@cloudflare/workers-types@4.20251014.0': {} - - '@cloudflare/workers-types@4.20251118.0': - optional: true + '@cloudflare/workers-types@4.20251118.0': {} '@commander-js/extra-typings@13.1.0(commander@13.1.0)': dependencies: @@ -10075,7 +10197,7 @@ snapshots: '@sveltejs/adapter-cloudflare@7.2.4(@sveltejs/kit@2.48.4(@opentelemetry/api@1.8.0)(@sveltejs/vite-plugin-svelte@6.2.1(svelte@5.43.6)(vite@7.2.2(@types/node@20.19.24)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)))(svelte@5.43.6)(vite@7.2.2(@types/node@20.19.24)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)))(wrangler@4.46.0(@cloudflare/workers-types@4.20251118.0))': dependencies: - '@cloudflare/workers-types': 4.20251014.0 + '@cloudflare/workers-types': 4.20251118.0 '@sveltejs/kit': 2.48.4(@opentelemetry/api@1.8.0)(@sveltejs/vite-plugin-svelte@6.2.1(svelte@5.43.6)(vite@7.2.2(@types/node@20.19.24)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)))(svelte@5.43.6)(vite@7.2.2(@types/node@20.19.24)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)) worktop: 0.8.0-next.18 wrangler: 4.46.0(@cloudflare/workers-types@4.20251118.0) @@ -10233,6 +10355,11 @@ snapshots: '@types/braces@3.0.5': {} + '@types/chai@5.2.3': + dependencies: + '@types/deep-eql': 4.0.2 + assertion-error: 2.0.1 + '@types/cookie@0.6.0': {} '@types/d3-color@3.1.3': {} @@ -10260,6 +10387,8 @@ snapshots: dependencies: '@types/ms': 2.1.0 + '@types/deep-eql@4.0.2': {} + '@types/deno@2.5.0': {} '@types/estree-jsx@1.0.5': @@ -10667,22 +10796,58 @@ snapshots: '@vitest/utils': 1.3.1 chai: 4.5.0 + '@vitest/expect@3.2.7': + dependencies: + '@types/chai': 5.2.3 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + tinyrainbow: 2.0.0 + + '@vitest/mocker@3.2.7(vite@7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1))': + dependencies: + '@vitest/spy': 3.2.7 + estree-walker: 3.0.3 + magic-string: 0.30.21 + optionalDependencies: + vite: 7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1) + + '@vitest/pretty-format@3.2.7': + dependencies: + tinyrainbow: 2.0.0 + '@vitest/runner@1.3.1': dependencies: '@vitest/utils': 1.3.1 p-limit: 5.0.0 pathe: 1.1.2 + '@vitest/runner@3.2.7': + dependencies: + '@vitest/utils': 3.2.7 + pathe: 2.0.3 + strip-literal: 3.1.0 + '@vitest/snapshot@1.3.1': dependencies: magic-string: 0.30.21 pathe: 1.1.2 pretty-format: 29.7.0 + '@vitest/snapshot@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + magic-string: 0.30.21 + pathe: 2.0.3 + '@vitest/spy@1.3.1': dependencies: tinyspy: 2.2.1 + '@vitest/spy@3.2.7': + dependencies: + tinyspy: 4.0.6 + '@vitest/utils@1.3.1': dependencies: diff-sequences: 29.6.3 @@ -10690,6 +10855,12 @@ snapshots: loupe: 2.3.7 pretty-format: 29.7.0 + '@vitest/utils@3.2.7': + dependencies: + '@vitest/pretty-format': 3.2.7 + loupe: 3.2.1 + tinyrainbow: 2.0.0 + '@volar/kit@2.4.23(typescript@5.9.3)': dependencies: '@volar/language-service': 2.4.23 @@ -10771,7 +10942,7 @@ snapshots: de-indent: 1.0.2 he: 1.2.0 - '@vue/language-core@1.8.27(typescript@5.8.3)': + '@vue/language-core@1.8.27(typescript@5.9.3)': dependencies: '@volar/language-core': 1.11.1 '@volar/source-map': 1.11.1 @@ -10783,7 +10954,7 @@ snapshots: path-browserify: 1.0.1 vue-template-compiler: 2.7.16 optionalDependencies: - typescript: 5.8.3 + typescript: 5.9.3 '@vue/language-core@2.2.0(typescript@5.8.3)': dependencies: @@ -10988,6 +11159,8 @@ snapshots: assertion-error@1.1.0: {} + assertion-error@2.0.1: {} + astring@1.9.0: {} astro-d2@0.8.1(astro@5.15.9(@netlify/blobs@10.0.7)(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(rollup@4.53.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(typescript@5.9.3)(yaml@2.8.1)): @@ -11306,6 +11479,14 @@ snapshots: pathval: 1.1.1 type-detect: 4.1.0 + chai@5.3.3: + dependencies: + assertion-error: 2.0.1 + check-error: 2.1.3 + deep-eql: 5.0.2 + loupe: 3.2.1 + pathval: 2.0.1 + chalk@4.1.2: dependencies: ansi-styles: 4.3.0 @@ -11327,6 +11508,8 @@ snapshots: dependencies: get-func-name: 2.0.2 + check-error@2.1.3: {} + chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -11539,6 +11722,8 @@ snapshots: dependencies: type-detect: 4.1.0 + deep-eql@5.0.2: {} + deep-is@0.1.4: {} deepmerge@4.3.1: {} @@ -12009,6 +12194,8 @@ snapshots: exit-hook@2.2.1: {} + expect-type@1.4.0: {} + expressive-code@0.41.3: dependencies: '@expressive-code/core': 0.41.3 @@ -12946,6 +13133,8 @@ snapshots: dependencies: get-func-name: 2.0.2 + loupe@3.2.1: {} + lru-cache@10.4.3: {} lru-cache@5.1.1: @@ -13938,6 +14127,8 @@ snapshots: pathval@1.1.1: {} + pathval@2.0.1: {} + picocolors@1.1.1: {} picomatch@2.3.1: {} @@ -14790,6 +14981,10 @@ snapshots: dependencies: js-tokens: 9.0.1 + strip-literal@3.1.0: + dependencies: + js-tokens: 9.0.1 + strnum@2.1.1: {} strong-log-transformer@2.1.0: @@ -14941,6 +15136,8 @@ snapshots: tinybench@2.9.0: {} + tinyexec@0.3.2: {} + tinyexec@1.0.2: {} tinyglobby@0.2.15: @@ -14950,8 +15147,14 @@ snapshots: tinypool@0.8.4: {} + tinypool@1.1.1: {} + + tinyrainbow@2.0.0: {} + tinyspy@2.2.1: {} + tinyspy@4.0.6: {} + tmp-promise@3.0.3: dependencies: tmp: 0.2.5 @@ -15307,16 +15510,37 @@ snapshots: - supports-color - terser - vite-plugin-dts@3.8.3(@types/node@22.19.1)(rollup@4.53.2)(typescript@5.8.3)(vite@7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)): + vite-node@3.2.4(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1): + dependencies: + cac: 6.7.14 + debug: 4.4.3 + es-module-lexer: 1.7.0 + pathe: 2.0.3 + vite: 7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1) + transitivePeerDependencies: + - '@types/node' + - jiti + - less + - lightningcss + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + + vite-plugin-dts@3.8.3(@types/node@22.19.1)(rollup@4.53.2)(typescript@5.9.3)(vite@7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)): dependencies: '@microsoft/api-extractor': 7.43.0(@types/node@22.19.1) '@rollup/pluginutils': 5.3.0(rollup@4.53.2) - '@vue/language-core': 1.8.27(typescript@5.8.3) + '@vue/language-core': 1.8.27(typescript@5.9.3) debug: 4.4.3 kolorist: 1.8.0 magic-string: 0.30.21 - typescript: 5.8.3 - vue-tsc: 1.8.27(typescript@5.8.3) + typescript: 5.9.3 + vue-tsc: 1.8.27(typescript@5.9.3) optionalDependencies: vite: 7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1) transitivePeerDependencies: @@ -15456,7 +15680,6 @@ snapshots: terser: 5.43.1 tsx: 4.20.6 yaml: 2.8.1 - optional: true vitefu@1.1.1(vite@6.4.1(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)): optionalDependencies: @@ -15536,6 +15759,49 @@ snapshots: - supports-color - terser + vitest@3.2.7(@types/debug@4.1.12)(@types/node@22.19.1)(jiti@2.6.1)(jsdom@22.1.0)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1): + dependencies: + '@types/chai': 5.2.3 + '@vitest/expect': 3.2.7 + '@vitest/mocker': 3.2.7(vite@7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1)) + '@vitest/pretty-format': 3.2.7 + '@vitest/runner': 3.2.7 + '@vitest/snapshot': 3.2.7 + '@vitest/spy': 3.2.7 + '@vitest/utils': 3.2.7 + chai: 5.3.3 + debug: 4.4.3 + expect-type: 1.4.0 + magic-string: 0.30.21 + pathe: 2.0.3 + picomatch: 4.0.3 + std-env: 3.10.0 + tinybench: 2.9.0 + tinyexec: 0.3.2 + tinyglobby: 0.2.15 + tinypool: 1.1.1 + tinyrainbow: 2.0.0 + vite: 7.2.2(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1) + vite-node: 3.2.4(@types/node@22.19.1)(jiti@2.6.1)(less@4.1.3)(lightningcss@1.30.2)(sass-embedded@1.93.3)(sass@1.94.1)(stylus@0.64.0)(terser@5.43.1)(tsx@4.20.6)(yaml@2.8.1) + why-is-node-running: 2.3.0 + optionalDependencies: + '@types/debug': 4.1.12 + '@types/node': 22.19.1 + jsdom: 22.1.0 + transitivePeerDependencies: + - jiti + - less + - lightningcss + - msw + - sass + - sass-embedded + - stylus + - sugarss + - supports-color + - terser + - tsx + - yaml + volar-service-css@0.0.66(@volar/language-service@2.4.23): dependencies: vscode-css-languageservice: 6.3.8 @@ -15638,12 +15904,12 @@ snapshots: de-indent: 1.0.2 he: 1.2.0 - vue-tsc@1.8.27(typescript@5.8.3): + vue-tsc@1.8.27(typescript@5.9.3): dependencies: '@volar/typescript': 1.11.1 - '@vue/language-core': 1.8.27(typescript@5.8.3) + '@vue/language-core': 1.8.27(typescript@5.9.3) semver: 7.7.3 - typescript: 5.8.3 + typescript: 5.9.3 w3c-xmlserializer@4.0.0: dependencies: diff --git a/tsconfig.base.json b/tsconfig.base.json index a5c22f971..51b6b4237 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -25,7 +25,7 @@ "noUnusedLocals": true, "pretty": true, "removeComments": false, - "resolveJsonModule": false, + "resolveJsonModule": true, "skipDefaultLibCheck": false, "skipLibCheck": true, "sourceMap": false, From 4d10e7fa6bd2f04e9a8333d3570ddf60892dabbf Mon Sep 17 00:00:00 2001 From: Agent Date: Fri, 18 Sep 2026 15:50:49 +0000 Subject: [PATCH 2/2] fix(telemetry-worker): override inferred typecheck with tsc --noEmit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI build-and-test failed on @pgflow/telemetry-worker:typecheck with TS5069: the @nx/js plugin infers 'tsc --build --emitDeclarationOnly' for every root-tsconfig project, but this wrangler-deployed worker's tsconfig sets noEmit and never emits declarations. A package.json 'typecheck' script overrides the inferred target with tsc --noEmit — the same per-project override pattern pkgs/client uses (project.json noEmit target) instead of touching nx.json. Verified: focused typecheck exit=0; nx affected -t lint typecheck test green except the known unrelated demo Groq tests (sandbox GROQ_API_KEY, decommissioned llama-3.1-8b-instant). --- apps/telemetry-worker/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/telemetry-worker/package.json b/apps/telemetry-worker/package.json index 2b4eab28f..8ef8f8f3e 100644 --- a/apps/telemetry-worker/package.json +++ b/apps/telemetry-worker/package.json @@ -3,6 +3,7 @@ "private": true, "type": "module", "scripts": { + "typecheck": "tsc --noEmit", "test": "vitest run", "deploy": "wrangler deploy" },