From 1da45ff25e17b7caa25b5a039423fd83a7bf37a3 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 21:35:37 -0700 Subject: [PATCH 01/14] feat(setup): add privacy-scoped PostHog funnel telemetry --- .github/workflows/release-setup-sourcebot.yml | 17 +- .github/workflows/setup-wizard-e2e.yml | 57 + docs/docs/misc/telemetry.mdx | 24 +- packages/setupWizard/README.md | 37 +- packages/setupWizard/bin.cjs | 10 + packages/setupWizard/package.json | 22 +- packages/setupWizard/src/azuredevops.ts | 24 +- packages/setupWizard/src/bitbucket.ts | 70 +- packages/setupWizard/src/docker.ts | 196 ++ packages/setupWizard/src/genericGit.ts | 13 +- packages/setupWizard/src/gerrit.ts | 14 +- packages/setupWizard/src/gitea.ts | 22 +- packages/setupWizard/src/github.ts | 96 +- packages/setupWizard/src/gitlab.ts | 135 +- packages/setupWizard/src/index.ts | 608 +++-- packages/setupWizard/src/lifecycle.ts | 211 ++ packages/setupWizard/src/localRepos.ts | 99 +- packages/setupWizard/src/models.ts | 145 +- packages/setupWizard/src/prompts.ts | 37 + packages/setupWizard/src/spinner.ts | 9 + packages/setupWizard/src/telemetry.ts | 182 ++ packages/setupWizard/src/telemetryEvents.ts | 332 +++ packages/setupWizard/src/telemetrySummary.ts | 228 ++ packages/setupWizard/src/utils.ts | 4 +- packages/setupWizard/telemetry.html | 2090 +++++++++++++++++ .../setupWizard/tests/approvedSchema.json | 599 +++++ packages/setupWizard/tests/e2e/baseline.mjs | 53 + .../setupWizard/tests/e2e/collectors.test.mjs | 223 ++ .../setupWizard/tests/e2e/docker.test.mjs | 142 ++ packages/setupWizard/tests/e2e/fakeDocker.cjs | 31 + packages/setupWizard/tests/e2e/harness.mjs | 326 +++ packages/setupWizard/tests/e2e/linux.mjs | 27 + packages/setupWizard/tests/e2e/liveSmoke.mjs | 14 + packages/setupWizard/tests/e2e/network.mjs | 22 + .../setupWizard/tests/e2e/packageManagers.mjs | 57 + .../setupWizard/tests/e2e/platform.test.mjs | 53 + .../setupWizard/tests/e2e/runtime.test.mjs | 92 + .../setupWizard/tests/e2e/runtimeFixture.mjs | 37 + .../setupWizard/tests/e2e/safety.test.mjs | 149 ++ .../setupWizard/tests/e2e/unsupportedNode.mjs | 19 + .../setupWizard/tests/e2e/wizard.test.mjs | 143 ++ .../tests/integration/sdk.test.mjs | 47 + packages/setupWizard/tests/schemaSnapshot.mjs | 56 + .../setupWizard/tests/unit/schema.test.mjs | 9 + .../setupWizard/tests/unit/telemetry.test.mjs | 165 ++ yarn.lock | 79 +- 46 files changed, 6569 insertions(+), 456 deletions(-) create mode 100644 .github/workflows/setup-wizard-e2e.yml create mode 100755 packages/setupWizard/bin.cjs create mode 100644 packages/setupWizard/src/docker.ts create mode 100644 packages/setupWizard/src/lifecycle.ts create mode 100644 packages/setupWizard/src/prompts.ts create mode 100644 packages/setupWizard/src/spinner.ts create mode 100644 packages/setupWizard/src/telemetry.ts create mode 100644 packages/setupWizard/src/telemetryEvents.ts create mode 100644 packages/setupWizard/src/telemetrySummary.ts create mode 100644 packages/setupWizard/telemetry.html create mode 100644 packages/setupWizard/tests/approvedSchema.json create mode 100644 packages/setupWizard/tests/e2e/baseline.mjs create mode 100644 packages/setupWizard/tests/e2e/collectors.test.mjs create mode 100644 packages/setupWizard/tests/e2e/docker.test.mjs create mode 100644 packages/setupWizard/tests/e2e/fakeDocker.cjs create mode 100644 packages/setupWizard/tests/e2e/harness.mjs create mode 100644 packages/setupWizard/tests/e2e/linux.mjs create mode 100644 packages/setupWizard/tests/e2e/liveSmoke.mjs create mode 100644 packages/setupWizard/tests/e2e/network.mjs create mode 100644 packages/setupWizard/tests/e2e/packageManagers.mjs create mode 100644 packages/setupWizard/tests/e2e/platform.test.mjs create mode 100644 packages/setupWizard/tests/e2e/runtime.test.mjs create mode 100644 packages/setupWizard/tests/e2e/runtimeFixture.mjs create mode 100644 packages/setupWizard/tests/e2e/safety.test.mjs create mode 100644 packages/setupWizard/tests/e2e/unsupportedNode.mjs create mode 100644 packages/setupWizard/tests/e2e/wizard.test.mjs create mode 100644 packages/setupWizard/tests/integration/sdk.test.mjs create mode 100644 packages/setupWizard/tests/schemaSnapshot.mjs create mode 100644 packages/setupWizard/tests/unit/schema.test.mjs create mode 100644 packages/setupWizard/tests/unit/telemetry.test.mjs diff --git a/.github/workflows/release-setup-sourcebot.yml b/.github/workflows/release-setup-sourcebot.yml index 657f2b25e..484fe5926 100644 --- a/.github/workflows/release-setup-sourcebot.yml +++ b/.github/workflows/release-setup-sourcebot.yml @@ -60,7 +60,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '20.x' + node-version: '24.x' - name: Install dependencies working-directory: . @@ -103,11 +103,21 @@ jobs: # publishing. yarn pack --out /tmp/setup-sourcebot.tgz + - name: Verify the exact publish artifact + working-directory: . + env: + PACKAGE_TRACKER_ANALYTICS: 'false' + SETUP_TEST_TARBALL: /tmp/setup-sourcebot.tgz + run: | + docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest + yarn workspace setup-sourcebot test + yarn workspace setup-sourcebot test:e2e + yarn workspace setup-sourcebot test:node22 + - name: Upgrade npm for Trusted Publishing working-directory: . run: | - # OIDC Trusted Publishing requires npm >= 11.5.1; Node 20 ships an - # older npm. + # Keep npm current for OIDC Trusted Publishing. npm install -g npm@latest npm --version @@ -132,4 +142,3 @@ jobs: git tag -a "setup-sourcebot-v$VERSION" -m "setup-sourcebot v$VERSION" git push origin HEAD:main git push origin "setup-sourcebot-v$VERSION" - diff --git a/.github/workflows/setup-wizard-e2e.yml b/.github/workflows/setup-wizard-e2e.yml new file mode 100644 index 000000000..5cf589b45 --- /dev/null +++ b/.github/workflows/setup-wizard-e2e.yml @@ -0,0 +1,57 @@ +name: Setup wizard verification + +on: + pull_request: + paths: + - 'packages/setupWizard/**' + - 'packages/schemas/**' + - 'entrypoint.sh' + - 'yarn.lock' + - '.github/workflows/setup-wizard-e2e.yml' + - '.github/workflows/release-setup-sourcebot.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + platform: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + env: + PACKAGE_TRACKER_ANALYTICS: 'false' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: '24.x' + - run: corepack enable + - run: yarn install --immutable --mode=skip-build + - run: yarn rebuild node-pty + - run: yarn workspace @sourcebot/schemas build + - run: yarn workspace setup-sourcebot build + - run: yarn workspace setup-sourcebot test + - run: yarn workspace setup-sourcebot test:platform + - name: Linux packed-artifact and runtime checks + if: runner.os == 'Linux' + run: | + docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest + yarn workspace setup-sourcebot test:e2e + yarn workspace setup-sourcebot test:node22 + yarn workspace setup-sourcebot test:baseline + node packages/setupWizard/tests/e2e/packageManagers.mjs + + setup-wizard-e2e: + if: always() + needs: [platform] + runs-on: ubuntu-latest + steps: + - name: Require every platform job + env: + RESULT: ${{ needs.platform.result }} + run: test "$RESULT" = success diff --git a/docs/docs/misc/telemetry.mdx b/docs/docs/misc/telemetry.mdx index 4a77674dd..47ee445b8 100644 --- a/docs/docs/misc/telemetry.mdx +++ b/docs/docs/misc/telemetry.mdx @@ -6,7 +6,7 @@ By default, Sourcebot collects anonymized usage data through [PostHog](https://p The data we collect includes general usage statistics and metadata such as query performance (e.g., search duration, error rates) to monitor the application's health and functionality. This information helps us better understand how Sourcebot is used and where improvements can be made. -If you'd like to disable all telemetry, you can do so by setting the environment variable `SOURCEBOT_TELEMETRY_DISABLED` to `true`: +If you'd like to disable telemetry from the deployed Sourcebot application, you can do so by setting the environment variable `SOURCEBOT_TELEMETRY_DISABLED` to `true`: ```bash docker run \ @@ -20,3 +20,25 @@ If you disabled telemetry correctly, you'll see the following log when starting ```sh Disabling telemetry since SOURCEBOT_TELEMETRY_DISABLED was set. ``` + +## Setup wizard + +The `setup-sourcebot` npm wizard separately sends high-level progress events to +PostHog. These include selected code-host and AI-provider types, configuration +counts, coarse OS/architecture/Node information, and completion, cancellation, +or failure categories. The wizard does not send credentials, emails, repository +or model names, URLs, hostnames, local paths, configuration contents, or raw errors. +GeoIP enrichment is disabled. + +Each invocation generates a random UUID in memory, used as its PostHog identity. +For a new deployment, the same ID is saved as `SOURCEBOT_INSTALL_ID` in the existing +`.env` file, allowing setup and deployment events to be associated. Valid existing +deployment IDs are preserved without sending them in the new setup session. +PostHog's default Person processing is retained, but no real-world identity or +custom person properties are attached. No telemetry-only file or disk queue is created. + +Setup-wizard PostHog analytics has no opt-out and is independent of the deployed +application's `SOURCEBOT_TELEMETRY_DISABLED` setting. Delivery is best-effort; +telemetry failures do not prevent setup or keep the CLI running indefinitely. +The npm package's existing Reo installation tracking remains independent and +continues to use its own `PACKAGE_TRACKER_ANALYTICS` control. diff --git a/packages/setupWizard/README.md b/packages/setupWizard/README.md index 4224b2fc2..796483b5c 100644 --- a/packages/setupWizard/README.md +++ b/packages/setupWizard/README.md @@ -17,9 +17,44 @@ The wizard walks you through: ## Requirements -- Node.js 18+ +- Node.js 24+ - Docker and Docker Compose +## Setup analytics + +The wizard sends high-level setup progress to Sourcebot's PostHog project: selected +code-host/provider types, counts, coarse system properties, and setup outcomes. +It does not send access tokens, repository/model names, email addresses, URLs, +hostnames, local paths, or raw errors. GeoIP enrichment is disabled. + +Each invocation creates a random UUID in memory. New deployments receive that +same UUID as `SOURCEBOT_INSTALL_ID` in the existing `.env` file; a valid existing +ID is preserved. No telemetry state, identifier file, or disk queue is created. +Analytics failures do not prevent setup. The package's existing Reo installation +tracking remains separate; its `PACKAGE_TRACKER_ANALYTICS` setting and the +deployment's `SOURCEBOT_TELEMETRY_DISABLED` setting do not control wizard analytics. + +Ctrl+C exits with status 130. Once foreground Docker has spawned, setup is recorded +as completed; interrupting it cleans up the CLI without changing that setup outcome. +Completion means configuration handoff, not that Sourcebot is healthy or ready. + +## Development tests + +From the repository root, under Node 24: + +```bash +yarn workspace @sourcebot/schemas build +yarn workspace setup-sourcebot build +yarn workspace setup-sourcebot test +yarn workspace setup-sourcebot test:e2e +``` + +The E2E tests compile and pack the package, install it outside the repository, +drive its published binary in a PTY, inspect real SDK requests through a local TLS +collector, and clean up temporary installations. OpenSSL and Docker are required. +The runtime suite uses `docker.sourcebot.dev/sourcebot-dev/sourcebot:latest` +(override only the test image with `SETUP_TEST_SOURCEBOT_IMAGE`). + ## Docs Full deployment guide: [docs.sourcebot.dev/docs/deployment/docker-compose](https://docs.sourcebot.dev/docs/deployment/docker-compose) diff --git a/packages/setupWizard/bin.cjs b/packages/setupWizard/bin.cjs new file mode 100755 index 000000000..9c5cd3791 --- /dev/null +++ b/packages/setupWizard/bin.cjs @@ -0,0 +1,10 @@ +#!/usr/bin/env node +if (Number(process.versions.node.split('.')[0]) < 24) { + console.error('setup-sourcebot requires Node.js 24 or newer. Please upgrade Node.js.'); + process.exitCode = 1; +} else { + import('./dist/index.js').catch(() => { + console.error('Unable to start setup-sourcebot. Please reinstall the package.'); + process.exitCode = 1; + }); +} diff --git a/packages/setupWizard/package.json b/packages/setupWizard/package.json index 464d21b53..201a5cc94 100644 --- a/packages/setupWizard/package.json +++ b/packages/setupWizard/package.json @@ -8,31 +8,43 @@ "url": "git+https://github.com/sourcebot-dev/sourcebot.git", "directory": "packages/setupWizard" }, - "bin": "./dist/index.js", + "bin": "./bin.cjs", "scripts": { "build": "tsc", "watch": "tsc --watch", "dev": "tsx src/index.ts", - "prepublishOnly": "yarn build" + "prepublishOnly": "yarn build", + "test": "node --test tests/unit/*.test.mjs tests/integration/*.test.mjs", + "test:e2e": "node --test --test-concurrency=1 tests/e2e/*.test.mjs", + "test:platform": "node --test tests/e2e/platform.test.mjs", + "test:linux": "node tests/e2e/linux.mjs", + "test:runtime": "node --test tests/e2e/runtime.test.mjs", + "test:node22": "node tests/e2e/unsupportedNode.mjs", + "test:baseline": "node tests/e2e/baseline.mjs", + "test:live": "node tests/e2e/liveSmoke.mjs" }, "dependencies": { "@inquirer/prompts": "^8.4.3", "chalk": "^5.6.2", "inquirer-select-pro": "^1.0.0-alpha.9", "ora": "^9.4.0", + "posthog-node": "5.52.1", "reo-census": "^1.2.10" }, "devDependencies": { "@sourcebot/schemas": "workspace:^", - "@types/node": "^22.7.5", + "@types/node": "^24.0.0", + "node-pty": "^1.1.0", "tsx": "^4.21.0", - "typescript": "^5.6.2" + "typescript": "^5.6.2", + "undici": "^7" }, "engines": { - "node": ">=18" + "node": ">=24.0.0" }, "files": [ "dist", + "bin.cjs", "README.md" ] } diff --git a/packages/setupWizard/src/azuredevops.ts b/packages/setupWizard/src/azuredevops.ts index fead965c9..b60356919 100644 --- a/packages/setupWizard/src/azuredevops.ts +++ b/packages/setupWizard/src/azuredevops.ts @@ -1,5 +1,6 @@ -import { confirm, input, password, select } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { sourceSummary } from './telemetrySummary.js'; +import { confirm, input, password, select } from './prompts.js'; +import { checkbox } from './prompts.js'; import type { AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/azuredevops.type'; import type { CollectResult, EnvVars } from './utils.js'; import { multiInput, note, toEnvKey } from './utils.js'; @@ -60,7 +61,7 @@ export async function collectAzureDevOpsConfig(connectionName: string): Promise< const token = await password({ message: `Azure DevOps Personal Access Token (stored locally in .env as ${envKey})`, mask: true, - validate: (v) => !v?.trim() ? 'Token is required' : true, + validate: (v) => (!v?.trim() ? 'Token is required' : true), }); env[envKey] = token; config.token = { env: envKey }; @@ -96,5 +97,20 @@ export async function collectAzureDevOpsConfig(connectionName: string): Promise< }); } - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('azure_devops', { + deploymentType: deploymentType === 'cloud' ? 'cloud' : 'self_hosted', + credentialMode: 'personal_access_token', + scopeTypes: [ + ...(targets.includes('orgs') ? ['organizations' as const] : []), + ...(targets.includes('projects') ? ['projects' as const] : []), + ...(targets.includes('repos') ? ['repositories' as const] : []), + ], + organizationCount: config.orgs?.length ?? 0, + projectCount: config.projects?.length ?? 0, + repositoryCount: config.repos?.length ?? 0, + }), + }; } diff --git a/packages/setupWizard/src/bitbucket.ts b/packages/setupWizard/src/bitbucket.ts index 4e547fc83..def0bf02a 100644 --- a/packages/setupWizard/src/bitbucket.ts +++ b/packages/setupWizard/src/bitbucket.ts @@ -1,5 +1,6 @@ -import { confirm, input, password, select } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { sourceSummary } from './telemetrySummary.js'; +import { confirm, input, password, select } from './prompts.js'; +import { checkbox } from './prompts.js'; import type { BitbucketConnectionConfig } from '@sourcebot/schemas/v3/bitbucket.type'; import type { CollectResult, EnvVars } from './utils.js'; import { multiInput, note, toEnvKey } from './utils.js'; @@ -41,14 +42,11 @@ async function collectBitbucketCloud( }); if (authMethod === 'api-token') { - note( - 'The email you use to sign in to Atlassian (e.g. you@example.com).', - 'Atlassian account email', - ); + note('The email you use to sign in to Atlassian (e.g. you@example.com).', 'Atlassian account email'); const email = await input({ message: 'Atlassian account email', - validate: (v) => !v?.trim() ? 'Email is required' : true, + validate: (v) => (!v?.trim() ? 'Email is required' : true), }); config.user = email; @@ -62,7 +60,7 @@ async function collectBitbucketCloud( const gitUser = await input({ message: 'Bitbucket username', - validate: (v) => !v?.trim() ? 'Username is required' : true, + validate: (v) => (!v?.trim() ? 'Username is required' : true), }); config.gitUser = gitUser; @@ -81,7 +79,7 @@ async function collectBitbucketCloud( const token = await password({ message: `API Token (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => !v?.trim() ? 'Token is required' : true, + validate: (v) => (!v?.trim() ? 'Token is required' : true), }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; @@ -98,7 +96,7 @@ async function collectBitbucketCloud( const token = await password({ message: `Access Token (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => !v?.trim() ? 'Token is required' : true, + validate: (v) => (!v?.trim() ? 'Token is required' : true), }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; @@ -116,7 +114,7 @@ async function collectBitbucketCloud( const username = await input({ message: 'Bitbucket username', - validate: (v) => !v?.trim() ? 'Username is required' : true, + validate: (v) => (!v?.trim() ? 'Username is required' : true), }); config.user = username; @@ -124,7 +122,7 @@ async function collectBitbucketCloud( const token = await password({ message: `Bitbucket App Password (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => !v?.trim() ? 'App Password is required' : true, + validate: (v) => (!v?.trim() ? 'App Password is required' : true), }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; @@ -151,7 +149,25 @@ async function collectBitbucketCloud( }); } - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('bitbucket', { + deploymentType: 'cloud', + credentialMode: + authMethod === 'api-token' + ? 'api_token' + : authMethod === 'access-token' + ? 'access_token' + : 'app_password', + scopeTypes: [ + ...(targets.includes('workspaces') ? ['workspaces' as const] : []), + ...(targets.includes('repos') ? ['repositories' as const] : []), + ], + workspaceCount: config.workspaces?.length ?? 0, + repositoryCount: config.repos?.length ?? 0, + }), + }; } async function collectBitbucketServer( @@ -196,7 +212,7 @@ async function collectBitbucketServer( const token = await password({ message: `Bitbucket HTTP Access Token (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => !v?.trim() ? 'Token is required' : true, + validate: (v) => (!v?.trim() ? 'Token is required' : true), }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; @@ -208,7 +224,16 @@ async function collectBitbucketServer( if (indexAll) { config.all = true; - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('bitbucket', { + deploymentType: 'self_hosted', + credentialMode: 'http_access_token', + indexAll: true, + scopeTypes: ['all'], + }), + }; } const targets = await checkbox({ @@ -232,5 +257,18 @@ async function collectBitbucketServer( }); } - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('bitbucket', { + deploymentType: 'self_hosted', + credentialMode: 'http_access_token', + scopeTypes: [ + ...(targets.includes('projects') ? ['projects' as const] : []), + ...(targets.includes('repos') ? ['repositories' as const] : []), + ], + projectCount: config.projects?.length ?? 0, + repositoryCount: config.repos?.length ?? 0, + }), + }; } diff --git a/packages/setupWizard/src/docker.ts b/packages/setupWizard/src/docker.ts new file mode 100644 index 000000000..4b41b6961 --- /dev/null +++ b/packages/setupWizard/src/docker.ts @@ -0,0 +1,196 @@ +import { spawn } from 'node:child_process'; +import { lifecycle } from './lifecycle.js'; +import type { FailureCategory } from './telemetryEvents.js'; + +export type DockerResult = + | { ok: true; value: T } + | { ok: false; failureCategory: FailureCategory }; +export type ComposeContainer = { Name: string; Service: string; State: string }; +export class Docker { + status: 'available' | 'unavailable' | 'error' | 'not_checked' = + 'not_checked'; + failed = false; + constructor( + private readonly onFailure: (category: FailureCategory) => void, + ) {} + private failure(failureCategory: FailureCategory): DockerResult { + lifecycle.check(); + this.failed = true; + this.onFailure(failureCategory); + return { ok: false, failureCategory }; + } + private async execute(args: string[]) { + lifecycle.check(); + const result = await new Promise<{ + code: number | null; + stdout: string; + missing: boolean; + stderr: string; + }>((resolve) => { + const child = lifecycle.child( + spawn('docker', args, { + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + }), + ); + let stdout = ''; + let stderr = ''; + child.stdout?.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr?.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.once('error', (error: NodeJS.ErrnoException) => + resolve({ + code: null, + stdout: '', + stderr: '', + missing: ['ENOENT', 'EACCES'].includes(error.code ?? ''), + }), + ); + child.once('close', (code) => + resolve({ code, stdout, stderr, missing: false }), + ); + }); + lifecycle.check(); + return result; + } + async run(args: string[]): Promise> { + const result = await this.execute(args); + if (result.code !== 0) { + if (result.missing) { + this.status = 'unavailable'; + } else if ( + this.status !== 'available' && + this.status !== 'unavailable' + ) { + if (result.code === null) { + this.status = 'error'; + } else { + // Diagnose availability using fixed commands, never by parsing user stderr. + // These probes classify this single failed operation; they do not emit more failures. + const engine = await this.execute([ + 'info', + '--format', + '{{.ServerVersion}}', + ]); + const compose = + args[0] === 'compose' && engine.code === 0 + ? await this.execute([ + 'compose', + 'version', + '--short', + ]) + : undefined; + this.status = + engine.code === 0 && (!compose || compose.code === 0) + ? 'available' + : 'unavailable'; + } + } + // Human diagnostics stay local; never use stderr for analytics classification. + if ( + ['stop', 'rm'].includes(args[0]) || + ['down', 'rm'].includes(args[1]) + ) { + if (result.stderr.trim()) { + console.error(result.stderr.trim()); + } + } + return this.failure( + this.status === 'unavailable' + ? 'docker_unavailable' + : 'docker_command', + ); + } + // A working volume command does not repair a known missing Compose plugin. + if (this.status !== 'unavailable') { + this.status = 'available'; + } + return { ok: true, value: result.stdout }; + } + async containers(): Promise> { + const result = await this.run([ + 'compose', + 'ps', + '-a', + '--format', + 'json', + ]); + if (!result.ok) { + return result; + } + try { + const text = result.value.trim(); + const items: unknown = !text + ? [] + : text.startsWith('[') + ? JSON.parse(text) + : text + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); + if ( + !Array.isArray(items) || + items.some( + (c) => + !c || + typeof c.Name !== 'string' || + typeof c.Service !== 'string' || + typeof c.State !== 'string', + ) + ) { + throw new Error('Invalid container inventory'); + } + return { ok: true, value: items }; + } catch { + return this.failure('docker_command'); + } + } + async volumes(expected: string[]): Promise> { + if (!expected.length) { + return { ok: true, value: [] }; + } + const result = await this.run([ + 'volume', + 'ls', + '--format', + '{{.Name}}', + ]); + if (!result.ok) { + return result; + } + const names = new Set( + result.value + .split('\n') + .map((s) => s.trim()) + .filter(Boolean), + ); + return { ok: true, value: expected.filter((name) => names.has(name)) }; + } + async portOwners(): Promise>> { + const result = await this.run([ + 'ps', + '--format', + '{{.Names}}\t{{.Ports}}', + ]); + if (!result.ok) { + return result; + } + const owners = new Map(); + for (const line of result.value.split('\n').filter(Boolean)) { + const [name, ports] = line.split('\t'); + if (ports === undefined) { + return this.failure('docker_command'); + } + for (const match of ports.matchAll(/(\d+)->/g)) { + const port = Number(match[1]); + owners.set(port, [ + ...new Set([...(owners.get(port) ?? []), name]), + ]); + } + } + return { ok: true, value: owners }; + } +} diff --git a/packages/setupWizard/src/genericGit.ts b/packages/setupWizard/src/genericGit.ts index 5f636220e..f5deabe01 100644 --- a/packages/setupWizard/src/genericGit.ts +++ b/packages/setupWizard/src/genericGit.ts @@ -1,4 +1,5 @@ -import { input } from '@inquirer/prompts'; +import { sourceSummary } from './telemetrySummary.js'; +import { input } from './prompts.js'; import type { GenericGitHostConnectionConfig } from '@sourcebot/schemas/v3/genericGitHost.type'; import type { CollectResult } from './utils.js'; @@ -21,5 +22,13 @@ export async function collectGenericGitConfig(): Promise { url, }; - return { connections: [{ config }], env: {} }; + return { + connections: [{ config }], + env: {}, + telemetry: sourceSummary('remote_git', { + deploymentType: 'remote', + repositoryCount: 1, + scopeTypes: ['repositories'], + }), + }; } diff --git a/packages/setupWizard/src/gerrit.ts b/packages/setupWizard/src/gerrit.ts index 51ccd97e0..873fb5ed6 100644 --- a/packages/setupWizard/src/gerrit.ts +++ b/packages/setupWizard/src/gerrit.ts @@ -1,4 +1,5 @@ -import { confirm, input } from '@inquirer/prompts'; +import { sourceSummary } from './telemetrySummary.js'; +import { confirm, input } from './prompts.js'; import type { GerritConnectionConfig } from '@sourcebot/schemas/v3/gerrit.type'; import type { CollectResult } from './utils.js'; import { multiInput } from './utils.js'; @@ -33,5 +34,14 @@ export async function collectGerritConfig(): Promise { }); } - return { connections: [{ config }], env: {} }; + return { + connections: [{ config }], + env: {}, + telemetry: sourceSummary('gerrit', { + deploymentType: 'self_hosted', + indexAll, + scopeTypes: indexAll ? ['all'] : ['projects'], + projectCount: config.projects?.length ?? 0, + }), + }; } diff --git a/packages/setupWizard/src/gitea.ts b/packages/setupWizard/src/gitea.ts index 050a55e51..c2f56f267 100644 --- a/packages/setupWizard/src/gitea.ts +++ b/packages/setupWizard/src/gitea.ts @@ -1,5 +1,6 @@ -import { input, password } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { sourceSummary, deployment } from './telemetrySummary.js'; +import { input, password } from './prompts.js'; +import { checkbox } from './prompts.js'; import type { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; import type { CollectResult, EnvVars } from './utils.js'; import { INPUT_THEME, multiInput, toEnvKey } from './utils.js'; @@ -64,5 +65,20 @@ export async function collectGiteaConfig(connectionName: string): Promise }; - - const literalFallback = (): SearchOption | null => { - return { name: query, value: query }; - }; + const url = + type === 'repo' + ? `${apiBase}/search/repositories?q=${encodeURIComponent(query)}&per_page=8` + : `${apiBase}/search/users?q=${encodeURIComponent(query)}+type:${type}&per_page=8`; + try { + const res = await wizardFetch(url, { headers, signal: AbortSignal.timeout(8000) }); + const data = (await res.json()) as { items?: Array<{ login?: string; full_name?: string }> }; - if (!res.ok) { - const warning = - (res.status === 403 && res.headers.get('x-ratelimit-remaining') === '0') - ? '⚠ Autocomplete disabled — GitHub rate limit exceeded.' - : '⚠ Autocomplete disabled — authentication failed, check your PAT.'; - const fallback = literalFallback(); - return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; - } + const literalFallback = (): SearchOption | null => { + return { name: query, value: query }; + }; - const results: SearchOption[] = (data.items ?? []).map((item) => { - const value = type === 'repo' ? item.full_name! : item.login!; - return { name: value, value }; - }); - if (results.length === 0) { - const fallback = literalFallback(); - return fallback ? [fallback] : []; + if (!res.ok) { + lifecycle.fail('network', true); + const warning = + res.status === 403 && res.headers.get('x-ratelimit-remaining') === '0' + ? '⚠ Autocomplete disabled — GitHub rate limit exceeded.' + : '⚠ Autocomplete disabled — authentication failed, check your PAT.'; + const fallback = literalFallback(); + return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; + } + + if ( + !data || + (data.items !== undefined && + (!Array.isArray(data.items) || + data.items.some((item) => typeof (type === 'repo' ? item?.full_name : item?.login) !== 'string'))) + ) { + throw new Error('Invalid search response'); + } + const results: SearchOption[] = (data.items ?? []).map((item) => { + const value = type === 'repo' ? item.full_name! : item.login!; + return { name: value, value }; + }); + if (results.length === 0) { + const fallback = literalFallback(); + return fallback ? [fallback] : []; + } + githubSearchCache.set(cacheKey, results); + return results; + } catch { + lifecycle.check(); + lifecycle.fail('network', true); + const fallback = true; + const warning = new Separator('⚠ Autocomplete unavailable — enter a value manually.'); + return fallback ? [{ name: query, value: query }, warning] : [warning]; } - githubSearchCache.set(cacheKey, results); - return results; } export async function collectGitHubConfig(connectionName: string): Promise { @@ -218,5 +239,20 @@ export async function collectGitHubConfig(connectionName: string): Promise { + if (type === 'project') { + return PROJECT_PATTERN.test(query) ? { name: query, value: query } : null; + } + return { name: query, value: query }; + }; - const literalFallback = (): SearchOption | null => { - if (type === 'project') { - return PROJECT_PATTERN.test(query) ? { name: query, value: query } : null; + if (!res.ok) { + lifecycle.fail('network', true); + const warning = + res.status === 401 + ? '⚠ Autocomplete disabled — authentication failed, check your PAT.' + : `⚠ Autocomplete disabled — GitLab API error (${res.status}).`; + const fallback = literalFallback(); + return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; } - return { name: query, value: query }; - }; - if (!res.ok) { - const warning = res.status === 401 - ? '⚠ Autocomplete disabled — authentication failed, check your PAT.' - : `⚠ Autocomplete disabled — GitLab API error (${res.status}).`; - const fallback = literalFallback(); - return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; - } + const data = (await res.json()) as Array<{ + full_path?: string; + path_with_namespace?: string; + username?: string; + }>; - const data = await res.json() as Array<{ - full_path?: string; - path_with_namespace?: string; - username?: string; - }>; - - const results: SearchOption[] = data.map((item) => { - let value: string; - if (type === 'group') { - value = item.full_path!; - } else if (type === 'project') { - value = item.path_with_namespace!; - } else { - value = item.username!; + if ( + !Array.isArray(data) || + data.some( + (item) => + typeof (type === 'group' + ? item?.full_path + : type === 'project' + ? item?.path_with_namespace + : item?.username) !== 'string', + ) + ) { + throw new Error('Invalid search response'); } - return { name: value, value }; - }); + const results: SearchOption[] = data.map((item) => { + let value: string; + if (type === 'group') { + value = item.full_path!; + } else if (type === 'project') { + value = item.path_with_namespace!; + } else { + value = item.username!; + } + return { name: value, value }; + }); - if (results.length === 0) { - const fallback = literalFallback(); - return fallback ? [fallback] : []; - } + if (results.length === 0) { + const fallback = literalFallback(); + return fallback ? [fallback] : []; + } - gitlabSearchCache.set(cacheKey, results); - return results; + gitlabSearchCache.set(cacheKey, results); + return results; + } catch { + lifecycle.check(); + lifecycle.fail('network', true); + const fallback = type !== 'project' || PROJECT_PATTERN.test(query); + const warning = new Separator('⚠ Autocomplete unavailable — enter a value manually.'); + return fallback ? [{ name: query, value: query }, warning] : [warning]; + } } export async function collectGitLabConfig(connectionName: string): Promise { @@ -106,11 +132,7 @@ export async function collectGitLabConfig(connectionName: string): Promise 0 && (current.length + 1 + word.length) > width) { + if (current.length > 0 && current.length + 1 + word.length > width) { lines.push(indent + current); current = word; } else { @@ -59,26 +72,28 @@ function wrapText(text: string, indent: string, width: number): string[] { } function openBrowser(url: string): void { - const cmd = process.platform === 'darwin' ? 'open' - : process.platform === 'win32' ? 'cmd' - : 'xdg-open'; + const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'cmd' : 'xdg-open'; const args = process.platform === 'win32' ? ['/c', 'start', '""', url] : [url]; - spawn(cmd, args, { stdio: 'ignore', detached: true }).unref(); + lifecycle.check(); + const browser = spawn(cmd, args, { stdio: 'ignore', detached: true }); + browser.on('error', () => {}); + browser.unref(); } -async function openBrowserWhenReady(url: string, timeoutMs = 120_000): Promise { +async function openBrowserWhenReady(url: string, signal: AbortSignal, timeoutMs = 120_000): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { try { - const res = await fetch(url, { signal: AbortSignal.timeout(2000) }); + const res = await wizardFetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(2000)]) }); if (res.status < 500) { + signal.throwIfAborted(); openBrowser(url); return; } } catch { // not yet ready } - await new Promise((r) => setTimeout(r, 2000)); + await sleep(2000, undefined, { signal }); } } @@ -119,9 +134,9 @@ type PublishedPort = { host: string; port: number }; // undefined for specs with no fixed host port (container-only, ranges, ${VAR}). function parseHostPortSpec(spec: string): PublishedPort | undefined { let s = spec.trim(); - s = s.replace(/\s+#.*$/, '').trim(); // strip inline comment - s = s.replace(/^["']|["']$/g, '').trim(); // strip surrounding quotes - s = s.replace(/\/(tcp|udp|sctp)$/i, ''); // strip protocol suffix + s = s.replace(/\s+#.*$/, '').trim(); // strip inline comment + s = s.replace(/^["']|["']$/g, '').trim(); // strip surrounding quotes + s = s.replace(/\/(tcp|udp|sctp)$/i, ''); // strip protocol suffix const parts = s.split(':'); let host = '0.0.0.0'; let hostPort: string; @@ -189,11 +204,21 @@ function parsePublishedHostPorts(composeYaml: string): PublishedPort[] { // 5432), which is the actual failure mode `docker compose up` hits. function isPortInUse({ host, port }: PublishedPort): Promise { return new Promise((resolve) => { + lifecycle.check(); const server = net.createServer(); + const release = lifecycle.own(() => server.close()); + server.once('close', release); server.once('error', (err: NodeJS.ErrnoException) => { - server.close(() => { /* noop */ }); + server.close(() => { + /* noop */ + }); // EADDRINUSE = taken. Other errors (e.g. EACCES on privileged ports) aren't // a "someone else has it" conflict we can meaningfully report, so treat as free. + if (err.code !== 'EADDRINUSE') { + portInspectionFailed = true; + docker.failed = true; + lifecycle.fail('validation', true); + } resolve(err.code === 'EADDRINUSE'); }); server.once('listening', () => { @@ -207,183 +232,14 @@ function isPortInUse({ host, port }: PublishedPort): Promise { }); } -// Best-effort: maps a host port to the running Docker container(s) publishing it, so -// a conflict can name the offender. Returns an empty map if Docker isn't available. -async function getDockerPublishedPortOwners(): Promise> { - return new Promise>((resolve) => { - const child = spawn('docker', ['ps', '--format', '{{.Names}}\t{{.Ports}}'], { - stdio: ['ignore', 'pipe', 'ignore'], - }); - let out = ''; - child.stdout?.on('data', (chunk: Buffer) => { - out += chunk.toString(); - }); - child.on('exit', (code) => { - const map = new Map(); - if (code !== 0) { - resolve(map); - return; - } - for (const line of out.split('\n')) { - const [name, portsStr] = line.split('\t'); - if (!name || !portsStr) { - continue; - } - // e.g. "0.0.0.0:5432->5432/tcp, :::5432->5432/tcp" — the number before - // each "->" is the published host port. - for (const m of portsStr.matchAll(/(\d+)->/g)) { - const port = Number(m[1]); - const list = map.get(port) ?? []; - if (!list.includes(name)) { - list.push(name); - } - map.set(port, list); - } - } - resolve(map); - }); - child.on('error', () => resolve(new Map())); - }); -} - -// Stops the given containers (by name). Returns true only if all stopped cleanly. -async function stopDockerContainers(names: string[]): Promise { - if (names.length === 0) { - return true; - } - return new Promise((resolve) => { - const child = spawn('docker', ['stop', ...names], { stdio: ['ignore', 'ignore', 'pipe'] }); - let err = ''; - child.stderr?.on('data', (chunk: Buffer) => { - err += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0 && err.trim()) { - console.error(chalk.red('✗ ') + err.trim()); - } - resolve(code === 0); - }); - child.on('error', () => resolve(false)); - }); -} - -// Mirrors Docker Compose's project-name normalization for the default case -// where the project name is derived from the working directory basename. +const docker = new Docker((category) => lifecycle.fail(category, true)); +let portInspectionFailed = false; function dockerComposeProjectName(): string { return basename(process.cwd()) .toLowerCase() .replace(/[^a-z0-9_-]/g, ''); } -async function listExistingDockerVolumes(expectedNames: string[]): Promise { - if (expectedNames.length === 0) { - return []; - } - return new Promise((resolve) => { - const child = spawn('docker', ['volume', 'ls', '--format', '{{.Name}}'], { - stdio: ['ignore', 'pipe', 'ignore'], - }); - let out = ''; - child.stdout?.on('data', (chunk: Buffer) => { - out += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0) { - resolve([]); - return; - } - const existing = new Set(out.split('\n').map((l) => l.trim()).filter(Boolean)); - resolve(expectedNames.filter((name) => existing.has(name))); - }); - child.on('error', () => resolve([])); - }); -} - -async function removeDockerVolumes(volumes: string[]): Promise { - if (volumes.length === 0) { - return true; - } - return new Promise((resolve) => { - const child = spawn('docker', ['volume', 'rm', ...volumes], { stdio: ['ignore', 'ignore', 'pipe'] }); - let err = ''; - child.stderr?.on('data', (chunk: Buffer) => { - err += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0 && err.trim()) { - console.error(chalk.red('✗ ') + err.trim()); - } - resolve(code === 0); - }); - child.on('error', () => resolve(false)); - }); -} - -type ComposeContainer = { Name: string; Service: string; State: string }; - -function parseComposePsOutput(output: string): ComposeContainer[] { - const trimmed = output.trim(); - if (!trimmed) { - return []; - } - if (trimmed.startsWith('[')) { - try { - return JSON.parse(trimmed) as ComposeContainer[]; - } catch { - // fall through to line-based parse - } - } - const containers: ComposeContainer[] = []; - for (const line of trimmed.split('\n')) { - if (!line.trim()) { - continue; - } - try { - containers.push(JSON.parse(line) as ComposeContainer); - } catch { - // skip unparseable line - } - } - return containers; -} - -async function listComposeContainers(): Promise { - return new Promise((resolve) => { - const child = spawn('docker', ['compose', 'ps', '-a', '--format', 'json'], { - stdio: ['ignore', 'pipe', 'ignore'], - }); - let out = ''; - child.stdout?.on('data', (chunk: Buffer) => { - out += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0) { - resolve([]); - return; - } - resolve(parseComposePsOutput(out)); - }); - child.on('error', () => resolve([])); - }); -} - -async function runComposeCommand(args: string[], label: string): Promise { - return new Promise((resolve) => { - const child = spawn('docker', ['compose', ...args], { stdio: ['ignore', 'ignore', 'pipe'] }); - let err = ''; - child.stderr?.on('data', (chunk: Buffer) => { - err += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0 && err.trim()) { - console.error(chalk.red('✗ ') + `${label}: ` + err.trim()); - } - resolve(code === 0); - }); - child.on('error', () => resolve(false)); - }); -} - const PLATFORM_LABELS: Record = { github: 'GitHub', gitlab: 'GitLab', @@ -396,6 +252,9 @@ const PLATFORM_LABELS: Record = { }; async function main() { + lifecycle.install(); + lifecycle.capture('started', { invocationMethod: invocationMethod(), isInteractive: !!process.stdin.isTTY }); + lifecycle.failureCategory = 'filesystem'; console.log(String.raw` ███████╗ ██████╗ ██╗ ██╗██████╗ ██████╗███████╗██████╗ ██████╗ ████████╗ ██╔════╝██╔═══██╗██║ ██║██╔══██╗██╔════╝██╔════╝██╔══██╗██╔═══██╗╚══██╔══╝ @@ -417,7 +276,8 @@ async function main() { }, }); - if (existsSync(setupDir)) { + const selectedDirectoryExisted = existsSync(setupDir); + if (selectedDirectoryExisted) { const overwrite = await confirm({ message: `Directory '${setupDir}' already exists. Do you want to overwrite it?`, default: false, @@ -425,21 +285,29 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + 'Setup cancelled.'); - process.exit(0); + await lifecycle.decline('existing_directory_declined'); + return; } } else { mkdirSync(setupDir, { recursive: true }); } + lifecycle.check(); process.chdir(setupDir); + lifecycle.capture('chose_setup_directory', { + usedDefaultDirectory: setupDir === 'sourcebot', + directoryExisted: selectedDirectoryExisted, + directoryAction: selectedDirectoryExisted ? 'existing_directory_accepted' : 'created', + }); + lifecycle.stage = 'code_sources'; + lifecycle.failureCategory = 'unknown'; + const sourceSummaries: CodeSourceSummary[] = []; const connections: Record = {}; const allEnv: EnvVars = {}; const localRepoIndex = new Map(); - note( - 'Code is cloned and indexed locally on this machine. No code is ever transmitted to Sourcebot.', - ); + note('Code is cloned and indexed locally on this machine. No code is ever transmitted to Sourcebot.'); // eslint-disable-next-line no-constant-condition while (true) { @@ -447,9 +315,21 @@ async function main() { message: 'Which code host do you want to connect?', loop: false, choices: [ - { value: 'github', name: 'GitHub', description: 'github.com, GitHub Enterprise Server, or GitHub Enterprise Cloud' }, - { value: 'gitlab', name: 'GitLab', description: 'gitlab.com, GitLab Self Managed, or GitLab Dedicated' }, - { value: 'local', name: 'Local git repositories', description: 'git repositories in a local directory' }, + { + value: 'github', + name: 'GitHub', + description: 'github.com, GitHub Enterprise Server, or GitHub Enterprise Cloud', + }, + { + value: 'gitlab', + name: 'GitLab', + description: 'gitlab.com, GitLab Self Managed, or GitLab Dedicated', + }, + { + value: 'local', + name: 'Local git repositories', + description: 'git repositories in a local directory', + }, { value: 'git', name: 'Remote git repository', description: 'Arbitrary git URL' }, { value: 'azuredevops', name: 'Azure DevOps', description: 'dev.azure.com or Azure Devops Server' }, { value: 'bitbucket', name: 'Bitbucket', description: 'Bitbucket Cloud or Bitbucket Data Center' }, @@ -493,10 +373,14 @@ async function main() { continue; } + lifecycle.check(); + sourceSummaries.push(result.telemetry); + lifecycle.capture('configured_code_source', { + configurationIndex: sourceSummaries.length, + ...result.telemetry, + }); for (const { name, config } of result.connections) { - const finalName = name - ? generateConnectionName(name, connections) - : connectionName; + const finalName = name ? generateConnectionName(name, connections) : connectionName; connections[finalName] = config; } Object.assign(allEnv, result.env); @@ -511,7 +395,20 @@ async function main() { } } - const { models, env: modelEnv } = await collectModels(); + const sourceSummary = aggregateSources(sourceSummaries); + lifecycle.capture('configured_code_sources', sourceSummary); + lifecycle.stage = 'ai_setup'; + let modelIndex = 0; + const { + models, + env: modelEnv, + telemetry: modelSummaries, + } = await collectModels((summary) => { + lifecycle.capture('configured_ai_provider', { configurationIndex: ++modelIndex, ...summary }); + }); + const aiSummary = aggregateAi(modelSummaries); + lifecycle.capture('ai_setup_completed', aiSummary); + lifecycle.stage = 'hosted_url'; Object.assign(allEnv, modelEnv); const authUrl = await input({ @@ -529,8 +426,17 @@ async function main() { }, }); allEnv.AUTH_URL = authUrl; + lifecycle.capture('configured_hosted_url', { + usedDefaultUrl: authUrl === SOURCEBOT_URL, + protocol: authUrl.startsWith('https:') ? 'https' : 'http', + hostCategory: hostCategory(authUrl), + }); + lifecycle.stage = 'config_overwrite'; + lifecycle.failureCategory = 'filesystem'; + const overwritten: Events['generated_configs']['overwroteExistingFiles'] = []; if (existsSync('config.json')) { + overwritten.push('config_json'); const overwrite = await confirm({ message: 'config.json already exists. Overwrite?', default: true, @@ -538,11 +444,13 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + 'config.json was not overwritten.'); - process.exit(0); + await lifecycle.decline('config_overwrite_declined'); + return; } } if (existsSync('.env')) { + overwritten.push('env'); const overwrite = await confirm({ message: '.env already exists. Overwrite?', default: true, @@ -550,11 +458,13 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + '.env was not overwritten.'); - process.exit(0); + await lifecycle.decline('config_overwrite_declined'); + return; } } if (localRepoIndex.size > 0 && existsSync('docker-compose.override.yml')) { + overwritten.push('compose_override'); const overwrite = await confirm({ message: 'docker-compose.override.yml already exists. Overwrite?', default: true, @@ -562,11 +472,18 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + 'docker-compose.override.yml was not overwritten.'); - process.exit(0); + await lifecycle.decline('config_overwrite_declined'); + return; } } - const s = ora('Writing configuration files...').start(); + lifecycle.check(); + const deploymentIdentity = selectInstallId( + existsSync('.env') ? readFileSync('.env', 'utf8') : '', + lifecycle.telemetry.setupSessionId, + ); + const s = spinner('Writing configuration files...'); + const releaseWriter = lifecycle.own(() => s.stop()); const configOutput: Record = { $schema: 'https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v3/index.json', @@ -579,10 +496,19 @@ async function main() { const TOP_LEVEL_ENV_KEYS = ['AUTH_URL']; const connectionEnv = Object.fromEntries( - Object.entries(allEnv).filter(([k]) => !Object.values(PROVIDER_ENV_KEYS).includes(k) && !['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k) && !TOP_LEVEL_ENV_KEYS.includes(k)) + Object.entries(allEnv).filter( + ([k]) => + !Object.values(PROVIDER_ENV_KEYS).includes(k) && + !['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k) && + !TOP_LEVEL_ENV_KEYS.includes(k), + ), ); const aiEnv = Object.fromEntries( - Object.entries(allEnv).filter(([k]) => Object.values(PROVIDER_ENV_KEYS).includes(k) || ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k)) + Object.entries(allEnv).filter( + ([k]) => + Object.values(PROVIDER_ENV_KEYS).includes(k) || + ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k), + ), ); const envLines: string[] = [ @@ -596,6 +522,10 @@ async function main() { `AUTH_URL=${allEnv.AUTH_URL}`, ]; + if (deploymentIdentity.id) { + envLines.push('', '# Deployment identifier', `SOURCEBOT_INSTALL_ID=${deploymentIdentity.id}`); + } + if (Object.keys(connectionEnv).length > 0) { envLines.push('', '# Code host credentials'); for (const [key, value] of Object.entries(connectionEnv)) { @@ -610,6 +540,7 @@ async function main() { } } + lifecycle.check(); writeFileSync('config.json', configJson + '\n'); writeFileSync('.env', envLines.join('\n') + '\n'); @@ -632,19 +563,35 @@ async function main() { writtenFiles.push('docker-compose.override.yml'); } + lifecycle.capture('generated_configs', { + filesWritten: ['config_json', 'env', ...(localRepoIndex.size > 0 ? ['compose_override' as const] : [])], + overwroteExistingFiles: overwritten, + wroteComposeOverride: localRepoIndex.size > 0, + localMountCount: localRepoIndex.size, + generatedConnectionCount: sourceSummary.generatedConnectionCount, + aiConfigurationCount: aiSummary.aiConfigurationCount, + credentialVariableCount: + Object.keys(connectionEnv).filter((k) => !['GOOGLE_VERTEX_PROJECT', 'GOOGLE_VERTEX_REGION'].includes(k)) + .length + Object.keys(aiEnv).length, + deploymentIdentityAction: deploymentIdentity.action, + }); + releaseWriter(); const fileInfo: Record = { 'config.json': { - description: 'The Sourcebot configuration file. This controls which repos Sourcebot indexes and which language models it connects to.', + description: + 'The Sourcebot configuration file. This controls which repos Sourcebot indexes and which language models it connects to.', docsLabel: 'Configuration file docs', docsUrl: 'https://docs.sourcebot.dev/docs/configuration/config-file', }, '.env': { - description: 'The environment file your Sourcebot deployment will load. This includes any of the access tokens you provided here, as well as generated secrets required to run Sourcebot.', + description: + 'The environment file your Sourcebot deployment will load. This includes any of the access tokens you provided here, as well as generated secrets required to run Sourcebot.', docsLabel: 'Environment variables docs', docsUrl: 'https://docs.sourcebot.dev/docs/configuration/environment-variables', }, 'docker-compose.override.yml': { - description: 'Mounts your local repositories into the Sourcebot container so they can be indexed. Merged with docker-compose.yml at `docker compose up` time.', + description: + 'Mounts your local repositories into the Sourcebot container so they can be indexed. Merged with docker-compose.yml at `docker compose up` time.', }, }; @@ -669,25 +616,48 @@ async function main() { s.succeed(chalk.bold('Wrote the following files:')); console.log(['', ...fileLines].join('\n')); + lifecycle.stage = 'compose_file'; let downloadedCompose = false; + const compose: Events['resolved_compose_file'] = { + outcome: 'already_present', + composeAvailable: true, + downloadPromptShown: false, + downloadAttempted: false, + failureCategory: null, + }; if (!existsSync('docker-compose.yml')) { + compose.downloadPromptShown = true; + compose.outcome = 'declined'; + compose.composeAvailable = false; const download = await confirm({ message: 'Download docker-compose.yml?', default: true, }); if (download) { - const ds = ora('Downloading docker-compose.yml...').start(); + compose.downloadAttempted = true; + let downloadFailure: NonNullable = 'network'; + const ds = spinner('Downloading docker-compose.yml...'); try { - const res = await fetch(DOCKER_COMPOSE_URL); + const res = await wizardFetch(DOCKER_COMPOSE_URL); + downloadFailure = res.status >= 500 ? 'http_5xx' : res.status >= 400 ? 'http_4xx' : 'network'; if (!res.ok) { throw new Error(`HTTP ${res.status}`); } - await writeFile('docker-compose.yml', await res.text()); + const body = await res.text(); + lifecycle.check(); + downloadFailure = 'filesystem'; + await writeFile('docker-compose.yml', body); ds.succeed('Downloaded docker-compose.yml'); downloadedCompose = true; + compose.outcome = 'downloaded'; + compose.composeAvailable = true; } catch { + lifecycle.check(); + compose.outcome = 'download_failed'; + compose.failureCategory = downloadFailure; + lifecycle.fail(downloadFailure === 'filesystem' ? 'filesystem' : 'network', true); ds.fail('Download failed — you can get it manually (see next steps)'); } } @@ -695,12 +665,28 @@ async function main() { downloadedCompose = true; } + lifecycle.capture('resolved_compose_file', compose); + lifecycle.stage = 'docker_validation'; + lifecycle.failureCategory = 'docker_command'; + const dockerSummary = emptyDockerSummary(); let leftDeploymentRunning = false; if (downloadedCompose) { - const containers = await listComposeContainers(); + const containerResult = await docker.containers(); + const containers = containerResult.ok ? containerResult.value : []; const running = containers.filter((c) => c.State === 'running'); const stopped = containers.filter((c) => c.State !== 'running'); + if (containerResult.ok) { + dockerSummary.runningComposeContainerCount = running.length; + dockerSummary.stoppedComposeContainerCount = stopped.length; + dockerSummary.composeContainerState = running.length + ? stopped.length + ? 'mixed' + : 'running' + : stopped.length + ? 'stopped' + : 'none'; + } if (running.length > 0) { console.log(); @@ -709,12 +695,14 @@ async function main() { console.log(' ' + chalk.dim('- ') + `${c.Name} ${chalk.dim(`(${c.Service})`)}`); } const stop = await confirm({ - message: 'Stop and remove the running deployment? (required before any volume changes or restart can apply)', + message: + 'Stop and remove the running deployment? (required before any volume changes or restart can apply)', default: true, }); if (stop) { - const ds = ora('Stopping deployment...').start(); - const ok = await runComposeCommand(['down'], 'docker compose down'); + const ds = spinner('Stopping deployment...'); + const ok = (await docker.run(['compose', 'down'])).ok; + dockerSummary.existingDeploymentAction = ok ? 'stopped' : 'stop_failed'; if (ok) { ds.succeed('Stopped deployment'); } else { @@ -722,11 +710,14 @@ async function main() { leftDeploymentRunning = true; } } else { + dockerSummary.existingDeploymentAction = 'left_running'; leftDeploymentRunning = true; } } else if (stopped.length > 0) { console.log(); - console.log(chalk.yellow('⚠ ') + 'Stopped containers from a previous run exist and will conflict on next start:'); + console.log( + chalk.yellow('⚠ ') + 'Stopped containers from a previous run exist and will conflict on next start:', + ); for (const c of stopped) { console.log(' ' + chalk.dim('- ') + `${c.Name} ${chalk.dim(`(${c.Service})`)}`); } @@ -734,9 +725,11 @@ async function main() { message: 'Remove them now to prevent name conflicts when Sourcebot starts?', default: true, }); + dockerSummary.stoppedContainerAction = 'kept'; if (remove) { - const rs = ora('Removing containers...').start(); - const ok = await runComposeCommand(['rm', '-f'], 'docker compose rm'); + const rs = spinner('Removing containers...'); + const ok = (await docker.run(['compose', 'rm', '-f'])).ok; + dockerSummary.stoppedContainerAction = ok ? 'removed' : 'remove_failed'; if (ok) { rs.succeed('Removed containers'); } else { @@ -751,7 +744,9 @@ async function main() { const declaredVolumes = parseTopLevelVolumes(readFileSync('docker-compose.yml', 'utf-8')); const project = dockerComposeProjectName(); const expectedNames = declaredVolumes.map((v) => `${project}_${v}`); - const existing = await listExistingDockerVolumes(expectedNames); + const volumeResult = await docker.volumes(expectedNames); + const existing = volumeResult.ok ? volumeResult.value : []; + dockerSummary.existingVolumeCount = volumeResult.ok ? existing.length : null; if (existing.length > 0) { console.log(); @@ -763,9 +758,11 @@ async function main() { message: 'Wipe these volumes? This will permanently delete any existing Sourcebot data in them.', default: false, }); + dockerSummary.volumeAction = 'kept'; if (wipe) { - const ws = ora('Removing volumes...').start(); - const ok = await removeDockerVolumes(existing); + const ws = spinner('Removing volumes...'); + const ok = (await docker.run(['volume', 'rm', ...existing])).ok; + dockerSummary.volumeAction = ok ? 'removed' : 'remove_failed'; if (ok) { ws.succeed(`Removed ${existing.length} volume${existing.length === 1 ? '' : 's'}`); } else { @@ -786,20 +783,38 @@ async function main() { } const publishedPorts = parsePublishedHostPorts(composeYaml); + if (publishedPorts.length === 0) { + dockerSummary.initialPortConflictCount = 0; + dockerSummary.remainingPortConflictCount = 0; + dockerSummary.portConflictSource = 'none'; + } if (publishedPorts.length > 0) { - const ps = ora('Checking for port conflicts...').start(); + const ps = spinner('Checking for port conflicts...'); // Detect via two complementary sources: `docker ps` (authoritative for ports // published by other containers — a plain socket bind can't see those reliably, // e.g. Docker Desktop on macOS lets us bind a port it already forwards), and a // socket bind (catches non-Docker processes like a local Postgres/Redis). - const owners = await getDockerPublishedPortOwners(); + const ownersResult = await docker.portOwners(); + const owners = ownersResult.ok ? ownersResult.value : new Map(); const inUse: PublishedPort[] = []; for (const p of publishedPorts) { const ownedByContainer = (owners.get(p.port)?.length ?? 0) > 0; - if (ownedByContainer || await isPortInUse(p)) { + if (ownedByContainer || (await isPortInUse(p))) { inUse.push(p); } } + if (ownersResult.ok && !portInspectionFailed) { + dockerSummary.initialPortConflictCount = inUse.length; + dockerSummary.remainingPortConflictCount = inUse.length; + const dockerOwned = inUse.filter((p) => owners.has(p.port)).length; + dockerSummary.portConflictSource = !inUse.length + ? 'none' + : dockerOwned === inUse.length + ? 'docker' + : dockerOwned === 0 + ? 'non_docker' + : 'mixed'; + } if (inUse.length === 0) { ps.succeed('No port conflicts detected'); } else { @@ -810,16 +825,13 @@ async function main() { for (const p of inUse) { const display = p.host === '0.0.0.0' ? `${p.port}` : `${p.host}:${p.port}`; const by = owners.get(p.port); - const suffix = by && by.length > 0 - ? chalk.dim(` (in use by Docker container ${by.join(', ')})`) - : ''; + const suffix = + by && by.length > 0 ? chalk.dim(` (in use by Docker container ${by.join(', ')})`) : ''; console.log(' ' + chalk.dim('- ') + display + suffix); } // Containers we can stop ourselves; ports held by non-Docker processes we can't. - const conflictingContainers = [...new Set( - inUse.flatMap((p) => owners.get(p.port) ?? []), - )]; + const conflictingContainers = [...new Set(inUse.flatMap((p) => owners.get(p.port) ?? []))]; if (conflictingContainers.length > 0) { console.log(); @@ -827,9 +839,11 @@ async function main() { message: `Stop ${conflictingContainers.length === 1 ? 'this container' : 'these containers'} (${conflictingContainers.join(', ')}) to free the ports?`, default: true, }); + dockerSummary.portConflictAction = 'kept'; if (stop) { - const ss = ora('Stopping containers...').start(); - const ok = await stopDockerContainers(conflictingContainers); + const ss = spinner('Stopping containers...'); + const ok = (await docker.run(['stop', ...conflictingContainers])).ok; + dockerSummary.portConflictAction = ok ? 'containers_stopped' : 'stop_failed'; if (ok) { ss.succeed(`Stopped ${conflictingContainers.join(', ')}`); } else { @@ -837,19 +851,25 @@ async function main() { } // Re-check the conflicting ports now that the containers are stopped. const stillInUse: PublishedPort[] = []; - const freshOwners = await getDockerPublishedPortOwners(); + portInspectionFailed = false; + const freshResult = await docker.portOwners(); + const freshOwners = freshResult.ok ? freshResult.value : new Map(); for (const p of inUse) { const ownedByContainer = (freshOwners.get(p.port)?.length ?? 0) > 0; - if (ownedByContainer || await isPortInUse(p)) { + if (ownedByContainer || (await isPortInUse(p))) { stillInUse.push(p); } } + dockerSummary.remainingPortConflictCount = + freshResult.ok && !portInspectionFailed ? stillInUse.length : null; if (stillInUse.length === 0) { hasPortConflicts = false; console.log(chalk.green('✓ ') + 'All required ports are now free'); } else { console.log(); - console.log(chalk.yellow('⚠ ') + 'These ports are still in use (likely a non-Docker process):'); + console.log( + chalk.yellow('⚠ ') + 'These ports are still in use (likely a non-Docker process):', + ); for (const p of stillInUse) { const display = p.host === '0.0.0.0' ? `${p.port}` : `${p.host}:${p.port}`; console.log(' ' + chalk.dim('- ') + display); @@ -860,13 +880,39 @@ async function main() { if (hasPortConflicts) { console.log(); - console.log(chalk.dim(' Free these ports (stop the process or container using them), or change the host')); + console.log( + chalk.dim(' Free these ports (stop the process or container using them), or change the host'), + ); console.log(chalk.dim(' port mappings in docker-compose.yml, before starting Sourcebot.')); } } } } + dockerSummary.dockerStatus = docker.status; + dockerSummary.leftExistingDeploymentRunning = leftDeploymentRunning; + dockerSummary.outcome = dockerOutcome(dockerSummary, downloadedCompose, docker.failed); + lifecycle.capture('validated_docker_state', dockerSummary); + lifecycle.stage = 'start'; + lifecycle.failureCategory = 'process_spawn'; + const completion: Events['completed'] = { + completionMode: leftDeploymentRunning ? 'existing_deployment_left_running' : 'manual_start_required', + sourcebotStartOffered: downloadedCompose && !leftDeploymentRunning, + sourcebotStartRequested: false, + sourcebotStartOutcome: 'not_offered', + composeAvailable: downloadedCompose, + dockerValidationOutcome: dockerSummary.outcome, + remainingPortConflictCount: dockerSummary.remainingPortConflictCount, + generatedConnectionCount: sourceSummary.generatedConnectionCount, + codeHostTypes: sourceSummary.codeHostTypes, + repositoryCount: sourceSummary.repositoryCount, + aiConfigured: aiSummary.aiConfigured, + aiConfigurationCount: aiSummary.aiConfigurationCount, + providerTypes: aiSummary.providerTypes, + deploymentIdentityAction: deploymentIdentity.action, + totalDurationMs: 0, + }; + const complete = () => lifecycle.complete({ ...completion, totalDurationMs: lifecycle.telemetry.elapsed() }); if (downloadedCompose && !leftDeploymentRunning) { const startNow = await confirm({ message: hasPortConflicts @@ -880,16 +926,56 @@ async function main() { `Sourcebot will open at ${SOURCEBOT_URL} once it's ready.\nPress Ctrl+C to stop.`, 'Starting Sourcebot', ); - void openBrowserWhenReady(SOURCEBOT_URL).catch(() => { /* best effort */ }); + lifecycle.check(); + completion.sourcebotStartRequested = true; + const readiness = new AbortController(); + const releaseReadiness = lifecycle.own(() => readiness.abort()); + let spawned = false; await new Promise((resolve) => { - const child = spawn('docker', ['compose', 'up'], { stdio: 'inherit' }); - child.on('exit', () => resolve()); - child.on('error', (err) => { - console.error(chalk.red('✗ ') + 'Failed to run `docker compose up`: ' + (err instanceof Error ? err.message : String(err))); + const child = lifecycle.child( + spawn('docker', ['compose', 'up'], { stdio: 'inherit', detached: process.platform !== 'win32' }), + ); + child.once('spawn', () => { + if (lifecycle.interrupted) { + child.kill(); + return; + } + spawned = true; + completion.sourcebotStartOutcome = 'spawned'; + completion.completionMode = 'sourcebot_start_spawned'; + void complete(); + void openBrowserWhenReady( + SOURCEBOT_URL, + AbortSignal.any([lifecycle.signal, readiness.signal]), + ).catch(() => {}); + }); + child.once('close', () => { + readiness.abort(); + resolve(); + }); + child.once('error', (error: NodeJS.ErrnoException) => { + readiness.abort(); + if (!lifecycle.interrupted) { + lifecycle.fail( + error.code === 'ENOENT' || error.code === 'EACCES' ? 'docker_unavailable' : 'process_spawn', + true, + ); + completion.sourcebotStartOutcome = 'spawn_failed'; + completion.completionMode = 'sourcebot_start_failed'; + console.error(chalk.red('✗ ') + 'Failed to run docker compose up.'); + } resolve(); }); }); - return; + readiness.abort(); + releaseReadiness(); + lifecycle.check(); + if (spawned) { + await lifecycle.telemetry.shutdown(); + return; + } + } else { + completion.sourcebotStartOutcome = 'declined'; } } @@ -904,6 +990,7 @@ async function main() { nextSteps.push(`${step++}. To apply your new configuration, restart Sourcebot:`); nextSteps.push(' docker compose down && docker compose up'); note(nextSteps.join('\n'), 'Sourcebot is already running'); + await complete(); return; } @@ -914,7 +1001,9 @@ async function main() { } if (hasPortConflicts) { - nextSteps.push(`${step++}. Free the host ports listed above (or change the host port mappings in docker-compose.yml).`); + nextSteps.push( + `${step++}. Free the host ports listed above (or change the host port mappings in docker-compose.yml).`, + ); nextSteps.push(''); } @@ -924,16 +1013,31 @@ async function main() { nextSteps.push(`${step}. Open ${SOURCEBOT_URL}`); note(nextSteps.join('\n'), 'Next steps'); + await complete(); } -main().catch(err => { - const isExitPrompt = err instanceof Error - && (err.name === 'ExitPromptError' || err.message?.startsWith('User force closed the prompt')); - if (isExitPrompt) { - console.log(); - console.log(chalk.red('✗ ') + 'Setup cancelled.'); - process.exit(0); - } - console.error(err); - process.exit(1); -}); +main() + .catch(async (error) => { + if (lifecycle.interrupted) { + return; + } + if (error instanceof Error && error.name === 'ExitPromptError') { + lifecycle.interrupt(); + return; + } + const code = error && typeof error === 'object' ? error.code : undefined; + const category = ['ENOENT', 'ENOTDIR', 'EISDIR', 'EROFS', 'ENOSPC', 'EACCES', 'EPERM'].includes(code) + ? 'filesystem' + : error instanceof Error && error.name === 'ValidationError' + ? 'validation' + : lifecycle.failureCategory; + lifecycle.fail(category, false); + console.error(error); + await lifecycle.telemetry.shutdown(); + process.exitCode = 1; + }) + .finally(() => { + if (!lifecycle.interrupted) { + lifecycle.exit(Number(process.exitCode ?? 0)); + } + }); diff --git a/packages/setupWizard/src/lifecycle.ts b/packages/setupWizard/src/lifecycle.ts new file mode 100644 index 000000000..a15846694 --- /dev/null +++ b/packages/setupWizard/src/lifecycle.ts @@ -0,0 +1,211 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { join } from 'node:path'; +import { Telemetry } from './telemetry.js'; +import type { + Events, + EventName, + Stage, + FailureCategory, +} from './telemetryEvents.js'; + +export class Lifecycle { + readonly controller = new AbortController(); + readonly children = new Set(); + private readonly cleanups = new Set<() => void>(); + stage: Stage = 'setup_directory'; + failureCategory: FailureCategory = 'unknown'; + terminal?: 'completed' | 'cancelled' | 'failed'; + interrupted = false; + private installed = false; + constructor(readonly telemetry = new Telemetry()) {} + get signal(): AbortSignal { + return this.controller.signal; + } + check(): void { + this.signal.throwIfAborted(); + } + install(): void { + if (!this.installed) { + this.installed = true; + process.on('SIGINT', this.interrupt); + } + } + dispose(): void { + process.off('SIGINT', this.interrupt); + this.installed = false; + } + exit(code: number): never { + this.dispose(); + this.controller.abort(); + for (const cleanup of this.cleanups) { + try { + cleanup(); + } catch { + /* Cleanup is independent of telemetry. */ + } + } + this.killChildren(true); + // SDK retry sockets/timers can outlive its bounded shutdown promise. + // Only call after main returns, never at the foreground Docker handoff. + process.exit(code); + } + own(cleanup: () => void): () => void { + this.cleanups.add(cleanup); + return () => this.cleanups.delete(cleanup); + } + child(child: ChildProcess): ChildProcess { + this.children.add(child); + child.once('close', () => { + // A client can exit before descendants that ignored its interrupt. + // Tear down its dedicated group before relinquishing ownership. + if (process.platform !== 'win32' && child.pid) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { /* The group normally no longer exists. */ } + } + this.children.delete(child); + }); + if (this.interrupted) { + child.kill(); + } + return child; + } + capture(name: K, props: Events[K]): void { + if (!this.terminal && !this.interrupted) { + this.telemetry.capture(name, props); + } + } + fail(category: FailureCategory, recoverable: boolean): void { + if (this.interrupted || this.terminal) { + return; + } + if (!recoverable) { + this.terminal = 'failed'; + } + this.telemetry.capture('failed', { + stage: this.stage, + failureCategory: category, + recoverable, + }); + } + async complete(properties: Events['completed']): Promise { + if (this.terminal || this.interrupted) { + return; + } + this.terminal = 'completed'; + this.telemetry.capture('completed', properties); + await this.telemetry.shutdown(); + } + async decline(reason: Events['cancelled']['reason']): Promise { + if (!this.terminal) { + this.terminal = 'cancelled'; + this.telemetry.capture('cancelled', { stage: this.stage, reason }); + } + await this.telemetry.shutdown(); + } + private killChildren(force: boolean): void { + for (const child of this.children) { + if ( + child.exitCode !== null || + child.signalCode !== null || + !child.pid + ) { + continue; + } + try { + if (process.platform === 'win32') { + const taskkill = join( + process.env.SystemRoot ?? 'C:\\Windows', + 'System32', + 'taskkill.exe', + ); + const killer = spawn( + taskkill, + [ + '/pid', + String(child.pid), + '/T', + ...(force ? ['/F'] : []), + ], + { stdio: 'ignore' }, + ); + killer.on('error', () => {}); + killer.unref(); + } else { + // Owned Docker clients run in their own POSIX group, never the user's shell group. + process.kill(-child.pid, force ? 'SIGKILL' : 'SIGINT'); + } + } catch { + /* Child may have exited concurrently. */ + } + } + } + interrupt = (): void => { + if (this.interrupted) { + this.killChildren(true); + process.exit(130); + } + this.interrupted = true; + if (!this.terminal) { + this.terminal = 'cancelled'; + this.telemetry.capture('cancelled', { + stage: this.stage, + reason: 'keyboard_interrupt', + }); + } + this.controller.abort(); + for (const cleanup of this.cleanups) { + try { + cleanup(); + } catch { + /* Independent cleanup must proceed. */ + } + } + this.killChildren(false); + const force = setTimeout(() => this.killChildren(true), 2000); + const deadline = setTimeout(() => { + this.killChildren(true); + process.exit(130); + }, 3000); + void (async () => { + await this.telemetry.shutdown(); + while ( + [...this.children].some( + (c) => c.exitCode === null && c.signalCode === null, + ) + ) { + await sleep(20); + } + clearTimeout(force); + clearTimeout(deadline); + if (process.stdin.isTTY) { + process.stdin.setRawMode(false); + } + process.stdin.pause(); + process.stdout.write('\u001b[?25h\n'); + process.exit(130); + })().catch(() => { + /* Deadline still forces exit. */ + }); + }; +} +export const lifecycle = new Lifecycle(); +export async function wizardFetch( + input: string | URL, + init: RequestInit = {}, +): Promise { + lifecycle.check(); + const signal = init.signal + ? AbortSignal.any([lifecycle.signal, init.signal]) + : lifecycle.signal; + try { + const result = await fetch(input, { ...init, signal }); + lifecycle.check(); + return result; + } catch (error) { + lifecycle.check(); + lifecycle.failureCategory = 'network'; + throw error; + } +} diff --git a/packages/setupWizard/src/localRepos.ts b/packages/setupWizard/src/localRepos.ts index f22c17155..30eefd654 100644 --- a/packages/setupWizard/src/localRepos.ts +++ b/packages/setupWizard/src/localRepos.ts @@ -1,5 +1,7 @@ -import { input } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { lifecycle } from './lifecycle.js'; +import { sourceSummary, discoveredBucket } from './telemetrySummary.js'; +import { input } from './prompts.js'; +import { checkbox } from './prompts.js'; import { existsSync, statSync } from 'fs'; import { readdir } from 'fs/promises'; import { homedir } from 'os'; @@ -11,16 +13,7 @@ import { note } from './utils.js'; const MAX_DEPTH = 5; -const SKIP_DIRS = new Set([ - 'node_modules', - 'dist', - 'build', - 'out', - 'target', - 'vendor', - 'coverage', - '__pycache__', -]); +const SKIP_DIRS = new Set(['node_modules', 'dist', 'build', 'out', 'target', 'vendor', 'coverage', '__pycache__']); function expandHostPath(p: string): string { const trimmed = p.trim(); @@ -34,6 +27,7 @@ async function findGitRepos(root: string, maxDepth: number): Promise { const repos: string[] = []; async function walk(dir: string, depth: number): Promise { + lifecycle.check(); if (existsSync(join(dir, '.git'))) { repos.push(dir); return; @@ -65,9 +59,7 @@ async function findGitRepos(root: string, maxDepth: number): Promise { return repos.sort(); } -export async function collectLocalReposConfig( - localRepoIndex: Map, -): Promise { +export async function collectLocalReposConfig(localRepoIndex: Map): Promise { note( [ 'Point at a directory on your machine that contains git repositories.', @@ -102,7 +94,14 @@ export async function collectLocalReposConfig( hostPath = expandHostPath(rawPath); const spinner = ora(`Scanning ${hostPath} for git repositories...`).start(); - repos = await findGitRepos(hostPath, MAX_DEPTH); + const releaseSpinner = lifecycle.own(() => spinner.stop()); + try { + repos = await findGitRepos(hostPath, MAX_DEPTH); + lifecycle.check(); + } finally { + spinner.stop(); + releaseSpinner(); + } if (repos.length === 0) { spinner.fail(`No git repositories found under ${hostPath}`); continue; @@ -121,15 +120,23 @@ export async function collectLocalReposConfig( const hostPathIsRepo = repos.length === 1 && repos[0] === hostPath; if (hostPathIsRepo) { return { - connections: [{ - name: basename(hostPath), - config: { - type: 'git', - url: `file://${containerRoot}`, - } satisfies GenericGitHostConnectionConfig, - }], + connections: [ + { + name: basename(hostPath), + config: { + type: 'git', + url: `file://${containerRoot}`, + } satisfies GenericGitHostConnectionConfig, + }, + ], env: {}, localRepoHostPath: hostPath, + telemetry: sourceSummary('local_git', { + deploymentType: 'local', + scopeTypes: ['repositories'], + repositoryCount: 1, + localDiscoveredRepoCountBucket: '1', + }), }; } @@ -152,20 +159,34 @@ export async function collectLocalReposConfig( const allSelected = selected.length === repos.length; const allAtDepthOne = repos.every((p) => !posixRel(p).includes('/')); - const connections = allSelected && allAtDepthOne - ? [{ - config: { - type: 'git', - url: `file://${containerRoot}/*`, - } satisfies GenericGitHostConnectionConfig, - }] - : selected.map((repoPath) => { - const config: GenericGitHostConnectionConfig = { - type: 'git', - url: `file://${containerRoot}/${posixRel(repoPath)}`, - }; - return { name: basename(repoPath), config }; - }); - - return { connections, env: {}, localRepoHostPath: hostPath }; + const connections = + allSelected && allAtDepthOne + ? [ + { + config: { + type: 'git', + url: `file://${containerRoot}/*`, + } satisfies GenericGitHostConnectionConfig, + }, + ] + : selected.map((repoPath) => { + const config: GenericGitHostConnectionConfig = { + type: 'git', + url: `file://${containerRoot}/${posixRel(repoPath)}`, + }; + return { name: basename(repoPath), config }; + }); + + return { + connections, + env: {}, + localRepoHostPath: hostPath, + telemetry: sourceSummary('local_git', { + deploymentType: 'local', + scopeTypes: ['repositories'], + repositoryCount: selected.length, + generatedConnectionCount: connections.length, + localDiscoveredRepoCountBucket: discoveredBucket(repos.length), + }), + }; } diff --git a/packages/setupWizard/src/models.ts b/packages/setupWizard/src/models.ts index 50faea65f..f28069aea 100644 --- a/packages/setupWizard/src/models.ts +++ b/packages/setupWizard/src/models.ts @@ -1,5 +1,7 @@ -import { confirm, input, password, select } from '@inquirer/prompts'; -import { select as searchSelect } from 'inquirer-select-pro'; +import { wizardFetch, lifecycle } from './lifecycle.js'; +import type { AiSummary } from './telemetryEvents.js'; +import { confirm, input, password, select } from './prompts.js'; +import { searchSelect } from './prompts.js'; import type { AmazonBedrockLanguageModel, AzureLanguageModel, @@ -13,15 +15,15 @@ import { INPUT_THEME, note, type EnvVars } from './utils.js'; type Provider = LanguageModel['provider']; export const PROVIDER_ENV_KEYS: Record = { - 'anthropic': 'ANTHROPIC_API_KEY', - 'openai': 'OPENAI_API_KEY', + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', 'google-generative-ai': 'GOOGLE_GENERATIVE_AI_API_KEY', - 'deepseek': 'DEEPSEEK_API_KEY', - 'mistral': 'MISTRAL_API_KEY', - 'xai': 'XAI_API_KEY', - 'openrouter': 'OPENROUTER_API_KEY', + deepseek: 'DEEPSEEK_API_KEY', + mistral: 'MISTRAL_API_KEY', + xai: 'XAI_API_KEY', + openrouter: 'OPENROUTER_API_KEY', 'openai-compatible': 'OPENAI_COMPATIBLE_API_KEY', - 'azure': 'AZURE_OPENAI_API_KEY', + azure: 'AZURE_OPENAI_API_KEY', }; // ─── models.dev catalog ──────────────────────────────────────────────────── @@ -59,14 +61,21 @@ async function loadCatalog(): Promise { if (!catalogPromise) { catalogPromise = (async () => { try { - const response = await fetch(MODELS_DEV_API_URL, { + const response = await wizardFetch(MODELS_DEV_API_URL, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), }); if (!response.ok) { + lifecycle.fail('network', true); return null; } - return await response.json() as ModelsDevCatalog; + const catalog = await response.json(); + if (!catalog || typeof catalog !== 'object' || Array.isArray(catalog)) { + throw new Error('Invalid model catalog'); + } + return catalog as ModelsDevCatalog; } catch { + lifecycle.check(); + lifecycle.fail('network', true); return null; } })(); @@ -88,6 +97,19 @@ async function getModelOptionsForProvider(providerKey: string): Promise + !m || + typeof m.id !== 'string' || + !m.id || + (m.name !== undefined && typeof m.name !== 'string') || + (m.release_date !== undefined && typeof m.release_date !== 'string'), + ) + ) { + lifecycle.fail('network', true); + return null; + } return models .map((m) => ({ id: m.id, @@ -110,10 +132,7 @@ async function getModelOptionsForProvider(providerKey: string): Promise { +async function searchModel(options: { message: string; models: ModelOption[] }): Promise { const choices = options.models.map((m) => ({ name: m.name === m.id ? m.id : `${m.id} · ${m.name}`, value: m.id, @@ -131,8 +150,8 @@ async function searchModel(options: { return choices; } const lowered = trimmed.toLowerCase(); - const filtered = choices.filter((c) => - c.value.toLowerCase().includes(lowered) || c.name.toLowerCase().includes(lowered), + const filtered = choices.filter( + (c) => c.value.toLowerCase().includes(lowered) || c.name.toLowerCase().includes(lowered), ); const hasExact = choices.some((c) => c.value === trimmed); if (!hasExact) { @@ -153,7 +172,7 @@ async function ensureApiKey(provider: Provider, env: EnvVars): Promise { const apiKey = await password({ message: `API key (stored locally in .env as ${envKey})`, mask: true, - validate: (v) => !v?.trim() ? 'API key is required' : true, + validate: (v) => (!v?.trim() ? 'API key is required' : true), }); env[envKey] = apiKey; } @@ -164,6 +183,7 @@ async function collectModelConfig( provider: Provider, model: string, env: EnvVars, + setCredentialMode: (mode: AiSummary['credentialMode']) => void, ): Promise { switch (provider) { case 'anthropic': @@ -201,13 +221,13 @@ async function collectModelConfig( case 'azure': { const resourceName = await input({ message: 'Azure resource name', - validate: (v) => !v?.trim() ? 'Resource name is required' : true, + validate: (v) => (!v?.trim() ? 'Resource name is required' : true), }); const apiVersion = await input({ message: 'API version', default: '2024-08-01-preview', theme: INPUT_THEME, - validate: (v) => !v?.trim() ? 'API version is required' : true, + validate: (v) => (!v?.trim() ? 'API version is required' : true), }); const envKey = await ensureApiKey(provider, env); const config: AzureLanguageModel = { @@ -224,6 +244,7 @@ async function collectModelConfig( message: 'Use the default AWS credential chain? (No to provide Access Key ID and Secret explicitly)', default: true, }); + setCredentialMode(useDefaultChain ? 'aws_default_chain' : 'aws_explicit_keys'); const config: AmazonBedrockLanguageModel = { provider, model }; @@ -231,7 +252,7 @@ async function collectModelConfig( if (!env['AWS_ACCESS_KEY_ID']) { env['AWS_ACCESS_KEY_ID'] = await input({ message: 'AWS Access Key ID (stored locally in .env as AWS_ACCESS_KEY_ID)', - validate: (v) => !v?.trim() ? 'Access Key ID is required' : true, + validate: (v) => (!v?.trim() ? 'Access Key ID is required' : true), }); } config.accessKeyId = { env: 'AWS_ACCESS_KEY_ID' }; @@ -240,7 +261,7 @@ async function collectModelConfig( env['AWS_SECRET_ACCESS_KEY'] = await password({ message: 'AWS Secret Access Key (stored locally in .env as AWS_SECRET_ACCESS_KEY)', mask: true, - validate: (v) => !v?.trim() ? 'Secret Access Key is required' : true, + validate: (v) => (!v?.trim() ? 'Secret Access Key is required' : true), }); } config.accessKeySecret = { env: 'AWS_SECRET_ACCESS_KEY' }; @@ -250,7 +271,7 @@ async function collectModelConfig( message: 'AWS region', default: 'us-east-1', theme: INPUT_THEME, - validate: (v) => !v?.trim() ? 'Region is required' : true, + validate: (v) => (!v?.trim() ? 'Region is required' : true), }); return config; } @@ -259,7 +280,7 @@ async function collectModelConfig( if (!env['GOOGLE_VERTEX_PROJECT']) { env['GOOGLE_VERTEX_PROJECT'] = await input({ message: 'Google Cloud project ID (stored locally in .env as GOOGLE_VERTEX_PROJECT)', - validate: (v) => !v?.trim() ? 'Project ID is required' : true, + validate: (v) => (!v?.trim() ? 'Project ID is required' : true), }); } if (!env['GOOGLE_VERTEX_REGION']) { @@ -267,7 +288,7 @@ async function collectModelConfig( message: 'Google Cloud region (stored locally in .env as GOOGLE_VERTEX_REGION)', default: 'us-central1', theme: INPUT_THEME, - validate: (v) => !v?.trim() ? 'Region is required' : true, + validate: (v) => (!v?.trim() ? 'Region is required' : true), }); } @@ -275,6 +296,7 @@ async function collectModelConfig( message: 'Use Application Default Credentials? (No to provide a service account credentials file path)', default: true, }); + setCredentialMode(useAppDefault ? 'google_application_default_credentials' : 'google_credentials_file'); const config: GoogleVertexLanguageModel | GoogleVertexAnthropicLanguageModel = { provider, @@ -284,8 +306,9 @@ async function collectModelConfig( if (!useAppDefault) { if (!env['GOOGLE_APPLICATION_CREDENTIALS']) { env['GOOGLE_APPLICATION_CREDENTIALS'] = await input({ - message: 'Path to service account credentials JSON (stored locally in .env as GOOGLE_APPLICATION_CREDENTIALS)', - validate: (v) => !v?.trim() ? 'Credentials path is required' : true, + message: + 'Path to service account credentials JSON (stored locally in .env as GOOGLE_APPLICATION_CREDENTIALS)', + validate: (v) => (!v?.trim() ? 'Credentials path is required' : true), }); } config.credentials = { env: 'GOOGLE_APPLICATION_CREDENTIALS' }; @@ -295,9 +318,12 @@ async function collectModelConfig( } } -export async function collectModels(): Promise<{ models: LanguageModel[]; env: EnvVars }> { +export async function collectModels( + onAccepted: (summary: AiSummary) => void = () => {}, +): Promise<{ models: LanguageModel[]; env: EnvVars; telemetry: AiSummary[] }> { const models: LanguageModel[] = []; const env: EnvVars = {}; + const telemetry: AiSummary[] = []; note( [ @@ -305,7 +331,7 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E 'in natural language and get answers grounded in your indexed code.', ' https://docs.sourcebot.dev/docs/features/ask/ask-sourcebot', '', - 'You\'ll need an API key from at least one supported provider', + "You'll need an API key from at least one supported provider", '(Anthropic, OpenAI, Google, etc.) to enable these features.', ].join('\n'), 'AI features', @@ -317,7 +343,7 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E }); if (!wantsAI) { - return { models, env }; + return { models, env, telemetry }; } // eslint-disable-next-line no-constant-condition @@ -332,7 +358,11 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E { value: 'amazon-bedrock', name: 'Amazon Bedrock' }, { value: 'google-generative-ai', name: 'Google Gemini' }, { value: 'google-vertex', name: 'Google Vertex AI', description: 'Gemini via Vertex' }, - { value: 'google-vertex-anthropic', name: 'Google Vertex AI (Anthropic)', description: 'Claude via Vertex' }, + { + value: 'google-vertex-anthropic', + name: 'Google Vertex AI (Anthropic)', + description: 'Claude via Vertex', + }, { value: 'azure', name: 'Azure OpenAI' }, { value: 'deepseek', name: 'DeepSeek' }, { value: 'mistral', name: 'Mistral' }, @@ -341,28 +371,45 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E ], }); - const modelOptions = provider === 'openai-compatible' - ? null - : await getModelOptionsForProvider(provider); - const model = modelOptions && modelOptions.length > 0 - ? await searchModel({ - message: 'Model name', - models: modelOptions, - }) - : await input({ - message: 'Model name', - validate: (v) => !v?.trim() ? 'Model name is required' : true, - }); - - const config = await collectModelConfig(provider, model, env); + const modelOptions = provider === 'openai-compatible' ? null : await getModelOptionsForProvider(provider); + const model = + modelOptions && modelOptions.length > 0 + ? await searchModel({ + message: 'Model name', + models: modelOptions, + }) + : await input({ + message: 'Model name', + validate: (v) => (!v?.trim() ? 'Model name is required' : true), + }); + + let credentialMode: AiSummary['credentialMode'] = 'api_key'; + const config = await collectModelConfig(provider, model, env, (mode) => { + credentialMode = mode; + }); - const displayName = (await input({ - message: 'Display name (optional, press enter to skip)', - })).trim(); + const displayName = ( + await input({ + message: 'Display name (optional, press enter to skip)', + }) + ).trim(); if (displayName) { config.displayName = displayName; } models.push(config); + const summary: AiSummary = { + provider, + credentialMode, + usesCustomEndpoint: provider === 'openai-compatible', + hasDisplayName: !!displayName, + modelSelectionMethod: modelOptions?.length + ? modelOptions.some((option) => option.id === model) + ? 'catalog' + : 'custom_entry' + : 'manual_fallback', + }; + telemetry.push(summary); + onAccepted(summary); const addAnother = await confirm({ message: 'Add another model?', @@ -374,5 +421,5 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E } } - return { models, env }; + return { models, env, telemetry }; } diff --git a/packages/setupWizard/src/prompts.ts b/packages/setupWizard/src/prompts.ts new file mode 100644 index 000000000..fcb190aeb --- /dev/null +++ b/packages/setupWizard/src/prompts.ts @@ -0,0 +1,37 @@ +import { + confirm as rawConfirm, + input as rawInput, + password as rawPassword, + select as rawSelect, +} from '@inquirer/prompts'; +import { select as rawSearchSelect } from 'inquirer-select-pro'; +import { tabCheckbox } from './tabCheckbox.js'; +import { lifecycle } from './lifecycle.js'; + +// Preserve the library's generic call signatures, including multi-select return types. +function cancellable(prompt: T): T { + return (async (config: unknown, context: Record = {}) => { + lifecycle.check(); + try { + const result = await ( + prompt as ( + config: unknown, + context: unknown, + ) => Promise + )(config, { ...context, signal: lifecycle.signal }); + lifecycle.check(); + return result; + } catch (error) { + if (error instanceof Error && error.name === 'ExitPromptError') { + lifecycle.interrupt(); + } + throw error; + } + }) as T; +} +export const confirm = cancellable(rawConfirm); +export const input = cancellable(rawInput); +export const password = cancellable(rawPassword); +export const select = cancellable(rawSelect); +export const searchSelect = cancellable(rawSearchSelect); +export const checkbox = cancellable(tabCheckbox); diff --git a/packages/setupWizard/src/spinner.ts b/packages/setupWizard/src/spinner.ts new file mode 100644 index 000000000..fa807f435 --- /dev/null +++ b/packages/setupWizard/src/spinner.ts @@ -0,0 +1,9 @@ +import ora from 'ora'; +import { lifecycle } from './lifecycle.js'; + +export function spinner(text: string) { + lifecycle.check(); + const result = ora(text).start(); + lifecycle.own(() => result.stop()); + return result; +} diff --git a/packages/setupWizard/src/telemetry.ts b/packages/setupWizard/src/telemetry.ts new file mode 100644 index 000000000..f10325101 --- /dev/null +++ b/packages/setupWizard/src/telemetry.ts @@ -0,0 +1,182 @@ +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { PostHog } from 'posthog-node'; +import { + eventSchemas, + validateFields, + type Events, + type EventName, +} from './telemetryEvents.js'; + +export const INSTALL_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +// Same public ingestion token as packages/shared/src/env.server.ts. Never read user overrides. +export const POSTHOG_PROJECT_TOKEN = + 'phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW'; +export const POSTHOG_OPTIONS = { + host: 'https://us.i.posthog.com', + flushAt: 1, + flushInterval: 0, + disableGeoip: true, + isServer: false, +} as const; +function bestEffort(read: () => T, fallback: T): T { + try { + return read(); + } catch { + return fallback; + } +} +export function systemProperties(runtime = process) { + return { + platform: bestEffort( + () => + ['darwin', 'linux', 'win32'].includes(runtime.platform) + ? runtime.platform + : 'other', + 'other', + ), + arch: bestEffort( + () => + ['arm64', 'x64'].includes(runtime.arch) + ? runtime.arch + : 'other', + 'other', + ), + nodeMajorVersion: bestEffort(() => { + const v = Number(runtime.versions.node.split('.')[0]); + return Number.isSafeInteger(v) && v > 0 ? v : null; + }, null), + packageManager: bestEffort( + () => + /^(npm|yarn|pnpm|bun)\//.exec( + runtime.env.npm_config_user_agent ?? '', + )?.[1] ?? 'unknown', + 'unknown', + ), + isCI: bestEffort( + () => + ['CI', 'CONTINUOUS_INTEGRATION', 'BUILD_NUMBER', 'RUN_ID'].some( + (k) => !!runtime.env[k] && runtime.env[k] !== 'false', + ), + null, + ), + }; +} +export function invocationMethod(): Events['started']['invocationMethod'] { + return bestEffort(() => { + if ( + process.env.npm_command === 'exec' || + process.argv[1]?.includes('_npx') + ) { + return 'npx'; + } + if (process.env.npm_lifecycle_event) { + return 'workspace'; + } + return process.argv[1]?.includes('node_modules') + ? 'local_binary' + : 'unknown'; + }, 'unknown'); +} +type Client = Pick; +export class Telemetry { + readonly setupSessionId: string | undefined; + private client?: Client; + private readonly began: number; + private shutdownPromise?: Promise; + private closed = false; + private readonly wallClockStart = Date.now(); + private lastTimestamp = 0; + constructor( + createClient: () => Client = () => + new PostHog(POSTHOG_PROJECT_TOKEN, POSTHOG_OPTIONS), + uuid: () => string = randomUUID, + private readonly clock = () => performance.now(), + ) { + this.began = clock(); + this.setupSessionId = bestEffort(() => { + const id = uuid(); + return INSTALL_ID_PATTERN.test(id) ? id : undefined; + }, undefined); + if (this.setupSessionId) { + this.client = bestEffort(() => createClient(), undefined); + bestEffort(() => this.client?.on('error', () => {}), undefined); + } + } + elapsed(): number { + return Math.max(0, Math.round(this.clock() - this.began)); + } + capture(event: K, values: Events[K]): void { + if (this.closed || !this.client || !this.setupSessionId) { + return; + } + try { + const properties = validateFields( + eventSchemas[event], + values as never, + ); + const version: unknown = bestEffort( + () => + JSON.parse( + readFileSync( + new URL('../package.json', import.meta.url), + 'utf8', + ), + ).version, + 'unknown', + ); + // Immediate independent requests can be ingested out of order. Preserve + // session order even across wall-clock adjustments and same-ms checkpoints. + this.lastTimestamp = Math.max( + this.lastTimestamp + 1, + this.wallClockStart + this.elapsed(), + ); + this.client.capture({ + distinctId: this.setupSessionId, + event: `setup_sourcebot_${event}`, + groups: { company: this.setupSessionId }, + timestamp: new Date(this.lastTimestamp), + properties: { + ...properties, + ...systemProperties(), + schemaVersion: 1, + source: 'setup-sourcebot-cli', + setupSourcebotVersion: + typeof version === 'string' && + /^\d+\.\d+\.\d+(?:[-+][\w.-]+)?$/.test(version) + ? version + : 'unknown', + setupSessionId: this.setupSessionId, + install_id: this.setupSessionId, + elapsedMs: this.elapsed(), + $geoip_disable: true, + $ignore_sent_at: true, + }, + }); + } catch { + /* Telemetry never participates in setup success. */ + } + } + shutdown(): Promise { + if (!this.shutdownPromise) { + this.closed = true; + this.shutdownPromise = (async () => { + let timer: ReturnType | undefined; + try { + await Promise.race([ + this.client?.shutdown(1000), + new Promise((resolve) => { + timer = setTimeout(resolve, 1000); + }), + ]); + } catch { + /* Best effort. */ + } finally { + clearTimeout(timer); + } + })(); + } + return this.shutdownPromise; + } +} diff --git a/packages/setupWizard/src/telemetryEvents.ts b/packages/setupWizard/src/telemetryEvents.ts new file mode 100644 index 000000000..ac198b29b --- /dev/null +++ b/packages/setupWizard/src/telemetryEvents.ts @@ -0,0 +1,332 @@ +// Runtime allowlists are also the TypeScript event contract. Never accept arbitrary maps. +type Rule = { parse: (value: unknown) => T }; +type Value = R extends Rule ? T : never; +type Shape = Record>; +type Fields = { [K in keyof S]: Value }; +function rule(check: (v: unknown) => boolean): Rule { + return { + parse(value) { + if (!check(value)) { + throw new Error('Invalid telemetry property'); + } + return value as T; + }, + }; +} +export function choice( + ...values: T +): Rule { + return rule((v) => typeof v === 'string' && values.includes(v)); +} +const boolean = rule((v) => typeof v === 'boolean'); +const count = rule( + (v) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 0, +); +const position = rule( + (v) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 1, +); +const nullable = (inner: Rule): Rule => ({ + parse: (v) => (v === null ? null : inner.parse(v)), +}); +const array = (inner: Rule): Rule => ({ + parse(v) { + if (!Array.isArray(v)) { + throw new Error('Invalid telemetry array'); + } + return v.map((item) => inner.parse(item)); + }, +}); +export const codeHost = choice( + 'github', + 'gitlab', + 'bitbucket', + 'gitea', + 'azure_devops', + 'gerrit', + 'local_git', + 'remote_git', +); +export const provider = choice( + 'anthropic', + 'openai', + 'openai-compatible', + 'amazon-bedrock', + 'google-generative-ai', + 'google-vertex', + 'google-vertex-anthropic', + 'azure', + 'deepseek', + 'mistral', + 'openrouter', + 'xai', +); +export const deploymentType = choice( + 'cloud', + 'self_hosted', + 'local', + 'remote', + 'unknown', +); +const sourceCredential = choice( + 'none', + 'personal_access_token', + 'api_token', + 'access_token', + 'app_password', + 'http_access_token', +); +const aiCredential = choice( + 'api_key', + 'aws_default_chain', + 'aws_explicit_keys', + 'google_application_default_credentials', + 'google_credentials_file', +); +const modelSelection = choice('catalog', 'custom_entry', 'manual_fallback'); +const scope = choice( + 'all', + 'repositories', + 'organizations', + 'users', + 'groups', + 'projects', + 'workspaces', +); +const file = choice('config_json', 'env', 'compose_override'); +const identity = choice('created_from_setup_session', 'preserved_existing'); +const stage = choice( + 'setup_directory', + 'code_sources', + 'ai_setup', + 'hosted_url', + 'config_overwrite', + 'compose_file', + 'docker_validation', + 'start', +); +const category = choice( + 'validation', + 'network', + 'filesystem', + 'docker_unavailable', + 'docker_command', + 'process_spawn', + 'unknown', +); +const dockerOutcome = choice( + 'passed', + 'passed_after_cleanup', + 'unresolved_conflicts', + 'skipped_no_compose', + 'skipped_existing_deployment_running', + 'validation_failed', +); +const entityCounts = { + repositoryCount: count, + organizationCount: count, + userCount: count, + groupCount: count, + projectCount: count, + workspaceCount: count, +}; +export const codeSourceSchema = { + codeHost, + deploymentType, + credentialMode: sourceCredential, + scopeTypes: array(scope), + indexAll: boolean, + ...entityCounts, + generatedConnectionCount: count, + localDiscoveredRepoCountBucket: nullable( + choice('1', '2-5', '6-20', '21-100', '101+'), + ), +}; +export const aiSchema = { + provider, + modelSelectionMethod: modelSelection, + credentialMode: aiCredential, + usesCustomEndpoint: boolean, + hasDisplayName: boolean, +}; +export const eventSchemas = { + started: { + invocationMethod: choice( + 'npx', + 'global_binary', + 'local_binary', + 'workspace', + 'unknown', + ), + isInteractive: boolean, + }, + chose_setup_directory: { + usedDefaultDirectory: boolean, + directoryExisted: boolean, + directoryAction: choice('created', 'existing_directory_accepted'), + }, + configured_code_source: { + configurationIndex: position, + ...codeSourceSchema, + }, + configured_code_sources: { + codeSourceConfigurationCount: count, + generatedConnectionCount: count, + uniqueCodeHostCount: count, + codeHostTypes: array(codeHost), + credentialedCodeSourceCount: count, + cloudCodeSourceCount: count, + selfHostedCodeSourceCount: count, + localCodeSourceCount: count, + indexAllCodeSourceCount: count, + ...entityCounts, + }, + configured_ai_provider: { configurationIndex: position, ...aiSchema }, + ai_setup_completed: { + aiConfigured: boolean, + aiConfigurationCount: count, + uniqueProviderCount: count, + providerTypes: array(provider), + usesCustomEndpoint: boolean, + credentialModes: array(aiCredential), + modelSelectionMethods: array(modelSelection), + }, + configured_hosted_url: { + usedDefaultUrl: boolean, + protocol: choice('http', 'https'), + hostCategory: choice('localhost', 'address', 'unknown'), + }, + generated_configs: { + filesWritten: array(file), + overwroteExistingFiles: array(file), + wroteComposeOverride: boolean, + localMountCount: count, + generatedConnectionCount: count, + aiConfigurationCount: count, + credentialVariableCount: count, + deploymentIdentityAction: identity, + }, + resolved_compose_file: { + outcome: choice( + 'downloaded', + 'already_present', + 'declined', + 'download_failed', + ), + composeAvailable: boolean, + downloadPromptShown: boolean, + downloadAttempted: boolean, + failureCategory: nullable( + choice( + 'network', + 'http_4xx', + 'http_5xx', + 'filesystem', + 'timeout', + 'unknown', + ), + ), + }, + validated_docker_state: { + outcome: dockerOutcome, + dockerStatus: choice( + 'available', + 'unavailable', + 'error', + 'not_checked', + ), + composeContainerState: choice( + 'none', + 'running', + 'stopped', + 'mixed', + 'unknown', + ), + runningComposeContainerCount: nullable(count), + stoppedComposeContainerCount: nullable(count), + existingVolumeCount: nullable(count), + initialPortConflictCount: nullable(count), + remainingPortConflictCount: nullable(count), + portConflictSource: choice( + 'none', + 'docker', + 'non_docker', + 'mixed', + 'unknown', + ), + existingDeploymentAction: choice( + 'none', + 'stopped', + 'left_running', + 'stop_failed', + ), + stoppedContainerAction: choice( + 'none', + 'removed', + 'kept', + 'remove_failed', + ), + volumeAction: choice('none', 'removed', 'kept', 'remove_failed'), + portConflictAction: choice( + 'none', + 'containers_stopped', + 'kept', + 'stop_failed', + ), + leftExistingDeploymentRunning: boolean, + }, + completed: { + completionMode: choice( + 'sourcebot_start_spawned', + 'sourcebot_start_failed', + 'existing_deployment_left_running', + 'manual_start_required', + ), + sourcebotStartOffered: boolean, + sourcebotStartRequested: boolean, + sourcebotStartOutcome: choice( + 'spawned', + 'declined', + 'not_offered', + 'spawn_failed', + ), + composeAvailable: boolean, + dockerValidationOutcome: dockerOutcome, + remainingPortConflictCount: nullable(count), + generatedConnectionCount: count, + codeHostTypes: array(codeHost), + repositoryCount: count, + aiConfigured: boolean, + aiConfigurationCount: count, + providerTypes: array(provider), + deploymentIdentityAction: identity, + totalDurationMs: count, + }, + cancelled: { + stage, + reason: choice( + 'keyboard_interrupt', + 'existing_directory_declined', + 'config_overwrite_declined', + ), + }, + failed: { stage, failureCategory: category, recoverable: boolean }, +}; +export type EventName = keyof typeof eventSchemas; +export type Events = { [K in EventName]: Fields<(typeof eventSchemas)[K]> }; +export type CodeSourceSummary = Fields; +export type AiSummary = Fields; +export type Stage = Value; +export type FailureCategory = Value; +export type DockerSummary = Events['validated_docker_state']; +export function validateFields( + schema: S, + input: Fields, +): Fields { + // Read only explicitly approved fields; the source object is never spread or serialized. + return Object.fromEntries( + Object.entries(schema).map(([key, validator]) => [ + key, + validator.parse(input[key]), + ]), + ) as Fields; +} diff --git a/packages/setupWizard/src/telemetrySummary.ts b/packages/setupWizard/src/telemetrySummary.ts new file mode 100644 index 000000000..764ee8d06 --- /dev/null +++ b/packages/setupWizard/src/telemetrySummary.ts @@ -0,0 +1,228 @@ +import { isIP } from 'node:net'; +import { INSTALL_ID_PATTERN } from './telemetry.js'; +import type { + CodeSourceSummary, + AiSummary, + Events, + DockerSummary, +} from './telemetryEvents.js'; + +export function sourceSummary( + codeHost: CodeSourceSummary['codeHost'], + fields: Partial> = {}, +): CodeSourceSummary { + return { + codeHost, + deploymentType: 'unknown', + credentialMode: 'none', + scopeTypes: [], + indexAll: false, + repositoryCount: 0, + organizationCount: 0, + userCount: 0, + groupCount: 0, + projectCount: 0, + workspaceCount: 0, + generatedConnectionCount: 1, + localDiscoveredRepoCountBucket: null, + ...fields, + }; +} +export function normalizeHost(value: string): string | undefined { + try { + const input = value.trim(); + if (!input) { + return undefined; + } + const url = new URL( + /^[a-z][a-z\d+.-]*:\/\//i.test(input) ? input : `https://${input}`, + ); + if (!['http:', 'https:'].includes(url.protocol)) { + return undefined; + } + return url.hostname + .toLowerCase() + .replace(/\.+$/, '') + .replace(/^www\./, ''); + } catch { + return undefined; + } +} +export function deployment( + host: 'github' | 'gitlab' | 'gitea', + url: string, +): CodeSourceSummary['deploymentType'] { + const name = normalizeHost(url); + if (!name) { + return 'unknown'; + } + if (host === 'github') { + return name === 'github.com' || name.endsWith('.ghe.com') + ? 'cloud' + : 'self_hosted'; + } + if (host === 'gitlab') { + return name === 'gitlab.com' || + name.endsWith('.gitlab-dedicated.com') || + name.endsWith('.gitlab-dedicated.systems') + ? 'cloud' + : 'unknown'; + } + return name === 'gitea.com' ? 'cloud' : 'self_hosted'; +} +export function hostCategory( + value: string, +): Events['configured_hosted_url']['hostCategory'] { + try { + const url = new URL(value); + const host = url.hostname + .toLowerCase() + .replace(/\.+$/, '') + .replace(/^\[|\]$/g, ''); + if (!host) { + return 'unknown'; + } + return host === 'localhost' || + host.endsWith('.localhost') || + host === '::1' || + (isIP(host) === 4 && host.startsWith('127.')) + ? 'localhost' + : 'address'; + } catch { + return 'unknown'; + } +} +export function discoveredBucket( + n: number, +): CodeSourceSummary['localDiscoveredRepoCountBucket'] { + return n <= 1 + ? '1' + : n <= 5 + ? '2-5' + : n <= 20 + ? '6-20' + : n <= 100 + ? '21-100' + : '101+'; +} +const unique = (items: T[]): T[] => + [...new Set(items)].sort(); +export function aggregateSources( + sources: CodeSourceSummary[], +): Events['configured_code_sources'] { + const sum = ( + key: + | 'generatedConnectionCount' + | 'repositoryCount' + | 'organizationCount' + | 'userCount' + | 'groupCount' + | 'projectCount' + | 'workspaceCount', + ) => sources.reduce((n, s) => n + s[key], 0); + const codeHostTypes = unique(sources.map((s) => s.codeHost)); + return { + codeSourceConfigurationCount: sources.length, + generatedConnectionCount: sum('generatedConnectionCount'), + uniqueCodeHostCount: codeHostTypes.length, + codeHostTypes, + credentialedCodeSourceCount: sources.filter( + (s) => s.credentialMode !== 'none', + ).length, + cloudCodeSourceCount: sources.filter( + (s) => s.deploymentType === 'cloud', + ).length, + selfHostedCodeSourceCount: sources.filter( + (s) => s.deploymentType === 'self_hosted', + ).length, + localCodeSourceCount: sources.filter( + (s) => s.deploymentType === 'local', + ).length, + indexAllCodeSourceCount: sources.filter((s) => s.indexAll).length, + repositoryCount: sum('repositoryCount'), + organizationCount: sum('organizationCount'), + userCount: sum('userCount'), + groupCount: sum('groupCount'), + projectCount: sum('projectCount'), + workspaceCount: sum('workspaceCount'), + }; +} +export function aggregateAi(models: AiSummary[]): Events['ai_setup_completed'] { + const providerTypes = unique(models.map((m) => m.provider)); + return { + aiConfigured: models.length > 0, + aiConfigurationCount: models.length, + uniqueProviderCount: providerTypes.length, + providerTypes, + usesCustomEndpoint: models.some((m) => m.usesCustomEndpoint), + credentialModes: unique(models.map((m) => m.credentialMode)), + modelSelectionMethods: unique( + models.map((m) => m.modelSelectionMethod), + ), + }; +} +export function selectInstallId( + contents: string, + sessionId: string | undefined, +) { + // Parse only this setting. Accept normal dotenv quoting; never evaluate or expand it. + const lines = contents + .split(/\r?\n/) + .filter((l) => /^\s*(?:export\s+)?SOURCEBOT_INSTALL_ID\s*=/.test(l)); + const raw = lines + .at(-1) + ?.replace(/^\s*(?:export\s+)?SOURCEBOT_INSTALL_ID\s*=\s*/, '') + .trim(); + const existing = raw + ?.match(/^(?:"([0-9a-f-]+)"|'([0-9a-f-]+)'|([0-9a-f-]+))\s*(?:#.*)?$/) + ?.slice(1) + .find(Boolean); + return existing && INSTALL_ID_PATTERN.test(existing) + ? { id: existing, action: 'preserved_existing' as const } + : { id: sessionId, action: 'created_from_setup_session' as const }; +} +export function emptyDockerSummary(): DockerSummary { + return { + outcome: 'skipped_no_compose', + dockerStatus: 'not_checked', + composeContainerState: 'unknown', + runningComposeContainerCount: null, + stoppedComposeContainerCount: null, + existingVolumeCount: null, + initialPortConflictCount: null, + remainingPortConflictCount: null, + portConflictSource: 'unknown', + existingDeploymentAction: 'none', + stoppedContainerAction: 'none', + volumeAction: 'none', + portConflictAction: 'none', + leftExistingDeploymentRunning: false, + }; +} +export function dockerOutcome( + summary: DockerSummary, + composeAvailable: boolean, + failed: boolean, +): DockerSummary['outcome'] { + if (!composeAvailable) { + return 'skipped_no_compose'; + } + if (failed) { + return 'validation_failed'; + } + if (summary.leftExistingDeploymentRunning) { + return 'skipped_existing_deployment_running'; + } + if ( + (summary.remainingPortConflictCount ?? 0) > 0 || + summary.stoppedContainerAction === 'kept' + ) { + return 'unresolved_conflicts'; + } + return summary.existingDeploymentAction === 'stopped' || + summary.stoppedContainerAction === 'removed' || + summary.volumeAction === 'removed' || + summary.portConflictAction === 'containers_stopped' + ? 'passed_after_cleanup' + : 'passed'; +} diff --git a/packages/setupWizard/src/utils.ts b/packages/setupWizard/src/utils.ts index 8a7b08f1f..415ab0740 100644 --- a/packages/setupWizard/src/utils.ts +++ b/packages/setupWizard/src/utils.ts @@ -1,6 +1,7 @@ +import type { CodeSourceSummary } from './telemetryEvents.js'; import chalk from 'chalk'; import { randomBytes } from 'crypto'; -import { select as searchSelect } from 'inquirer-select-pro'; +import { searchSelect } from './prompts.js'; import type { ConnectionConfig } from '@sourcebot/schemas/v3/index.type'; export type { ConnectionConfig }; @@ -13,6 +14,7 @@ export type CollectResult = { */ connections: Array<{ name?: string; config: ConnectionConfig }>; env: EnvVars; + telemetry: CodeSourceSummary; /** * Optional host path that needs to be mounted into the Sourcebot container. * Surfaced in the wizard's next-steps so users get the matching volume mount line. diff --git a/packages/setupWizard/telemetry.html b/packages/setupWizard/telemetry.html new file mode 100644 index 000000000..c817bff0a --- /dev/null +++ b/packages/setupWizard/telemetry.html @@ -0,0 +1,2090 @@ + + + + + + + setup-sourcebot telemetry proposal + + + +
+ +
+
+ + +
+

setup-sourcebot PostHog telemetry proposal

+

Status: Draft for review · Canonical plan

+

Source of truth: this HTML document is the authoritative proposal. Make future plan changes here first. telemetry.md is a convenience mirror and should only be regenerated from this page when explicitly needed.

+

Implementation status (September 11, 2026): the required repository-root entrypoint.sh change was merged through Sourcebot PR #1648 as commit a0ee2233, with all automated checks passing. Runtime support is therefore landed in the Sourcebot repository; setup-to-deployment continuity becomes available to users once the wizard selects a published Sourcebot image containing that merge. A container-level compatibility test verified first boot, same-version restart, upgrade restart, generated-ID fallback, telemetry-disabled behavior, PostHog-compatible HTTPS payloads, and stable identity propagation using the merged entrypoint. The setup-wizard implementation is in progress in the separate codex/setup-wizard-posthog worktree. It is not release-complete until every mandatory verification gate below has passed.

+

Executive summary

+

Instrument the setup-sourcebot CLI so Sourcebot can measure progression from wizard start through a completed setup, identify where users leave the flow, and understand high-level configuration choices without collecting repository information, credentials, real-world user identifiers, or other sensitive values.

+

The canonical product funnel is:

+
started
+  -> chose_setup_directory
+  -> configured_code_sources
+  -> ai_setup_completed (configured or explicitly skipped)
+  -> configured_hosted_url
+  -> generated_configs
+  -> resolved_compose_file
+  -> validated_docker_state
+  -> completed
+
+

The AI checkpoint must fire whether AI is configured or skipped. The compose and Docker checkpoints must likewise fire with an outcome describing success, an existing resource, an explicit skip, or a failure. This keeps valid branches in the funnel instead of reporting them as abandonment.

+

The recommended implementation upgrades setup-sourcebot to Node 24 LTS and uses the official posthog-node SDK behind a small package-owned privacy wrapper. At process start, the wizard creates one canonical lowercase UUIDv4 setupSessionId with crypto.randomUUID() and uses it as the PostHog distinctId. The normative application format is exactly ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$: 36 ASCII characters, lowercase hexadecimal, canonical hyphens, version 4, and the IETF UUID variant. RFC 9562 is the current UUID standard and supersedes RFC 4122; Node’s current API documentation still describes randomUUID() as generating an “RFC 4122 version 4 UUID.” These descriptions are compatible for this value, but the regex and byte-for-byte reuse rule are authoritative for this plan.

+

For a new generated deployment, that exact, unmodified value is written to .env as SOURCEBOT_INSTALL_ID. The required Sourcebot repository change to the root entrypoint.sh is merged in PR #1648. When built into a published Sourcebot container image, the script preserves the supplied ID on first boot. It does not validate, normalize, regenerate, uppercase, or otherwise transform a non-empty supplied value; the wizard is responsible for supplying the canonical UUIDv4. The setup wizard does not download, patch, or rewrite entrypoint.sh at runtime. Together, these changes allow setup events and resulting deployment telemetry to share the same installation identity without a telemetry-only state file. No email, name, account, or other PII is associated with it.

+

Goals

+
    +
  • Measure conversion from wizard start through completion.
    • +
  • Explain conversion using only coarse product choices, counts, branch outcomes, and runtime compatibility information.
    • +
  • Associate a successful first-time setup with the resulting Sourcebot deployment without storing telemetry-only state or linking the random ID to a real person, organization, repository, or machine.
    • +
  • Make the schema stable enough to build PostHog funnels and breakdowns before implementation ships.
    • +
  • Require a comprehensive isolated E2E suite against the compiled npm artifact—including all setup branches, exact PostHog contracts, behavioral regression checks, Docker identity continuity, and cleanup—before the feature can be marked complete.
    • +
+

Non-goals

+
    +
  • Product analytics after the deployed Sourcebot instance starts. Existing Sourcebot telemetry owns that lifecycle.
    • +
  • Capturing prompt text, user-entered values, configuration objects, errors, logs, or debugging traces.
    • +
  • Proving that Sourcebot became healthy. The v1 completion event records the wizard’s terminal handoff, not application readiness.
    • +
  • Exactly-once delivery. A CLI can be killed before a best-effort request completes.
    • +
  • Building a durable workflow or adding a queue. This is a local interactive CLI whose telemetry must remain non-blocking.
    • +
+

Decisions and rationale

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DecisionRecommendationRationale
Installation boundaryDo not emit a Sourcebot PostHog event from package installation. Begin the funnel at started.This removes the need to correlate two processes or persist telemetry state. Reo’s existing installation tracker remains independent and unchanged.
Setup identityCreate one canonical lowercase UUIDv4 setupSessionId in memory at wizard startup with crypto.randomUUID(); require the exact plan regex and use the same string as the setup PostHog distinctId.One identifier is sufficient for the complete started -> completed funnel. It requires no telemetry-only persistence and cannot survive or combine separate wizard invocations.
Install-ID formatRequire exactly xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx, where every character is lowercase hexadecimal and y is 8, 9, a, or b. The complete validation expression is ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$.This is the canonical 36-character UUIDv4 representation produced by Node’s crypto.randomUUID() and observed from Sourcebot’s container uuidgen fallback. RFC 9562 is the current standard; it preserves the version-4 and IETF-variant layout formerly specified by RFC 4122. Defining the regex explicitly prevents prefixes, braces, uppercase conversion, hyphen removal, or other transformations.
PostHog person handlingUse setupSessionId as distinctId and keep PostHog’s default person-profile processing. Do not call identify or send $set/$set_once person properties.For a new setup, the same UUID becomes the deployment’s SOURCEBOT_INSTALL_ID. The profile remains pseudonymous and must never be enriched with email, name, organization, repository, credentials, or other PII/sensitive data.
ID storageDo not create any telemetry state outside the selected setup directory. Keep the ID in memory and write it only into the generated .env as SOURCEBOT_INSTALL_ID.The .env is already a required Sourcebot file. Reusing it avoids a second state location while giving the deployment the same identity as its setup session.
Deployment handoffUse a published Sourcebot image containing merged PR #1648, whose repository-root entrypoint.sh preserves a non-empty pre-supplied SOURCEBOT_INSTALL_ID on first boot and generates one with uuidgen only when none was supplied.This is a Sourcebot runtime/container-image change, not a script the wizard modifies. Released images without the PR overwrite the value on first boot and break setup-to-deployment correlation. After first boot, the existing /data/.installedv3 file remains authoritative.
TransportUse the official posthog-node SDK behind a package-owned typed privacy wrapper.The SDK owns delivery mechanics. The wrapper remains responsible for event allowlists, identity, privacy, and failure isolation.
Node runtimeRaise setup-sourcebot from the stale >=18 declaration to >=24.0.0, update @types/node to 24, and run its build, E2E tests, and release job on Node 24.Node 24 is the highest current LTS line. The wizard’s existing dependencies already require newer Node versions than its declared Node 18 floor, and all current runtime dependencies support Node 24. Staying on an LTS line avoids the churn of Node 26 Current while removing the need for a custom PostHog transport.
DeliveryUse posthog-node's asynchronous capture and transport defaults, then perform a bounded SDK shutdown when the process exits.No custom delivery policy is needed. The exit bound is the one CLI-specific choice because the SDK's default shutdown wait can be too long for an interactive command.
Stage timingEmit a checkpoint only when that stage reaches a terminal outcome.Prompt views or button clicks would overstate progress and make branches difficult to compare.
Optional stagesAlways emit the AI, compose, and Docker checkpoints with an explicit outcome.A valid skip or pre-existing resource should remain in the canonical funnel rather than appear as abandonment.
Repository countsSend exact counts for repositories the user selected; bucket only the total discovered by a local filesystem scan.Selected counts answer configuration-depth questions; discovered totals can expose unusually distinctive local environments without adding comparable value.
Completion boundaryEmit completion after Docker’s child process emits spawn, or after manual next steps are printed.Waiting for foreground docker compose up to exit can delay conversion for hours and still does not prove readiness.
ReadinessDefer a separate readiness event until there is a defined health-check requirement.Process launch and application health are different semantics and should not be mixed in one event.
Existing trackerRetain reo-census unchanged alongside the new PostHog telemetry.Reo remains an independent installation tracker with its own endpoint, payload, and PACKAGE_TRACKER_ANALYTICS control. Its events must not be imported into or treated as PostHog funnel events.
PostHog opt-outDo not provide an environment-variable or wizard-level opt-out for the new setup-wizard PostHog telemetry.npm does not require packages to provide a telemetry opt-out. PACKAGE_TRACKER_ANALYTICS remains Reo-only, and SOURCEBOT_TELEMETRY_DISABLED continues to govern deployed Sourcebot telemetry rather than this setup funnel. This policy still requires privacy/legal review and clear documentation before release.
+

Funnel strategy

+

Timestamp ordering: validated during implementation

+

The live dev-project smoke exposed an ordering issue: with immediate independent requests, PostHog adjusts each event by its request's sent_at clock skew, which can reorder closely spaced checkpoints. Set the documented $ignore_sent_at: true control on every setup event. Pass an SDK-envelope timestamp anchored to invocation-start wall time plus monotonic elapsed time, increasing by at least 1 ms per event. This preserves within-run order without delaying setup or serializing network requests. It adds no custom wall-clock application property and does not change transport retry defaults.

+

Tradeoff: absolute dates depend on the invoking machine's clock; durations and within-run order do not. Cross-machine chronological ordering against deployment events is not guaranteed when clocks are badly skewed. Verify stored event order in the live smoke, not only capture-call order. Reference: PostHog timestamp processing.

+

Event semantics

+
    +
  • Event names use snake_case, matching existing Sourcebot PostHog conventions.
    • +
  • Events use the setup_sourcebot_ prefix. The wa_ prefix is not appropriate because these events come from the CLI, not the web app.
    • +
  • Product funnel events fire at most once per setup session, after the corresponding stage reaches a terminal outcome.
    • +
  • Repeated detail events may fire once per configured code source or AI model configuration.
    • +
  • A telemetry failure must never block, delay materially, or fail setup.
    • +
  • Events use PostHog’s default person-profile behavior, keyed by the random setupSessionId. The CLI must never call identify, set custom person properties, or associate the profile with PII or sensitive data.
    • +
+

Feasibility finding

+

This identity handoff fits the existing Sourcebot path with one small deployment-runtime change:

+
    +
  • The generated deployment already loads .env into the Sourcebot container through docker-compose.yml’s env_file, so the wizard can pass SOURCEBOT_INSTALL_ID without changing the compose schema.
    • +
  • Sourcebot images built before PR #1648 replace SOURCEBOT_INSTALL_ID with uuidgen whenever /data/.installedv3 does not exist. The merged repository-root entrypoint.sh now lets a non-empty supplied value win on first boot. The image selected by the wizard must contain the merge before identity continuity is available to users.
    • +
  • Sourcebot’s current runtime and Lighthouse schemas accept a general non-empty string for the ID; they do not impose a stricter UUID validator. This proposal deliberately adopts the canonical format produced by Sourcebot’s own first-boot generator as the setup-wizard contract.
    • +
  • Backend deployment telemetry already uses SOURCEBOT_INSTALL_ID as distinctId, the install_id event property, and the PostHog company group.
    • +
  • Some web telemetry uses a browser- or user-level distinctId, but it still attaches the deployment install_id and company group. Cross-surface deployment analysis should therefore use those deployment dimensions.
    • +
+

Scope boundary: the wizard change ends after writing SOURCEBOT_INSTALL_ID to the generated .env. The root entrypoint.sh change is merged through Sourcebot PR #1648 and ships through the normal Sourcebot container-image build. The wizard never fetches or edits that script.

+

Conclusion: full identity continuity is achievable for deployments newly generated by the wizard. It requires coordinated setup-wizard and Sourcebot container-runtime changes, but does not require a second persisted ID, PostHog aliases, or changes to the deployed telemetry schema.

+

Normative install-ID compatibility contract

+

The wizard creates one canonical lowercase UUIDv4 before emitting started. It remains in memory for that invocation and is reused byte-for-byte as the resulting deployment ID for a new setup.

+

This is a MUST-level implementation requirement: setupSessionId must be the direct string returned by Node’s crypto.randomUUID() and must match ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$. The plan calls this a canonical lowercase UUIDv4 under RFC 9562, compatible with the RFC 4122 UUIDv4 terminology used by Node. The exact regex is normative if terminology differs between documentation versions.

+

The value must remain byte-for-byte identical across this chain:

+
crypto.randomUUID()
+  = setupSessionId
+  = every setup event distinctId
+  = every setup event install_id
+  = every setup event company group key
+  = generated .env SOURCEBOT_INSTALL_ID
+  = first-boot container SOURCEBOT_INSTALL_ID
+  = deployment install event distinct_id
+  = /data/.installedv3 install_id
+  = SOURCEBOT_INSTALL_ID restored on every later boot
+
+

No boundary may add a prefix, braces, quotes, whitespace, or a trailing newline; change letter case; remove hyphens; hash, encode, parse and reformat, or regenerate the value. The generated .env line is exactly SOURCEBOT_INSTALL_ID=<setupSessionId>. Canonical UUID characters require no shell or dotenv escaping.

+

Responsibility boundary: the wizard enforces and tests the UUIDv4 format. PR #1648 intentionally keeps the runtime handoff simple: on first boot, entrypoint.sh preserves any non-empty supplied value and calls uuidgen only when the value is missing or empty. It then safely serializes the selected value with jq. On later boots, /data/.installedv3 wins over the environment. Because the entrypoint does not normalize the supplied value, a conforming wizard value passes through unchanged.

+

Release compatibility gate: the default downloaded Compose file must select a published Sourcebot image containing PR #1648. Existing or user-customized Compose files are outside this compatibility guarantee: reuse them without image inspection, rewriting, warnings, or blocking setup. The wizard still writes the install ID normally, but setup-to-deployment continuity is not guaranteed if a user-selected image replaces it. This exception applies to all image-compatibility and deployment-continuity requirements below.

+

Identity and funnel correlation

+ + + + + + + + + + + + + + + + + + + + + + + + + +
IdentifierLifetimePurpose
setupSessionIdOne CLI invocationCanonical lowercase 36-character UUIDv4 created with crypto.randomUUID() before started, conforming to the exact plan regex. It correlates every event in this wizard run.
PostHog distinctIdOne CLI invocation, then deployment identitySet to setupSessionId for every setup event. For a new setup, deployment backend and first-run events later use the same value as SOURCEBOT_INSTALL_ID.
PostHog install_id and company groupOne CLI invocation, then deployment identitySet to setupSessionId on setup events, matching Sourcebot’s existing deployment telemetry convention and enabling group-level continuity even where web-user distinctId values differ.
+

This supports two deliberately simple analyses:

+
    +
  1. Setup funnel: started to completed using setupSessionId as the PostHog person/distinct identity.
    2. +
  2. Setup-to-deployment continuity for a new setup: query the same UUID through setup events and later deployment events/properties/groups.
    3. +
+

Separate wizard invocations intentionally receive different IDs and are not merged. The plan does not attempt eventual conversion across retries because doing so would require persisted telemetry state or identity merging.

+

Recommended PostHog analyses

+
    +
  1. Canonical product funnel: started -> chose_setup_directory -> configured_code_sources -> ai_setup_completed -> configured_hosted_url -> generated_configs -> resolved_compose_file -> validated_docker_state -> completed.
    2. +
  2. Setup-to-deployment continuity: filter or break down setup and deployment events by the shared install_id property or company group.
    3. +
  3. AI branch comparison: break down ai_setup_completed by aiConfigured, then compare downstream completion rather than removing the AI checkpoint.
    4. +
  4. Completion breakdown: break down completed by completionMode, codeHostTypes, aiConfigured, and dockerValidationOutcome.
    5. +
  5. Drop-off diagnosis: compare the next missing checkpoint with cancelled and failed where recoverable: false, broken down only by their fixed stage and reason/category enums. Analyze failed with recoverable: true separately as friction that may precede further progress, completion, cancellation, or a later fatal error.
    6. +
+

The canonical setup funnel can use a standard ordered PostHog funnel because every event in one run has the same distinctId. For a new deployment, backend and container first-run events also use that ID. Browser events may use an anonymous browser or authenticated-user distinctId, but Sourcebot already attaches install_id and the company group, so deployment-level analysis should use those dimensions rather than assuming every web event shares the setup person ID.

+

PostHog event schema

+

Common properties

+

These properties are included on every event unless explicitly noted.

+

The system-derived properties are platform, arch, nodeMajorVersion, packageManager, and isCI. Collect each one independently and best-effort. Failure to read, detect, or parse any system-derived property must never suppress the event, throw into the wizard, change its exit status, or affect setup behavior. Use the fallback defined for that property below while retaining every other successfully collected property.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeDescription
schemaVersion1Version of this event contract.
source"setup-sourcebot-cli"Stable event source.
setupSourcebotVersionstringPublished setup-sourcebot package version.
setupSessionIdcanonical lowercase UUIDv4 stringCanonical lowercase UUIDv4 under RFC 9562, compatible with Node’s RFC 4122 terminology, matching ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$. The exact same string is also used as PostHog distinctId and, for a new generated deployment, SOURCEBOT_INSTALL_ID.
install_idstringSet to setupSessionId, matching Sourcebot deployment telemetry’s existing property name.
platform"darwin" | "linux" | "win32" | "other"Coarse operating-system family. Use "other" when unavailable or outside the allowlist.
arch"arm64" | "x64" | "other"Coarse processor architecture. Use "other" when unavailable or outside the allowlist.
nodeMajorVersionnumber | nullNode.js major version only. Use null if the value cannot be read or parsed; do not substitute 0 or a string.
packageManager"npm" | "yarn" | "pnpm" | "bun" | "unknown"Detected package manager. Use "unknown" when it cannot be detected or is outside the allowlist.
isCIboolean | nullWhether a recognized CI environment is active. Use null when detection cannot determine the answer; do not treat detection failure as false. Do not include CI vendor names or environment values.
elapsedMsnumberMilliseconds since the CLI invocation began.
$ignore_sent_attruePreserve ordered SDK-envelope timestamps instead of per-request clock-skew adjustment. Fixed ingestion control, never a wizard answer.
$geoip_disabletruePrevent IP-based geolocation enrichment.
+

Every capture also sets PostHog distinctId: setupSessionId and groups: { company: setupSessionId }. This mirrors Sourcebot’s deployment-level grouping without adding another identifier.

+

Do not add working directory, OS release, CPU count, hostname, IP-derived location, Git configuration, npm username, or arbitrary environment variables as common properties.

+

Product funnel event catalog

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Funnel conceptProposed eventFires when
startedsetup_sourcebot_startedThe CLI process starts, before the first prompt.
chose setup directorysetup_sourcebot_chose_setup_directoryThe directory is created or the user agrees to reuse an existing directory.
configured code sourcessetup_sourcebot_configured_code_sourcesThe user finishes the one-or-more code-source loop.
optional AI setupsetup_sourcebot_ai_setup_completedThe user finishes AI configuration or explicitly skips it.
configured hosted URLsetup_sourcebot_configured_hosted_urlA valid hosted URL is accepted.
generated configssetup_sourcebot_generated_configsAll required configuration files are written successfully.
downloaded compose filesetup_sourcebot_resolved_compose_fileThe compose-file stage ends with a download, existing file, explicit decline, or failure.
Docker state validatedsetup_sourcebot_validated_docker_stateDocker validation and optional cleanup finish, or validation is skipped for a known reason.
complete setupsetup_sourcebot_completedThe wizard hands off to Docker or prints actionable manual next steps.
+

setup_sourcebot_resolved_compose_file is intentionally named “resolved” rather than “downloaded.” A pre-existing compose file is a successful outcome, while a declined or failed download still allows the wizard to finish with manual next steps.

+

Event definitions

+

setup_sourcebot_started

+

Purpose: establish the start of the setup funnel and its in-memory session/deployment identity.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
invocationMethod"npx" | "global_binary" | "local_binary" | "workspace" | "unknown"yesHow the CLI appears to have been launched.
isInteractivebooleanyesWhether stdin and stdout are interactive terminals.
+

setup_sourcebot_chose_setup_directory

+

Purpose: measure progression through the first prompt and whether setup is new or overwriting an existing directory.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
usedDefaultDirectorybooleanyesWhether the default sourcebot directory was accepted.
directoryExistedbooleanyesWhether the selected directory already existed.
directoryAction"created" | "existing_directory_accepted"yesThe resulting directory path branch.
+

Never include the entered path, its basename, its parent, or any information about files already in the directory.

+

setup_sourcebot_configured_code_source

+

This is a repeated diagnostic event, emitted once after each code-source configuration is accepted. It is not a required step in the product funnel, but it makes the aggregate checkpoint explainable.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
configurationIndexnumberyesOne-based position in the code-source configuration loop.
codeHost"github" | "gitlab" | "bitbucket" | "gitea" | "azure_devops" | "gerrit" | "local_git" | "remote_git"yesConfigured source type.
deploymentType"cloud" | "self_hosted" | "local" | "remote" | "unknown"yesCoarse hosting mode derived locally from the source branch and existing wizard answers. Use "unknown" when the available answers cannot distinguish the mode.
credentialMode"none" | "personal_access_token" | "api_token" | "access_token" | "app_password" | "http_access_token"yesCredential mechanism, never the credential value.
scopeTypesfixed enum arrayyesSubset of all, repositories, organizations, users, groups, projects, and workspaces.
indexAllbooleanyesWhether every repository visible to the credential was selected.
repositoryCountnumberyesExplicit remote repositories or selected local repositories; use 0 when not applicable.
organizationCountnumberyesSelected organizations; use 0 when not applicable.
userCountnumberyesSelected users; use 0 when not applicable.
groupCountnumberyesSelected groups; use 0 when not applicable.
projectCountnumberyesSelected projects; use 0 when not applicable.
workspaceCountnumberyesSelected Bitbucket workspaces; use 0 when not applicable.
generatedConnectionCountnumberyesNumber of config connections generated by this selection.
localDiscoveredRepoCountBucket"1" | "2-5" | "6-20" | "21-100" | "101+" | nullyesBucketed number of discovered local repositories. null for non-local sources.
+

Rules by source:

+
    +
  • For URL-based classification, trim surrounding whitespace. If the value has no URI scheme, prepend https:// for classification only; this does not rewrite the user’s configuration. Accept only http: or https:, parse with Node’s URL, lowercase URL.hostname, remove trailing DNS dots, and remove one leading www.. URL.hostname naturally excludes the protocol, credentials, port, path, query, and fragment, so inputs such as github.com, www.github.com, https://github.com, and https://www.github.com:443/path all classify from the same normalized hostname. Do not perform DNS, HTTP, IP, or ownership lookups. Never retain or send the hostname, URL, path, port, or parse error.
    • +
  • Match provider domains only by exact normalized hostname or a dot-boundary suffix that includes the leading dot, such as .ghe.com. Never use substring matching: a hostname such as notgithub.com must not match github.com.
    • +
  • GitHub is cloud when the normalized hostname is exactly github.com or ends in .ghe.com; other valid hostnames are self_hosted. The .ghe.com rule covers GitHub Enterprise Cloud with data residency.
    • +
  • GitLab is cloud when the normalized hostname is exactly gitlab.com, ends in .gitlab-dedicated.com, or ends in .gitlab-dedicated.systems. Other valid hostnames are unknown, because GitLab Dedicated supports arbitrary custom domains that cannot be distinguished locally from GitLab Self-Managed without adding a new prompt or network lookup.
    • +
  • Bitbucket uses the existing explicit Cloud versus Data Center wizard answer: Cloud maps to cloud and Data Center maps to self_hosted.
    • +
  • Azure DevOps uses the existing explicit Cloud versus Server wizard answer: Cloud maps to cloud and Server maps to self_hosted. Do not report organization names, server URLs, or whether /tfs appears in a real URL.
    • +
  • Gitea is cloud when the normalized hostname is exactly gitea.com; other valid hostnames are self_hosted.
    • +
  • Gerrit always maps to self_hosted.
    • +
  • The Local Git source branch always maps to local. Report selected repository counts and a bucketed discovered count, never paths or repository names.
    • +
  • The arbitrary Remote Git source branch always maps to remote. Report one repository, never its clone URL.
    • +
  • An unparseable URL, missing collector answer, unsupported source type, or unexpected value maps to unknown. The event must still be captured.
    • +
+

setup_sourcebot_configured_code_sources

+

Purpose: product-funnel checkpoint after all code sources have been configured.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
codeSourceConfigurationCountnumberyesNumber of passes through the code-source loop.
generatedConnectionCountnumberyesTotal generated config.json connections. This can exceed the loop count for local repositories.
uniqueCodeHostCountnumberyesNumber of unique code-host types.
codeHostTypesfixed enum arrayyesDeduplicated codeHost values, sorted for stable payloads.
credentialedCodeSourceCountnumberyesConfigured sources with a credential present.
cloudCodeSourceCountnumberyesSources configured against known cloud services.
selfHostedCodeSourceCountnumberyesSources configured against self-hosted services.
localCodeSourceCountnumberyesLocal directory selections.
indexAllCodeSourceCountnumberyesSources configured to index everything visible.
repositoryCountnumberyesTotal explicit remote and selected local repositories.
organizationCountnumberyesTotal selected organizations.
userCountnumberyesTotal selected users.
groupCountnumberyesTotal selected groups.
projectCountnumberyesTotal selected projects.
workspaceCountnumberyesTotal selected Bitbucket workspaces.
+

Do not serialize or spread connections into telemetry. Construct this event from an explicit allowlisted summary object.

+

setup_sourcebot_configured_ai_provider

+

This is a repeated diagnostic event, emitted once per configured model. A user can configure multiple models for the same provider.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
configurationIndexnumberyesOne-based position in the model configuration loop.
providerfixed enumyesOne of anthropic, openai, openai-compatible, amazon-bedrock, google-generative-ai, google-vertex, google-vertex-anthropic, azure, deepseek, mistral, openrouter, or xai.
modelSelectionMethod"catalog" | "custom_entry" | "manual_fallback"yesWhether the model came from the catalog, a custom value entered through catalog search, or manual input after catalog failure/unavailability.
credentialMode"api_key" | "aws_default_chain" | "aws_explicit_keys" | "google_application_default_credentials" | "google_credentials_file"yesCoarse credential strategy.
usesCustomEndpointbooleanyesTrue only for an OpenAI-compatible custom endpoint.
hasDisplayNamebooleanyesWhether the optional display-name field was populated.
+

Never include model names or IDs, display names, API keys, resource names, base URLs, cloud project IDs, regions, API versions, access-key IDs, or credential-file paths.

+

setup_sourcebot_ai_setup_completed

+

Purpose: preserve a linear funnel while distinguishing configured and intentionally skipped AI setup.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
aiConfiguredbooleanyesWhether at least one model was configured.
aiConfigurationCountnumberyesNumber of configured models; 0 when skipped.
uniqueProviderCountnumberyesNumber of unique provider types; 0 when skipped.
providerTypesfixed enum arrayyesDeduplicated provider values, sorted; empty when skipped.
usesCustomEndpointbooleanyesWhether any OpenAI-compatible endpoint was configured.
credentialModesfixed enum arrayyesDeduplicated credential strategies, sorted; empty when skipped.
modelSelectionMethodsfixed enum arrayyesDeduplicated selection methods, sorted; empty when skipped.
+

setup_sourcebot_configured_hosted_url

+

Purpose: measure progression through deployment URL configuration without collecting the URL.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
usedDefaultUrlbooleanyesWhether http://localhost:3000 was accepted unchanged.
protocol"http" | "https"yesURL protocol only.
hostCategory"localhost" | "address" | "unknown"yesLocal structural classification that distinguishes confirmed loopback hosts from every other successfully parsed address.
+

Determine hostCategory only from the already-entered hosted URL. Parse it with Node’s URL, lowercase the hostname, remove trailing DNS dots, and remove IPv6 URL brackets before applying these rules:

+
    +
  • localhost: the parsed hostname is exactly localhost, ends in .localhost, is an IPv4 address in 127.0.0.0/8, or is the IPv6 loopback address ::1.
    • +
  • address: the URL parses successfully and has any other non-empty hostname or IP address. This value makes no claim about whether the address is public, private, reachable, or resolvable.
    • +
  • unknown: the URL or hostname is unavailable, cannot be parsed, or the classifier fails unexpectedly. The event must still be captured.
    • +
+

This classifier performs no DNS, HTTP, socket, IP ownership, or reachability lookup. It must discard the parsed value after producing the fixed category.

+

Never include the URL, hostname, domain, path, query, fragment, or port.

+

setup_sourcebot_generated_configs

+

Purpose: record that the configuration-writing stage completed successfully.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
filesWrittenfixed enum arrayyesSubset of config_json, env, and compose_override.
overwroteExistingFilesfixed enum arrayyesExisting fixed file types the user agreed to overwrite. Empty for a new setup.
wroteComposeOverridebooleanyesWhether local-repository mounts required an override file.
localMountCountnumberyesNumber of local root directories mounted.
generatedConnectionCountnumberyesNumber of connections written to config.json.
aiConfigurationCountnumberyesNumber of model configurations written.
credentialVariableCountnumberyesCount of credential environment variables written, never their names or values.
deploymentIdentityAction"created_from_setup_session" | "preserved_existing"yesWhether the generated deployment received this run’s setupSessionId or retained a valid ID from an existing .env. The existing ID itself is never copied into setup telemetry.
+

The event fires only after every required write succeeds. It must not include file paths, file contents, environment-variable names, or generated secrets.

+

setup_sourcebot_resolved_compose_file

+

Purpose: represent every terminal branch of the compose-file stage.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
outcome"downloaded" | "already_present" | "declined" | "download_failed"yesResult of the compose-file stage.
composeAvailablebooleanyesWhether a compose file is available for later validation and startup.
downloadPromptShownbooleanyesWhether the download prompt was shown.
downloadAttemptedbooleanyesWhether an HTTP download was attempted.
failureCategory"network" | "http_4xx" | "http_5xx" | "filesystem" | "timeout" | "unknown" | nullyesCoarse failure reason. null unless outcome is download_failed.
+

Never include the download URL, HTTP body, destination path, or raw error message.

+

setup_sourcebot_validated_docker_state

+

Purpose: measure whether the generated setup can be started immediately and how often existing Docker state requires intervention.

+

This event must fire even when the phase is skipped because no compose file is available or because an existing deployment is intentionally left running.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
outcome"passed" | "passed_after_cleanup" | "unresolved_conflicts" | "skipped_no_compose" | "skipped_existing_deployment_running" | "validation_failed"yesOverall result of the Docker validation phase.
dockerStatus"available" | "unavailable" | "error" | "not_checked"yesWhether Docker commands could be executed successfully.
composeContainerState"none" | "running" | "stopped" | "mixed" | "unknown"yesAggregate state, never container names.
runningComposeContainerCountnumber | nullyesCount of running containers belonging to the compose project.
stoppedComposeContainerCountnumber | nullyesCount of stopped containers belonging to the compose project.
existingVolumeCountnumber | nullyesCount of matching existing volumes.
initialPortConflictCountnumber | nullyesConflicts found before remediation. Do not include port numbers.
remainingPortConflictCountnumber | nullyesConflicts remaining after remediation.
portConflictSource"none" | "docker" | "non_docker" | "mixed" | "unknown"yesCoarse owner category.
existingDeploymentAction"none" | "stopped" | "left_running" | "stop_failed"yesOutcome of the running-deployment prompt.
stoppedContainerAction"none" | "removed" | "kept" | "remove_failed"yesOutcome of stopped-container cleanup.
volumeAction"none" | "removed" | "kept" | "remove_failed"yesOutcome of existing-volume cleanup.
portConflictAction"none" | "containers_stopped" | "kept" | "stop_failed"yesOutcome of port-conflict cleanup.
leftExistingDeploymentRunningbooleanyesWhether an existing deployment was intentionally or unsuccessfully left running.
+

Unknown and skipped measurements: all five Docker count properties remain required, but use null when their check was skipped, failed, or could not produce a complete result. Use 0 only after a successful check confirmed zero matching resources/conflicts (including successfully parsing a Compose file with no published ports). Never coerce unknown values to zero in completion summaries or PostHog analyses. Preserve independent successful measurements when another check fails.

+
    +
  • Container counts and composeContainerState describe the initial successful Compose inventory, before cleanup. Volume count describes the inventory before volume cleanup. An unavailable inventory produces null counts and, for containers, unknown state.
    • +
  • initialPortConflictCount and portConflictSource describe the first complete port check, after deployment/volume cleanup and before port-conflict remediation. If the necessary Docker-owner or socket checks fail, use null and unknown rather than claiming a complete inventory. remainingPortConflictCount describes the recheck after attempted port remediation; use null if that recheck fails. If no port remediation is attempted, reuse the initial count, including null.
    • +
  • With no Compose file, emit skipped_no_compose, dockerStatus: "not_checked", all five counts as null, container state and conflict source as unknown, actions as none, and leftExistingDeploymentRunning: false.
    • +
  • When an existing deployment is left running, retain the measured initial container inventory; skipped volume and port checks stay null and conflict source stays unknown. Set leftExistingDeploymentRunning: true and retain the actual deployment action, including stop_failed when applicable.
    • +
  • Action none means no action was offered or attempted, including skipped checks; it does not assert that no resources existed. kept and left_running mean the user explicitly declined the corresponding action.
    • +
  • dockerStatus is not_checked if no Docker check ran, unavailable if a check established a missing/inaccessible executable, Compose plugin, daemon, or socket, error if attempted checks cannot establish availability, and available when availability was confirmed. An individual cleanup command failure does not by itself mean Docker is unavailable.
    • +
+

Overall outcome precedence: first use skipped_no_compose when applicable. Otherwise, any required inspection or attempted cleanup failure yields validation_failed, even when later checks succeed or conflicts also exist. With no such failure, use skipped_existing_deployment_running for a deliberately retained running deployment, then unresolved_conflicts for remaining measured port conflicts, then passed_after_cleanup if cleanup occurred successfully, otherwise passed. Action fields and counts retain the additional detail. This classification does not alter prompts, Docker actions, or continuation behavior; operation failures also emit the nonterminal diagnostics defined below.

+

Never include container names, service names, volume names, Docker project names, port numbers, process information, compose contents, or raw command output.

+

setup_sourcebot_completed

+

Purpose: final funnel conversion event. “Completed” means the wizard finished its work and either handed off to Docker or printed sufficient manual next steps; it does not claim that Sourcebot became healthy.

+

Additional properties:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
completionMode"sourcebot_start_spawned" | "sourcebot_start_failed" | "existing_deployment_left_running" | "manual_start_required"yesHow the wizard ended.
sourcebotStartOfferedbooleanyesWhether the final start prompt was shown.
sourcebotStartRequestedbooleanyesWhether the user chose to run docker compose up.
sourcebotStartOutcome"spawned" | "declined" | "not_offered" | "spawn_failed"yesResult of the start handoff. This describes process launch, not application readiness.
composeAvailablebooleanyesWhether the compose file was available at completion.
dockerValidationOutcomeenum from validation eventyesFinal Docker validation result.
remainingPortConflictCountnumber | nullyesCopy the Docker validation value unchanged, including null when unmeasured. Do not replace unknown with zero.
generatedConnectionCountnumberyesTotal generated code connections.
codeHostTypesfixed enum arrayyesDeduplicated configured code-host types.
repositoryCountnumberyesTotal explicit remote and selected local repositories.
aiConfiguredbooleanyesWhether any AI model was configured.
aiConfigurationCountnumberyesNumber of configured models.
providerTypesfixed enum arrayyesDeduplicated AI provider types.
deploymentIdentityAction"created_from_setup_session" | "preserved_existing"yesSafe summary of whether this run established deployment identity continuity. This repeats the generated-config value so completion can be filtered directly.
totalDurationMsnumberyesDuration from CLI start to this terminal handoff.
+

When the user chooses to start Sourcebot, emit this event after the child process successfully emits spawn, not after foreground docker compose up exits. Waiting for process exit can delay the event until the user stops Sourcebot hours later or presses Ctrl+C.

+

If actual readiness is important, add a separate setup_sourcebot_became_ready event when the local readiness poll succeeds. Do not overload setup_sourcebot_completed with a health claim.

+

Drop-off diagnostic events

+

These events are not funnel checkpoints, but they are necessary to explain missing next-step events.

+

setup_sourcebot_cancelled

+ + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
stagefixed enumyesOne of setup_directory, code_sources, ai_setup, hosted_url, config_overwrite, compose_file, docker_validation, or start.
reason"keyboard_interrupt" | "existing_directory_declined" | "config_overwrite_declined"yesAllowlisted cancellation reason.
+

The config-overwrite case may additionally include fileType: "config_json" | "env" | "compose_override". Never include a path or filename supplied by the user.

+

setup_sourcebot_failed

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PropertyTypeRequiredDescription
stagesame stage enum as cancellationyesStage active when the failure occurred.
failureCategoryfixed enumyesOne of validation, network, filesystem, docker_unavailable, docker_command, process_spawn, or unknown.
recoverablebooleanyestrue when the wizard handles the error and continues to further setup work or manual next steps; false only when the error causes the wizard to terminate. This describes actual control flow, not whether the underlying problem could theoretically be fixed.
+

Failure-event lifecycle: setup_sourcebot_failed with recoverable: true is a nonterminal diagnostic event. Emit it once per distinct handled operation failure; multiple such events may occur in one stage or session, including with identical stage/category values. Keep the same session ID and the SDK active so later checkpoints and diagnostics can be captured. A stage outcome may summarize the same failure without emitting a second failure event for that operation. Only failed with recoverable: false ends the setup telemetry chain due to an error.

+

These categories intentionally stay coarse and cover the wizard’s current fatal and recoverable failure paths:

+
    +
  • validation: malformed or unsupported user/configuration data that escapes normal prompt validation, including local parsing or invariant failures. Ordinary rejected prompt input is not a failure event because the prompt remains open.
    • +
  • network: a required fetch, HTTP response, timeout, or response-decoding operation fails. Best-effort autocomplete/model-catalog failures that successfully fall back do not emit setup_sourcebot_failed.
    • +
  • filesystem: directory creation/change, local repository inspection, configuration or compose-file read/write, permissions, missing files, or storage-capacity operations fail.
    • +
  • docker_unavailable: the Docker executable, Compose plugin, daemon, or daemon socket is unavailable or inaccessible.
    • +
  • docker_command: Docker is available, but a Docker/Compose inspect, stop, remove, volume, or startup command fails or returns unusable output.
    • +
  • process_spawn: a required child process fails to spawn and the failure is not more specifically docker_unavailable. A missing Docker executable therefore remains docker_unavailable.
    • +
  • unknown: any remaining unexpected runtime, prompt-library, cryptography, serialization, dependency, or programming failure. This catch-all ensures telemetry classification never replaces or masks the original wizard behavior.
    • +
+

Classify from the operation that failed and fixed error metadata such as a known operation result or error code; never inspect or transmit free-form exception messages, command output, paths, URLs, or user input. When more than one category appears applicable, prefer the most specific category above rather than emitting multiple failure events.

+

Do not send exception messages, stack traces, command lines, stderr, HTTP bodies, or arbitrary error codes. If more detail is needed, add a reviewed fixed enum rather than forwarding runtime text.

+

Privacy and data minimization

+

Explicitly allowed

+
    +
  • Fixed enums describing product choices.
    • +
  • Booleans describing whether optional capabilities or credentials were configured.
    • +
  • Counts of selected configuration entities.
    • +
  • Bucketed counts where the exact value is not needed, such as repositories discovered during a local filesystem scan.
    • +
  • Coarse runtime compatibility fields: OS family, architecture family, Node major version, and package manager.
    • +
  • Random UUIDs generated solely for pseudonymous telemetry correlation and PostHog’s default Person profile.
    • +
+

Explicitly prohibited

+
    +
  • Access tokens, API keys, passwords, generated secrets, credential contents, or arbitrary environment variables.
    • +
  • Repository, organization, group, project, workspace, or user names.
    • +
  • Search/autocomplete input.
    • +
  • Git clone URLs, code-host URLs, hosted Sourcebot URLs, custom AI endpoint URLs, domains, hostnames, IP addresses, or port numbers.
    • +
  • Setup directories, working directories, local repository paths, credential-file paths, filenames derived from user input, or file contents.
    • +
  • Email addresses or domains, Git usernames, npm usernames, OS usernames, or machine hostnames.
    • +
  • Model names/IDs, display names, cloud project/resource names, regions, or API versions.
    • +
  • Container, service, volume, process, or Docker project names.
    • +
  • Raw errors, stack traces, command output, or HTTP response bodies.
    • +
  • Sourcebot configuration objects or environment maps, even after attempted redaction.
    • +
  • PostHog identify, alias, $set, $set_once, or any custom Person property that could enrich or link the random session/deployment profile to a real person, organization, repository, or machine.
    • +
+

Every event must be built from an event-specific allowlisted object. Code must never spread prompt results, connection configs, model configs, environment maps, process.env, errors, or Docker command results into event properties.

+

The SDK wrapper does not deliberately send an IP address, hostname, or user agent as an event property, and disableGeoip: true prevents PostHog GeoIP enrichment. As with any direct HTTPS request, the receiving endpoint can observe the source IP at the transport layer. If policy requires that PostHog never receive the client IP at all, direct client-side capture is insufficient and the design must instead use a reviewed first-party relay. That stricter requirement is not assumed by this proposal.

+

Telemetry controls and delivery

+
    +
  • The new setup-wizard PostHog telemetry has no package-level opt-out. It does not inspect SOURCEBOT_TELEMETRY_DISABLED or PACKAGE_TRACKER_ANALYTICS before creating its random ID or sending events.
    • +
  • PACKAGE_TRACKER_ANALYTICS remains owned and interpreted only by reo-census. Setting it to false disables Reo’s tracking but does not disable PostHog setup-wizard telemetry.
    • +
  • SOURCEBOT_TELEMETRY_DISABLED remains applicable to telemetry from the deployed Sourcebot product. It does not disable the setup-wizard PostHog funnel proposed here.
    • +
  • Use the SDK's non-blocking capture queue during the wizard and a bounded SDK shutdown at terminal events.
    • +
  • Do not add a custom telemetry transport around posthog-node.
    • +
  • Suppress telemetry transport errors unless verbose debugging is explicitly enabled; never print payloads because future payload changes could expose data in terminal logs.
    • +
  • Document the always-on setup-wizard PostHog telemetry, its collected data categories, Person-profile behavior, in-memory session identity, deployment-ID handoff, and separation from Reo before release.
    • +
+

Existing reo-census tracker

+

The package currently depends on reo-census, whose install hook sends data to Reo rather than PostHog. Its default payload includes the working directory, Git username, email domain, npm username when available, detailed OS information, and CPU count. Full-data mode can include a full email address and dependency lists.

+

Reo remains installed and continues its existing behavior, including its own PACKAGE_TRACKER_ANALYTICS variable. The PostHog allowlist in this proposal governs only the new Sourcebot-owned PostHog events; it does not modify or make claims about Reo’s separate payload. The two systems must use separate event definitions and reporting so Reo installation data is not mistaken for the Sourcebot PostHog funnel.

+

Implementation proposal

+

Architecture

+

Keep telemetry isolated from prompt and configuration objects:

+
setup-sourcebot CLI
+        |
+        +--- create in-memory UUIDv4 setupSessionId
+        |
+        +--- typed, allowlisted PostHog events
+        |          distinctId = setupSessionId
+        |          install_id/company = setupSessionId
+        |
+        +--- generated .env
+                   SOURCEBOT_INSTALL_ID = setupSessionId
+                              |
+                              +--- Sourcebot first boot preserves the ID
+                              +--- deployed telemetry continues on that ID
+
+

The implementation does not need a postinstall hook, telemetry state file, background daemon, local queue, or server-side proxy. Delivery is best-effort from the short-lived wizard process. The only persisted identifier is SOURCEBOT_INSTALL_ID in the generated deployment configuration, where it is operational Sourcebot state rather than setup-wizard telemetry state.

+

Proposed files and responsibilities

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FileChangeResponsibility
src/telemetry.tsaddCreate the in-memory setupSessionId, create common properties, initialize posthog-node, send allowlisted events, and perform bounded shutdown.
src/telemetryEvents.tsaddDefine event-name-to-property TypeScript types, fixed enums, schema version, and pure event payload builders.
bin.cjsaddPerform a dependency-free Node 24 runtime check before dynamically importing dist/index.js, ensuring unsupported runtimes fail with a clear message before prompts, telemetry, file writes, or other side effects.
src/index.tsmodifyCreate the setup session, track the active stage, emit aggregate checkpoints, write or preserve SOURCEBOT_INSTALL_ID in .env, and route terminal paths through bounded flush helpers.
src/models.ts and code-host collectorsmodifyReturn non-sensitive telemetry summaries beside generated config, built from already-known branch choices and counts.
Docker helper module(s)modifyReturn structured outcomes instead of swallowing command failures that need to be distinguished from an empty Docker state. Never expose stderr to telemetry.
Repository-root entrypoint.shmergedPR #1648 implements the Sourcebot runtime change built into subsequent published container images: on first boot, preserve a non-empty pre-supplied SOURCEBOT_INSTALL_ID; generate a UUID only when one was not supplied. It safely serializes the selected ID into /data/.installedv3 and telemetry JSON, propagates uuidgen failures, and includes restart regression coverage for JSON-special characters. This file is not owned, downloaded, or modified by the setup wizard at runtime.
package.jsonmodifySet engines.node to >=24.0.0, point bin at bin.cjs, update @types/node to 24, add posthog-node, publish the runtime bootstrap, and retain reo-census and its existing hook unchanged.
packages/setupWizard/tests/e2e/addOwn the compiled-artifact builder, PTY driver, scenario manifest, prompt fixtures, PostHog-compatible TLS capture service, fake Docker executable/state machine, filesystem fixtures, payload oracle, privacy scanner, cleanup audit, and redacted completion reporter.
packages/setupWizard/tests/integration/addExercise the real posthog-node client and package-owned wrapper against local capture fixtures, including transport degradation and exact envelope/property validation. These tests complement but do not replace packed CLI E2E runs.
.github/workflows/setup-wizard-e2e.ymlmodifyAdd the required setup-wizard-e2e jobs: exhaustive Linux/Node 24 packed-artifact and Docker coverage, macOS/Windows Node 24 smoke coverage, and intentional Node 22.22 rejection. Upload only redacted failure diagnostics and cleanup reports.
.github/workflows/release-setup-sourcebot.ymlmodifyChange the setup-sourcebot release runtime from Node 20 to Node 24, build one candidate tarball, and require packed-artifact smoke, identity-continuity, PostHog contract, and cleanup gates against that artifact before publishing.
package README and docs/docs/misc/telemetry.mdxmodifyDescribe the events at a category level, pseudonymous ID/profile behavior, deployment-ID handoff, always-on setup-wizard PostHog policy, and the independent Reo/deployed-product controls.
+

The build and files configuration must include bin.cjs and the compiled wizard. No Sourcebot-owned postinstall entrypoint is added. npm pack --dry-run should verify the executable package contents before release.

+

Node 24 migration and PostHog SDK

+

Use Node 24 LTS as the package’s minimum and CI/release runtime. This is the highest production LTS line that does not require an architectural migration for this CLI: the code already targets ES2022, TypeScript is already configured with Node typings, native APIs used by the wizard remain available, and the current runtime dependencies accept Node 24. Node 26 is a Current release rather than LTS and is therefore not selected for a published setup tool.

+

The existing >=18 declaration is already inaccurate: @inquirer/prompts 8.4.3 requires at least Node 20.12 on the Node 20 line, and ora 9.4.0 requires Node 20. Raising the floor makes the supported runtime truthful. Updating @types/node from 22 to 24 keeps compile-time APIs aligned with the declared runtime; no application rewrite is expected.

+

Initialize the official SDK once per process:

+
const posthog = new PostHog(projectToken, {
+    host: "https://us.i.posthog.com",
+    flushAt: 1,
+    flushInterval: 0,
+    disableGeoip: true,
+    isServer: false,
+});
+
+

PostHog's Node guide recommends flushAt: 1 and flushInterval: 0 when a short-lived runtime should send queued events immediately. Leave the SDK's request and retry options unspecified so its maintained defaults apply. disableGeoip: true prevents location enrichment, and isServer: false tells PostHog this is a CLI-like runtime. Use the same public project token already configured by Sourcebot, defined as a setup-package constant with a comment linking it to the canonical shared default. Do not source the token or host from a user’s POSTHOG_* environment variables.

+

The package-owned wrapper should:

+
    +
  1. Initialize independently of Reo and deployed-Sourcebot telemetry environment variables.
    2. +
  2. Accept only a typed event name and the matching allowlisted property type.
    3. +
  3. Add common properties and use PostHog’s default person-profile processing without sending $set, $set_once, or other person properties.
    4. +
  4. Call posthog.capture() without forwarding configuration objects, prompt values, exceptions, or arbitrary environment data.
    5. +
  5. Swallow SDK capture and shutdown errors without printing payloads or changing setup behavior.
    6. +
+

At normal completion and handled-exit boundaries, call await posthog.shutdown(1_000). PostHog recommends awaiting shutdown() in short-lived runtimes so queued events are sent; supplying 1,000 ms instead of accepting its longer default caps the effect on CLI exit. Validate that budget in E2E tests. No additional delivery machinery is required.

+

setupSessionId creation and deployment handoff

+
    +
  1. At the start of main(), before setup_sourcebot_started, create setupSessionId directly with Node’s crypto.randomUUID(). Do not use a custom random-byte encoder or derive it from any user, repository, path, machine, or environment value.
    2. +
  2. Assert in tests that the generated value matches the exact canonical contract: ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$. This enforces 36 ASCII characters, lowercase hexadecimal, standard hyphen positions, UUID version 4, and the IETF variant defined by RFC 9562 (historically called the RFC 4122 variant).
    3. +
  3. Keep the value only in process memory while the wizard runs. Every setup event uses the same string, without normalization or transformation, as distinctId, setupSessionId, the install_id property, and the PostHog company group key.
    4. +
  4. When generating a new .env, write the same string byte-for-byte as SOURCEBOT_INSTALL_ID=<setupSessionId> alongside the other generated Sourcebot settings. Do not uppercase it, add braces or a prefix, remove hyphens, hash it, or encode it again. Docker Compose already loads this file through env_file, so no compose-file change is required.
    5. +
  5. Require a published Sourcebot image containing merged PR #1648. In that image, first boot uses a non-empty supplied SOURCEBOT_INSTALL_ID when present and calls uuidgen only as a fallback. The entrypoint intentionally does not reformat or regenerate a supplied value; the wizard guarantees that it already matches the canonical UUIDv4 contract. The existing deployment install event and /data/.installedv3 record then use that same ID. The wizard itself never modifies this script.
    6. +
  6. On subsequent container boots, keep the current behavior: the ID persisted in /data/.installedv3 is authoritative. This prevents an edited environment file from silently changing the identity of an existing deployment.
    7. +
+

Recommended implementation:

+
import { randomUUID } from "node:crypto";
+
+const setupSessionId = randomUUID();
+
+

No conversion step is needed or allowed between generation and use. randomUUID() already returns the canonical lowercase hyphenated UUIDv4 string required by this plan. A runtime reformatter would add risk without adding validation; enforce the invariant at the generation boundary and in tests instead.

+

No telemetry-only file or registry entry is created. If the wizard exits before configuration generation, the ID disappears with the process. Once configuration is generated, the value exists only as the deployment’s required SOURCEBOT_INSTALL_ID in .env and later in Sourcebot’s existing .installedv3 deployment record.

+

Existing setup directories: before overwriting an existing .env, read only the exact SOURCEBOT_INSTALL_ID key and preserve it if it matches the canonical UUIDv4 expression above. Do not send that existing value in setup telemetry or change the current session’s distinctId. Record only a safe enum such as deploymentIdentityAction: "preserved_existing". Therefore end-to-end identity continuity is guaranteed for newly generated deployments; a rerun against an existing deployment remains a separate setup session and is not aliased or merged.

+

Tracker-specific controls

+ + + + + + + + + +
TrackerControlEffect
Setup-wizard PostHogNoneAlways attempts capture when the relevant wizard checkpoint executes.
ReoPACKAGE_TRACKER_ANALYTICS=falseInterpreted only by reo-census; it has no effect on PostHog.
Deployed SourcebotSOURCEBOT_TELEMETRY_DISABLED=trueApplies to product telemetry after Sourcebot is deployed; it has no effect on setup-wizard PostHog events.
+

No Sourcebot code should intercept, override, remove, or reinterpret Reo’s variable. Conversely, the setup-wizard PostHog wrapper must not use either variable as an early return. A setup event can still be absent because the wizard never reached that checkpoint, the process was terminated, the network was blocked, or best-effort delivery failed.

+

Collector result contract

+

Collectors currently return generated configuration. Extend their result type with an explicitly constructed summary:

+
type CollectResult<TConfig, TSummary> = {
+    config: TConfig;
+    env: Record<string, string>;
+    telemetry: TSummary;
+};
+
+

telemetry must be built from booleans, fixed enums, and counts at the point where the collector already knows the user’s branch. It must never be produced by serializing, cloning, redacting, or spreading config, env, a prompt result, or an exception. The aggregate code-source and AI summaries should be computed from these safe summaries, not reconstructed from final configuration files.

+

Use exhaustive TypeScript unions for codeHost, providerType, stage, outcomes, credential modes, and failure categories. A new provider or branch should fail type checking until its safe telemetry mapping is explicitly selected.

+

Instrumentation points

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
EventExact implementation point
setup_sourcebot_startedAt the beginning of main(), after telemetry/session initialization and before the banner or first prompt.
setup_sourcebot_chose_setup_directoryAfter directory creation succeeds or reuse is confirmed. A declined existing-directory prompt emits cancelled instead.
setup_sourcebot_configured_code_sourceImmediately after each collector returns an accepted safe summary.
setup_sourcebot_configured_code_sourcesOnce after the add-another-source loop ends, using aggregates of per-source safe summaries.
setup_sourcebot_configured_ai_providerAfter each selected model/provider configuration is accepted.
setup_sourcebot_ai_setup_completedOnce when the AI loop returns, including the zero-configuration skip branch.
setup_sourcebot_configured_hosted_urlAfter URL validation succeeds; derive only fixed booleans such as HTTPS/default acceptance, never preserve the URL.
setup_sourcebot_generated_configsAfter all required writes complete, including writing this session’s ID for a new setup or preserving a valid existing deployment ID. If any write fails, emit failed at config_overwrite or generation stage and do not emit this checkpoint.
setup_sourcebot_resolved_compose_fileOnce after the existing/download/decline/failure branch resolves. Record only the fixed outcome and coarse failure category.
setup_sourcebot_validated_docker_stateOnce after validation/cleanup, or immediately with an explicit skip outcome when compose is unavailable or an existing deployment is left running.
setup_sourcebot_completedAfter manual instructions are printed, after an existing deployment is intentionally retained, after a Docker child emits spawn, or after a spawn failure is converted to manual instructions. Begin bounded SDK shutdown without waiting for the foreground child to exit; the spawned child may already be running and producing output.
setup_sourcebot_cancelledIn centralized handling for Ctrl+C and known declined terminal prompts, using the current fixed stage.
setup_sourcebot_failedAt the handling site for each recoverable infrastructure operation failure, with recoverable: true, even if later recovery succeeds. Emit recoverable: false at centralized fatal-error handling only when the error terminates the wizard. Do not recapture the same handled failure at the stage boundary.
+

Maintain a currentStage enum in main() and update it immediately before each stage begins. This makes Ctrl+C and top-level failures classifiable without inspecting prompt text or exception messages.

+

Docker and process handling

+

Docker helpers must distinguish command succeeded and found zero resources from command failed. Return internal structured results such as { ok, value, failureCategory }; keep stdout/stderr and command details local. This is necessary for truthful dockerStatus, cleanup actions, and validation_failed outcomes.

+

For docker compose up, attach spawn and error listeners before deciding the completion event:

+
    +
  • spawn: commit completion with sourcebotStartOutcome: "spawned" and begin bounded SDK shutdown while the foreground process continues. Keep child exit/error listeners and process cleanup active.
    • +
  • error: capture failed with recoverable: true and the applicable failure category, print existing/manual recovery guidance, and emit completed with sourcebotStartOutcome: "spawn_failed" and completionMode: "sourcebot_start_failed".
    • +
  • Later child exit: do not change the setup completion event. Runtime uptime and health belong to deployed Sourcebot observability.
    • +
+

Terminal paths and error policy

+

Use one lifecycle coordinator with separate setup-outcome and process-shutdown state. The mutually exclusive setup terminal events are completed, cancelled, and failed with recoverable: false. Commit the outcome synchronously before awaiting SDK shutdown so competing callbacks cannot emit another terminal event. Process cleanup must still run on Ctrl+C even when the setup outcome is already committed. Replace immediate exits with this coordinator; after main has returned and bounded SDK shutdown has finished, explicitly exit to prevent SDK retry sockets/timers from keeping the process alive. Never perform this final exit at the foreground Docker handoff; continue supervising Docker until it exits or Ctrl+C is handled.

+

Recoverable failure capture must bypass the terminal helper: it does not mark the session finished, call shutdown(), or change the wizard’s existing continuation or exit behavior. For example, a failed stopped-container removal followed by a failed volume removal emits two failed events with recoverable: true; the run can then emit validated_docker_state and completed, or end with cancelled or a later fatal failed. Capture each failure only once for that operation; do not deduplicate distinct failures just because their stage/category values match.

+

Telemetry code is outside the setup success/failure contract:

+
    +
  • Session-ID generation failure drops telemetry initialization and setup continues; no fallback file is created.
    • +
  • Failure to collect one or more system-derived properties uses the schema-defined fallback values and the event is still captured.
    • +
  • Payload construction failure drops that event.
    • +
  • Network, HTTP, timeout, and flush failure are swallowed.
    • +
  • Telemetry never changes generated files, Docker decisions, wizard exit status, or normal terminal messaging.
    • +
  • Debug logs, if added for development, report only event name and fixed failure category and are disabled in published normal operation.
    • +
+

Completion, Ctrl+C, and process cleanup

+

Code findings: src/index.ts currently catches Inquirer ExitPromptError and exits immediately, starts openBrowserWhenReady() before Docker emits spawn, and leaves its fetch/sleep loop active for up to 120 seconds. Docker helpers create child processes, port checks create temporary servers, and GitHub/GitLab autocomplete fetches have no shared cancellation signal. The custom tabCheckbox uses Inquirer core; prompt interruption can arrive through readline rather than the process-level signal listener. These resources need explicit ownership and cancellation.

+ + + + + + + + + + +
SituationTelemetry decisionProcess behavior
Manual instructions printed, or existing deployment retainedCommit and capture completed once.Finish bounded SDK shutdown and release wizard resources; exit normally.
Docker emits spawn while setup is activeCommit and capture completed immediately, then begin the one bounded SDK shutdown.Keep waiting for foreground Docker. Begin cancellable readiness polling only after spawn; do not wait for readiness to count completion.
Ctrl+C before any setup terminal outcomeCommit and capture cancelled with reason: "keyboard_interrupt" and the active stage, once.Abort setup work immediately, begin resource cleanup and bounded telemetry shutdown in parallel, and exit within the interrupt deadline.
Ctrl+C after completion, including during its flushKeep completed; do not emit cancelled or reopen the SDK. Reuse the pending shutdown promise if present.Stop the foreground CLI and readiness work within the same interrupt deadline. An already-finished setup is not cancelled by stopping its foreground runtime.
Spawn error before completionCapture a recoverable failure, print manual instructions, then commit completed with the existing spawn-failed properties.Do not start readiness polling; clean up and exit. If cancellation already won the race, suppress this recovery/completion path.
Foreground Docker exits after completionNo new setup event.Cancel readiness polling and release handles immediately. Preserve the current normal child-exit behavior; do not wait out the readiness timeout.
+

Implementation: introduce a small package-owned lifecycle helper (for example src/lifecycle.ts) used by main(), prompts, collectors, and Docker helpers. Track an initially unset terminal outcome, one shared SDK-shutdown promise, an interrupt flag, an AbortController for setup work, and a registry of owned child processes, probe servers, and timers. Check cancellation after awaited operations and before starting another prompt, writing files, spawning Docker, opening a browser, or emitting a checkpoint. A late callback must not resume setup after cancellation. Keep this lifecycle functioning when SDK initialization or capture fails.

+
    +
  • Install the process SIGINT listener before the first prompt. Route it and typed Inquirer ExitPromptError into the same idempotent interrupt handler. Pass the shared abort signal through the prompt context, including the custom checkbox and search prompts where supported. Treat a prompt abort caused by this controller as cancellation already in progress, not another failure. Do not add a competing raw-stdin reader. Ensure prompt cleanup restores terminal mode/cursor and releases readline; test third-party prompt behavior through the PTY.
    • +
  • Abort pending fetches in autocomplete, model catalog, Compose download, and readiness checks. Combine existing request timeouts with lifecycle cancellation; never let a catch/fallback restart work after the lifecycle signal is aborted. Make readiness sleeps abortable and cancel them on interruption, spawn error, or child exit. Stop local repository traversal at asynchronous boundaries, close probe servers, and stop spinners. Synchronous file writes already underway cannot be interrupted mid-call; do not start another write after interruption is handled, and do not delete user files as cancellation cleanup.
    • +
  • Use a maximum 3,000 ms process-shutdown budget from the first handled Ctrl+C. Start SDK shutdown (at most 1,000 ms) and resource cleanup concurrently. Signal only live subprocesses owned by this invocation; allow up to 2,000 ms for graceful termination, then force termination of those owned processes/descendants within the overall budget. Use a tested platform-specific child cleanup adapter: POSIX signal behavior and Windows process-tree termination differ. Do not assume child.killed proves exit; observe exit/close. Handle both terminal-delivered interruption and a signal sent only to the parent. Do not signal unrelated processes or run volume deletion, broad Docker cleanup, or extra destructive Compose commands.
    • +
  • After cleanup, release stdin and owned handles and exit. A deadline watchdog must force the CLI to exit if a dependency leaves handles open; it must not rely only on process.exitCode. A second Ctrl+C escalates immediately without capturing another event or restarting any deadline. Resource cleanup proceeds even if telemetry throws or times out. Killing a Docker client does not prove its daemon-side operation or containers have stopped; retain Docker's normal interrupt behavior and do not promise deployment shutdown as part of the telemetry contract.
    • +
  • Standardize Ctrl+C exit status to 130 before and after completion, including forced exit. This is an intentional change from the current prompt-cancellation status 0; list it in the behavioral regression allowlist. Explicit decline prompts and ordinary completion keep status 0, and fatal setup errors keep status 1. An interrupt during an already-committed fatal/completion flush changes the process exit reason, not the recorded setup outcome.
    • +
+

Race and delivery contract: whichever handler first commits a setup outcome wins. If cancellation commits before Docker's spawn callback, clean up any subsequently spawned child and emit no completion; if spawn commits completion first, Ctrl+C performs process cleanup without cancellation telemetry. Capture always precedes the shutdown attempt. A healthy collector must receive the selected terminal event in tests; network failure, immediate repeated interruption, or an uncatchable kill can prevent delivery. Never extend the exit deadline or claim guaranteed receipt to compensate.

+

These are lifecycle changes required to satisfy prompt cancellation and cleanup; they do not add application-readiness telemetry. Node's signal documentation explains why installing a signal handler replaces default exit behavior; child-process documentation describes platform differences and why sending a signal alone does not establish process termination. Inquirer documents prompt cancellation through AbortSignal.

+

Reo coexistence

+

Keep reo-census in runtime dependencies without modifying its code, install hook, or PACKAGE_TRACKER_ANALYTICS behavior. The new Sourcebot PostHog telemetry starts only when the interactive wizard runs; no Sourcebot-owned PostHog postinstall hook is added.

+

The Sourcebot PostHog wrapper must not read Reo’s variable, reuse Reo payloads, or attempt to combine the two trackers. Reo installation tracking remains a separate data source and is not a stage in the PostHog setup funnel.

+

The release notes and telemetry documentation should state that setup-wizard PostHog telemetry is always on when its code executes, list the categories collected, explain that PostHog creates a Person profile keyed only by a random pseudonymous session/deployment ID, explain the new-deployment ID handoff, and distinguish PostHog from Reo’s separate tracking and variable. They must not describe the project ingestion token as a secret.

+

Testing plan

+

Implementation verification status

+

The feature is not release-complete until the full completion gate below is satisfied. Executable tests cover packed-package collectors, terminal paths, real SDK payloads, Docker branches, and actual container identity continuity. CI definitions are not evidence that Windows or other remote jobs have run.

+

Implementation-discovered changes for the behavioral baseline allowlist: status 130 for Ctrl+C; cancellation-aware readiness cleanup; explicit final process exit after bounded telemetry shutdown; recoverable autocomplete fallback on network/decode failure with an 8-second timeout; and fixed, credential-free SDK shutdown-timeout diagnostics where PostHog itself emits them. Do not patch SDK internals or silence the global console. Empty search/catalog results are not failures; actual infrastructure errors can emit recoverable diagnostics while retaining manual fallback.

+

Docker availability failures can trigger fixed docker info/docker compose version probes for classification. These probes classify the original failed operation and do not create extra failure events. The dedicated workflow is .github/workflows/setup-wizard-e2e.yml.

+
+Local implementation evidence — September 11, 2026; release gate still incomplete +

Implementation is isolated in the msukkari/setup-wizard-telemetry-SOU-2211 branch and dedicated worktree. The original planning checkout is unchanged. The verified release-shaped package has SHA-256 c4c6157ceacf224e023bf8233dc79830b6683d05fcd614493071037f464eddab. Test infrastructure, this plan, certificates, transcripts, and temporary installations are excluded from the tarball.

+ + + + + + + + + + + +
CheckObserved result
macOS, Node 24.21106 unit/integration/packed-artifact tests passed, zero failures/skips. The subsequently expanded platform suite passed all four tests, adding foreground Docker spawn/interrupt coverage to the three existing platform cases.
Isolated Linux, Node 2491 packed-CLI cases passed, zero failures/skips, including all code-host collectors, all AI providers, recoverable failures, fatal writes, Ctrl+C races, stubborn subprocesses, and rejected/stalled/reset telemetry transport.
Package-manager launchersnpm 12.0.2, Yarn 4.7.0, pnpm 12.4.1, and Bun 1.4.2 passed on macOS and Linux using the same installed tarball. Bun uses bunx --no-install, which launches the Node CLI; direct execution with the Bun runtime is not this supported launcher test. pnpm's own store/state is allowed only in its known test-owned directories; direct-binary tests still require an empty per-user home.
Unsupported NodeNode 22.22 exits before importing the wizard, with the Node 24 requirement and no generated files.
Baseline differentialTwo representative manual/downloaded-Compose flows match base 31734dc2 for generated configuration and Docker commands, normalizing generated secrets and the intentional install-ID addition. The baseline uses current resolved dependencies and Node 24, not a reconstructed historical dependency/runtime environment.
Real runtime identityEight disposable Sourcebot containers passed first boot, conflicting/missing env on restart, upgrade, generated-ID fallback, and deployment telemetry opt-out. The actual repository entrypoint and image's curl/jq/uuidgen run; database migration and supervisor are fixture stubs. This verifies entrypoint identity/telemetry, not full application health.
Live dev PostHogTwelve events were accepted and queried back in project 323169 for synthetic ID 519ed47f-27e2-45dc-a672-0960af758639: all ten minimal setup checkpoints in order, followed by the real entrypoint's install and upgrade under the same distinct ID. The test relay changes only the project token after local payload assertions; normal tests do not enable this relay.
+

Still required before declaring completion: execute the Windows CI job and resolve any platform failures; finish the complete scenario-manifest/enum/emission-site accounting and compiled branch-coverage review; close the remaining exhaustive fault-injection and behavioral-differential gaps against the full matrix below; and attach passing CI evidence for the final commit. The independent tests/approvedSchema.json snapshot prevents silent field/enum changes, but is not proof of 100% scenario coverage. No claim of full completion or 100% coverage is made by these local results.

+

Test-harness caveat: an exploratory direct-Bun-runtime launch did not produce local collector evidence and is excluded from the passing matrix. Its outbound delivery was not verified, so it must not be used as evidence of network isolation; it may have bypassed the Node-only transport shim. Use the verified bunx Node launcher, and require container-level egress enforcement before repeating unsupported-runtime experiments. Bun's launcher documentation describes respecting the executable's Node shebang.

+
+

Unit tests

+
    +
  • Inject filesystem, environment, clock, UUID generator, package metadata, and transport dependencies; never contact real PostHog in tests.
    • +
  • Verify one ID is created before started, matches ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$, remains stable for all events in that invocation, and is never written outside the selected setup directory.
    • +
  • Verify that setting either SOURCEBOT_TELEMETRY_DISABLED=true or PACKAGE_TRACKER_ANALYTICS=false does not prevent setup-wizard SDK initialization or event capture.
    • +
  • Verify a new .env receives SOURCEBOT_INSTALL_ID=setupSessionId; an existing valid value is preserved; a missing or invalid existing value is replaced with the current session ID; and telemetry records only the approved action enum.
    • +
  • Verify first-boot entrypoint identity selection: preserve a non-empty supplied ID, generate when missing or empty, persist the chosen value in .installedv3, and prefer .installedv3 on subsequent boots. Format enforcement for wizard-generated values belongs to the wizard tests.
    • +
  • Snapshot the exact property keys for every event. Seed collectors with token-, URL-, email-, repository-, model-, path-, hostname-, port-, and raw-error-shaped values and assert none occur anywhere in serialized payloads.
    • +
  • Verify each system-property collector independently handles missing values, unsupported values, malformed values, and thrown errors; the event must still be captured with "other", "unknown", or null as defined in the schema, while successfully collected properties remain intact.
    • +
  • Verify deployment classification with and without an HTTP(S) protocol, optional leading www., mixed hostname case, trailing DNS dots, ports, paths, and surrounding whitespace. Cover exact public-service domains, dot-boundary managed-service suffixes, custom GitLab domains, malformed URLs, and lookalike domains such as notgithub.com. Assert only the fixed deploymentType value is captured and the input URL and normalized hostname are absent.
    • +
  • Verify hosted-URL classification maps localhost, *.localhost, every IPv4 127.0.0.0/8 loopback address, and IPv6 ::1 to localhost; maps every other successfully parsed non-empty host to address; and maps unavailable, malformed, or unexpectedly failing inputs to unknown without suppressing the event. Assert no URL or hostname is captured.
    • +
  • Verify every other fixed enum rejects or maps unknown runtime input to its schema-defined fallback rather than forwarding it.
    • +
  • Inject representative failures from the current wizard operations and verify the coarse mapping: prompt/config parsing to validation; required fetch/HTTP/decode failures to network; directory and file operations to filesystem; missing/inaccessible Docker or Compose to docker_unavailable; non-zero or unusable Docker results to docker_command; other child-process spawn errors to process_spawn; and uncategorized runtime errors to unknown. Assert only one failure event is emitted and no raw error data is captured.
    • +
  • Verify application duration properties use an injected monotonic clock. Standard SDK envelope timestamps are permitted and validated separately; do not add custom wall-clock properties to the application schema.
    • +
  • Verify the exact posthog-node options, use of SDK transport defaults, absence of $process_person_profile: false, absence of custom person properties, and bounded shutdown behavior.
    • +
+

Mandatory end-to-end completion gate

+

The E2E suite is part of the feature implementation, not deferred follow-up work. Implement and run it after the telemetry and Node changes are code-complete but before the feature is marked complete, merged, or released. The feature is incomplete while any required scenario is missing, skipped, quarantined, allowed to fail, or failing. Unit tests and source-level integration tests cannot substitute for this gate.

+

Every required E2E scenario must execute the newly compiled and packed npm artifact through its published binary entrypoint. Tests must not import src/, run tsx, or execute a repository source entrypoint as a substitute. Build @sourcebot/schemas, build setup-sourcebot, create the same .tgz shape used by the release workflow, record its SHA-256 digest, install that tarball with lifecycle scripts enabled into a clean temporary project, and use that exact digest for every scenario in the run.

+

A required CI check named setup-wizard-e2e must gate the feature PR. The release workflow must rebuild and rerun the packed-artifact smoke and identity-continuity tests before publishing. A release must stop before npm publish if the artifact, E2E suite, cleanup audit, or PostHog contract validation fails.

+ +

E2E harness and isolation architecture

+
    +
  1. Artifact builder: compile dependencies and the wizard, pack the npm tarball using the release workflow’s Yarn command, inspect the archive, and install it into a new temporary project with npm lifecycle scripts enabled. Assert the published binary resolves and starts without repository-only dependencies.
    2. +
  2. PTY driver: launch node_modules/.bin/setup-sourcebot in a pseudoterminal, wait for each real prompt, submit answers or signals, enforce a per-prompt and per-scenario timeout, and retain a redacted transcript only as an ephemeral CI artifact on failure.
    3. +
  3. Scenario fixture: define each run as declarative prompt answers, filesystem seed state, fixture-server responses, Docker behavior, expected exit result, expected files, and expected ordered telemetry. Generate scenario names from fixed identifiers rather than user data.
    4. +
  4. PostHog capture service: run a local HTTPS endpoint that accepts the actual posthog-node request paths and records raw request bodies. In the isolated test network, resolve the fixed production ingestion hostname to this service and trust a test-only CA through the test environment. Do not add a production environment-variable host override or replace the SDK with a mock in packed-artifact tests.
    5. +
  5. Code-host and catalog fixtures: serve deterministic GitHub-, GitLab-, models.dev-, and compose-download-compatible responses locally. Deny all other outbound traffic during execution so a missed fixture fails the test rather than contacting a real service.
    6. +
  6. Deterministic Docker layer: place a stateful fake docker executable first on PATH for the exhaustive branch matrix. It must emulate command exit codes and stdout/stderr for Compose state, cleanup, volumes, and startup while recording only fixed test metadata.
    7. +
  7. Real Docker layer: separately run the generated files and repository-root entrypoint.sh in disposable Docker containers with unique Compose project names, networks, images, and data volumes. The fake Docker layer does not replace these container-level identity and restart tests.
    8. +
  8. Filesystem sandbox: give every scenario a fresh temporary home, working directory, setup directory, npm cache, and XDG state/config/cache directories. Afterward, assert that no telemetry file or unexpected setup artifact exists inside or outside the selected setup directory.
    9. +
  9. Secret canaries: use unique synthetic token-, email-, repository-, organization-, URL-, model-, path-, hostname-, and error-shaped values for every sensitive prompt and fixture response. Recursively scan captured PostHog bodies, terminal diagnostics, and retained test artifacts to prove none were transmitted through telemetry.
    10. +
  10. Cleanup controller: use try/finally cleanup for processes, PTYs, servers, temporary installations, Docker resources, and certificates. Cleanup runs after success, assertion failure, timeout, or signal and fails the suite if a labeled resource or temporary artifact remains.
    11. +
+ +

Required isolated environment matrix

+ + + + + + + + + +
EnvironmentRequired coveragePurpose
Linux container, Node 24Full scenario matrix, fake Docker matrix, real Docker suite, PostHog capture, packaging, and cleanup auditPrimary deterministic release gate.
Linux package-manager smoke, Node 24Install/invoke the same packed tarball through npm/npx, Yarn, pnpm, and Bun’s package launcherVerify the reported packageManager/invocationMethod, lifecycle behavior, binary resolution, and graceful unknown fallback without duplicating the full branch matrix.
macOS, Node 24Packed install, minimal happy path, local-repository path handling, cancellation, system properties, and spawn behaviorProtect Darwin-specific paths, process behavior, and package execution.
Windows, Node 24Packed install, minimal happy path, local-repository path handling, cancellation, system properties, and spawn behaviorProtect Win32 paths, executable resolution, signals available on Windows, and package execution.
Linux container, Node 22.22Unsupported-runtime rejection onlyProve the Node 24 boundary fails early without prompts, telemetry, files, or Docker work.
+

The Linux run is authoritative for exhaustive branch coverage. Platform smoke jobs must use the same tarball build recipe and must not be silently omitted from the required check; if CI infrastructure cannot support a platform temporarily, the feature remains incomplete until an equivalent isolated runner is available and passing.

+ +

Required setup-flow scenario matrix

+ + + + + + + + + + + + + + + + + + + + + + +
AreaScenarios that must execute through the packed CLIPrimary assertions
Installation and invocationFresh tarball install with lifecycle scripts; direct package binary; npm-exec/npx-style invocation; package path containing spaces; read-only package installation after installNo setup PostHog event during installation, no module-resolution failure, correct packageManager/invocationMethod classification or fallback, and no writes into the installed package.
Setup directoryNew relative path, new absolute path, existing directory accepted, existing directory declined, nested creation, spaces/Unicode, creation failure, and chdir failureCorrect checkpoint/cancellation/failure event, unchanged exit semantics, no path leakage, and no writes outside the chosen sandbox.
GitHubDefault cloud and custom/GHE hosts; no token and token; repository, organization, and user scopes; autocomplete success, empty result, authentication/rate-limit response, malformed response, timeout, and connection failureCorrect safe summary and deployment classification, exact counts, successful literal fallback where supported, and no URL, token, search text, owner, or repository leakage.
GitLabGitLab.com, known Dedicated domain, ambiguous custom domain, and self-managed-shaped input; all, group, project, and user scopes; token/no-token; autocomplete success and every fallback/failure classCorrect cloud/unknown rule, scope/count properties, no hostname or selected-name leakage, and continued setup for supported fallback paths.
BitbucketCloud API token, access token, and app password; workspace and repository scopes; Data Center index-all and selected-project paths; cleanup of all credential-shaped canariesCorrect deployment and credential enums, counts, and complete exclusion of emails, usernames, hosts, and credentials.
Azure DevOpsCloud and Server, optional TFS path, organization/project/repository scopes, and multiple selectionsCorrect deployment/scope/count properties without organization, collection, server, URL, or token values.
Gitea and GerritGitea.com and custom Gitea; token/no-token; organization/repository/user selections; Gerrit index-all and selected projectsCorrect fixed deployment type, credential mode, counts, and no host/project leakage.
Local and remote GitDirectory that is itself a repository; all depth-one repositories collapsed to a wildcard; subset and nested repositories generating multiple connections; unreadable directory; no repositories; arbitrary remote Git URLCorrect repository count, bucket, generatedConnectionCount, local/remote deployment type, mount file behavior, and no local path, basename, or clone URL leakage.
Multiple code sourcesOne source, repeated provider, mixed providers, and three-or-more-source loop with both add-another choicesOne repeated event per accepted source, one aggregate checkpoint, one-based indexes, deduplicated host types, exact totals, and stable ordering.
AI setupSkip; every provider offered by the wizard; one and multiple models; OpenAI-compatible endpoint; AWS default chain and explicit keys; Vertex ADC and credential-file path; catalog success, empty, malformed, HTTP failure, timeout, and manual-model fallbackProvider events and aggregate checkpoint are correct, skip remains in the funnel, counts match generated models, and no model, endpoint, region, resource, project, credential path, key, or display name is captured.
Hosted URLDefault URL; localhost, subdomain of localhost, IPv4 loopback range, IPv6 loopback, other hostname/IP address, invalid input followed by valid input, and classifier failure injectionCorrect usedDefaultUrl, protocol, and hostCategory; invalid prompt input does not create a failure event; URL, hostname, port, path, query, and fragment remain absent.
Configuration generationNew files; each existing file accepted for overwrite; each overwrite declined; valid/missing/invalid existing install ID; write failure for each output; generated-secret failureExisting behavior and exit codes remain stable, files parse, only approved files are listed, UUID handoff follows policy, no partial-success checkpoint is emitted, and failures use coarse categories without content leakage.
Compose-file resolutionExisting file; download accepted and successful; download declined; HTTP 4xx/5xx; timeout/disconnect; malformed body where relevant; destination write failureEvery terminal branch emits the compose checkpoint with the right outcome/category, manual instructions remain correct, and setup continues only where it did before instrumentation.
Docker validationDocker executable missing; Compose missing; daemon/socket unavailable; empty state; running, stopped, and mixed containers; cleanup accepted/declined/success/failed; volumes absent/present/remove success/failure; no-compose skipSuccess is not confused with command failure, outcomes and failure categories match, no container/volume names or stderr enter telemetry, and existing user-facing recovery behavior is preserved.
Port conflictsNo published ports; free ports; Docker-owned conflict; non-Docker conflict; mixed conflict; stop accepted/declined/success/failed; conflict remains after cleanupCorrect aggregate conflict fields and completion mode, no port numbers/process/container names in telemetry, and temporary listening sockets are closed.
Completion and startupManual completion, compose unavailable, existing deployment retained, start declined, docker compose up spawn success, spawn error, and later child exitExactly one completion/failure terminal decision, completion fires after spawn rather than child exit, and Sourcebot-start outcome is truthful.
Cancellation and fatal errorsSIGINT at every named stage and during shutdown; repeated signals; every explicit decline; one injected unexpected failure at each stage boundary; cancellation or fatal error after multiple recoverable failuresAt most one terminal event across completion, cancellation, and fatal failure; recoverable diagnostics remain nonterminal. Correct stage/reason/category and exit semantics, bounded shutdown, no hanging PTY, and no corrupted partial files.
Telemetry degradationSDK initialization/capture/shutdown throw; collector returns 4xx/5xx; connection refusal; response timeout; connection reset; malformed response; system-property collector failureThe same prompts, files, Docker actions, terminal output, completion mode, and exit code as the matching telemetry-healthy control; system properties use approved fallbacks and no telemetry state file is created.
+ +

Packed-artifact, container, and identity-continuity suite

+

Add a dedicated E2E suite that runs from the packed npm artifact under Node 24, rather than importing TypeScript source. This is the regression gate for the runtime upgrade, telemetry integration, generated files, Docker orchestration, and setup-to-deployment identity handoff.

+
    +
  1. Build and pack setup-sourcebot, install the tarball into an isolated temporary project with PACKAGE_TRACKER_ANALYTICS=false, and assert installation succeeds under Node 24 without warnings or module-resolution errors. Assert no Sourcebot PostHog request occurs during package installation. Using Reo’s opt-out keeps the test from contacting Reo and does not affect wizard telemetry.
    2. +
  2. Redirect the fixed PostHog ingestion hostname at the isolated test-network boundary to the local TLS capture service, leaving the packed production code and official SDK unchanged. Launch the wizard with both PACKAGE_TRACKER_ANALYTICS=false and SOURCEBOT_TELEMETRY_DISABLED=true; assert started is still captured, contains no $process_person_profile: false, PII, sensitive values, or custom person properties, and uses bounded shutdown. Production code must continue using the fixed Sourcebot host, and the E2E design must not add a user-accessible host override.
    3. +
  3. Launch the packed binary through a PTY under Node 24 and drive a deterministic minimal happy path: choose a temporary setup directory, configure a fixture code source, skip AI, accept the hosted URL, generate configuration, resolve the compose branch, skip or stub Docker validation, and decline foreground startup.
    4. +
  4. Assert the wizard exits successfully, generated files parse correctly, the expected ordered started -> completed events arrive, each event contains only its approved keys, and secret-shaped fixture values are absent from serialized requests.
    5. +
  5. Assert the generated .env value for SOURCEBOT_INSTALL_ID matches the canonical UUIDv4 expression and exactly equals, byte-for-byte, the setup events’ distinctId, setupSessionId, install_id, and company group key.
    6. +
  6. Run the actual repository-root entrypoint.sh containing PR #1648 in an isolated Sourcebot container with an empty data volume and the wizard-produced UUID. Stub only external dependencies and long-running processes after identity initialization. Capture the HTTPS request at a local PostHog-compatible endpoint and assert the deployment install event’s distinct_id, the process environment, and .installedv3.install_id all equal the wizard value byte-for-byte.
    7. +
  7. Recreate the container against the same data volume through the complete identity matrix: same-version restart with a conflicting environment value, upgrade restart with a conflicting value, another same-version restart with the environment value absent, first boot with the ID absent so uuidgen is exercised, and telemetry-disabled first boot/restart. Assert the persisted value always wins after first boot, upgrade telemetry retains the same distinct_id, same-version restarts do not duplicate install/upgrade events, generated fallback IDs match the canonical UUIDv4 expression, and telemetry-disabled runs emit no events.
    8. +
  8. Run the wizard over a fixture existing .env with a valid deployment ID. Assert the ID is preserved, setup events remain keyed to the new session UUID, and deploymentIdentityAction is preserved_existing without transmitting the existing ID.
    9. +
  9. Run a second packed invocation that sends SIGINT during a prompt. With a healthy collector, assert exactly one allowlisted cancelled event, bounded SDK shutdown, exit status 130 under the lifecycle policy, and no partial configuration corruption. This protects signal and process behavior that can differ across Node majors.
    10. +
  10. Run the packed binary under Node 22.22 in a separate CI container and assert bin.cjs fails immediately with a clear Node 24 requirement before prompts, UUID generation, telemetry, file writes, or Docker commands. This verifies the migration boundary is intentional rather than a late dependency/runtime crash.
    11. +
+ +

PostHog E2E contract assertions

+

The packed-artifact tests must use the real posthog-node dependency shipped in the tarball. Mocking the package-owned typed wrapper is appropriate for unit tests but does not satisfy E2E completion.

+
    +
  1. Decode every request accepted by the local PostHog-compatible capture service, including SDK batching or compression, and validate the actual ingestion envelope rather than only the arguments passed to posthog.capture().
    2. +
  2. For each scenario, compare the received logical event sequence with an explicit oracle. Assert no missing or unexpected event, correct repeated-event count, checkpoint order, and at most one terminal event across completed, cancelled, and failed with recoverable: false. Recoverable failure events do not count toward this limit. Assert one logical failure capture per failed operation, allowing distinct operation failures with identical stage/category values. Transport retries caused by an induced ambiguous network failure are not treated as duplicate instrumentation.
    3. +
  3. Run packed-CLI scenarios with both stopped-container removal and volume removal failing in the same session, followed respectively by manual completion, cancellation at a later prompt, and a later fatal error. Assert both recoverable events arrive under the same setup ID, subsequent reached checkpoints are still captured, and exactly one appropriate terminal event arrives. Add a spawn-error-to-manual-completion scenario. Verify the SDK remains active after recoverable failures and shuts down only at the terminal boundary; repeated signals or callbacks must not add a terminal event after that boundary.
    4. +
  4. Verify Docker measurement semantics for no Compose file, deliberately retained deployment, unavailable Docker, partial inspection failure, confirmed empty inventories, and failed port rechecks. Assert skipped/failed counts are null, confirmed empty counts are 0, independent measurements survive other failures, and completion copies the remaining-conflict value exactly. Combine cleanup failure with remaining port conflicts and with a left-running deployment to verify validation_failed takes precedence while action fields preserve the details.
    5. +
  5. Validate setup event names against the 13-event allowlist, and validate application property keys/types against the common and event-specific schema. Separately validate SDK-generated properties and transport envelopes using the versioned allowlist below. Existing deployment install/upgrade events in the identity suite use their own runtime contract, not the setup-event allowlist.
    6. +
  6. Assert one canonical UUIDv4 is reused for distinct_id, setupSessionId, install_id, and the company group throughout a new setup. Assert separate invocations receive different setup IDs and are not accidentally merged.
    7. +
  7. Assert schemaVersion, package version, platform, architecture, Node major, package manager, CI state, elapsed timing, GeoIP disablement, and source attribution are correct for the test environment. Inject each permitted system-property failure and verify the documented fallback without dropping the event.
    8. +
  8. Assert PostHog’s default Person behavior is preserved: no $process_person_profile: false, identify, alias, $set, or $set_once mutation is emitted, while the random setup identity and company group remain present.
    9. +
  9. Recursively inspect both keys and values in the serialized requests. Reject exact canaries, substrings, URL-encoded forms, JSON-escaped forms, and accidental nested configuration objects containing credentials, repositories, emails, URLs, paths, model identifiers, hostnames, ports, command output, errors, or environment data.
    10. +
  10. Exercise a successful collector, HTTP rejection, connection refusal/reset, and non-responsive collector. Assert capture remains non-blocking during prompts and the explicit shutdown deadline bounds normal completion, cancellation, and failure exits without changing setup correctness.
    11. +
  11. Assert package installation itself produces no Sourcebot PostHog request, and assert the wizard creates no telemetry-only file, durable queue, identifier, cache entry, or state directory.
    12. +
+ +

SDK metadata and transport-envelope allowlist

+

Keep two explicit contracts: the application schema above, and a transport contract for the exact posthog-node/@posthog/core versions resolved in the tested artifact. Inspect those resolved versions during implementation and record the enabled wire format in the fixtures. The inspected Node SDK 5.52.1/core 1.53.2 adds library metadata, event timestamps, event UUIDs, and grouping metadata. These fields must pass tests when correctly generated; they are not additional wizard answers.

+ + + + + + + +
LayerPermitted metadata and assertions
SDK event properties$lib equals posthog-node; $lib_version equals the resolved SDK version; $groups contains exactly { company: setupSessionId }; $geoip_disable and $ignore_sent_at are true. Group and GeoIP assertions complement the application's existing contract. With the planned isServer: false, $is_server is absent.
Event envelopeValidate the observed SDK fields such as event, distinct_id, properties, timestamp, and uuid. Timestamps must parse, increase strictly within the invocation, and fall within the test's allowed time window. The SDK's event UUID is separate from the setup UUID: the inspected core generates UUIDv7 event IDs, so do not require these to equal setupSessionId or match the setup UUIDv4 regex.
Batch and HTTP transportAllow the actual resolved format's project token field, batch array, send/creation timestamp, and content-type/compression/SDK-identification headers. For a capture-v1 format, validate its options object and SDK metadata in PostHog-Sdk-Info rather than requiring $lib/$lib_version inside every event. Only enable these alternatives when the tested artifact actually uses that format; do not accept arbitrary extra envelope keys.
+

The wire allowlist is implemented in test fixtures, not by stripping SDK-generated fields in production. SDK upgrades must update the fixture contract after inspecting their actual output. Do not blanket-allow $* properties: unexpected user identifiers, person mutations, feature-flag/session context, location data, raw errors, or arbitrary SDK enrichment must still fail. Scan both permitted metadata and application properties for sensitive canaries. The live PostHog smoke may observe additional server-generated ingestion metadata; distinguish that from outbound capture data and review unexpected enrichment rather than comparing a stored event object directly to the wire schema.

+

Behavioral regression comparison

+

Required lifecycle regressions: through the packed CLI, exercise real PTY Ctrl+C and parent-directed SIGINT at prompts, stalled autocomplete/catalog/Compose fetches, repository scanning, Docker commands, port checks, immediately before/after spawn, readiness fetch/sleep, and pending terminal shutdown. Cover multiple recoverable errors followed by interruption, spawn failure, immediate child exit, and repeated Ctrl+C. For each interrupted case, assert no subsequent setup work, CLI exit within the 3-second budget plus a small documented CI scheduling tolerance, expected exit code, released terminal state, and no surviving owned test subprocesses/timers/listeners. With a healthy collector, assert exactly one cancelled event before completion, or exactly one completed event and no cancellation after completion. Repeat with a stalled collector and stubborn child to verify the bounded fallback. Verify Windows console/PTY interruption separately using supported platform mechanisms. Add the intentional status-130 and readiness-cleanup differences to the baseline comparison allowlist.

+

Run representative non-telemetry setup scenarios against both the feature artifact and an artifact built from the PR base revision. Normalize only nondeterministic values such as temporary paths, ANSI timing, generated secrets, and the intentionally added SOURCEBOT_INSTALL_ID. Maintain a small reviewed allowlist of intentional differences; an unlisted difference fails the gate.

+
    +
  • Compare prompt order, prompt defaults, validation behavior, cancellation points, terminal success/failure messages, and process exit codes.
    • +
  • Compare generated config.json, .env, and compose override semantics after redacting generated secrets. Validate JSON against the Sourcebot schema, parse compose YAML, and confirm the only telemetry-related persisted change is the install-ID entry in the existing .env.
    • +
  • Compare fake-Docker command order and arguments, cleanup choices, port-conflict decisions, and whether foreground startup occurs.
    • +
  • Run every happy-path scenario twice—once with a healthy local collector and once with telemetry transport forced to fail—and require identical wizard files, Docker actions, user-facing behavior, and exit status.
    • +
  • Exercise the Node-migration-sensitive APIs used by the package, including ESM loading, global fetch, AbortSignal.timeout, cryptography, child-process events, filesystem/path behavior, and signal handling through the compiled package.
    • +
+ +

Coverage accounting and scenario completeness

+

Store a reviewed, machine-readable scenario manifest with the E2E tests. It must map every wizard stage, prompt branch, code-host collector, AI-provider branch, compose outcome, Docker outcome, terminal path, event name, event enum value, and failure category to at least one scenario ID. CI must fail when an implementation adds or changes a branch/event enum without updating the manifest and its assertions.

+
    +
  • Require 100% coverage of the manifest, all telemetry emission sites, all terminal-event paths, all event names, and every documented outcome enum before completion.
    • +
  • Collect JavaScript branch coverage from the compiled dist execution as supporting evidence and review uncovered setup logic. Numeric source coverage alone is not a substitute for manifest coverage or behavioral assertions.
    • +
  • Generate a compact machine-readable report containing the tarball digest, Node/OS matrix, scenario totals, pass/fail/skip counts, event names observed, enum values observed, cleanup result, and duration. Do not include prompt answers, request bodies, paths, credentials, or other canary values.
    • +
  • Use zero test-level retries. A failed scenario must remain a failure until its cause is understood and fixed; rerunning an entire CI job for diagnosed infrastructure failure does not waive the original result.
    • +
+ +

Execution order, cleanup, and completion evidence

+
    +
  1. After implementation is ready, run formatting, type checking, unit tests, and source-level integration tests.
    2. +
  2. Build and pack one candidate tarball, record its digest, and run the full isolated Linux E2E matrix against that artifact.
    3. +
  3. Run the required macOS, Windows, unsupported-Node, fake-Docker, real-Docker, entrypoint identity, and telemetry-degradation jobs.
    4. +
  4. Run one final PostHog ingestion smoke in the supplied dev project using the external relay described below and synthetic, non-sensitive fixture data. Query the resulting events by the known generated setup/deployment UUID and verify event visibility, ordering, property types, Person behavior, company grouping, and setup-to-deployment identity continuity. Delete or expire test data according to the test project’s retention policy; never send fixture secrets.
    5. +
  5. Run the cleanup audit and repository-dirtiness check after every job. No temporary tarball, certificate, npm cache, setup directory, telemetry file, process, listener, Docker container, image, network, or volume may remain.
    6. +
  6. Attach the redacted completion report and required CI links to the feature PR. Mark the feature complete only when every required scenario and environment passes against the final commit and artifact, the manifest reports no uncovered entry, cleanup passes, and the PostHog smoke is verified.
    7. +
+

All E2E infrastructure and tests land with the feature changes. None of these checks may be converted into a post-merge task merely to unblock completion.

+

Dev-project PostHog smoke routing

+

Use the user-supplied dev-project ingestion token phc_EJR6BsaBbvIKhM4t4zp1boYC92Tpp5Fgb9Csa9Us5aw for live test ingestion. Keep the production package token unchanged. Run the same packed artifact and digest used by the local E2E suite; the existing external HTTPS capture service acts as a forwarding relay only for the final live smoke. It substitutes the dev ingestion token in the SDK request’s project-authentication field, forwards to the dev project’s verified PostHog ingestion host, and preserves event names, UUIDs, timestamps, distinct IDs, groups, and properties. Apply the same routing to container install/upgrade telemetry so both surfaces reach the dev project.

+

The relay must validate the original and forwarded envelopes and assert that only project-authentication metadata changes. Support the shipped SDK’s batching/compression and the entrypoint’s capture payload. Resolve the upstream host outside the local hostname override to avoid a forwarding loop. Confirm the dev project’s region/ingestion host before running; do not infer it from the token. The ingestion token does not grant event-query access: use the authenticated PostHog connection or a separately configured read credential to verify stored events. Record the observed results; request acceptance alone does not satisfy ingestion verification.

+

Only the final synthetic live smoke permits this relay to reach PostHog. Exhaustive canary and failure tests remain on the local collector with external traffic denied. Never embed the dev token or relay controls into the published CLI; test routing is entirely external. The live smoke, including event-query verification, remains part of the mandatory development completion gate.

+ +

Packaging and integration tests

+
    +
  • Use a local HTTP capture server with the real posthog-node client to verify the final JSON envelope, SDK options, default profile behavior, and absence of PII, sensitive fields, and custom person-property mutations.
    • +
  • Add explicit package scripts for unit, integration, packed-artifact E2E, platform smoke, and cleanup-audit jobs so local development, pull-request CI, and release CI invoke the same commands.
    • +
  • Run the packed CLI against deterministic local service fixtures and the fake Docker executable for exhaustive branches, then run the separate real-Docker identity suite; neither layer may be represented as covering the other.
    • +
  • Run npm pack --dry-run and assert bin.cjs and the compiled wizard are included and executable through the package entrypoint, with no new Sourcebot-owned postinstall entrypoint.
    • +
  • Inspect the actual release-shaped tarball and assert test harnesses, fixture secrets, certificates, transcripts, coverage output, and temporary files are excluded from the published package.
    • +
  • Assert reo-census remains in the package manifest and packed dependency graph, and that Sourcebot code neither removes nor rewrites PACKAGE_TRACKER_ANALYTICS.
    • +
  • Assert no test-only PostHog host/token override, fake transport, deterministic UUID hook, or failure-injection switch is reachable through the published CLI. Test redirection belongs to the external harness or internal dependency injection exercised outside the production entrypoint.
    • +
  • Verify formatting, Node 24 type checking/build, the setup-wizard unit suite, integration suite, full packed-package E2E matrix, platform/package-manager smoke jobs, and cleanup audit in CI.
    • +
+

Rollout and operational validation

+
    +
  1. PR #1648 is merged for the repository-root entrypoint.sh first-boot behavior. Land the setup-wizard telemetry and .env handoff in packages/setupWizard.
    2. +
  2. Publish a Sourcebot container image containing the updated root entrypoint.sh before or alongside the telemetry-enabled wizard. Older images will overwrite the wizard-supplied ID and therefore cannot provide setup-to-deployment continuity.
    3. +
  3. Include the Node 24 runtime floor, Node typings, runtime bootstrap, posthog-node wrapper, typed schema, in-memory session identity, instrumentation, E2E tests, documentation, and explicit Reo coexistence behavior in the setup-wizard change.
    4. +
  4. Run the packed-package E2E suite under Node 24 and its intentional Node 22.22 rejection case before testing production ingestion.
    5. +
  5. After every required local packed-artifact scenario passes, run the final synthetic smoke in a non-production PostHog project and verify ingestion, Person/group behavior, and identity continuity by the generated UUID. Keep exhaustive sensitive-canary and failure testing on the isolated local collector.
    6. +
  6. Point the production build at the existing Sourcebot PostHog project and release the Node requirement as a clearly documented breaking package version rather than a silent minor runtime change.
    7. +
  7. Build the ordered started -> completed funnel and setup-to-deployment analysis from the names and shared identity fields in this document; save breakdowns only on allowlisted properties.
    8. +
  8. During the first release, monitor event counts, property-key cardinality, unknown enum rates, and impossible orderings such as completed without started in the same session. Do not add payload logging to diagnose delivery.
    9. +
  9. If telemetry causes a setup regression, ship a patch that repairs or removes the faulty integration path; setup behavior and exit status must remain independent of telemetry delivery success.
    10. +
+

Schema changes after release require incrementing schemaVersion when meaning, type, or enum interpretation changes. Additive event properties still require privacy review and exact-key tests. Renaming an event or changing checkpoint timing should use a new event/schema version rather than silently changing the existing funnel.

+

Acceptance criteria for implementation

+
    +
  • Every product-funnel checkpoint fires exactly once per setup session after its stage resolves.
    • +
  • Skipping AI emits setup_sourcebot_ai_setup_completed with zero counts and aiConfigured: false.
    • +
  • Existing, downloaded, declined, and failed compose-file branches all emit setup_sourcebot_resolved_compose_file with distinct outcomes.
    • +
  • Docker validation emits an outcome even when skipped for a known reason. Skipped or failed measurements use required nullable counts, never fabricated zeroes; initial/final measurements, status values, action semantics, and outcome precedence follow the Docker schema and are covered by E2E tests.
    • +
  • Completion is captured at the defined handoff without waiting for foreground Docker to exit. Ctrl+C before completion captures cancellation; Ctrl+C after completion cleans up and exits without changing the successful setup outcome. The lifecycle coordinator cancels owned background work, restores terminal state, and enforces the 3-second interrupt exit budget independently of telemetry success.
    • +
  • Cancelling with Ctrl+C or declining an overwrite emits only fixed stage/reason values.
    • +
  • Telemetry failure cannot alter wizard output files, Docker actions, exit status, or user-visible success.
    • +
  • Every reported setup failure maps to exactly one coarse failureCategory; expected prompt rejection, cancellation, and best-effort autocomplete/model-catalog fallback are not misreported as failures. Handled infrastructure errors remain reportable as recoverable failures even when the wizard later completes successfully.
    • +
  • Each handled infrastructure operation failure emits one nonterminal failed event with recoverable: true and leaves the telemetry session active. Multiple recoverable failures may precede further checkpoints. Only completion, cancellation, or an error that actually terminates the wizard ends the chain; at most one terminal event is emitted and SDK shutdown occurs only at that boundary.
    • +
  • Automated tests assert separate exact application and resolved-SDK/envelope allowlists for every setup event and seed inputs with token-, URL-, email-, repository-, model-, path-, and error-shaped values to prove none enter captured payloads. Legitimate SDK-generated library metadata, timestamps, event UUIDs, and grouping fields pass the versioned transport contract.
    • +
  • Deployment classification normalizes common URL formatting variants, uses exact or dot-boundary hostname matches, emits unknown when the mode is ambiguous, and never sends the entered or normalized hostname.
    • +
  • Hosted-URL classification reports only confirmed loopback, another parsed address, or unknown; it performs no network lookup and makes no public/private or reachability claim.
    • +
  • reo-census remains installed and retains sole ownership of PACKAGE_TRACKER_ANALYTICS.
    • +
  • A UUIDv4 setupSessionId is created in memory at wizard startup regardless of Reo or deployed-product telemetry variables; no telemetry-only state is persisted.
    • +
  • The generated setupSessionId is exactly 36 ASCII characters and matches ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$; no code path transforms it before use. This is the plan’s normative UUIDv4 contract under current RFC 9562 and is compatible with Node’s RFC 4122 terminology.
    • +
  • For a new setup, SOURCEBOT_INSTALL_ID in .env, every setup event identity field, the deployment’s first-run event, and .installedv3 all use the same UUID.
    • +
  • The identity-continuity E2E test runs against Sourcebot runtime code containing the committed repository-root entrypoint.sh change; the setup wizard never edits or substitutes that script.
    • +
  • Existing valid deployment IDs are preserved without being sent in setup telemetry; .installedv3 remains authoritative after first boot.
    • +
  • package.json declares Node >=24.0.0, Node typings use major 24, and setup-sourcebot CI/release jobs run on Node 24.
    • +
  • The packed-package E2E suite passes on Node 24 and verifies a clear, side-effect-free failure on Node 22.22.
    • +
  • The required setup-wizard-e2e check executes the final compiled npm tarball across the complete scenario manifest and required OS/package-manager environments with no skipped, quarantined, retried, or allowed-failure scenario.
    • +
  • The E2E completion report proves exact event/property contracts, privacy canary exclusion, baseline behavioral equivalence, telemetry-failure isolation, real Docker identity continuity, non-production PostHog ingestion, and complete resource cleanup for the final feature commit.
    • +
  • The official posthog-node SDK is configured with immediate flush, SDK-maintained transport defaults, a bounded shutdown timeout, GeoIP disabled, and CLI runtime attribution.
    • +
  • PostHog uses default Person profiles keyed by random setupSessionId values, with GeoIP disabled and no identify, $set, or $set_once calls.
    • +
  • Documentation explains collection categories, ephemeral setup identity, new-deployment ID handoff, always-on setup-wizard PostHog telemetry, Reo’s independent variable, and the deployed product’s separate telemetry setting.
    • +
+

References

+
    +
  • PostHog Node.js SDK: short-lived process shutdown/flush behavior and current Node runtime guidance.
    • +
  • PostHog Node SDK reference: shutdown(timeoutMs) behavior and its default timeout.
    • +
  • PostHog anonymous vs. identified events: default identified-event profile processing and the distinction between event correlation and person properties.
    • +
  • PostHog capture API: raw event ingestion envelope and endpoint.
    • +
  • GitHub Enterprise Cloud with data residency: managed GitHub Enterprise Cloud instances use dedicated ghe.com subdomains.
    • +
  • GitLab Dedicated: default managed-instance domains and support for arbitrary custom domains, which makes some URL-only classifications ambiguous.
    • +
  • Node.js release schedule: Node 24 LTS status and the production recommendation to use an Active or Maintenance LTS release.
    • +
  • Node.js crypto.randomUUID(): generates a cryptographically random version-4 UUID and returns it as a string.
    • +
  • RFC 9562: the current UUID standard, including the canonical hex-and-dash text representation, IETF variant bits, and UUIDv4 version bits; it obsoletes RFC 4122.
    • +
  • Alpine Linux 3.23 uuidgen package: confirms the Sourcebot image uses the uuidgen implementation from util-linux.
    • +
  • util-linux uuidgen source and UUID formatting source: the random generator emits a standard UUID and the default formatter uses lowercase hexadecimal with canonical hyphens.
    • +
  • Existing Sourcebot identity flow: entrypoint.sh, docker-compose.yml, packages/backend/src/posthog.ts, packages/web/src/lib/posthog.ts, packages/shared/src/env.server.ts, and docs/docs/misc/telemetry.mdx.
    • +
+

Reviewer checklist

+
    +
  • [ ] Approve started -> completed as the canonical and only Sourcebot PostHog setup funnel.
    • +
  • [ ] Approve setupSessionId as the in-memory setup distinctId and, for new deployments, the resulting SOURCEBOT_INSTALL_ID.
    • +
  • [ ] Approve the exact canonical lowercase UUIDv4 regex under RFC 9562, its compatibility with Node’s RFC 4122 terminology, and byte-for-byte reuse across all setup and deployment identity fields.
    • +
  • [ ] Approve each event name, checkpoint timing, property type, and fixed enum in PostHog event schema.
    • +
  • [ ] Approve no telemetry-only persistence and the explicit existing-deployment behavior.
    • +
  • [ ] Approve exact selected-repository counts and bucketed local-discovery totals.
    • +
  • [ ] Approve Node >=24.0.0, Node 24 CI/release execution, and the explicit Node 22.22 rejection test.
    • +
  • [ ] Approve the official posthog-node transport defaults, privacy wrapper, immediate-flush configuration, and bounded shutdown timing.
    • +
  • [ ] Approve PostHog’s default Person profile keyed only by the random setupSessionId, with no identify, aliases, custom person properties, PII, or sensitive data.
    • +
  • [ ] Approve retaining reo-census and leaving PACKAGE_TRACKER_ANALYTICS exclusively under Reo’s control.
    • +
  • [ ] Approve no package-level opt-out for setup-wizard PostHog telemetry, subject to privacy/legal review and clear release documentation.
    • +
  • [ ] Approve completion as successful handoff/manual guidance rather than application readiness.
    • +
  • [ ] Confirm that a separate readiness event is out of scope for v1.
    • +
  • [ ] Approve the mandatory compiled-artifact E2E matrix, scenario-manifest coverage, platform/package-manager jobs, PostHog contract oracle, behavioral differential checks, Docker identity suite, cleanup audit, and no-deferral completion gate.
    • +
+
+
+
+ + + diff --git a/packages/setupWizard/tests/approvedSchema.json b/packages/setupWizard/tests/approvedSchema.json new file mode 100644 index 000000000..e1c7b9725 --- /dev/null +++ b/packages/setupWizard/tests/approvedSchema.json @@ -0,0 +1,599 @@ +{ + "started": { + "invocationMethod": { + "enum": [ + "npx", + "global_binary", + "local_binary", + "workspace", + "unknown" + ] + }, + "isInteractive": { + "type": "boolean" + } + }, + "chose_setup_directory": { + "usedDefaultDirectory": { + "type": "boolean" + }, + "directoryExisted": { + "type": "boolean" + }, + "directoryAction": { + "enum": [ + "created", + "existing_directory_accepted" + ] + } + }, + "configured_code_source": { + "configurationIndex": { + "type": "number" + }, + "codeHost": { + "enum": [ + "github", + "gitlab", + "bitbucket", + "gitea", + "azure_devops", + "gerrit", + "local_git", + "remote_git" + ] + }, + "deploymentType": { + "enum": [ + "cloud", + "self_hosted", + "local", + "remote", + "unknown" + ] + }, + "credentialMode": { + "enum": [ + "none", + "personal_access_token", + "api_token", + "access_token", + "app_password", + "http_access_token" + ] + }, + "scopeTypes": { + "array": { + "enum": [ + "all", + "repositories", + "organizations", + "users", + "groups", + "projects", + "workspaces" + ] + } + }, + "indexAll": { + "type": "boolean" + }, + "repositoryCount": { + "type": "number" + }, + "organizationCount": { + "type": "number" + }, + "userCount": { + "type": "number" + }, + "groupCount": { + "type": "number" + }, + "projectCount": { + "type": "number" + }, + "workspaceCount": { + "type": "number" + }, + "generatedConnectionCount": { + "type": "number" + }, + "localDiscoveredRepoCountBucket": { + "nullable": { + "enum": [ + "1", + "2-5", + "6-20", + "21-100", + "101+" + ] + } + } + }, + "configured_code_sources": { + "codeSourceConfigurationCount": { + "type": "number" + }, + "generatedConnectionCount": { + "type": "number" + }, + "uniqueCodeHostCount": { + "type": "number" + }, + "codeHostTypes": { + "array": { + "enum": [ + "github", + "gitlab", + "bitbucket", + "gitea", + "azure_devops", + "gerrit", + "local_git", + "remote_git" + ] + } + }, + "credentialedCodeSourceCount": { + "type": "number" + }, + "cloudCodeSourceCount": { + "type": "number" + }, + "selfHostedCodeSourceCount": { + "type": "number" + }, + "localCodeSourceCount": { + "type": "number" + }, + "indexAllCodeSourceCount": { + "type": "number" + }, + "repositoryCount": { + "type": "number" + }, + "organizationCount": { + "type": "number" + }, + "userCount": { + "type": "number" + }, + "groupCount": { + "type": "number" + }, + "projectCount": { + "type": "number" + }, + "workspaceCount": { + "type": "number" + } + }, + "configured_ai_provider": { + "configurationIndex": { + "type": "number" + }, + "provider": { + "enum": [ + "anthropic", + "openai", + "openai-compatible", + "amazon-bedrock", + "google-generative-ai", + "google-vertex", + "google-vertex-anthropic", + "azure", + "deepseek", + "mistral", + "openrouter", + "xai" + ] + }, + "modelSelectionMethod": { + "enum": [ + "catalog", + "custom_entry", + "manual_fallback" + ] + }, + "credentialMode": { + "enum": [ + "api_key", + "aws_default_chain", + "aws_explicit_keys", + "google_application_default_credentials", + "google_credentials_file" + ] + }, + "usesCustomEndpoint": { + "type": "boolean" + }, + "hasDisplayName": { + "type": "boolean" + } + }, + "ai_setup_completed": { + "aiConfigured": { + "type": "boolean" + }, + "aiConfigurationCount": { + "type": "number" + }, + "uniqueProviderCount": { + "type": "number" + }, + "providerTypes": { + "array": { + "enum": [ + "anthropic", + "openai", + "openai-compatible", + "amazon-bedrock", + "google-generative-ai", + "google-vertex", + "google-vertex-anthropic", + "azure", + "deepseek", + "mistral", + "openrouter", + "xai" + ] + } + }, + "usesCustomEndpoint": { + "type": "boolean" + }, + "credentialModes": { + "array": { + "enum": [ + "api_key", + "aws_default_chain", + "aws_explicit_keys", + "google_application_default_credentials", + "google_credentials_file" + ] + } + }, + "modelSelectionMethods": { + "array": { + "enum": [ + "catalog", + "custom_entry", + "manual_fallback" + ] + } + } + }, + "configured_hosted_url": { + "usedDefaultUrl": { + "type": "boolean" + }, + "protocol": { + "enum": [ + "http", + "https" + ] + }, + "hostCategory": { + "enum": [ + "localhost", + "address", + "unknown" + ] + } + }, + "generated_configs": { + "filesWritten": { + "array": { + "enum": [ + "config_json", + "env", + "compose_override" + ] + } + }, + "overwroteExistingFiles": { + "array": { + "enum": [ + "config_json", + "env", + "compose_override" + ] + } + }, + "wroteComposeOverride": { + "type": "boolean" + }, + "localMountCount": { + "type": "number" + }, + "generatedConnectionCount": { + "type": "number" + }, + "aiConfigurationCount": { + "type": "number" + }, + "credentialVariableCount": { + "type": "number" + }, + "deploymentIdentityAction": { + "enum": [ + "created_from_setup_session", + "preserved_existing" + ] + } + }, + "resolved_compose_file": { + "outcome": { + "enum": [ + "downloaded", + "already_present", + "declined", + "download_failed" + ] + }, + "composeAvailable": { + "type": "boolean" + }, + "downloadPromptShown": { + "type": "boolean" + }, + "downloadAttempted": { + "type": "boolean" + }, + "failureCategory": { + "nullable": { + "enum": [ + "network", + "http_4xx", + "http_5xx", + "filesystem", + "timeout", + "unknown" + ] + } + } + }, + "validated_docker_state": { + "outcome": { + "enum": [ + "passed", + "passed_after_cleanup", + "unresolved_conflicts", + "skipped_no_compose", + "skipped_existing_deployment_running", + "validation_failed" + ] + }, + "dockerStatus": { + "enum": [ + "available", + "unavailable", + "error", + "not_checked" + ] + }, + "composeContainerState": { + "enum": [ + "none", + "running", + "stopped", + "mixed", + "unknown" + ] + }, + "runningComposeContainerCount": { + "nullable": { + "type": "number" + } + }, + "stoppedComposeContainerCount": { + "nullable": { + "type": "number" + } + }, + "existingVolumeCount": { + "nullable": { + "type": "number" + } + }, + "initialPortConflictCount": { + "nullable": { + "type": "number" + } + }, + "remainingPortConflictCount": { + "nullable": { + "type": "number" + } + }, + "portConflictSource": { + "enum": [ + "none", + "docker", + "non_docker", + "mixed", + "unknown" + ] + }, + "existingDeploymentAction": { + "enum": [ + "none", + "stopped", + "left_running", + "stop_failed" + ] + }, + "stoppedContainerAction": { + "enum": [ + "none", + "removed", + "kept", + "remove_failed" + ] + }, + "volumeAction": { + "enum": [ + "none", + "removed", + "kept", + "remove_failed" + ] + }, + "portConflictAction": { + "enum": [ + "none", + "containers_stopped", + "kept", + "stop_failed" + ] + }, + "leftExistingDeploymentRunning": { + "type": "boolean" + } + }, + "completed": { + "completionMode": { + "enum": [ + "sourcebot_start_spawned", + "sourcebot_start_failed", + "existing_deployment_left_running", + "manual_start_required" + ] + }, + "sourcebotStartOffered": { + "type": "boolean" + }, + "sourcebotStartRequested": { + "type": "boolean" + }, + "sourcebotStartOutcome": { + "enum": [ + "spawned", + "declined", + "not_offered", + "spawn_failed" + ] + }, + "composeAvailable": { + "type": "boolean" + }, + "dockerValidationOutcome": { + "enum": [ + "passed", + "passed_after_cleanup", + "unresolved_conflicts", + "skipped_no_compose", + "skipped_existing_deployment_running", + "validation_failed" + ] + }, + "remainingPortConflictCount": { + "nullable": { + "type": "number" + } + }, + "generatedConnectionCount": { + "type": "number" + }, + "codeHostTypes": { + "array": { + "enum": [ + "github", + "gitlab", + "bitbucket", + "gitea", + "azure_devops", + "gerrit", + "local_git", + "remote_git" + ] + } + }, + "repositoryCount": { + "type": "number" + }, + "aiConfigured": { + "type": "boolean" + }, + "aiConfigurationCount": { + "type": "number" + }, + "providerTypes": { + "array": { + "enum": [ + "anthropic", + "openai", + "openai-compatible", + "amazon-bedrock", + "google-generative-ai", + "google-vertex", + "google-vertex-anthropic", + "azure", + "deepseek", + "mistral", + "openrouter", + "xai" + ] + } + }, + "deploymentIdentityAction": { + "enum": [ + "created_from_setup_session", + "preserved_existing" + ] + }, + "totalDurationMs": { + "type": "number" + } + }, + "cancelled": { + "stage": { + "enum": [ + "setup_directory", + "code_sources", + "ai_setup", + "hosted_url", + "config_overwrite", + "compose_file", + "docker_validation", + "start" + ] + }, + "reason": { + "enum": [ + "keyboard_interrupt", + "existing_directory_declined", + "config_overwrite_declined" + ] + } + }, + "failed": { + "stage": { + "enum": [ + "setup_directory", + "code_sources", + "ai_setup", + "hosted_url", + "config_overwrite", + "compose_file", + "docker_validation", + "start" + ] + }, + "failureCategory": { + "enum": [ + "validation", + "network", + "filesystem", + "docker_unavailable", + "docker_command", + "process_spawn", + "unknown" + ] + }, + "recoverable": { + "type": "boolean" + } + } +} diff --git a/packages/setupWizard/tests/e2e/baseline.mjs b/packages/setupWizard/tests/e2e/baseline.mjs new file mode 100644 index 000000000..426f9224f --- /dev/null +++ b/packages/setupWizard/tests/e2e/baseline.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, writeFileSync, readFileSync, symlinkSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { join, dirname, resolve, delimiter } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { artifact, scenario, minimal, defaultCompose } from './harness.mjs'; +const require = createRequire(import.meta.url); +const repo = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..'); +const packed = artifact(); +const base = process.env.SETUP_TEST_BASE_REF ?? '31734dc2'; +try { + const checkout = join(packed.root, 'baseline'); + mkdirSync(checkout); + const archive = execFileSync('git', ['archive', base, 'packages/setupWizard'], { cwd: repo }); + execFileSync('tar', ['-x', '-C', checkout], { input: archive }); + const pkg = join(checkout, 'packages/setupWizard'); + symlinkSync(join(repo, 'node_modules'), join(checkout, 'node_modules'), 'junction'); + symlinkSync(join(repo, 'packages/setupWizard/node_modules'), join(pkg, 'node_modules'), 'junction'); + execFileSync(process.execPath, [require.resolve('typescript/bin/tsc'), '-p', pkg], { stdio: 'inherit' }); + const manifest = JSON.parse(readFileSync(join(pkg, 'package.json'), 'utf8')); + // Yarn pack rewrites this dev-only workspace reference; mirror that transformation. + manifest.devDependencies['@sourcebot/schemas'] = JSON.parse(readFileSync(join(repo, 'packages/schemas/package.json'), 'utf8')).version; + writeFileSync(join(pkg, 'package.json'), JSON.stringify(manifest)); + const env = { ...process.env, PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false' }; + const tgz = execFileSync('npm', ['pack', '--ignore-scripts', '--silent'], { cwd: pkg, env, encoding: 'utf8' }).trim(); + const install = join(packed.root, 'baseline-installed'); + mkdirSync(install); + writeFileSync(join(install, 'package.json'), '{"private":true,"allowScripts":{"reo-census":true}}'); + execFileSync('npm', ['install', '--no-audit', '--no-fund', join(pkg, tgz)], { cwd: install, env }); + const prior = { ...packed, bin: join(install, 'node_modules/.bin/setup-sourcebot'), installed: join(install, 'node_modules/setup-sourcebot') }; + const normalize = files => Object.fromEntries(Object.entries(files).map(([name, contents]) => [name, name === '.env' + ? contents.replace(/^SOURCEBOT_INSTALL_ID=.*\n/gm, '').replace(/^# Deployment identifier\n/gm, '').replace(/^(AUTH_SECRET|SOURCEBOT_ENCRYPTION_KEY)=.*$/gm, '$1=').replace(/\n{3,}/g, '\n\n').trim() + : contents])); + for (const download of [false, true]) { + const run = (target, baseline) => scenario(target, { baseline }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', download ? 'y' : 'n'); + if (download) { + await d.answer('Start Sourcebot now?', 'n'); + } + }); + const original = await run(prior, true); + const feature = await run(packed, false); + assert.deepEqual(original.events, []); + assert.deepEqual(normalize(feature.files), normalize(original.files)); + assert.deepEqual(feature.dockerCalls, original.dockerCalls); + assert.equal(feature.exitCode, original.exitCode); + } + console.log(`Baseline ${base}: generated config/env and Docker operations match (manual and downloaded Compose); only UUID persistence and generated secrets normalized.`); +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/collectors.test.mjs b/packages/setupWizard/tests/e2e/collectors.test.mjs new file mode 100644 index 000000000..e6de1202c --- /dev/null +++ b/packages/setupWizard/tests/e2e/collectors.test.mjs @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { artifact, scenario, minimal, canary } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Collector artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const props = (r, name) => r.events.filter(e => e.event === `setup_sourcebot_${name}`).map(e => e.properties); +const end = async d => { + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); +}; + +for (const [host, index] of [['github', 0], ['gitlab', 1], ['gitea', 6]]) { + for (const token of [false, true]) { + test(`${host}: all explicit scopes, ${token ? 'credentialed' : 'public'}`, async () => { + const label = host === 'github' ? 'GitHub' : host === 'gitlab' ? 'GitLab' : 'Gitea'; + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', index); + await d.answer(`${label} URL`); + await d.answer(`${label} ${host === 'gitea' ? 'Access Token' : 'Personal Access Token'}`, token ? `${canary}-token` : ''); + await d.check('What do you want to index?', [0, 1, 2]); + if (host === 'github') { + await d.multi('Repositories to index'); + await d.multi('Organizations to index', canary); + await d.multi('GitHub users to index', canary); + } else if (host === 'gitlab') { + await d.multi('Groups to index', canary); + await d.multi('Projects to index'); + await d.multi('Users to index', canary); + } else { + await d.multi('Organizations to index', canary); + await d.multi('Repositories to index'); + await d.multi('Users to index', canary); + } + await end(d); + }); + const summary = props(result, 'configured_code_source')[0]; + assert.equal(summary.codeHost, host); + assert.equal(summary.deploymentType, 'cloud'); + assert.equal(summary.scopeTypes.length, 3); + assert.equal(summary.userCount, 1); + assert.equal(summary.credentialMode === 'none', !token); + assert.equal(summary.repositoryCount, host === 'gitlab' ? 0 : 1); + assert.equal(summary.projectCount, host === 'gitlab' ? 1 : 0); + }); + } +} + +for (const all of [true, false]) { + test(`Gerrit: ${all ? 'all' : 'projects'}`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 7); + await d.answer('Gerrit URL', `https://${canary}.example.invalid`); + await d.answer('Index all projects?', all ? 'y' : 'n'); + if (!all) { + await d.multi('Projects to index', canary); + } + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.indexAll, all); + assert.equal(s.projectCount, all ? 0 : 1); + assert.equal(s.deploymentType, 'self_hosted'); + }); +} + +for (const server of [false, true]) { + test(`Azure DevOps ${server ? 'Server/TFS' : 'Cloud'}: all scopes`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 4); + await d.select('Which Azure DevOps deployment?', server ? 1 : 0); + if (server) { + await d.answer('Azure DevOps Server URL', `https://${canary}.example.invalid`); + await d.answer('Use legacy TFS path format', 'y'); + } + await d.answer('Azure DevOps Personal Access Token', `${canary}-token`); + await d.check('What do you want to index?', [0, 1, 2]); + await d.multi(server ? 'Collections to index' : 'Organizations to index', canary); + await d.multi('Projects to index'); + await d.multi('Repositories to index', `${canary}/project/repo`); + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.deploymentType, server ? 'self_hosted' : 'cloud'); + assert.equal(s.organizationCount, 1); + assert.equal(s.projectCount, 1); + assert.equal(s.repositoryCount, 1); + }); +} + +for (const auth of [0, 1, 2]) { + test(`Bitbucket Cloud auth ${auth}: workspaces and repositories`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 5); + await d.select('Which Bitbucket deployment?'); + await d.select('How will you authenticate?', auth); + if (auth === 0) { + await d.answer('Atlassian account email', `${canary}@example.invalid`); + await d.answer('Bitbucket username', canary); + await d.answer('API Token (', `${canary}-token`); + } else if (auth === 1) { + await d.answer('Access Token (', `${canary}-token`); + } else { + await d.answer('Bitbucket username', canary); + await d.answer('Bitbucket App Password', `${canary}-password`); + } + await d.check('What do you want to index?', [0, 1]); + await d.multi('Workspaces to index', canary); + await d.multi('Repositories to index'); + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.credentialMode, ['api_token', 'access_token', 'app_password'][auth]); + assert.equal(s.workspaceCount, 1); + assert.equal(s.repositoryCount, 1); + }); +} + +for (const all of [false, true]) { + test(`Bitbucket Data Center: ${all ? 'all' : 'selected'}`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 5); + await d.select('Which Bitbucket deployment?', 1); + await d.answer('Bitbucket Data Center URL', `https://${canary}.example.invalid`); + await d.answer('Bitbucket username'); + await d.answer('Bitbucket HTTP Access Token', `${canary}-token`); + await d.answer('Index every repository visible to the token?', all ? 'y' : 'n'); + if (!all) { + await d.check('What do you want to index?', [0, 1]); + await d.multi('Project keys to index', canary); + await d.multi('Repositories to index'); + } + await end(d); + }); + assert.equal(props(result, 'configured_code_source')[0].indexAll, all); + assert.equal(props(result, 'configured_code_source')[0].deploymentType, 'self_hosted'); + }); +} + +for (const shape of ['root', 'wildcard', 'nested']) { + test(`Local repositories: ${shape}`, async () => { + const result = await scenario(packed, { prepare({ cwd }) { + const root = join(cwd, canary); + if (shape === 'root') { + mkdirSync(join(root, '.git'), { recursive: true }); + } else { + mkdirSync(join(root, 'one/.git'), { recursive: true }); + mkdirSync(join(root, shape === 'wildcard' ? 'two/.git' : 'nested/two/.git'), { recursive: true }); + } + } }, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 2); + await d.answer('Path to your repos directory', join(d.cwd, canary)); + if (shape !== 'root') { + await d.answer('Which repositories should be indexed?'); + } + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.repositoryCount, shape === 'root' ? 1 : 2); + assert.equal(s.generatedConnectionCount, shape === 'nested' ? 2 : 1); + assert.equal(props(result, 'generated_configs')[0].wroteComposeOverride, true); + assert.ok(result.files['docker-compose.override.yml']); + }); +} + +const providers = ['anthropic', 'openai', 'openai-compatible', 'amazon-bedrock', 'google-generative-ai', 'google-vertex', 'google-vertex-anthropic', 'azure', 'deepseek', 'mistral', 'openrouter', 'xai']; +for (const provider of providers) { + for (const explicit of (provider === 'amazon-bedrock' || provider.startsWith('google-vertex') ? [false, true] : [false])) { + test(`AI ${provider}${explicit ? ' explicit credentials' : ''}`, async () => { + const result = await scenario(packed, {}, async d => { + await minimal(d, { ai: true }); + await d.select('Which AI provider?', providers.indexOf(provider)); + await d.answer('Model name', `${canary}-model`); + if (provider === 'openai-compatible') { + await d.answer('Base URL', `https://${canary}.example.invalid/v1`); + } + if (provider === 'azure') { + await d.answer('Azure resource name', canary); + await d.answer('API version'); + } + if (provider === 'amazon-bedrock') { + await d.answer('Use the default AWS credential chain?', explicit ? 'n' : 'y'); + if (explicit) { + await d.answer('AWS Access Key ID', canary); + await d.answer('AWS Secret Access Key', `${canary}-secret`); + } + await d.answer('AWS region'); + } else if (provider.startsWith('google-vertex')) { + await d.answer('Google Cloud project ID', canary); + await d.answer('Google Cloud region'); + await d.answer('Use Application Default Credentials?', explicit ? 'n' : 'y'); + if (explicit) { + await d.answer('Path to service account credentials JSON', `/${canary}/credentials.json`); + } + } else { + await d.answer('API key (', `${canary}-api-key`); + } + await d.answer('Display name', `${canary}-display`); + await d.answer('Add another model?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + const s = props(result, 'configured_ai_provider')[0]; + assert.equal(s.provider, provider); + assert.equal(s.hasDisplayName, true); + assert.equal(s.usesCustomEndpoint, provider === 'openai-compatible'); + assert.equal(s.modelSelectionMethod, 'manual_fallback'); + assert.deepEqual(props(result, 'ai_setup_completed')[0].providerTypes, [provider]); + assert.equal(JSON.parse(result.files['config.json']).models[0].provider, provider); + }); + } +} diff --git a/packages/setupWizard/tests/e2e/docker.test.mjs b/packages/setupWizard/tests/e2e/docker.test.mjs new file mode 100644 index 000000000..9d3843bac --- /dev/null +++ b/packages/setupWizard/tests/e2e/docker.test.mjs @@ -0,0 +1,142 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { artifact, scenario, minimal, defaultCompose } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Docker artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const p = (r, name) => r.events.find(e => e.event === `setup_sourcebot_${name}`)?.properties; +const running = { Name: 'canary-sensitive-running', Service: 'sourcebot', State: 'running' }; +const stopped = { Name: 'canary-sensitive-stopped', Service: 'sourcebot', State: 'exited' }; +const initial = async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'y'); }; + +for (const action of ['stop', 'keep', 'fail']) { + test(`running deployment: ${action}`, async () => { + const result = await scenario(packed, { docker: { containers: [running], fail: action === 'fail' ? ['compose down'] : [] } }, async d => { + await initial(d); + await d.answer('Stop and remove the running deployment?', action === 'keep' ? 'n' : 'y'); + if (action === 'stop') { + await d.answer('Start Sourcebot now?', 'n'); + } + }); + assert.equal(p(result, 'validated_docker_state').existingDeploymentAction, { stop: 'stopped', keep: 'left_running', fail: 'stop_failed' }[action]); + assert.equal(p(result, 'validated_docker_state').outcome, { stop: 'passed_after_cleanup', keep: 'skipped_existing_deployment_running', fail: 'validation_failed' }[action]); + assert.equal(p(result, 'completed').sourcebotStartOffered, action === 'stop'); + if (action !== 'stop') { + assert.equal(p(result, 'validated_docker_state').existingVolumeCount, null); + assert.equal(p(result, 'validated_docker_state').remainingPortConflictCount, null); + } + }); +} + +for (const action of ['remove', 'keep', 'fail']) { + test(`stopped containers and volumes: ${action}`, async () => { + const result = await scenario(packed, { docker: { containers: [stopped], volumes: ['sourcebot_cache'], fail: action === 'fail' ? ['compose rm', 'volume rm'] : [] } }, async d => { + await initial(d); + await d.answer('Remove them now', action === 'keep' ? 'n' : 'y'); + await d.answer('Wipe these volumes?', action === 'keep' ? 'n' : 'y'); + await d.answer('Start Sourcebot now?', 'n'); + }); + const state = p(result, 'validated_docker_state'); + assert.equal(state.stoppedComposeContainerCount, 1); + assert.equal(state.existingVolumeCount, 1); + assert.equal(state.stoppedContainerAction, { remove: 'removed', keep: 'kept', fail: 'remove_failed' }[action]); + assert.equal(state.volumeAction, { remove: 'removed', keep: 'kept', fail: 'remove_failed' }[action]); + const failures = result.events.filter(e => e.event === 'setup_sourcebot_failed'); + assert.equal(failures.length, action === 'fail' ? 2 : 0); + assert.ok(failures.every(e => e.properties.recoverable)); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_completed'); + }); +} + +for (const failure of ['missing', 'inventory', 'malformed', 'volumes']) { + test(`Docker failed measurement: ${failure}`, async () => { + const result = await scenario(packed, { dockerMissing: failure === 'missing', docker: { malformed: failure === 'malformed', fail: failure === 'inventory' ? ['compose ps'] : failure === 'volumes' ? ['volume ls'] : [] } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'n'); + }); + const state = p(result, 'validated_docker_state'); + assert.equal(state.outcome, 'validation_failed'); + if (failure !== 'volumes') { + assert.equal(state.runningComposeContainerCount, null); + assert.equal(state.composeContainerState, 'unknown'); + } else { + assert.equal(state.runningComposeContainerCount, 0); + } + if (failure === 'volumes' || failure === 'missing') { + assert.equal(state.existingVolumeCount, null); + } + assert.equal(state.dockerStatus, failure === 'missing' ? 'unavailable' : 'available'); + }); +} + +for (const action of ['keep', 'stop', 'fail', 'remain']) { + test(`Docker port conflict: ${action}`, async () => { + const result = await scenario(packed, { compose: 'services:\n sourcebot:\n ports:\n - "43187:3000"\n', docker: { ports: 'canary-sensitive-container\t0.0.0.0:43187->3000/tcp', fail: action === 'fail' ? ['stop canary-sensitive-container'] : [], keepPorts: action === 'remain' } }, async d => { + await initial(d); + await d.answer('Stop this container', action === 'keep' ? 'n' : 'y'); + await d.answer(action === 'stop' ? 'Start Sourcebot now?' : 'Start Sourcebot anyway?', 'n'); + }); + const state = p(result, 'validated_docker_state'); + assert.equal(state.initialPortConflictCount, 1); + assert.equal(state.remainingPortConflictCount, action === 'stop' ? 0 : 1); + assert.equal(state.portConflictSource, 'docker'); + assert.equal(state.outcome, action === 'fail' ? 'validation_failed' : action === 'stop' ? 'passed_after_cleanup' : 'unresolved_conflicts'); + }); +} + +test('failed spawn offers manual steps and completes after recoverable failures', async () => { + const result = await scenario(packed, { dockerMissing: true }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + }); + assert.equal(p(result, 'completed').sourcebotStartOutcome, 'spawn_failed'); + assert.equal(p(result, 'completed').completionMode, 'sourcebot_start_failed'); + assert.ok(result.events.filter(e => e.event === 'setup_sourcebot_failed').every(e => e.properties.recoverable)); +}); + +for (const stage of ['fetch', 'docker', 'after_failures']) { + test(`Ctrl+C during ${stage} cancels outstanding work`, async () => { + const result = await scenario(packed, { + composeStatus: stage === 'fetch' ? 'stall' : undefined, + docker: stage === 'docker' ? { stall: ['compose ps'], stubborn: true } : stage === 'after_failures' ? { containers: [stopped], volumes: ['sourcebot_cache'], fail: ['compose rm', 'volume rm'] } : {}, + }, async d => { + await initial(d); + if (stage === 'after_failures') { + await d.answer('Remove them now', 'y'); + await d.answer('Wipe these volumes?', 'y'); + await d.wait('Start Sourcebot now?'); + } else { + await sleep(300); + } + const began = Date.now(); + d.interrupt(); + await d.finish(130); + assert.ok(Date.now() - began < 3500); + }); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_cancelled'); + assert.equal(p(result, 'cancelled').stage, { fetch: 'compose_file', docker: 'docker_validation', after_failures: 'start' }[stage]); + assert.equal(result.events.some(e => e.event === 'setup_sourcebot_completed'), false); + }); +} + +test('Ctrl+C cleans stubborn descendants even if their parent exits first', async () => { + await scenario(packed, { docker: { stall: ['compose ps'], descendant: true } }, async d => { + await initial(d); + await sleep(350); + d.interrupt(); + await d.finish(130); + }); +}); + +for (const missing of ['info', 'compose version']) { + test(`availability probe distinguishes unavailable ${missing}`, async () => { + const result = await scenario(packed, { docker: { fail: ['compose ps', missing === 'info' ? 'info --format' : missing] } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'n'); + }); + assert.equal(p(result, 'validated_docker_state').dockerStatus, 'unavailable'); + assert.equal(p(result, 'failed').failureCategory, 'docker_unavailable'); + }); +} diff --git a/packages/setupWizard/tests/e2e/fakeDocker.cjs b/packages/setupWizard/tests/e2e/fakeDocker.cjs new file mode 100644 index 000000000..b162a3a4d --- /dev/null +++ b/packages/setupWizard/tests/e2e/fakeDocker.cjs @@ -0,0 +1,31 @@ +const fs = require('node:fs'); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.TEST_DOCKER_LOG, JSON.stringify(args) + '\n'); +const state = JSON.parse(fs.readFileSync(process.env.TEST_DOCKER_STATE, 'utf8')); +const command = args.slice(0, 2).join(' '); +fs.appendFileSync(process.env.TEST_DOCKER_PIDS, String(process.pid) + '\n'); +if (state.fail?.includes(command)) { + console.error('canary-sensitive-error'); + process.exit(1); +} +if (state.stall?.includes(command)) { + if (state.descendant) { + const child = require('node:child_process').spawn(process.execPath, ['-e', "process.on('SIGINT', () => {}); setInterval(() => {}, 1000)"], { stdio: 'ignore' }); + fs.appendFileSync(process.env.TEST_DOCKER_PIDS, String(child.pid) + '\n'); + } + if (state.stubborn) { + process.on('SIGINT', () => {}); + } + setInterval(() => {}, 1000); +} else if (command === 'compose ps') { + console.log(state.malformed ? 'malformed' : JSON.stringify(state.containers ?? [])); +} else if (command === 'volume ls') { + console.log((state.volumes ?? []).join('\n')); +} else if (args[0] === 'ps') { + console.log(state.ports ?? ''); +} else if (args[0] === 'stop') { + if (!state.keepPorts) { + state.ports = ''; + fs.writeFileSync(process.env.TEST_DOCKER_STATE, JSON.stringify(state)); + } +} diff --git a/packages/setupWizard/tests/e2e/harness.mjs b/packages/setupWizard/tests/e2e/harness.mjs new file mode 100644 index 000000000..d3ab62459 --- /dev/null +++ b/packages/setupWizard/tests/e2e/harness.mjs @@ -0,0 +1,326 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:https'; +import { once } from 'node:events'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, chmodSync, existsSync, readdirSync, copyFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, delimiter, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { stripVTControlCharacters } from 'node:util'; +import { gunzipSync } from 'node:zlib'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { createRequire } from 'node:module'; +import pty from 'node-pty'; +import { eventSchemas, validateFields } from '../../dist/telemetryEvents.js'; +import { INSTALL_ID_PATTERN } from '../../dist/telemetry.js'; + +const require = createRequire(import.meta.url); +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const commandEnv = { ...process.env, PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false' }; +export const down = '\x1b[B'; +export const canary = 'canary-sensitive'; +export const defaultCompose = 'services:\n sourcebot:\n image: sourcebot-test\nvolumes:\n cache:\n'; +const commonKeys = ['platform', 'arch', 'nodeMajorVersion', 'packageManager', 'isCI', 'schemaVersion', 'source', 'setupSourcebotVersion', 'setupSessionId', 'install_id', 'elapsedMs', '$geoip_disable', '$ignore_sent_at', '$groups', '$lib', '$lib_version']; +const approvedSchema = JSON.parse(readFileSync(new URL('../approvedSchema.json', import.meta.url), 'utf8')); + +export function contract(events, requireTerminal = true) { + assert.ok(events.length > 0); + const id = events[0].distinct_id; + assert.match(id, INSTALL_ID_PATTERN); + let elapsed = -1; + let timestamp = -Infinity; + for (const event of events) { + const name = event.event.replace(/^setup_sourcebot_/, ''); + assert.ok(approvedSchema[name], `Unexpected event ${name}`); + assert.deepEqual(Object.keys(event).sort(), ['distinct_id', 'event', 'properties', 'timestamp', 'uuid'].sort()); + assert.deepEqual(Object.keys(event.properties).sort(), [...commonKeys, ...Object.keys(approvedSchema[name])].sort()); + validateFields(eventSchemas[name], event.properties); + assert.equal(event.distinct_id, id); + assert.equal(event.properties.setupSessionId, id); + assert.equal(event.properties.install_id, id); + assert.deepEqual(event.properties.$groups, { company: id }); + assert.equal(event.properties.$lib, 'posthog-node'); + assert.equal(event.properties.$lib_version, '5.52.1'); + assert.equal(event.properties.$geoip_disable, true); + assert.equal(event.properties.$ignore_sent_at, true); + assert.equal(event.properties.nodeMajorVersion, 24); + assert.equal(event.properties.source, 'setup-sourcebot-cli'); + assert.equal(event.properties.schemaVersion, 1); + assert.ok(event.properties.elapsedMs >= elapsed); + elapsed = event.properties.elapsedMs; + assert.ok(Number.isFinite(Date.parse(event.timestamp))); + assert.ok(Date.parse(event.timestamp) > timestamp, 'Event timestamps must preserve funnel order'); + timestamp = Date.parse(event.timestamp); + assert.match(event.uuid, /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + } + assert.equal(JSON.stringify(events).includes(canary), false, 'Sensitive canary reached telemetry'); + const terminal = events.filter(e => ['setup_sourcebot_completed', 'setup_sourcebot_cancelled'].includes(e.event) || (e.event === 'setup_sourcebot_failed' && !e.properties.recoverable)); + if (requireTerminal) { + assert.equal(terminal.length, 1); + } + return id; +} + +export function artifact() { + assert.equal(Number(process.versions.node.split('.')[0]), 24, 'Run packed-artifact tests using Node 24'); + // node-pty 1.1.0 ships its macOS helper without its executable bit in the npm tarball. + if (process.platform === 'darwin') { + const helper = join(dirname(require.resolve('node-pty/package.json')), 'prebuilds', `darwin-${process.arch}`, 'spawn-helper'); + if (existsSync(helper)) { + chmodSync(helper, 0o755); + } + } + const root = mkdtempSync(join(tmpdir(), 'sourcebot-e2e-')); + // Windows cannot exec .cmd shims directly. Invoke the bundled JS entry points + // without a shell so temporary paths containing spaces remain literal. + const packageCommand = (name, args, options) => { + if (process.platform !== 'win32') { + return execFileSync(name, args, options); + } + const script = join(dirname(process.execPath), 'node_modules', name === 'npm' ? 'npm/bin/npm-cli.js' : 'corepack/dist/yarn.js'); + assert.ok(existsSync(script), `Missing package-manager entry point: ${script}`); + return execFileSync(process.execPath, [script, ...args], options); + }; + try { + const tarball = process.env.SETUP_TEST_TARBALL || join(root, 'setup-sourcebot.tgz'); + if (!process.env.SETUP_TEST_TARBALL) { + packageCommand('yarn', ['workspace', '@sourcebot/schemas', 'build'], { cwd: packageRoot, env: commandEnv }); + packageCommand('yarn', ['build'], { cwd: packageRoot, env: commandEnv }); + packageCommand('yarn', ['pack', '--out', tarball], { cwd: packageRoot, env: commandEnv }); + } + const digest = createHash('sha256').update(readFileSync(tarball)).digest('hex'); + const installation = join(root, 'packed install with spaces'); + mkdirSync(installation); + writeFileSync(join(installation, 'package.json'), '{"name":"isolated-setup-test","private":true,"allowScripts":{"reo-census":true}}'); + packageCommand('npm', ['install', '--no-audit', '--no-fund', '--cache', join(root, 'npm-cache'), tarball], { cwd: installation, env: commandEnv, timeout: 120000 }); + const installed = join(installation, 'node_modules/setup-sourcebot'); + assert.deepEqual(readdirSync(installed).sort(), ['README.md', 'bin.cjs', 'dist', 'package.json'].sort()); + const cert = join(root, 'cert.pem'); + const key = join(root, 'key.pem'); + const openssl = process.platform === 'win32' + ? [join(process.env.ProgramFiles ?? 'C:\\Program Files', 'Git', 'usr', 'bin', 'openssl.exe'), join(process.env.ProgramFiles ?? 'C:\\Program Files', 'Git', 'mingw64', 'bin', 'openssl.exe')].find(existsSync) ?? 'openssl' + : 'openssl'; + execFileSync(openssl, ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-keyout', key, '-out', cert, '-subj', '/CN=setup-fixture', '-addext', 'subjectAltName=DNS:us.i.posthog.com,DNS:raw.githubusercontent.com,DNS:models.dev,DNS:api.github.com,DNS:gitlab.com,DNS:*.example.invalid'], { stdio: 'ignore' }); + return { root, installed, cert, key, digest, tarball, bin: join(installation, 'node_modules/.bin/setup-sourcebot'), cleanup() { rmSync(root, { recursive: true, force: true }); assert.equal(existsSync(root), false); } }; + } catch (error) { + rmSync(root, { recursive: true, force: true }); + throw error; + } +} + +export async function scenario(artifact, options, drive) { + const root = mkdtempSync(join(artifact.root, 'scenario-')); + const events = []; + const requests = []; + const forwarding = []; + const cwd = join(root, 'work'); + const setup = join(cwd, 'sourcebot'); + const fakeBin = join(root, 'bin'); + const home = join(root, 'home'); + for (const dir of [cwd, fakeBin, home]) { + mkdirSync(dir); + } + if (options.files) { + mkdirSync(setup); + for (const [name, data] of Object.entries(options.files)) { + writeFileSync(join(setup, name), data); + } + } + options.prepare?.({ cwd, setup, root }); + writeFileSync(join(root, 'docker-state.json'), JSON.stringify(options.docker ?? {})); + if (!options.dockerMissing) { + if (process.platform === 'win32') { + // A real executable is needed: Windows spawn(shell:false) cannot + // execute Unix shebangs or .cmd wrappers. The external preload + // dispatches this dedicated Node copy to the Docker fixture. + copyFileSync(process.execPath, join(fakeBin, 'docker.exe')); + } else { + writeFileSync(join(fakeBin, 'docker'), `#!${process.execPath}\n` + readFileSync(new URL('./fakeDocker.cjs', import.meta.url), 'utf8'), { mode: 0o755 }); + } + } + const server = createServer({ key: readFileSync(artifact.key), cert: readFileSync(artifact.cert) }, async (req, res) => { + if (req.headers.host === 'us.i.posthog.com') { + const parts = []; + for await (const chunk of req) { + parts.push(chunk); + } + const bytes = Buffer.concat(parts); + const body = JSON.parse((req.headers['content-encoding'] === 'gzip' ? gunzipSync(bytes) : bytes).toString()); + requests.push(body); + if (options.live) { + // Only a separately invoked dev smoke enables network forwarding. + contract(body.batch, false); + assert.equal(body.api_key, 'phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW'); + assert.ok(process.env.SETUP_TEST_DEV_TOKEN, 'Dev project token required'); + const forwarded = { ...body, api_key: process.env.SETUP_TEST_DEV_TOKEN }; + assert.deepEqual({ ...forwarded, api_key: body.api_key }, body); + const request = fetch(`https://us.i.posthog.com${req.url}`, { + method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(forwarded), signal: AbortSignal.timeout(15000), + }).then(async response => { + assert.ok(response.ok, `Dev ingestion rejected: ${response.status}`); + await response.text(); + }); + forwarding.push(request); + request.catch(() => {}); + } + if (options.telemetry === 'stall') { + return; + } + if (options.telemetry === 'reset') { + req.socket.destroy(); + return; + } + if (options.telemetry === 'reject') { + res.writeHead(503).end('{}'); + return; + } + events.push(...body.batch); + res.writeHead(200, { 'content-type': 'application/json' }).end('{"status":1}'); + } else if (req.headers.host === 'raw.githubusercontent.com') { + if (options.composeStatus === 'stall') { + return; + } + res.writeHead(options.composeStatus ?? 200).end(options.compose ?? defaultCompose); + } else if (req.headers.host === 'models.dev') { + if (options.catalog === 'stall') { + return; + } + res.writeHead(options.catalogStatus ?? 200).end(JSON.stringify(options.catalog ?? {})); + } else if (req.headers.host === 'api.github.com') { + if (options.searchStatus === 'stall') { + return; + } + if (options.searchStatus === 'reset') { + req.socket.destroy(); + return; + } + res.writeHead(options.searchStatus ?? 200).end(JSON.stringify(options.search ?? { items: [] })); + } else if (req.headers.host === 'gitlab.com' || req.headers.host?.endsWith('.example.invalid')) { + res.writeHead(options.searchStatus ?? 200).end(JSON.stringify(options.search ?? [])); + } else { + res.writeHead(403).end('Unexpected test egress denied'); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + let child; + let transcript = ''; + let ended; + let cursor = 0; + const env = { + PATH: `${fakeBin}${delimiter}${dirname(process.execPath)}`, + HOME: home, USERPROFILE: home, TMPDIR: root, TEMP: root, TMP: root, + XDG_STATE_HOME: join(home, 'state'), XDG_CONFIG_HOME: join(home, 'config'), XDG_CACHE_HOME: join(home, 'cache'), + SYSTEMROOT: process.env.SYSTEMROOT ?? '', LANG: 'en_US.UTF-8', TERM: 'xterm-256color', + NODE_OPTIONS: `--import=${new URL('./network.mjs', import.meta.url).href}`, + NODE_EXTRA_CA_CERTS: artifact.cert, TEST_CAPTURE_PORT: String(server.address().port), + TEST_DOCKER_STATE: join(root, 'docker-state.json'), TEST_DOCKER_LOG: join(root, 'docker.log'), + TEST_DOCKER_PIDS: join(root, 'docker-pids'), + PACKAGE_TRACKER_ANALYTICS: 'false', SOURCEBOT_TELEMETRY_DISABLED: 'true', + ...options.environment, + }; + try { + child = options.launcher + ? pty.spawn(options.launcher.file, options.launcher.args, { name: 'xterm-256color', cols: 180, rows: 60, cwd, env }) + : process.platform === 'win32' + ? pty.spawn(process.execPath, [join(artifact.installed, 'bin.cjs')], { name: 'xterm-256color', cols: 180, rows: 60, cwd, env }) + : pty.spawn(artifact.bin, [], { name: 'xterm-256color', cols: 180, rows: 60, cwd, env }); + child.onData(data => { transcript += stripVTControlCharacters(data); }); + child.onExit(result => { ended = result; }); + const wait = async text => { + const started = Date.now(); + while (!transcript.slice(cursor).includes(text)) { + assert.equal(ended, undefined, `CLI exited before ${text}: ${transcript.slice(-1500)}`); + assert.ok(Date.now() - started < 12000, `Prompt timeout ${text}: ${transcript.slice(-1500)}`); + await sleep(10); + } + cursor = transcript.indexOf(text, cursor) + text.length; + await sleep(25); + }; + const answer = async (text, value = '') => { await wait(text); child.write(value + '\r'); }; + const select = async (text, index = 0) => { + await wait(text); + for (let n = 0; n < index; n++) { + child.write(down); + await sleep(15); + } + child.write('\r'); + }; + const multi = async (text, value = `${canary}/repository`) => { + await wait(text); + // select-pro initializes its asynchronous option loader after a debounce. + await sleep(300); + child.write(value); + await sleep(550); + child.write('\t'); + await sleep(50); + child.write('\r'); + }; + const check = async (text, indexes = [0]) => { + await wait(text); + let position = 0; + for (const index of indexes) { + while (position < index) { + child.write(down); + position++; + await sleep(15); + } + child.write('\t'); + await sleep(20); + } + child.write('\r'); + }; + const finish = async (code = 0) => { + const start = Date.now(); + while (!ended) { + assert.ok(Date.now() - start < 7000, `CLI did not exit: ${transcript.slice(-1000)}`); + await sleep(20); + } + assert.equal(ended.exitCode, code, transcript.slice(-1000)); + }; + await drive({ answer, select, multi, check, wait, write: data => child.write(data), interrupt: () => process.kill(child.pid, 'SIGINT'), finish, events, setup, root, cwd, home, get transcript() { return transcript; } }); + if (!ended) { + await finish(); + } + await Promise.all(forwarding); + if (!options.telemetry && !options.baseline) { + contract(events); + } + assert.equal(JSON.stringify(requests).includes(canary), false); + if (options.assertLauncherHome) { + options.assertLauncherHome(readdirSync(home, { recursive: true })); + } else { + assert.deepEqual(readdirSync(home), [], 'Wizard wrote unexpected per-user state'); + } + const files = existsSync(setup) ? Object.fromEntries(readdirSync(setup).filter(f => !options.ignoreFiles?.includes(f)).map(f => [f, readFileSync(join(setup, f), 'utf8')])) : {}; + const dockerCalls = existsSync(join(root, 'docker.log')) ? readFileSync(join(root, 'docker.log'), 'utf8').trim().split('\n').map(JSON.parse) : []; + if (existsSync(join(root, 'docker-pids'))) { + for (const pid of readFileSync(join(root, 'docker-pids'), 'utf8').trim().split('\n').map(Number)) { + assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }, 'Owned Docker fixture process survived CLI exit'); + } + } + return { events, files, dockerCalls, requests, exitCode: ended.exitCode }; + } finally { + if (child && !ended) { + child.kill('SIGKILL'); + } + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + rmSync(root, { recursive: true, force: true }); + assert.equal(existsSync(root), false); + } +} + +export async function minimal(driver, { existing = false, ai = false, pauseBeforeHosted = false } = {}) { + await driver.answer('What directory would you like'); + if (existing) { + await driver.answer('Do you want to overwrite it?', 'y'); + } + await driver.select('Which code host', 3); + await driver.answer('Git clone URL', `https://${canary}.example.invalid/repository`); + await driver.answer('Add another code host?', 'n'); + await driver.answer('Would you like to configure AI features?', ai ? 'y' : 'n'); + if (!ai && !pauseBeforeHosted) { + await driver.answer('What URL will Sourcebot be hosted at?'); + } +} diff --git a/packages/setupWizard/tests/e2e/linux.mjs b/packages/setupWizard/tests/e2e/linux.mjs new file mode 100644 index 000000000..40b2546f3 --- /dev/null +++ b/packages/setupWizard/tests/e2e/linux.mjs @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; +import { cpSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { randomUUID } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { artifact } from './harness.mjs'; + +const packed = artifact(); +const name = `sourcebot-linux-e2e-${randomUUID()}`; +const packageRoot = resolve(fileURLToPath(new URL('.', import.meta.url)), '../..'); +try { + const snapshot = join(packed.root, 'linux-package'); + mkdirSync(snapshot); + // Never mount the changing worktree into a running verification job. + for (const path of ['dist', 'tests', 'package.json']) { + cpSync(join(packageRoot, path), join(snapshot, path), { recursive: true }); + } + execFileSync('docker', ['run', '--rm', '--name', name, '-e', 'PACKAGE_TRACKER_ANALYTICS=false', '-e', 'SETUP_TEST_TARBALL=/fixture/setup-sourcebot.tgz', + '-v', `${snapshot}:/work/packages/setupWizard:ro`, '-v', `${packed.tarball}:/fixture/setup-sourcebot.tgz:ro`, '-w', '/work', 'node:24-bookworm', 'sh', '-c', + 'npm install --no-audit --no-fund posthog-node@5.52.1 node-pty@1.1.0 undici@7.29.1 && node --test --test-concurrency=1 packages/setupWizard/tests/e2e/wizard.test.mjs packages/setupWizard/tests/e2e/collectors.test.mjs packages/setupWizard/tests/e2e/docker.test.mjs packages/setupWizard/tests/e2e/safety.test.mjs && node packages/setupWizard/tests/e2e/packageManagers.mjs'], { stdio: 'inherit', timeout: 600000 }); + console.log(`Linux packed-artifact matrix passed: ${packed.digest}`); +} finally { + try { execFileSync('docker', ['rm', '-f', name], { stdio: 'ignore' }); } catch { /* --rm already handled completion. */ } + assert.equal(execFileSync('docker', ['ps', '-aq', '--filter', `name=^/${name}$`], { encoding: 'utf8' }).trim(), ''); + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/liveSmoke.mjs b/packages/setupWizard/tests/e2e/liveSmoke.mjs new file mode 100644 index 000000000..1e27759f3 --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveSmoke.mjs @@ -0,0 +1,14 @@ +import assert from 'node:assert/strict'; +import { artifact, scenario, minimal } from './harness.mjs'; +assert.ok(process.env.SETUP_TEST_DEV_TOKEN, 'Provide the designated dev project token; never use the production project.'); +assert.notEqual(process.env.SETUP_TEST_DEV_TOKEN, 'phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW'); +const packed = artifact(); +try { + const result = await scenario(packed, { live: true }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'n'); + }); + console.log(JSON.stringify({ artifactSha256: packed.digest, distinctId: result.events[0].distinct_id, eventsForwarded: result.events.length, ingestion: 'accepted; query verification still required' })); +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/network.mjs b/packages/setupWizard/tests/e2e/network.mjs new file mode 100644 index 000000000..778c87e90 --- /dev/null +++ b/packages/setupWizard/tests/e2e/network.mjs @@ -0,0 +1,22 @@ +// Test-only network boundary. This file is outside the installed tarball. It changes +// TCP routing, not the SDK, URL, headers, event payload, or production configuration. +import tls from 'node:tls'; +import { Agent, setGlobalDispatcher } from 'undici'; +import { basename } from 'node:path'; + +if (process.platform === 'win32' && basename(process.execPath).toLowerCase() === 'docker.exe') { + process.argv.splice(1, 0, 'fakeDocker.cjs'); + await import('./fakeDocker.cjs'); + // The fixture owns its exit/timers; never execute "compose" as a JS file. + await new Promise(() => {}); +} + +setGlobalDispatcher(new Agent({ connect(options, callback) { + if (options.protocol !== 'https:') { + callback(new Error('Non-HTTPS test egress denied'), null); + return; + } + const socket = tls.connect({ host: '127.0.0.1', port: Number(process.env.TEST_CAPTURE_PORT), servername: options.hostname }); + socket.once('secureConnect', () => callback(null, socket)); + socket.once('error', error => callback(error, null)); +} })); diff --git a/packages/setupWizard/tests/e2e/packageManagers.mjs b/packages/setupWizard/tests/e2e/packageManagers.mjs new file mode 100644 index 000000000..3d36b0b30 --- /dev/null +++ b/packages/setupWizard/tests/e2e/packageManagers.mjs @@ -0,0 +1,57 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, writeFileSync, readFileSync, symlinkSync } from 'node:fs'; +import { join, dirname, delimiter } from 'node:path'; +import { artifact, scenario, minimal } from './harness.mjs'; + +// Install launchers into this test's disposable root, never globally. Their +// executable scripts and the wizard are both run with the required Node 24. +const packed = artifact(); +try { + const managerRoot = join(packed.root, 'managers'); + mkdirSync(managerRoot); + writeFileSync(join(managerRoot, 'package.json'), JSON.stringify({ private: true, allowScripts: { bun: true, pnpm: true } })); + execFileSync('npm', ['install', '--no-audit', '--no-fund', '--cache', join(packed.root, 'manager-cache'), 'npm@12.0.2', '@yarnpkg/cli-dist@4.7.0', 'pnpm@12.4.1', 'bun@1.4.2'], { + cwd: managerRoot, env: { ...process.env, PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false' }, timeout: 120000, + }); + for (const [name, packageName, prefix] of [ + ['npm', 'npm', ['exec', '--offline', '--']], + ['yarn', '@yarnpkg/cli-dist', ['exec']], + ['pnpm', 'pnpm', ['exec']], + ['bun', 'bun', ['x', '--no-install']], + ]) { + const directory = join(managerRoot, 'node_modules', packageName); + const manifest = JSON.parse(readFileSync(join(directory, 'package.json'), 'utf8')); + const binary = join(directory, typeof manifest.bin === 'string' ? manifest.bin : manifest.bin[name]); + const managerEnv = { npm_config_cache: join(packed.root, 'invocation-cache'), npm_config_script_shell: '/bin/sh', YARN_ENABLE_TELEMETRY: '0', YARN_GLOBAL_FOLDER: join(packed.root, 'yarn-global'), YARN_CACHE_FOLDER: join(packed.root, 'yarn-cache') }; + const result = await scenario(packed, { + launcher: { file: binary, args: [...prefix, name === 'yarn' ? JSON.stringify(packed.bin) : name === 'bun' ? 'setup-sourcebot' : packed.bin] }, + environment: managerEnv, + assertLauncherHome(paths) { + // Package managers may write their own state before launching + // the wizard. Direct-binary suites still require an empty home. + const permitted = /^(Library(?:\/(?:Caches\/)?pnpm(?:\/.*)?)?|state(?:\/pnpm(?:\/.*)?)?|cache(?:\/pnpm(?:\/.*)?)?|\.local(?:\/share(?:\/pnpm(?:\/.*)?)?)?)$/; + const unexpected = paths.filter(path => name !== 'pnpm' || !permitted.test(path)); + assert.deepEqual(unexpected, [], 'Unexpected launcher state outside the known pnpm store/state directories'); + }, + prepare({ cwd }) { + writeFileSync(join(cwd, 'package.json'), JSON.stringify({ private: true, name: 'isolated-launcher', ...(name === 'yarn' ? { packageManager: 'yarn@4.7.0' } : {}) })); + writeFileSync(join(cwd, 'yarn.lock'), ''); + writeFileSync(join(cwd, '.yarnrc.yml'), 'nodeLinker: node-modules\n'); + if (name === 'yarn') { + execFileSync(process.execPath, [binary, 'install', '--mode=skip-build'], { cwd, env: { ...process.env, ...managerEnv }, timeout: 30000 }); + } + if (name === 'bun') { + symlinkSync(dirname(packed.installed), join(cwd, 'node_modules'), 'junction'); + } + }, + }, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + const started = result.events[0].properties; + assert.equal(started.packageManager, name); + assert.ok(['npx', 'workspace', 'local_binary', 'unknown'].includes(started.invocationMethod)); + console.log(`${name}@${manifest.version}: packed binary completed; packageManager=${started.packageManager}; invocationMethod=${started.invocationMethod}`); + } + console.log(`Package-manager launcher matrix passed: ${packed.digest}`); +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/platform.test.mjs b/packages/setupWizard/tests/e2e/platform.test.mjs new file mode 100644 index 000000000..cab90a2e5 --- /dev/null +++ b/packages/setupWizard/tests/e2e/platform.test.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { artifact, scenario, minimal, canary } from './harness.mjs'; +let packed; +before(() => { packed = artifact(); }); +after(() => packed?.cleanup()); + +test('platform: published bootstrap and minimal happy path', async () => { + const result = await scenario(packed, {}, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + assert.equal(result.events[0].properties.platform, process.platform); + assert.equal(result.events[0].properties.arch, process.arch); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_completed'); +}); +test('platform: real terminal Ctrl+C exits with cancellation', async () => { + const result = await scenario(packed, {}, async d => { + await d.wait('What directory would you like'); + d.write('\x03'); + await d.finish(130); + }); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_cancelled'); +}); +test('platform: local repository in a path with spaces and Unicode', async () => { + const result = await scenario(packed, { prepare({ cwd }) { mkdirSync(join(cwd, `${canary} space-é`, '.git'), { recursive: true }); } }, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 2); + await d.answer('Path to your repos directory', join(d.cwd, `${canary} space-é`)); + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.ok(result.files['docker-compose.override.yml']); + assert.equal(result.events.find(e => e.event === 'setup_sourcebot_configured_code_source').properties.codeHost, 'local_git'); +}); + +test('platform: foreground Docker spawn, completion and Ctrl+C cleanup', async () => { + const result = await scenario(packed, { docker: { stall: ['compose up'], stubborn: true } }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + const started = Date.now(); + while (!d.events.some(event => event.event === 'setup_sourcebot_completed')) { + assert.ok(Date.now() - started < 3000); + await new Promise(resolve => setTimeout(resolve, 20)); + } + d.write('\x03'); + await d.finish(130); + }); + assert.equal(result.events.filter(event => event.event === 'setup_sourcebot_completed').length, 1); + assert.equal(result.events.some(event => event.event === 'setup_sourcebot_cancelled'), false); +}); diff --git a/packages/setupWizard/tests/e2e/runtime.test.mjs b/packages/setupWizard/tests/e2e/runtime.test.mjs new file mode 100644 index 000000000..1b3e72249 --- /dev/null +++ b/packages/setupWizard/tests/e2e/runtime.test.mjs @@ -0,0 +1,92 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { randomUUID } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { writeFileSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { artifact, scenario, minimal } from './harness.mjs'; +import { INSTALL_ID_PATTERN } from '../../dist/telemetry.js'; + +test('real Sourcebot containers: packed wizard identity survives first boot, restart, upgrade and opt-out', async () => { + const packed = artifact(); + const image = process.env.SETUP_TEST_SOURCEBOT_IMAGE ?? 'docker.sourcebot.dev/sourcebot-dev/sourcebot:latest'; + const label = `setup-wizard-e2e-${randomUUID()}`; + const fixtures = fileURLToPath(new URL('.', import.meta.url)); + const entrypoint = resolve(fixtures, '../../../../entrypoint.sh'); + const containerNames = []; + const live = process.env.SETUP_TEST_LIVE === 'true'; + if (live) { + assert.equal(process.env.SETUP_TEST_DEV_TOKEN, 'phc_EJR6BsaBbvIKhM4t4zp1boYC92Tpp5Fgb9Csa9Us5aw', 'Live runtime verification is restricted to the approved dev project'); + } + try { + const setup = await scenario(packed, { live }, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + const setupId = setup.events[0].distinct_id; + const envFile = join(packed.root, 'deployment.env'); + writeFileSync(envFile, setup.files['.env']); + const versionFile = join(packed.root, 'version.ts'); + const boot = (data, version, options = {}) => { + mkdirSync(join(packed.root, data), { recursive: true }); + writeFileSync(versionFile, `export const SOURCEBOT_VERSION = "${version}";\n`); + const name = `${label}-${containerNames.length}`; + containerNames.push(name); + const output = execFileSync('docker', ['run', '--rm', '--name', name, '--label', label, '--network', 'none', '--add-host', 'us.i.posthog.com:127.0.0.1', + '--env-file', envFile, '-e', `SOURCEBOT_TELEMETRY_DISABLED=${options.disabled ? 'true' : 'false'}`, '-e', 'POSTHOG_PAPIK=phc_runtime_fixture', + ...(options.id !== undefined ? ['-e', `SOURCEBOT_INSTALL_ID=${options.id}`] : []), + ...(options.omit ? ['-e', 'TEST_OMIT_INSTALL_ID=true'] : []), + '-v', `${join(packed.root, data)}:/data`, '-v', `${join(fixtures, 'runtimeFixture.mjs')}:/fixture/runtimeFixture.mjs:ro`, + '-v', `${entrypoint}:/fixture/entrypoint.sh:ro`, '-v', `${packed.cert}:/fixture/cert.pem:ro`, '-v', `${packed.key}:/fixture/key.pem:ro`, + '-v', `${versionFile}:/app/packages/shared/src/version.ts:ro`, '--entrypoint', 'node', image, '/fixture/runtimeFixture.mjs'], { timeout: 45000, encoding: 'utf8' }); + return JSON.parse(output.trim()); + }; + const first = boot('supplied', 'v5.1.3'); + assert.equal(first.installId, setupId); + assert.equal(first.persisted.install_id, setupId); + assert.deepEqual(first.events.map(e => [e.event, e.distinct_id]), [['install', setupId]]); + const restart = boot('supplied', 'v5.1.3', { id: randomUUID() }); + assert.equal(restart.installId, setupId); + assert.deepEqual(restart.events, []); + const upgrade = boot('supplied', 'v5.1.4', { id: randomUUID() }); + assert.equal(upgrade.installId, setupId); + assert.deepEqual(upgrade.events.map(e => [e.event, e.distinct_id]), [['upgrade', setupId]]); + const omittedRestart = boot('supplied', 'v5.1.4', { omit: true }); + assert.equal(omittedRestart.installId, setupId); + assert.deepEqual(omittedRestart.events, []); + const generated = boot('generated', 'v5.1.3', { omit: true }); + assert.match(generated.installId, INSTALL_ID_PATTERN); + assert.notEqual(generated.installId, setupId); + assert.equal(generated.events[0].distinct_id, generated.installId); + const generatedRestart = boot('generated', 'v5.1.3', { omit: true }); + assert.equal(generatedRestart.installId, generated.installId); + const disabled = boot('disabled', 'v5.1.3', { disabled: true }); + assert.equal(disabled.installId, setupId); + assert.deepEqual(disabled.events, []); + const disabledRestart = boot('disabled', 'v5.1.4', { disabled: true, id: randomUUID() }); + assert.equal(disabledRestart.installId, setupId); + assert.deepEqual(disabledRestart.events, []); + if (live) { + // Forward only the two already-validated synthetic payloads captured + // from actual curl. The container itself never has external egress. + for (const payload of [first.events[0], upgrade.events[0]]) { + assert.equal(payload.distinct_id, setupId); + assert.equal(JSON.stringify(payload).includes('canary-sensitive'), false); + const forwarded = { ...payload, api_key: process.env.SETUP_TEST_DEV_TOKEN }; + assert.deepEqual({ ...forwarded, api_key: payload.api_key }, payload); + const response = await fetch('https://us.i.posthog.com/capture/', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify(forwarded), signal: AbortSignal.timeout(15000), + }); + assert.ok(response.ok, `Dev runtime ingestion rejected: ${response.status}`); + await response.text(); + } + console.log(JSON.stringify({ artifactSha256: packed.digest, distinctId: setupId, eventsForwarded: setup.events.length + 2, ingestion: 'accepted; query verification still required' })); + } + console.log(`Identity continuity verified across 8 containers; artifact SHA-256 ${packed.digest}`); + } finally { + for (const name of containerNames) { + try { execFileSync('docker', ['rm', '-f', name], { stdio: 'ignore' }); } catch { /* --rm already cleaned completed containers. */ } + } + assert.equal(execFileSync('docker', ['ps', '-aq', '--filter', `label=${label}`], { encoding: 'utf8' }).trim(), ''); + packed.cleanup(); + } +}); diff --git a/packages/setupWizard/tests/e2e/runtimeFixture.mjs b/packages/setupWizard/tests/e2e/runtimeFixture.mjs new file mode 100644 index 000000000..633458ae7 --- /dev/null +++ b/packages/setupWizard/tests/e2e/runtimeFixture.mjs @@ -0,0 +1,37 @@ +// Executed inside a disposable Sourcebot image. The repository entrypoint, real +// curl, jq and uuidgen are unchanged; only DB migration/supervisor are replaced. +import { createServer } from 'node:https'; +import { once } from 'node:events'; +import { spawn } from 'node:child_process'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +const root = mkdtempSync('/tmp/entrypoint-fixture-'); +writeFileSync(`${root}/yarn`, '#!/bin/sh\nexit 0\n', { mode: 0o755 }); +writeFileSync(`${root}/supervisord`, '#!/bin/sh\nprintf "%s" "$SOURCEBOT_INSTALL_ID" > "$TEST_ID_RESULT"\n', { mode: 0o755 }); +const events = []; +const server = createServer({ key: readFileSync('/fixture/key.pem'), cert: readFileSync('/fixture/cert.pem') }, async (req, res) => { + const parts = []; + for await (const part of req) { + parts.push(part); + } + events.push(JSON.parse(Buffer.concat(parts).toString())); + res.writeHead(200).end('{"status":1}'); +}).listen(443, '127.0.0.1'); +await once(server, 'listening'); +try { + const env = { ...process.env, PATH: `${root}:${process.env.PATH}`, CONFIG_PATH: '', DATA_CACHE_DIR: '/data', DATABASE_URL: 'postgresql://fixture', REDIS_URL: 'redis://fixture', CURL_CA_BUNDLE: '/fixture/cert.pem', TEST_ID_RESULT: `${root}/id` }; + if (env.TEST_OMIT_INSTALL_ID === 'true') { + delete env.SOURCEBOT_INSTALL_ID; + } + const child = spawn('/bin/sh', ['/fixture/entrypoint.sh'], { env, stdio: ['ignore', 'pipe', 'pipe'] }); + let diagnostics = ''; + child.stdout.on('data', b => { diagnostics += b; }); + child.stderr.on('data', b => { diagnostics += b; }); + const [code] = await once(child, 'close'); + if (code !== 0) { + throw Error(`Entrypoint failed (${code}): ${diagnostics}`); + } + console.log(JSON.stringify({ events, installId: readFileSync(`${root}/id`, 'utf8'), persisted: JSON.parse(readFileSync('/data/.installedv3', 'utf8')) })); +} finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); +} diff --git a/packages/setupWizard/tests/e2e/safety.test.mjs b/packages/setupWizard/tests/e2e/safety.test.mjs new file mode 100644 index 000000000..4a9e39b31 --- /dev/null +++ b/packages/setupWizard/tests/e2e/safety.test.mjs @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { artifact, scenario, minimal, canary } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Safety artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const prop = (r, n) => r.events.find(e => e.event === `setup_sourcebot_${n}`)?.properties; + +for (const stage of ['directory', 'config', 'env']) { + test(`explicit decline: ${stage}`, async () => { + const files = stage === 'directory' ? {} : stage === 'config' ? { 'config.json': 'original' } : { '.env': 'original' }; + const result = await scenario(packed, { files }, async d => { + if (stage === 'directory') { + await d.answer('What directory would you like'); + await d.answer('Do you want to overwrite it?', 'n'); + } else { + await minimal(d, { existing: true }); + await d.answer(stage === 'config' ? 'config.json already exists. Overwrite?' : '.env already exists. Overwrite?', 'n'); + } + }); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_cancelled'); + assert.deepEqual(result.files, files); + assert.equal(prop(result, 'cancelled').reason, stage === 'directory' ? 'existing_directory_declined' : 'config_overwrite_declined'); + }); +} + +test('fatal directory creation failure: failed without checkpoint or files', async () => { + const result = await scenario(packed, { prepare({ cwd }) { writeFileSync(join(cwd, 'blocked'), 'fixture'); } }, async d => { + await d.answer('What directory would you like', 'blocked/child'); + await d.finish(1); + }); + assert.deepEqual(result.events.map(e => e.event), ['setup_sourcebot_started', 'setup_sourcebot_failed']); + assert.equal(prop(result, 'failed').failureCategory, 'filesystem'); + assert.equal(prop(result, 'failed').recoverable, false); +}); + +test('fatal config write failure does not emit generated_configs', async () => { + const result = await scenario(packed, { files: {}, ignoreFiles: ['config.json'], prepare({ setup }) { mkdirSync(join(setup, 'config.json')); } }, async d => { + await minimal(d, { existing: true }); + await d.answer('config.json already exists. Overwrite?', 'y'); + await d.finish(1); + }); + assert.equal(prop(result, 'generated_configs'), undefined); + assert.equal(prop(result, 'failed').failureCategory, 'filesystem'); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_failed'); +}); + +test('multiple code-source loop emits sequential indexes and exact aggregate', async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + for (let i = 0; i < 3; i++) { + await d.select('Which code host', 3); + await d.answer('Git clone URL', `https://${canary}.example.invalid/repo-${i}`); + await d.answer('Add another code host?', i === 2 ? 'n' : 'y'); + } + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?', `https://${canary}.example.invalid/path?secret=yes`); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.deepEqual(result.events.filter(e => e.event === 'setup_sourcebot_configured_code_source').map(e => e.properties.configurationIndex), [1, 2, 3]); + assert.equal(prop(result, 'configured_code_sources').repositoryCount, 3); + assert.equal(prop(result, 'configured_code_sources').uniqueCodeHostCount, 1); + assert.equal(prop(result, 'configured_hosted_url').hostCategory, 'address'); +}); + +for (const status of [401, 403, 500, 'reset', 'malformed']) { + test(`GitHub autocomplete fallback: ${status}`, async () => { + const result = await scenario(packed, { searchStatus: status === 'malformed' ? 200 : status, search: status === 'malformed' ? { items: [null] } : undefined }, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host'); + await d.answer('GitHub URL'); + await d.answer('GitHub Personal Access Token', `${canary}-token`); + await d.check('What do you want to index?'); + await d.multi('Repositories to index'); + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.equal(prop(result, 'configured_code_source').repositoryCount, 1); + assert.equal(prop(result, 'failed').failureCategory, 'network'); + assert.equal(prop(result, 'failed').recoverable, true); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_completed'); + }); +} + +for (const mode of ['catalog', 'custom', 'malformed', 'http_failure']) { + test(`model catalog ${mode}, repeated-provider credential reuse`, async () => { + const catalog = { anthropic: { models: { fixture: { id: `${canary}-model`, name: `${canary}-name` } } } }; + if (mode === 'malformed') { + catalog.anthropic.models.fixture = null; + } + const result = await scenario(packed, { catalog, catalogStatus: mode === 'http_failure' ? 503 : 200 }, async d => { + await minimal(d, { ai: true }); + for (let i = 0; i < 2; i++) { + await d.select('Which AI provider?'); + if (mode === 'catalog' || mode === 'custom') { + await d.wait('Model name'); + await sleep(350); + if (mode === 'custom') { + d.write(`${canary}-custom`); + await sleep(350); + } + d.write('\r'); + } else { + await d.answer('Model name', `${canary}-fallback`); + } + if (!i) { + await d.answer('API key (', `${canary}-key`); + } + await d.answer('Display name'); + await d.answer('Add another model?', i ? 'n' : 'y'); + } + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.equal(prop(result, 'ai_setup_completed').aiConfigurationCount, 2); + assert.equal(prop(result, 'ai_setup_completed').uniqueProviderCount, 1); + assert.equal(prop(result, 'generated_configs').credentialVariableCount, 1); + assert.equal(prop(result, 'configured_ai_provider').modelSelectionMethod, mode === 'catalog' ? 'catalog' : mode === 'custom' ? 'custom_entry' : 'manual_fallback'); + }); +} + +for (const stage of ['code_sources', 'ai_setup', 'hosted_url', 'compose_file']) { + test(`Ctrl+C at stage ${stage}`, async () => { + const result = await scenario(packed, {}, async d => { + if (stage === 'code_sources') { + await d.answer('What directory would you like'); + await d.wait('Which code host'); + } else if (stage === 'ai_setup') { + await minimal(d, { ai: true }); + await d.wait('Which AI provider?'); + } else if (stage === 'hosted_url') { + await minimal(d, { pauseBeforeHosted: true }); + await d.wait('What URL will Sourcebot be hosted at?'); + } else { + await minimal(d); + await d.wait('Download docker-compose.yml?'); + } + d.write('\x03'); + await d.finish(130); + }); + assert.equal(prop(result, 'cancelled').stage, stage); + }); +} diff --git a/packages/setupWizard/tests/e2e/unsupportedNode.mjs b/packages/setupWizard/tests/e2e/unsupportedNode.mjs new file mode 100644 index 000000000..fe81593cc --- /dev/null +++ b/packages/setupWizard/tests/e2e/unsupportedNode.mjs @@ -0,0 +1,19 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdirSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { artifact } from './harness.mjs'; +const packed = artifact(); +try { + const node22 = process.env.SETUP_TEST_NODE22 ?? execFileSync('npm', ['exec', '--yes', '--package=node@22.22.0', '--', 'node', '-p', 'process.execPath'], { encoding: 'utf8' }).trim(); + const work = join(packed.root, 'node22'); + mkdirSync(work); + const result = spawnSync(node22, [join(packed.installed, 'bin.cjs')], { cwd: work, encoding: 'utf8', env: { PATH: '', HOME: work } }); + assert.equal(result.status, 1); + assert.match(result.stderr, /requires Node.js 24 or newer/); + assert.equal(result.stdout, ''); + assert.deepEqual(readdirSync(work), []); + console.log(`Node 22.22 rejected before loading the wizard; artifact ${packed.digest}`); +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/wizard.test.mjs b/packages/setupWizard/tests/e2e/wizard.test.mjs new file mode 100644 index 000000000..d53aa09c6 --- /dev/null +++ b/packages/setupWizard/tests/e2e/wizard.test.mjs @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { randomUUID } from 'node:crypto'; +import { artifact, scenario, minimal, defaultCompose } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Packed artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const names = result => result.events.map(e => e.event.replace('setup_sourcebot_', '')); +const props = (result, name) => result.events.find(e => e.event === `setup_sourcebot_${name}`)?.properties; +const funnel = ['started', 'chose_setup_directory', 'configured_code_source', 'configured_code_sources', 'ai_setup_completed', 'configured_hosted_url', 'generated_configs', 'resolved_compose_file', 'validated_docker_state', 'completed']; + +test('packed CLI: full manual funnel, exact identity and only Sourcebot files persisted', async () => { + const result = await scenario(packed, {}, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + assert.deepEqual(names(result), funnel); + assert.deepEqual(Object.keys(result.files).sort(), ['.env', 'config.json']); + assert.equal(JSON.parse(result.files['config.json']).connections.git.type, 'git'); + assert.equal(result.files['.env'].match(/^SOURCEBOT_INSTALL_ID=(.+)$/m)[1], result.events[0].distinct_id); + assert.equal(props(result, 'validated_docker_state').runningComposeContainerCount, null); + assert.equal(props(result, 'completed').sourcebotStartOutcome, 'not_offered'); + assert.equal(result.dockerCalls.length, 0); +}); + +for (const compose of ['download', 'existing', 'declined', 404, 500]) { + test(`packed compose resolution: ${compose}`, async () => { + const existing = compose === 'existing'; + const result = await scenario(packed, { files: existing ? { 'docker-compose.yml': defaultCompose } : undefined, composeStatus: typeof compose === 'number' ? compose : undefined }, async d => { + await minimal(d, { existing }); + if (!existing) { + await d.answer('Download docker-compose.yml?', compose === 'declined' ? 'n' : 'y'); + } + if (existing || compose === 'download') { + await d.answer('Start Sourcebot now?', 'n'); + } + }); + const expected = existing ? 'already_present' : compose === 'download' ? 'downloaded' : compose === 'declined' ? 'declined' : 'download_failed'; + assert.equal(props(result, 'resolved_compose_file').outcome, expected); + assert.equal(props(result, 'completed').sourcebotStartOutcome, existing || compose === 'download' ? 'declined' : 'not_offered'); + assert.equal(names(result).filter(n => n === 'failed').length, typeof compose === 'number' ? 1 : 0); + }); +} + +for (const identity of ['valid', 'invalid', 'absent']) { + test(`existing env identity: ${identity}`, async () => { + const old = randomUUID(); + const result = await scenario(packed, { files: { '.env': identity === 'absent' ? '' : `SOURCEBOT_INSTALL_ID=${identity === 'valid' ? old : 'invalid'}\n`, 'config.json': '{}' } }, async d => { + await minimal(d, { existing: true }); + await d.answer('config.json already exists. Overwrite?', 'y'); + await d.answer('.env already exists. Overwrite?', 'y'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + const id = result.files['.env'].match(/^SOURCEBOT_INSTALL_ID=(.+)$/m)[1]; + assert.equal(id, identity === 'valid' ? old : result.events[0].distinct_id); + assert.equal(JSON.stringify(result.events).includes(old), false); + assert.equal(props(result, 'generated_configs').deploymentIdentityAction, identity === 'valid' ? 'preserved_existing' : 'created_from_setup_session'); + }); +} + +for (const parentSignal of [false, true]) { + test(`prompt cancellation ${parentSignal ? 'parent SIGINT' : 'PTY Ctrl+C'}`, async () => { + const result = await scenario(packed, {}, async d => { + await d.wait('What directory would you like'); + const began = Date.now(); + if (parentSignal) { + d.interrupt(); + } else { + d.write('\x03'); + } + await d.finish(130); + assert.ok(Date.now() - began < 3500); + }); + assert.deepEqual(names(result), ['started', 'cancelled']); + assert.deepEqual(result.files, {}); + }); +} + +test('completion is committed at spawn; Ctrl+C still exits without a cancellation event', async () => { + const result = await scenario(packed, { docker: { stall: ['compose up'], stubborn: true } }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + const began = Date.now(); + while (!d.events.some(e => e.event === 'setup_sourcebot_completed')) { + assert.ok(Date.now() - began < 3000); + await new Promise(resolve => setTimeout(resolve, 20)); + } + d.interrupt(); + await d.finish(130); + }); + assert.deepEqual(names(result), funnel); + assert.equal(props(result, 'completed').sourcebotStartOutcome, 'spawned'); +}); + +for (const telemetry of ['reject', 'stall', 'reset']) { + test(`telemetry ${telemetry} does not change files or successful completion`, async () => { + const began = Date.now(); + const result = await scenario(packed, { telemetry }, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + assert.equal(result.exitCode, 0); + assert.ok(Date.now() - began < 10000); + assert.deepEqual(Object.keys(result.files).sort(), ['.env', 'config.json']); + }); +} + +test('Ctrl+C at start prompt cancels before Docker can spawn', async () => { + const result = await scenario(packed, {}, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.wait('Start Sourcebot now?'); + d.interrupt(); + await d.finish(130); + }); + assert.equal(props(result, 'cancelled').stage, 'start'); + assert.equal(props(result, 'completed'), undefined); + assert.equal(result.dockerCalls.some(args => args.join(' ') === 'compose up'), false); +}); + +test('repeated Ctrl+C forces prompt exit despite a stalled SDK shutdown', async () => { + const result = await scenario(packed, { telemetry: 'stall' }, async d => { + await d.wait('What directory would you like'); + const began = Date.now(); + d.interrupt(); + await new Promise(resolve => setTimeout(resolve, 50)); + d.interrupt(); + await d.finish(130); + assert.ok(Date.now() - began < 1500, 'Second interrupt must not restart the shutdown deadline'); + }); + const attempted = result.requests.flatMap(request => request.batch); + assert.ok(attempted.filter(event => event.event === 'setup_sourcebot_cancelled').length <= 1); + assert.deepEqual(result.files, {}); +}); + +test('Ctrl+C after completion still cleans stubborn Docker while telemetry stalls', async () => { + await scenario(packed, { telemetry: 'stall', docker: { stall: ['compose up'], stubborn: true } }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + await new Promise(resolve => setTimeout(resolve, 100)); + const began = Date.now(); + d.interrupt(); + await d.finish(130); + assert.ok(Date.now() - began < 3500); + }); +}); diff --git a/packages/setupWizard/tests/integration/sdk.test.mjs b/packages/setupWizard/tests/integration/sdk.test.mjs new file mode 100644 index 000000000..b60fd2a3b --- /dev/null +++ b/packages/setupWizard/tests/integration/sdk.test.mjs @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { gunzipSync } from 'node:zlib'; +import { PostHog } from 'posthog-node'; +import { Telemetry, POSTHOG_OPTIONS } from '../../dist/telemetry.js'; + +test('real SDK transport envelope keeps its permitted metadata and identity', async () => { + const requests = []; + const server = createServer(async (req, res) => { + const chunks = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const bytes = Buffer.concat(chunks); + requests.push(JSON.parse((req.headers['content-encoding'] === 'gzip' ? gunzipSync(bytes) : bytes).toString())); + res.writeHead(200, { 'content-type': 'application/json' }).end('{"status":1}'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + const telemetry = new Telemetry(() => new PostHog('phc_test', { ...POSTHOG_OPTIONS, host: `http://127.0.0.1:${server.address().port}` })); + telemetry.capture('started', { invocationMethod: 'unknown', isInteractive: true }); + telemetry.capture('cancelled', { stage: 'setup_directory', reason: 'keyboard_interrupt' }); + await telemetry.shutdown(); + const events = requests.flatMap(r => r.batch); + assert.equal(events.length, 2); + assert.ok(Date.parse(events[1].timestamp) > Date.parse(events[0].timestamp)); + for (const e of events) { + assert.equal(e.distinct_id, telemetry.setupSessionId); + assert.equal(e.properties.install_id, telemetry.setupSessionId); + assert.deepEqual(e.properties.$groups, { company: telemetry.setupSessionId }); + assert.equal(e.properties.$lib, 'posthog-node'); + assert.equal(e.properties.$lib_version, '5.52.1'); + assert.equal(e.properties.$geoip_disable, true); + assert.equal(e.properties.$ignore_sent_at, true); + assert.equal(e.properties.$is_server, undefined); + assert.equal(e.properties.$process_person_profile, undefined); + assert.equal(e.properties.$set, undefined); + assert.ok(!Number.isNaN(Date.parse(e.timestamp))); + assert.match(e.uuid, /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + } + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + } +}); diff --git a/packages/setupWizard/tests/schemaSnapshot.mjs b/packages/setupWizard/tests/schemaSnapshot.mjs new file mode 100644 index 000000000..cf8b1c145 --- /dev/null +++ b/packages/setupWizard/tests/schemaSnapshot.mjs @@ -0,0 +1,56 @@ +import ts from 'typescript'; +import { readFileSync } from 'node:fs'; + +// Extract the declared schema for comparison with an independently checked-in +// snapshot. Tests must not silently accept new properties just because the +// production allowlist also changed. +export function declaredSchema() { + const source = ts.createSourceFile('telemetryEvents.ts', readFileSync(new URL('../src/telemetryEvents.ts', import.meta.url), 'utf8'), ts.ScriptTarget.Latest, true); + const bindings = new Map(); + for (const statement of source.statements) { + if (ts.isVariableStatement(statement)) { + for (const declaration of statement.declarationList.declarations) { + bindings.set(declaration.name.getText(source), declaration.initializer); + } + } + } + function resolve(node) { + if (!node) { + throw Error('Unrecognized telemetry declaration'); + } + if (ts.isIdentifier(node)) { + return resolve(bindings.get(node.text)); + } + if (ts.isAsExpression(node) || ts.isParenthesizedExpression(node)) { + return resolve(node.expression); + } + if (ts.isObjectLiteralExpression(node)) { + return Object.assign({}, ...node.properties.map(property => { + if (ts.isSpreadAssignment(property)) { + return resolve(property.expression); + } + const name = property.name.getText(source); + return { [name]: resolve(ts.isShorthandPropertyAssignment(property) ? property.name : property.initializer) }; + })); + } + if (ts.isCallExpression(node)) { + const name = node.expression.getText(source); + if (name === 'choice') { + return { enum: node.arguments.map(value => { + if (!ts.isStringLiteral(value)) { + throw Error('Review nonliteral telemetry enum'); + } + return value.text; + }) }; + } + if (name === 'nullable' || name === 'array') { + return { [name]: resolve(node.arguments[0]) }; + } + if (name === 'rule') { + return { type: node.typeArguments?.[0]?.getText(source) ?? 'unknown' }; + } + } + throw Error(`Unrecognized telemetry schema expression: ${node.getText(source)}`); + } + return resolve(bindings.get('eventSchemas')); +} diff --git a/packages/setupWizard/tests/unit/schema.test.mjs b/packages/setupWizard/tests/unit/schema.test.mjs new file mode 100644 index 000000000..8009758a2 --- /dev/null +++ b/packages/setupWizard/tests/unit/schema.test.mjs @@ -0,0 +1,9 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; +import { declaredSchema } from '../schemaSnapshot.mjs'; + +test('telemetry fields and enum values match the reviewed independent schema snapshot', () => { + const approved = JSON.parse(readFileSync(new URL('../approvedSchema.json', import.meta.url), 'utf8')); + assert.deepEqual(declaredSchema(), approved, 'Review telemetry plan and update the approved snapshot when changing the wire contract'); +}); diff --git a/packages/setupWizard/tests/unit/telemetry.test.mjs b/packages/setupWizard/tests/unit/telemetry.test.mjs new file mode 100644 index 000000000..611ab71c7 --- /dev/null +++ b/packages/setupWizard/tests/unit/telemetry.test.mjs @@ -0,0 +1,165 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { randomUUID } from 'node:crypto'; +import { Telemetry, INSTALL_ID_PATTERN, POSTHOG_OPTIONS, systemProperties } from '../../dist/telemetry.js'; +import { eventSchemas, validateFields } from '../../dist/telemetryEvents.js'; +import { sourceSummary, aggregateSources, aggregateAi, deployment, hostCategory, selectInstallId, emptyDockerSummary, dockerOutcome, discoveredBucket } from '../../dist/telemetrySummary.js'; +import { Lifecycle } from '../../dist/lifecycle.js'; + +test('UUID contract and persisted identity precedence', () => { + for (let i = 0; i < 100; i++) { + const id = new Telemetry(() => { throw Error(); }).setupSessionId; + assert.match(id, INSTALL_ID_PATTERN); + assert.equal(id.length, 36); + } + const old = randomUUID(); + const next = randomUUID(); + for (const value of [old, `"${old}"`, `'${old}'`, `${old} # comment`]) { + assert.deepEqual(selectInstallId(`SOURCEBOT_INSTALL_ID=${value}`, next), { id: old, action: 'preserved_existing' }); + } + for (const value of ['', old.toUpperCase(), 'not-an-id', '$(danger)', `${old}\nSOURCEBOT_INSTALL_ID=invalid`]) { + assert.deepEqual(selectInstallId(`SOURCEBOT_INSTALL_ID=${value}`, next), { id: next, action: 'created_from_setup_session' }); + } +}); + +test('code host normalization matches boundaries and formatting, not arbitrary suffixes', () => { + for (const input of ['github.com', ' HTTPS://WWW.GITHUB.COM./path ', 'http://github.com:8080']) { + assert.equal(deployment('github', input), 'cloud'); + } + assert.equal(deployment('github', 'tenant.ghe.com'), 'cloud'); + assert.equal(deployment('github', 'github.com.attacker.invalid'), 'self_hosted'); + assert.equal(deployment('gitlab', 'gitlab.com'), 'cloud'); + assert.equal(deployment('gitlab', 'tenant.gitlab-dedicated.com'), 'cloud'); + assert.equal(deployment('gitlab', 'internal.example'), 'unknown'); + assert.equal(deployment('gitea', 'www.gitea.com'), 'cloud'); + assert.equal(deployment('gitea', 'internal.example'), 'self_hosted'); + for (const input of ['', '://', 'ssh://github.com']) { + assert.equal(deployment('github', input), 'unknown'); + } +}); + +test('host classification never performs DNS or exposes an address', () => { + for (const input of ['http://localhost', 'https://LOCALHOST.:3000', 'http://x.localhost', 'http://127.0.0.0', 'http://127.9.8.7', 'http://[::1]']) { + assert.equal(hostCategory(input), 'localhost'); + } + for (const input of ['http://192.168.0.1', 'https://secret.example', 'http://localhost.attacker.invalid', 'http://[::2]']) { + assert.equal(hostCategory(input), 'address'); + } + assert.equal(hostCategory('not a URL'), 'unknown'); +}); + +test('aggregates reflect selections, counts, unique providers, and local wildcard collapse', () => { + const aggregate = aggregateSources([ + sourceSummary('github', { deploymentType: 'cloud', repositoryCount: 2, credentialMode: 'personal_access_token' }), + sourceSummary('github', { deploymentType: 'self_hosted', organizationCount: 1, indexAll: true }), + sourceSummary('local_git', { deploymentType: 'local', repositoryCount: 9, generatedConnectionCount: 3 }), + ]); + assert.equal(aggregate.repositoryCount, 11); + assert.equal(aggregate.generatedConnectionCount, 5); + assert.equal(aggregate.codeSourceConfigurationCount, 3); + assert.deepEqual(aggregate.codeHostTypes, ['github', 'local_git']); + assert.equal(aggregate.credentialedCodeSourceCount, 1); + assert.equal(aggregate.indexAllCodeSourceCount, 1); + assert.deepEqual([1, 2, 5, 6, 20, 21, 100, 101].map(discoveredBucket), ['1', '2-5', '2-5', '6-20', '6-20', '21-100', '21-100', '101+']); + assert.deepEqual(aggregateAi([]), { aiConfigured: false, aiConfigurationCount: 0, uniqueProviderCount: 0, providerTypes: [], usesCustomEndpoint: false, credentialModes: [], modelSelectionMethods: [] }); + const model = { provider: 'openai', credentialMode: 'api_key', modelSelectionMethod: 'catalog', usesCustomEndpoint: false, hasDisplayName: true }; + assert.equal(aggregateAi([model, model]).uniqueProviderCount, 1); + assert.equal(aggregateAi([model, model]).aiConfigurationCount, 2); +}); + +test('Docker skipped/failed measurements are null and outcome precedence is explicit', () => { + const state = emptyDockerSummary(); + for (const key of Object.keys(state).filter(k => k.endsWith('Count'))) { + assert.equal(state[key], null); + } + assert.equal(dockerOutcome(state, false, false), 'skipped_no_compose'); + assert.equal(dockerOutcome(state, true, true), 'validation_failed'); + state.leftExistingDeploymentRunning = true; + assert.equal(dockerOutcome(state, true, false), 'skipped_existing_deployment_running'); + assert.equal(dockerOutcome(state, true, true), 'validation_failed'); + state.leftExistingDeploymentRunning = false; + state.remainingPortConflictCount = 2; + assert.equal(dockerOutcome(state, true, false), 'unresolved_conflicts'); + state.remainingPortConflictCount = 0; + state.volumeAction = 'removed'; + assert.equal(dockerOutcome(state, true, false), 'passed_after_cleanup'); +}); + +test('allowlists reject wrong types and strip arbitrary user values', () => { + const fields = { configurationIndex: 1, ...sourceSummary('github'), token: 'CANARY_SECRET', repository: 'CANARY_REPO' }; + const output = validateFields(eventSchemas.configured_code_source, fields); + assert.equal(JSON.stringify(output).includes('CANARY'), false); + for (const change of [{ configurationIndex: NaN }, { repositoryCount: -1 }, { codeHost: 'CANARY_HOST' }, { scopeTypes: ['CANARY_SCOPE'] }, { indexAll: 'true' }]) { + assert.throws(() => validateFields(eventSchemas.configured_code_source, { ...fields, ...change })); + } +}); + +test('telemetry is fail-open, uses exact identity, default SDK transport and no opt-out coupling', async () => { + const sent = []; + const client = { capture: e => sent.push(e), on() {}, async shutdown() {} }; + const previous = process.env.SOURCEBOT_TELEMETRY_DISABLED; + process.env.SOURCEBOT_TELEMETRY_DISABLED = 'true'; + try { + const telemetry = new Telemetry(() => client); + telemetry.capture('started', { invocationMethod: 'unknown', isInteractive: true }); + assert.equal(sent.length, 1); + const event = sent[0]; + assert.equal(event.distinctId, telemetry.setupSessionId); + assert.equal(event.properties.install_id, event.distinctId); + assert.deepEqual(event.groups, { company: event.distinctId }); + assert.equal(event.properties.$geoip_disable, true); + assert.equal(event.properties.$process_person_profile, undefined); + assert.ok(event.properties.elapsedMs >= 0); + await telemetry.shutdown(); + telemetry.capture('started', { invocationMethod: 'unknown', isInteractive: true }); + assert.equal(sent.length, 1); + for (const broken of [() => { throw Error(); }, () => ({ ...client, capture() { throw Error(); }, shutdown() { throw Error(); } })]) { + const t = new Telemetry(broken); + assert.doesNotThrow(() => t.capture('started', { invocationMethod: 'unknown', isInteractive: true })); + await t.shutdown(); + } + assert.deepEqual(POSTHOG_OPTIONS, { host: 'https://us.i.posthog.com', flushAt: 1, flushInterval: 0, disableGeoip: true, isServer: false }); + assert.ok(['npm', 'yarn', 'pnpm', 'bun', 'unknown'].includes(systemProperties().packageManager)); + } finally { + if (previous === undefined) { + delete process.env.SOURCEBOT_TELEMETRY_DISABLED; + } else { + process.env.SOURCEBOT_TELEMETRY_DISABLED = previous; + } + } +}); + +test('terminal latch is independent of multiple recoverable failures', async () => { + const events = []; + const t = { capture: (name, props) => events.push({ name, props }), shutdown: async () => {} }; + const life = new Lifecycle(t); + life.fail('docker_command', true); + life.fail('docker_unavailable', true); + await life.complete({}); + await life.decline('keyboard_interrupt'); + life.fail('unknown', false); + assert.deepEqual(events.map(e => e.name), ['failed', 'failed', 'completed']); + assert.equal(life.terminal, 'completed'); + const fatal = new Lifecycle(t); + fatal.fail('filesystem', false); + await fatal.complete({}); + assert.equal(events.at(-1).name, 'failed'); + assert.equal(events.at(-1).props.recoverable, false); +}); + +test('each system property falls back independently and elapsed time is monotonic', () => { + const runtime = { platform: 'linux', arch: 'arm64', versions: { node: '24.1.0' }, env: { npm_config_user_agent: 'pnpm/10.0', CI: 'true' } }; + assert.deepEqual(systemProperties(runtime), { platform: 'linux', arch: 'arm64', nodeMajorVersion: 24, packageManager: 'pnpm', isCI: true }); + for (const [key, field, fallback] of [['platform', 'platform', 'other'], ['arch', 'arch', 'other'], ['versions', 'nodeMajorVersion', null], ['env', 'packageManager', 'unknown']]) { + const broken = { ...runtime }; + Object.defineProperty(broken, key, { get() { throw Error('fixture'); } }); + assert.equal(systemProperties(broken)[field], fallback); + } + const brokenEnv = { ...runtime, get env() { throw Error(); } }; + assert.equal(systemProperties(brokenEnv).isCI, null); + assert.equal(systemProperties({ ...runtime, versions: { node: 'invalid' } }).nodeMajorVersion, null); + let now = 100; + const t = new Telemetry(() => { throw Error(); }, randomUUID, () => now); + now = 123.6; + assert.equal(t.elapsed(), 24); +}); diff --git a/yarn.lock b/yarn.lock index c07ff3465..71783e786 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4849,6 +4849,15 @@ __metadata: languageName: node linkType: hard +"@posthog/core@npm:^1.53.1": + version: 1.53.2 + resolution: "@posthog/core@npm:1.53.2" + dependencies: + "@posthog/types": "npm:^1.411.1" + checksum: 10c0/63248d839b140887dd7d4af81096909f9920a566f09bec945806c3ece7a8d047c3644569cee8962f100decf6b60a42188016319dd11ec87458eddbf7bcb5fd3a + languageName: node + linkType: hard + "@posthog/types@npm:1.369.0": version: 1.369.0 resolution: "@posthog/types@npm:1.369.0" @@ -4870,6 +4879,13 @@ __metadata: languageName: node linkType: hard +"@posthog/types@npm:^1.411.1": + version: 1.411.1 + resolution: "@posthog/types@npm:1.411.1" + checksum: 10c0/fd4cf6bd38abea89ff37fc2af8f329373ba6da8bfc99a1743f69f665428d43bcf6be5c3ef01196292e7b14f928df0b0de64ec3e401576ca15632c21b3b1b1703 + languageName: node + linkType: hard + "@preact/signals-core@npm:^1.7.0": version: 1.14.0 resolution: "@preact/signals-core@npm:1.14.0" @@ -10020,6 +10036,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^24.0.0": + version: 24.13.4 + resolution: "@types/node@npm:24.13.4" + dependencies: + undici-types: "npm:~7.18.0" + checksum: 10c0/a12196e984cb09ead549651217b4b395961ea011f3a3791a9f78311d70b5f3be343c6fc935376553c06a34c14d963d4f637ad7829837b0c88cf441919f422893 + languageName: node + linkType: hard + "@types/nodemailer@npm:^6.4.17": version: 6.4.17 resolution: "@types/nodemailer@npm:6.4.17" @@ -18473,6 +18498,15 @@ __metadata: languageName: node linkType: hard +"node-addon-api@npm:^7.1.0": + version: 7.1.1 + resolution: "node-addon-api@npm:7.1.1" + dependencies: + node-gyp: "npm:latest" + checksum: 10c0/fb32a206276d608037fa1bcd7e9921e177fe992fc610d098aa3128baca3c0050fc1e014fa007e9b3874cf865ddb4f5bd9f43ccb7cbbbe4efaff6a83e920b17e9 + languageName: node + linkType: hard + "node-cleanup@npm:^2.1.2": version: 2.1.2 resolution: "node-cleanup@npm:2.1.2" @@ -18545,6 +18579,16 @@ __metadata: languageName: node linkType: hard +"node-pty@npm:^1.1.0": + version: 1.1.0 + resolution: "node-pty@npm:1.1.0" + dependencies: + node-addon-api: "npm:^7.1.0" + node-gyp: "npm:latest" + checksum: 10c0/e527305263da36d554b7d2116aee6430b898675fc50e7bd82fbd762f14cb6b948fb8d997736af1f70d76bf9ffd4f5179bc1ed1d8002567dfcf3b29f039291798 + languageName: node + linkType: hard + "node-releases@npm:^2.0.53": version: 2.0.54 resolution: "node-releases@npm:2.0.54" @@ -19516,6 +19560,20 @@ __metadata: languageName: node linkType: hard +"posthog-node@npm:5.52.1": + version: 5.52.1 + resolution: "posthog-node@npm:5.52.1" + dependencies: + "@posthog/core": "npm:^1.53.1" + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true + checksum: 10c0/dfb7f71792156a817aaddf478e844f6d91f3f53c1f91f8aa496a1a4f30b414dfee40baec3cad35570ca383ef6e0a5ec316e59f687fa303960dedd2aeb2d09218 + languageName: node + linkType: hard + "posthog-node@npm:^5.51.2": version: 5.51.2 resolution: "posthog-node@npm:5.51.2" @@ -21240,15 +21298,18 @@ __metadata: dependencies: "@inquirer/prompts": "npm:^8.4.3" "@sourcebot/schemas": "workspace:^" - "@types/node": "npm:^22.7.5" + "@types/node": "npm:^24.0.0" chalk: "npm:^5.6.2" inquirer-select-pro: "npm:^1.0.0-alpha.9" + node-pty: "npm:^1.1.0" ora: "npm:^9.4.0" + posthog-node: "npm:5.52.1" reo-census: "npm:^1.2.10" tsx: "npm:^4.21.0" typescript: "npm:^5.6.2" + undici: "npm:^7" bin: - setup-sourcebot: ./dist/index.js + setup-sourcebot: ./bin.cjs languageName: unknown linkType: soft @@ -22971,6 +23032,20 @@ __metadata: languageName: node linkType: hard +"undici-types@npm:~7.18.0": + version: 7.18.2 + resolution: "undici-types@npm:7.18.2" + checksum: 10c0/85a79189113a238959d7a647368e4f7c5559c3a404ebdb8fc4488145ce9426fcd82252a844a302798dfc0e37e6fb178ff481ed03bc4caf634c5757d9ef43521d + languageName: node + linkType: hard + +"undici@npm:^7": + version: 7.29.1 + resolution: "undici@npm:7.29.1" + checksum: 10c0/5a419b1364531071ebdfd93748f7f2392d4eb6d68dd99814e1020f8fc6a517e3298129004c871516ee204bac68aba93541f587200b04a7449b8d45683a7d86f6 + languageName: node + linkType: hard + "unified@npm:^11.0.0": version: 11.0.5 resolution: "unified@npm:11.0.5" From 049014f693334fff9715bd5b48205fca10b200ee Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 21:36:13 -0700 Subject: [PATCH 02/14] docs: add setup telemetry changelog entry for #1653 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a14d69cd..e506048e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Added privacy-scoped setup wizard funnel telemetry with deployment identity handoff and Node 24 support. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) + ## [5.1.13] - 2026-09-12 ### Fixed From 5b186f97551c22f8d0588a23fc373c8146be7749 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 21:49:25 -0700 Subject: [PATCH 03/14] Fix setup wizard CI isolation and terminal test cleanup --- .github/workflows/setup-wizard-e2e.yml | 1 + .github/workflows/test.yml | 4 +- packages/setupWizard/telemetry.html | 1 + packages/setupWizard/tests/e2e/harness.mjs | 41 ++++++++++++++----- .../setupWizard/tests/e2e/packageManagers.mjs | 4 +- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.github/workflows/setup-wizard-e2e.yml b/.github/workflows/setup-wizard-e2e.yml index 5cf589b45..d220ae6d4 100644 --- a/.github/workflows/setup-wizard-e2e.yml +++ b/.github/workflows/setup-wizard-e2e.yml @@ -16,6 +16,7 @@ permissions: jobs: platform: + timeout-minutes: 20 strategy: fail-fast: false matrix: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3ac20e85..5bdac2e4e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,4 +41,6 @@ jobs: run: yarn install --frozen-lockfile - name: Test - run: yarn test + # The CLI requires Node 24 and is built/tested by setup-wizard-e2e. + # Keep the application workspaces on their existing runtime here. + run: yarn workspaces foreach --all --topological --exclude setup-sourcebot run test diff --git a/packages/setupWizard/telemetry.html b/packages/setupWizard/telemetry.html index c817bff0a..972a44c14 100644 --- a/packages/setupWizard/telemetry.html +++ b/packages/setupWizard/telemetry.html @@ -1743,6 +1743,7 @@

Reo coexistence

The release notes and telemetry documentation should state that setup-wizard PostHog telemetry is always on when its code executes, list the categories collected, explain that PostHog creates a Person profile keyed only by a random pseudonymous session/deployment ID, explain the new-deployment ID handoff, and distinguish PostHog from Reo’s separate tracking and variable. They must not describe the project ingestion token as a secret.

Testing plan

Implementation verification status

+

Live AI validation scope (user clarification): verify that the packaged wizard writes the selected provider/model configuration and credential references correctly, that the generated environment values reach the resulting Sourcebot container unchanged, and that Sourcebot starts successfully. Do not invoke Ask or make model-inference requests. A Pro license is not required for this test scope. Use available development credentials without printing them; where credentials are unavailable, distinguish configuration/startup coverage with synthetic values from actual credential validation. Continue real repository discovery, indexing, search, browser access, and restart tests independently.

The feature is not release-complete until the full completion gate below is satisfied. Executable tests cover packed-package collectors, terminal paths, real SDK payloads, Docker branches, and actual container identity continuity. CI definitions are not evidence that Windows or other remote jobs have run.

Implementation-discovered changes for the behavioral baseline allowlist: status 130 for Ctrl+C; cancellation-aware readiness cleanup; explicit final process exit after bounded telemetry shutdown; recoverable autocomplete fallback on network/decode failure with an 8-second timeout; and fixed, credential-free SDK shutdown-timeout diagnostics where PostHog itself emits them. Do not patch SDK internals or silence the global console. Empty search/catalog results are not failures; actual infrastructure errors can emit recoverable diagnostics while retaining manual fallback.

Docker availability failures can trigger fixed docker info/docker compose version probes for classification. These probes classify the original failed operation and do not create extra failure events. The dedicated workflow is .github/workflows/setup-wizard-e2e.yml.

diff --git a/packages/setupWizard/tests/e2e/harness.mjs b/packages/setupWizard/tests/e2e/harness.mjs index d3ab62459..a71b5fa1b 100644 --- a/packages/setupWizard/tests/e2e/harness.mjs +++ b/packages/setupWizard/tests/e2e/harness.mjs @@ -115,7 +115,7 @@ export async function scenario(artifact, options, drive) { const requests = []; const forwarding = []; const cwd = join(root, 'work'); - const setup = join(cwd, 'sourcebot'); + const setup = join(cwd, options.setupName ?? 'sourcebot'); const fakeBin = join(root, 'bin'); const home = join(root, 'home'); for (const dir of [cwd, fakeBin, home]) { @@ -129,7 +129,14 @@ export async function scenario(artifact, options, drive) { } options.prepare?.({ cwd, setup, root }); writeFileSync(join(root, 'docker-state.json'), JSON.stringify(options.docker ?? {})); - if (!options.dockerMissing) { + if (options.realDocker) { + writeFileSync(join(fakeBin, 'docker'), `#!${process.execPath}\n` + + `const {spawn}=require('node:child_process'); const args=process.argv.slice(2);\n` + + `if(!['compose','volume','ps','info'].includes(args[0])){throw Error('Live test Docker command not permitted');}\n` + + `if(args[0]==='volume' && args[1]!=='ls'){throw Error('Live test volume mutation not permitted');}\n` + + `const p=spawn(${JSON.stringify(options.realDocker)},args,{stdio:'inherit'});\n` + + `process.on('SIGINT',()=>{});p.on('error',()=>process.exit(1));p.on('close',c=>process.exit(c??130));\n`, { mode: 0o755 }); + } else if (!options.dockerMissing) { if (process.platform === 'win32') { // A real executable is needed: Windows spawn(shell:false) cannot // execute Unix shebangs or .cmd wrappers. The external preload @@ -155,7 +162,8 @@ export async function scenario(artifact, options, drive) { assert.ok(process.env.SETUP_TEST_DEV_TOKEN, 'Dev project token required'); const forwarded = { ...body, api_key: process.env.SETUP_TEST_DEV_TOKEN }; assert.deepEqual({ ...forwarded, api_key: body.api_key }, body); - const request = fetch(`https://us.i.posthog.com${req.url}`, { + assert.equal(req.url, '/batch/', 'Unexpected PostHog ingestion path'); + const request = fetch('https://us.i.posthog.com/batch/', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(forwarded), signal: AbortSignal.timeout(15000), }).then(async response => { assert.ok(response.ok, `Dev ingestion rejected: ${response.status}`); @@ -212,7 +220,7 @@ export async function scenario(artifact, options, drive) { HOME: home, USERPROFILE: home, TMPDIR: root, TEMP: root, TMP: root, XDG_STATE_HOME: join(home, 'state'), XDG_CONFIG_HOME: join(home, 'config'), XDG_CACHE_HOME: join(home, 'cache'), SYSTEMROOT: process.env.SYSTEMROOT ?? '', LANG: 'en_US.UTF-8', TERM: 'xterm-256color', - NODE_OPTIONS: `--import=${new URL('./network.mjs', import.meta.url).href}`, + NODE_OPTIONS: `--import=${new URL(options.networkModule ?? './network.mjs', import.meta.url).href}`, NODE_EXTRA_CA_CERTS: artifact.cert, TEST_CAPTURE_PORT: String(server.address().port), TEST_DOCKER_STATE: join(root, 'docker-state.json'), TEST_DOCKER_LOG: join(root, 'docker.log'), TEST_DOCKER_PIDS: join(root, 'docker-pids'), @@ -287,6 +295,9 @@ export async function scenario(artifact, options, drive) { contract(events); } assert.equal(JSON.stringify(requests).includes(canary), false); + for (const sensitive of options.sensitiveValues ?? []) { + assert.equal(JSON.stringify(requests).includes(sensitive), false, 'Live credential reached telemetry'); + } if (options.assertLauncherHome) { options.assertLauncherHome(readdirSync(home, { recursive: true })); } else { @@ -299,15 +310,25 @@ export async function scenario(artifact, options, drive) { assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }, 'Owned Docker fixture process survived CLI exit'); } } - return { events, files, dockerCalls, requests, exitCode: ended.exitCode }; + const result = { events, files, dockerCalls, requests, exitCode: ended.exitCode }; + await options.verifyDeployment?.({ ...result, setup, root }); + return result; } finally { - if (child && !ended) { + if (child && process.platform === 'win32') { + // node-pty's ConPTY worker survives a natural child exit unless + // the terminal is disposed; Windows kill() accepts no signal. + child.kill(); + } else if (child && !ended) { child.kill('SIGKILL'); } - server.closeAllConnections(); - await new Promise(resolve => server.close(resolve)); - rmSync(root, { recursive: true, force: true }); - assert.equal(existsSync(root), false); + try { + await options.cleanupDeployment?.({ setup, root }); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + rmSync(root, { recursive: true, force: true }); + assert.equal(existsSync(root), false); + } } } diff --git a/packages/setupWizard/tests/e2e/packageManagers.mjs b/packages/setupWizard/tests/e2e/packageManagers.mjs index 3d36b0b30..e9fd2d69d 100644 --- a/packages/setupWizard/tests/e2e/packageManagers.mjs +++ b/packages/setupWizard/tests/e2e/packageManagers.mjs @@ -39,7 +39,9 @@ try { writeFileSync(join(cwd, 'yarn.lock'), ''); writeFileSync(join(cwd, '.yarnrc.yml'), 'nodeLinker: node-modules\n'); if (name === 'yarn') { - execFileSync(process.execPath, [binary, 'install', '--mode=skip-build'], { cwd, env: { ...process.env, ...managerEnv }, timeout: 30000 }); + // This new disposable project intentionally has no lockfile + // yet; CI's immutable-install default cannot apply to it. + execFileSync(process.execPath, [binary, 'install', '--mode=skip-build'], { cwd, env: { ...process.env, ...managerEnv, YARN_ENABLE_IMMUTABLE_INSTALLS: 'false' }, timeout: 30000 }); } if (name === 'bun') { symlinkSync(dirname(packed.installed), join(cwd, 'node_modules'), 'junction'); From 0564ef87869d225e37975a4d1527980359b462a2 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 21:53:30 -0700 Subject: [PATCH 04/14] Keep root test orchestration out of the application workspace loop --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5bdac2e4e..15faf3056 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,4 +43,4 @@ jobs: - name: Test # The CLI requires Node 24 and is built/tested by setup-wizard-e2e. # Keep the application workspaces on their existing runtime here. - run: yarn workspaces foreach --all --topological --exclude setup-sourcebot run test + run: yarn workspaces foreach --all --topological --exclude setup-sourcebot --exclude 'root-workspace-*' run test From 7f491c19ecfb5a892a56bf010d66830f41aa7775 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 21:58:54 -0700 Subject: [PATCH 05/14] Add real setup CLI deployment and AI configuration end-to-end tests --- CHANGELOG.md | 1 + packages/setupWizard/telemetry.html | 1 + .../setupWizard/tests/e2e/liveDeployment.md | 52 +++ .../setupWizard/tests/e2e/liveDeployment.mjs | 392 ++++++++++++++++++ .../setupWizard/tests/e2e/liveNetwork.mjs | 21 + 5 files changed, 467 insertions(+) create mode 100644 packages/setupWizard/tests/e2e/liveDeployment.md create mode 100644 packages/setupWizard/tests/e2e/liveDeployment.mjs create mode 100644 packages/setupWizard/tests/e2e/liveNetwork.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index e506048e2..06f26598f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added privacy-scoped setup wizard funnel telemetry with deployment identity handoff and Node 24 support. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) +- Added isolated live setup CLI deployment tests covering public code hosts, local clones, AI configuration, search, and restart identity; repaired cross-platform test-runner cleanup. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) ## [5.1.13] - 2026-09-12 diff --git a/packages/setupWizard/telemetry.html b/packages/setupWizard/telemetry.html index 972a44c14..0d4271db7 100644 --- a/packages/setupWizard/telemetry.html +++ b/packages/setupWizard/telemetry.html @@ -1743,6 +1743,7 @@

Reo coexistence

The release notes and telemetry documentation should state that setup-wizard PostHog telemetry is always on when its code executes, list the categories collected, explain that PostHog creates a Person profile keyed only by a random pseudonymous session/deployment ID, explain the new-deployment ID handoff, and distinguish PostHog from Reo’s separate tracking and variable. They must not describe the project ingestion token as a secret.

Testing plan

Implementation verification status

+

Live deployment suite: tests/e2e/liveDeployment.mjs builds and installs the npm tarball, drives the real interactive CLI, and starts the released Sourcebot v5.1.13 image with real PostgreSQL, Redis, migrations, backend, and web processes. It exercises public GitHub repository/organization/user scopes, GitLab projects, Gitea repositories, Gerrit projects, generic Git URLs, three local-clone layouts, automatic startup/interruption, and AI configuration. Each deployment uses an isolated Compose project and loopback port; only the test Compose networking/image settings and deployment telemetry opt-out are changed. Generated configuration, environment values, local mounts, and install-ID handoff remain intact. See tests/e2e/liveDeployment.md for reproduction and limits. A final clean rerun of all thirteen live scenarios is in progress; this does not replace the still-required full scenario/branch-coverage accounting.

Live AI validation scope (user clarification): verify that the packaged wizard writes the selected provider/model configuration and credential references correctly, that the generated environment values reach the resulting Sourcebot container unchanged, and that Sourcebot starts successfully. Do not invoke Ask or make model-inference requests. A Pro license is not required for this test scope. Use available development credentials without printing them; where credentials are unavailable, distinguish configuration/startup coverage with synthetic values from actual credential validation. Continue real repository discovery, indexing, search, browser access, and restart tests independently.

The feature is not release-complete until the full completion gate below is satisfied. Executable tests cover packed-package collectors, terminal paths, real SDK payloads, Docker branches, and actual container identity continuity. CI definitions are not evidence that Windows or other remote jobs have run.

Implementation-discovered changes for the behavioral baseline allowlist: status 130 for Ctrl+C; cancellation-aware readiness cleanup; explicit final process exit after bounded telemetry shutdown; recoverable autocomplete fallback on network/decode failure with an 8-second timeout; and fixed, credential-free SDK shutdown-timeout diagnostics where PostHog itself emits them. Do not patch SDK internals or silence the global console. Empty search/catalog results are not failures; actual infrastructure errors can emit recoverable diagnostics while retaining manual fallback.

diff --git a/packages/setupWizard/tests/e2e/liveDeployment.md b/packages/setupWizard/tests/e2e/liveDeployment.md new file mode 100644 index 000000000..9218c2650 --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveDeployment.md @@ -0,0 +1,52 @@ +# Live deployment verification + +Run `liveDeployment.mjs` with Node 24 on macOS or Linux with Docker available. +It builds and installs the actual npm tarball in a disposable directory, drives +the interactive CLI through a PTY, starts the real Sourcebot/Postgres/Redis stack, +onboards a synthetic owner through Chromium, verifies indexing and authenticated +search, then verifies health, search, and install-ID continuity after restart. + +Pull `docker.sourcebot.dev/sourcebot-dev/sourcebot:v5.1.13`, `postgres:16`, and +`redis:8` first. The default scenario is `github_repo`. For the public/configuration +matrix, set: + +```sh +SETUP_TEST_LIVE_CASES=github_repo,github_org,github_user,gitlab_project,gitea_repo,gerrit_project,remote_git,local_root,local_wildcard,local_nested,auto_start,ai_all \ +SETUP_TEST_LIVE_OUTPUT=/absolute/path/outside/the/repository \ +node packages/setupWizard/tests/e2e/liveDeployment.mjs +``` + +The optional `anthropic` scenario reads `ANTHROPIC_API_KEY` from development env +files under `SETUP_TEST_CREDENTIAL_DIR`. Never pass the credential as a command +argument. `ai_all` configures all twelve providers with synthetic values. Neither +scenario calls Ask or an inference endpoint; neither proves credential validity +or requires a Pro license. The tests check generated model configuration, +credential references, container environment propagation, and successful startup. + +Every deployment uses a unique Compose project, random loopback HTTP port, and +unpublished database/Redis ports. The Compose download is intercepted only to +apply these isolation settings, pin the image, and disable deployment telemetry. +The wizard's generated config, `.env`, identity, and local-repository override +remain intact. Wizard PostHog requests use the real SDK and are routed to the +local TLS collector; live public autocomplete/model-catalog requests remain real. +This suite does not forward to production or dev PostHog. The separate dev smoke +test provides actual ingestion/query evidence. + +Public fixtures are bounded: one explicit repository/project per host, the +small `chalk` organization, and the `octocat` user. Organization/user tests obtain +the expected repository count before starting and require all repositories to +be discovered and indexed. Multi-local tests clone two distinct origins because +Sourcebot deduplicates multiple local clones of the same origin. + +Each scenario records success/failure and removes its containers, volumes, +network, generated secrets, and temporary files in cleanup. Only redacted reports +and browser screenshots remain when an output directory is supplied. Task-owned +image downloads should be removed by the operator afterward without removing +images that existed before the run. Do not commit generated artifacts. + +This live matrix supplements, not replaces, the deterministic collector, fault, +transport, lifecycle, and platform suites. It does not establish live access to +private/GHE/GitLab self-managed, Azure DevOps, or Bitbucket deployments, and does +not claim every possible code-host scope or credential mode has live coverage. +Unavailable integrations must remain explicitly unverified in the completion +report rather than being inferred from fixture-based coverage. diff --git a/packages/setupWizard/tests/e2e/liveDeployment.mjs b/packages/setupWizard/tests/e2e/liveDeployment.mjs new file mode 100644 index 000000000..90fcc252f --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveDeployment.mjs @@ -0,0 +1,392 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { randomUUID, createHash } from 'node:crypto'; +import { createServer } from 'node:net'; +import { once } from 'node:events'; +import { mkdirSync, readFileSync, writeFileSync, existsSync, chmodSync, realpathSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { artifact, scenario } from './harness.mjs'; + +const packed = artifact(); +const docker = execFileSync('which', ['docker'], { encoding: 'utf8' }).trim(); +const image = 'docker.sourcebot.dev/sourcebot-dev/sourcebot:v5.1.13'; +const reports = []; +const providers = ['anthropic', 'openai', 'openai-compatible', 'amazon-bedrock', 'google-generative-ai', 'google-vertex', 'google-vertex-anthropic', 'azure', 'deepseek', 'mistral', 'openrouter', 'xai']; +const output = process.env.SETUP_TEST_LIVE_OUTPUT; +if (output) { + mkdirSync(output, { recursive: true }); +} +let browser; +let parse, stringify, dotenv; +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +const liveMulti = async (driver, prompt, value) => { + await driver.wait(prompt); + await pause(350); + driver.write(value); + await driver.wait(`[ ] ${value}`); + driver.write('\t'); + await pause(100); + driver.write('\r'); +}; +const run = (args, options = {}) => execFileSync(docker, args, { encoding: 'utf8', timeout: 45000, stdio: ['ignore', 'pipe', 'pipe'], ...options }); +const freePort = async () => { + const server = createServer().listen(0, '127.0.0.1'); + await once(server, 'listening'); + const port = server.address().port; + await new Promise(resolve => server.close(resolve)); + return port; +}; +function credentials() { + assert.ok(process.env.SETUP_TEST_CREDENTIAL_DIR, 'Set SETUP_TEST_CREDENTIAL_DIR to the development env-file directory'); + for (const file of ['.env.local', '.env.development.local', '.env.local.development']) { + const path = join(process.env.SETUP_TEST_CREDENTIAL_DIR, file); + if (existsSync(path)) { + const values = dotenv(readFileSync(path)); + if (values.ANTHROPIC_API_KEY) { + return values.ANTHROPIC_API_KEY; + } + } + } +} + +try { + const dockerConfig = join(packed.root, 'docker-client'); + mkdirSync(dockerConfig); + const pluginDirectory = resolve(dirname(realpathSync(docker)), '../cli-plugins'); + writeFileSync(join(dockerConfig, 'config.json'), JSON.stringify({ cliPluginsExtraDirs: existsSync(pluginDirectory) ? [pluginDirectory] : [] })); + const dockerHost = run(['context', 'inspect', '--format', '{{.Endpoints.docker.Host}}']).trim(); + const tools = join(packed.root, 'browser-tools'); + mkdirSync(tools); + writeFileSync(join(tools, 'package.json'), '{"private":true}'); + process.env.PLAYWRIGHT_BROWSERS_PATH = join(packed.root, 'browsers'); + execFileSync('npm', ['install', '--no-audit', '--no-fund', 'playwright@1.63.0', 'yaml@2.8.3', 'dotenv@16.4.7'], { cwd: tools, timeout: 120000, stdio: 'ignore' }); + ({ parse, stringify } = await import(pathToFileURL(join(tools, 'node_modules/yaml/dist/index.js')).href)); + ({ parse: dotenv } = await import(pathToFileURL(join(tools, 'node_modules/dotenv/lib/main.js')).href)); + execFileSync(process.execPath, [join(tools, 'node_modules/playwright/cli.js'), 'install', 'chromium'], { timeout: 180000, stdio: 'ignore' }); + const { chromium } = await import(pathToFileURL(join(tools, 'node_modules/playwright/index.mjs')).href); + browser = await chromium.launch({ headless: true }); + const upstream = await fetch('https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/docker-compose.yml'); + assert.ok(upstream.ok); + const composeSource = await upstream.text(); + const cases = (process.env.SETUP_TEST_LIVE_CASES ?? 'github_repo').split(','); + const knownCases = new Set(['github_repo', 'github_org', 'github_user', 'gitlab_project', 'remote_git', 'local_root', 'local_wildcard', 'local_nested', 'anthropic', 'gitea_repo', 'gerrit_project', 'auto_start', 'ai_all']); + assert.ok(cases.every(name => knownCases.has(name)), 'Unknown live deployment scenario'); + for (const name of cases) { + const project = `sb-e2e-${randomUUID().slice(0, 8)}`; + const port = await freePort(); + const url = `http://localhost:${port}`; + let expectedRepositories = name === 'local_wildcard' || name === 'local_nested' ? 2 : 1; + if (name === 'github_org' || name === 'github_user') { + const endpoint = name === 'github_org' ? 'orgs/chalk' : 'users/octocat'; + const response = await fetch(`https://api.github.com/${endpoint}/repos?per_page=100`); + assert.ok(response.ok, 'Could not establish expected public repository count'); + const repositories = await response.json(); + assert.ok(repositories.length > 0 && repositories.length < 100, 'Public fixture must fit a single bounded page'); + expectedRepositories = repositories.length; + } + const compose = parse(composeSource); + // Isolation only: leave the generated config, secrets and identity intact. + compose.services.sourcebot.image = image; + compose.services.sourcebot.pull_policy = 'never'; + compose.services.sourcebot.container_name = `${project}-sourcebot`; + compose.services.sourcebot.ports = [`127.0.0.1:${port}:3000`]; + compose.services.sourcebot.environment.push('SOURCEBOT_TELEMETRY_DISABLED=true'); + for (const [service, config] of Object.entries(compose.services)) { + config.restart = 'no'; + if (service !== 'sourcebot') { + config.ports = []; + } + } + const report = { scenario: name, project, url, artifactSha256: packed.digest, composeSourceSha256: createHash('sha256').update(composeSource).digest('hex'), status: 'running', image }; + reports.push(report); + console.log(`Starting live scenario ${name} at ${url}`); + const aiKey = name === 'anthropic' ? credentials() : undefined; + const allAi = name === 'ai_all'; + const aiEnabled = Boolean(aiKey) || allAi; + const fixtureKey = 'fixture-ai-credential-not-real'; + if (name === 'anthropic') { + assert.ok(aiKey, 'Anthropic development credential missing'); + } + let context; + try { + await scenario(packed, { + setupName: project, + realDocker: docker, + compose: stringify(compose), + networkModule: './liveNetwork.mjs', + sensitiveValues: aiKey ? [aiKey] : allAi ? [fixtureKey] : [], + environment: { COMPOSE_PROJECT_NAME: project, DOCKER_CONFIG: dockerConfig, DOCKER_HOST: dockerHost }, + prepare({ cwd }) { + if (name.startsWith('local_')) { + const target = join(cwd, 'cloned repos'); + mkdirSync(target); + if (name === 'local_root') { + execFileSync('git', ['clone', '--depth=1', 'https://github.com/octocat/Hello-World.git', join(target, 'hello')], { stdio: 'ignore', timeout: 45000 }); + } else { + for (const [index, relative] of (name === 'local_nested' ? ['one', 'nested/two'] : ['one', 'two']).entries()) { + // Sourcebot deduplicates clones sharing the same + // origin; use two genuinely different repositories. + const remote = index === 0 ? 'Hello-World' : 'git-consortium'; + execFileSync('git', ['clone', '--depth=1', `https://github.com/octocat/${remote}.git`, join(target, relative)], { stdio: 'ignore', timeout: 45000 }); + } + } + } + }, + async verifyDeployment({ setup, files, events }) { + const props = event => events.find(item => item.event === `setup_sourcebot_${event}`)?.properties; + assert.equal(props('completed').sourcebotStartOutcome, name === 'auto_start' ? 'spawned' : 'declined'); + const source = props('configured_code_source'); + assert.equal(source.codeHost, name.startsWith('local_') ? 'local_git' : name === 'remote_git' ? 'remote_git' : name === 'gitlab_project' ? 'gitlab' : name === 'gitea_repo' ? 'gitea' : name === 'gerrit_project' ? 'gerrit' : 'github'); + assert.equal(source.credentialMode, 'none'); + assert.equal(source.configurationIndex, 1); + assert.equal(source.repositoryCount, ['github_org', 'github_user', 'gitlab_project', 'gerrit_project'].includes(name) ? 0 : name.startsWith('local_') ? expectedRepositories : 1); + assert.equal(source.organizationCount, name === 'github_org' ? 1 : 0); + assert.equal(source.userCount, name === 'github_user' ? 1 : 0); + assert.equal(source.projectCount, ['gitlab_project', 'gerrit_project'].includes(name) ? 1 : 0); + assert.equal(source.generatedConnectionCount, name === 'local_nested' ? 2 : 1); + assert.equal(props('ai_setup_completed').aiConfigured, aiEnabled); + const aiCount = allAi ? providers.length : aiEnabled ? 1 : 0; + assert.equal(props('ai_setup_completed').aiConfigurationCount, aiCount); + assert.deepEqual(events.filter(event => event.event !== 'setup_sourcebot_failed').map(event => event.event.replace('setup_sourcebot_', '')), [ + 'started', 'chose_setup_directory', 'configured_code_source', 'configured_code_sources', + ...Array(aiCount).fill('configured_ai_provider'), 'ai_setup_completed', 'configured_hosted_url', + 'generated_configs', 'resolved_compose_file', 'validated_docker_state', 'completed', + ]); + report.telemetryContract = 'passed'; + chmodSync(join(setup, '.env'), 0o600); + const command = ['compose', '-p', project, '--project-directory', setup]; + run([...command, 'up', '-d', '--pull', 'never']); + const began = Date.now(); + while (true) { + try { + const response = await fetch(`${url}/onboard`, { signal: AbortSignal.timeout(2000) }); + if (response.ok) { + break; + } + } catch { /* Startup is asynchronous. */ } + assert.ok(Date.now() - began < 120000, 'Sourcebot did not become HTTP-ready within two minutes'); + await pause(1000); + } + report.httpReady = true; + report.stage = 'identity'; + const persisted = JSON.parse(run(['exec', `${project}-sourcebot`, 'cat', '/data/.sourcebot/.installedv3'])); + assert.equal(persisted.install_id, events[0].distinct_id); + report.installIdPreserved = true; + context = await browser.newContext(); + const page = await context.newPage(); + report.stage = 'onboarding'; + await page.goto(`${url}/onboard`); + await page.getByRole('link', { name: /Get Started/ }).click(); + await page.getByLabel('Email', { exact: true }).fill(`owner-${project}@example.invalid`); + await page.getByLabel('Password', { exact: true }).fill(`Test-${randomUUID()}!`); + await page.getByRole('button', { name: 'Sign up with credentials' }).click(); + await page.getByRole('link', { name: /Continue/ }).click({ timeout: 30000 }); + await page.getByRole('button', { name: /Skip for now|Continue to Sourcebot/ }).click({ timeout: 30000 }); + await page.waitForURL(current => !current.pathname.startsWith('/onboard')); + report.onboarding = 'passed'; + report.stage = 'indexing'; + const database = run([...command, 'ps', '-q', 'postgres']).trim(); + const started = Date.now(); + let counts; + while (true) { + counts = JSON.parse(run(['exec', database, 'psql', '-U', 'postgres', '-d', 'postgres', '-Atc', `SELECT json_build_object('discovered',count(*),'indexed',count("indexedAt")) FROM "Repo";`])); + if (counts.indexed === expectedRepositories && counts.discovered === expectedRepositories) { + break; + } + assert.ok(Date.now() - started < 180000, `Repository indexing did not complete: discovered=${counts.discovered}, indexed=${counts.indexed}`); + await pause(1000); + } + report.repositories = counts; + report.expectedRepositories = expectedRepositories; + report.stage = 'search'; + const searchQuery = name === 'github_org' ? 'color' : name === 'gitlab_project' ? 'test' : name === 'gitea_repo' ? 'gitea' : name === 'gerrit_project' ? 'readonly' : 'hello'; + const response = await page.request.post(`${url}/api/search`, { data: { query: searchQuery, matches: 10 } }); + assert.equal(response.status(), 200, 'Authenticated search failed'); + const search = await response.json(); + assert.ok(search.stats?.fileCount > 0, `No search matches; response fields: ${Object.keys(search).join(',')}`); + report.searchMatches = search.stats.fileCount; + await page.goto(`${url}/search?query=${encodeURIComponent(searchQuery)}`); + await page.getByText(/Found \d+ match/).first().waitFor({ timeout: 20000 }); + report.searchUi = 'passed'; + if (output) { + mkdirSync(output, { recursive: true }); + await page.screenshot({ path: join(output, `${name}.png`), fullPage: true }); + } + if (name === 'anthropic') { + const config = JSON.parse(files['config.json']); + assert.equal(config.models[0].provider, 'anthropic'); + const expectedEnv = dotenv(files['.env']); + assert.ok(expectedEnv.ANTHROPIC_API_KEY === aiKey, 'Generated AI credential differs from the supplied credential'); + const actual = run(['exec', `${project}-sourcebot`, 'node', '-e', 'process.stdout.write(process.env.ANTHROPIC_API_KEY || "")']); + assert.ok(actual === aiKey, 'Generated AI credential did not reach the container'); + report.ai = { configured: true, environmentVerified: true, askRequest: 'not_requested' }; + } + if (allAi) { + const config = JSON.parse(files['config.json']); + assert.deepEqual(config.models.map(model => model.provider), providers); + assert.deepEqual(events.filter(event => event.event === 'setup_sourcebot_configured_ai_provider').map(event => event.properties.provider), providers); + const expectedEnv = dotenv(files['.env']); + const mountedConfig = JSON.parse(run(['exec', `${project}-sourcebot`, 'cat', '/data/config.json'])); + assert.deepEqual(mountedConfig, config); + const actualEnv = JSON.parse(run(['exec', `${project}-sourcebot`, 'node', '-e', 'process.stdout.write(JSON.stringify(process.env))'])); + for (const key of Object.keys(expectedEnv)) { + assert.ok(actualEnv[key] === expectedEnv[key], `Generated environment mismatch for ${key}`); + } + for (const model of config.models) { + assert.equal(model.model, `fixture-model-${model.provider}`); + if (model.token) { + assert.ok(expectedEnv[model.token.env] === fixtureKey, 'Model credential reference mismatch'); + } + } + report.ai = { providersConfigured: providers.length, environmentVerified: true, credentials: 'synthetic_not_validated', askRequest: 'not_requested' }; + } + run([...command, 'restart', 'sourcebot']); + const restartBegan = Date.now(); + while (true) { + try { + if ((await fetch(`${url}/api/health`, { signal: AbortSignal.timeout(2000) })).ok) { + break; + } + } catch { /* Wait for the restarted application. */ } + assert.ok(Date.now() - restartBegan < 120000, 'Restart did not become HTTP-ready'); + await pause(1000); + } + const afterRestart = JSON.parse(run(['exec', `${project}-sourcebot`, 'cat', '/data/.sourcebot/.installedv3'])); + assert.equal(afterRestart.install_id, persisted.install_id); + const restartedSearch = await page.request.post(`${url}/api/search`, { data: { query: searchQuery, matches: 10 } }); + assert.equal(restartedSearch.status(), 200); + assert.ok((await restartedSearch.json()).stats?.fileCount > 0, 'Search stopped working after restart'); + report.restartIdentity = 'passed'; + report.status = 'passed'; + }, + async cleanupDeployment({ setup }) { + if (output && context && report.status !== 'passed') { + await context.pages()[0]?.screenshot({ path: join(output, `${name}-failure.png`), fullPage: true }).catch(() => {}); + } + await context?.close(); + if (existsSync(join(setup, 'docker-compose.yml'))) { + run(['compose', '-p', project, '--project-directory', setup, 'down', '--volumes', '--remove-orphans'], { timeout: 45000 }); + } + assert.equal(run(['ps', '-aq', '--filter', `label=com.docker.compose.project=${project}`]).trim(), ''); + assert.equal(run(['volume', 'ls', '-q', '--filter', `label=com.docker.compose.project=${project}`]).trim(), ''); + report.cleaned = true; + }, + }, async d => { + await d.answer('What directory would you like', project); + if (name.startsWith('local_')) { + await d.select('Which code host', 2); + await d.answer('Path to your repos directory', join(d.cwd, 'cloned repos', ...(name === 'local_root' ? ['hello'] : []))); + if (name !== 'local_root') { + await d.answer('Which repositories should be indexed?'); + } + } else if (name === 'remote_git') { + await d.select('Which code host', 3); + await d.answer('Git clone URL', 'https://github.com/octocat/Hello-World.git'); + } else if (name === 'gitea_repo') { + await d.select('Which code host', 6); + await d.answer('Gitea URL'); + await d.answer('Gitea Access Token'); + await d.check('What do you want to index?', [1]); + await liveMulti(d, 'Repositories to index', 'gitea/go-sdk'); + } else if (name === 'gerrit_project') { + await d.select('Which code host', 7); + await d.answer('Gerrit URL', 'https://gerrit-review.googlesource.com'); + await d.answer('Index all projects?', 'n'); + await liveMulti(d, 'Projects to index', 'plugins/readonly'); + } else if (name === 'gitlab_project') { + await d.select('Which code host', 1); + await d.answer('GitLab URL'); + await d.answer('GitLab Personal Access Token'); + await d.check('What do you want to index?', [1]); + await liveMulti(d, 'Projects to index', 'gitlab-org/gitlab-test'); + } else { + await d.select('Which code host'); + await d.answer('GitHub URL'); + await d.answer('GitHub Personal Access Token'); + await d.check('What do you want to index?', [name === 'github_org' ? 1 : name === 'github_user' ? 2 : 0]); + await liveMulti(d, name === 'github_org' ? 'Organizations to index' : name === 'github_user' ? 'GitHub users to index' : 'Repositories to index', name === 'github_org' ? 'chalk' : name === 'github_user' ? 'octocat' : 'octocat/Hello-World'); + } + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', aiEnabled ? 'y' : 'n'); + if (allAi) { + for (const [index, provider] of providers.entries()) { + await d.select('Which AI provider?', index); + await d.wait('Model name'); + await pause(500); + d.write(`fixture-model-${provider}`); + await pause(700); + d.write('\r'); + if (provider === 'openai-compatible') { + await d.answer('Base URL', 'https://fixture-ai.example.invalid/v1'); + } + if (provider === 'azure') { + await d.answer('Azure resource name', 'fixture-resource'); + await d.answer('API version'); + } + if (provider === 'amazon-bedrock') { + await d.answer('Use the default AWS credential chain?', 'n'); + await d.answer('AWS Access Key ID', 'fixture-access-id'); + await d.answer('AWS Secret Access Key', fixtureKey); + await d.answer('AWS region'); + } else if (provider.startsWith('google-vertex')) { + if (provider === 'google-vertex') { + await d.answer('Google Cloud project ID', 'fixture-project'); + await d.answer('Google Cloud region'); + } + await d.answer('Use Application Default Credentials?', 'y'); + } else { + await d.answer('API key (', fixtureKey); + } + await d.answer('Display name'); + await d.answer('Add another model?', index === providers.length - 1 ? 'n' : 'y'); + } + } + if (aiKey) { + await d.select('Which AI provider?'); + await d.wait('Model name'); + await pause(500); + d.write('claude-haiku-4-5'); + await pause(500); + d.write('\r'); + await d.answer('API key (', aiKey); + await d.answer('Display name'); + await d.answer('Add another model?', 'n'); + } + await d.answer('What URL will Sourcebot be hosted at?', url); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', name === 'auto_start' ? 'y' : 'n'); + if (name === 'auto_start') { + const began = Date.now(); + while (true) { + try { + if ((await fetch(`${url}/api/health`, { signal: AbortSignal.timeout(2000) })).ok) { + break; + } + } catch { /* Compose is starting the actual application. */ } + assert.ok(Date.now() - began < 120000, 'CLI-started deployment did not become healthy'); + await pause(1000); + } + report.cliStartedDeployment = true; + d.write('\x03'); + await d.finish(130); + assert.equal(d.events.filter(event => event.event === 'setup_sourcebot_completed').length, 1); + assert.equal(d.events.some(event => event.event === 'setup_sourcebot_cancelled'), false); + } + }); + } catch (error) { + report.status = 'failed'; + report.failure = (aiKey ? error.message.replaceAll(aiKey, '[redacted]') : error.message).slice(0, 500); + } + console.log(JSON.stringify(report)); + if (output) { + const path = join(output, 'live-results.json'); + const previous = existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : []; + writeFileSync(path, JSON.stringify([...previous.filter(item => !reports.some(report => report.scenario === item.scenario)), ...reports], null, 2)); + } + } + process.exitCode = reports.some(report => report.status !== 'passed') ? 1 : 0; +} finally { + await browser?.close(); + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/liveNetwork.mjs b/packages/setupWizard/tests/e2e/liveNetwork.mjs new file mode 100644 index 000000000..385cc8cbc --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveNetwork.mjs @@ -0,0 +1,21 @@ +// External test routing only. Production PostHog is always captured locally; +// live public code hosts and model catalogs retain their real network behavior. +import tls from 'node:tls'; +import { Agent, buildConnector, setGlobalDispatcher } from 'undici'; + +if (process.versions.bun) { + throw Error('Live tests require the actual Node runtime'); +} +const connect = buildConnector({}); +const permitted = new Set(['api.github.com', 'github.com', 'gitlab.com', 'gitea.com', 'codeberg.org', 'models.dev', 'gerrit-review.googlesource.com']); +setGlobalDispatcher(new Agent({ connect(options, callback) { + if (options.hostname === 'us.i.posthog.com' || options.hostname === 'raw.githubusercontent.com') { + const socket = tls.connect({ host: '127.0.0.1', port: Number(process.env.TEST_CAPTURE_PORT), servername: options.hostname }); + socket.once('secureConnect', () => callback(null, socket)); + socket.once('error', error => callback(error, null)); + } else if (options.protocol === 'https:' && permitted.has(options.hostname)) { + connect(options, callback); + } else { + callback(new Error('Live wizard egress outside the approved code hosts denied'), null); + } +} })); From f3c8a64f963ca2bfbc222c42e10fbe7df26f78eb Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:03:09 -0700 Subject: [PATCH 06/14] Record verified live setup deployment E2E results and remaining gates --- packages/setupWizard/telemetry.html | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/setupWizard/telemetry.html b/packages/setupWizard/telemetry.html index 0d4271db7..011985747 100644 --- a/packages/setupWizard/telemetry.html +++ b/packages/setupWizard/telemetry.html @@ -1743,7 +1743,21 @@

Reo coexistence

The release notes and telemetry documentation should state that setup-wizard PostHog telemetry is always on when its code executes, list the categories collected, explain that PostHog creates a Person profile keyed only by a random pseudonymous session/deployment ID, explain the new-deployment ID handoff, and distinguish PostHog from Reo’s separate tracking and variable. They must not describe the project ingestion token as a secret.

Testing plan

Implementation verification status

-

Live deployment suite: tests/e2e/liveDeployment.mjs builds and installs the npm tarball, drives the real interactive CLI, and starts the released Sourcebot v5.1.13 image with real PostgreSQL, Redis, migrations, backend, and web processes. It exercises public GitHub repository/organization/user scopes, GitLab projects, Gitea repositories, Gerrit projects, generic Git URLs, three local-clone layouts, automatic startup/interruption, and AI configuration. Each deployment uses an isolated Compose project and loopback port; only the test Compose networking/image settings and deployment telemetry opt-out are changed. Generated configuration, environment values, local mounts, and install-ID handoff remain intact. See tests/e2e/liveDeployment.md for reproduction and limits. A final clean rerun of all thirteen live scenarios is in progress; this does not replace the still-required full scenario/branch-coverage accounting.

+

Live deployment suite: tests/e2e/liveDeployment.mjs builds and installs the npm tarball, drives the real interactive CLI, and starts the released Sourcebot v5.1.13 image with real PostgreSQL, Redis, migrations, backend, and web processes. It exercises public GitHub repository/organization/user scopes, GitLab projects, Gitea repositories, Gerrit projects, generic Git URLs, three local-clone layouts, automatic startup/interruption, and AI configuration. Each deployment uses an isolated Compose project and loopback port; only the test Compose networking/image settings and deployment telemetry opt-out are changed. Generated configuration, environment values, local mounts, and install-ID handoff remain intact. See tests/e2e/liveDeployment.md for reproduction and limits. The final clean rerun passed all thirteen scenarios against implementation/test commit 7f491c19 and package SHA-256 c4c6157ceacf224e023bf8233dc79830b6683d05fcd614493071037f464eddab. This does not replace the still-required full scenario/branch-coverage accounting.

+
+Live E2E evidence — thirteen scenarios passed; no Ask requests +
    +
  • GitHub: explicit repository, organization (all 16 repositories), and user (all 8 repositories). GitLab project, Gitea repository, Gerrit project, and generic remote Git each indexed successfully.
    • +
  • Local repositories: a single root, two sibling clones represented by a wildcard, and two nested clones represented by separate connections. All expected repositories were indexed; distinct origins avoid Sourcebot's existing clone deduplication.
    • +
  • Every scenario passed owner onboarding, authenticated search API and visible browser search results, HTTP readiness and working search after restart, and persistent install-ID equality with the wizard session.
    • +
  • Automatic docker compose up launched by the CLI reached readiness; Ctrl+C exited with 130, retained exactly one completed event, and emitted no cancelled event after completion.
    • +
  • All twelve AI providers were configured together using synthetic credentials, with exact generated/mounted model configuration and environment propagation verified. A separate Anthropic run verified the available development key without printing it. No Ask or inference calls were made.
    • +
  • The real SDK's captured payloads passed the strict transport/schema/privacy contract, expected checkpoint order, source/provider counts, and identity assertions. This suite captures locally; the separate already-verified dev PostHog smoke remains ingestion/query evidence.
    • +
  • All task-owned containers, volumes, networks, temporary installations, and credentials were removed. Redacted reports/screenshots are outside the repository. The task-downloaded Sourcebot image is retained for the separately requested Vertex-fix task to reuse and clean afterward.
    • +
+

Harness corrections, not production regressions: wait for live autocomplete results; use distinct repository origins in multi-clone fixtures; assert the actual search summary instead of assuming a README result; and supply a credential-free Docker plugin/context configuration when isolating HOME. Cross-platform CI fixes also dispose Windows ConPTY resources, permit initial lockfile creation only in disposable Yarn test projects, and separate Node 24 CLI tests from the application's Node 20 job.

+

Remaining limits: no live authenticated Azure DevOps/Bitbucket or private/self-hosted-host access was available; fixture coverage is not equivalent to live integration verification. AI credential validity, inference, default credential discovery, and service-account-file access are not implied by startup. The pre-existing Google Vertex host-file/mount limitation is assigned to a separate user-requested task, which must validate its fix E2E and open a separate PR with a changelog edit. Full manifest/branch-coverage accounting and final CI evidence still gate overall feature completion; this PR remains draft.

+

Live AI validation scope (user clarification): verify that the packaged wizard writes the selected provider/model configuration and credential references correctly, that the generated environment values reach the resulting Sourcebot container unchanged, and that Sourcebot starts successfully. Do not invoke Ask or make model-inference requests. A Pro license is not required for this test scope. Use available development credentials without printing them; where credentials are unavailable, distinguish configuration/startup coverage with synthetic values from actual credential validation. Continue real repository discovery, indexing, search, browser access, and restart tests independently.

The feature is not release-complete until the full completion gate below is satisfied. Executable tests cover packed-package collectors, terminal paths, real SDK payloads, Docker branches, and actual container identity continuity. CI definitions are not evidence that Windows or other remote jobs have run.

Implementation-discovered changes for the behavioral baseline allowlist: status 130 for Ctrl+C; cancellation-aware readiness cleanup; explicit final process exit after bounded telemetry shutdown; recoverable autocomplete fallback on network/decode failure with an 8-second timeout; and fixed, credential-free SDK shutdown-timeout diagnostics where PostHog itself emits them. Do not patch SDK internals or silence the global console. Empty search/catalog results are not failures; actual infrastructure errors can emit recoverable diagnostics while retaining manual fallback.

From 43417abf3033ed67e83865a40633a991841cb5b5 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:05:13 -0700 Subject: [PATCH 07/14] Remove analytics documentation and keep telemetry planning local --- docs/docs/misc/telemetry.mdx | 24 +- packages/setupWizard/README.md | 18 - packages/setupWizard/telemetry.html | 2106 --------------------------- 3 files changed, 1 insertion(+), 2147 deletions(-) delete mode 100644 packages/setupWizard/telemetry.html diff --git a/docs/docs/misc/telemetry.mdx b/docs/docs/misc/telemetry.mdx index 47ee445b8..4a77674dd 100644 --- a/docs/docs/misc/telemetry.mdx +++ b/docs/docs/misc/telemetry.mdx @@ -6,7 +6,7 @@ By default, Sourcebot collects anonymized usage data through [PostHog](https://p The data we collect includes general usage statistics and metadata such as query performance (e.g., search duration, error rates) to monitor the application's health and functionality. This information helps us better understand how Sourcebot is used and where improvements can be made. -If you'd like to disable telemetry from the deployed Sourcebot application, you can do so by setting the environment variable `SOURCEBOT_TELEMETRY_DISABLED` to `true`: +If you'd like to disable all telemetry, you can do so by setting the environment variable `SOURCEBOT_TELEMETRY_DISABLED` to `true`: ```bash docker run \ @@ -20,25 +20,3 @@ If you disabled telemetry correctly, you'll see the following log when starting ```sh Disabling telemetry since SOURCEBOT_TELEMETRY_DISABLED was set. ``` - -## Setup wizard - -The `setup-sourcebot` npm wizard separately sends high-level progress events to -PostHog. These include selected code-host and AI-provider types, configuration -counts, coarse OS/architecture/Node information, and completion, cancellation, -or failure categories. The wizard does not send credentials, emails, repository -or model names, URLs, hostnames, local paths, configuration contents, or raw errors. -GeoIP enrichment is disabled. - -Each invocation generates a random UUID in memory, used as its PostHog identity. -For a new deployment, the same ID is saved as `SOURCEBOT_INSTALL_ID` in the existing -`.env` file, allowing setup and deployment events to be associated. Valid existing -deployment IDs are preserved without sending them in the new setup session. -PostHog's default Person processing is retained, but no real-world identity or -custom person properties are attached. No telemetry-only file or disk queue is created. - -Setup-wizard PostHog analytics has no opt-out and is independent of the deployed -application's `SOURCEBOT_TELEMETRY_DISABLED` setting. Delivery is best-effort; -telemetry failures do not prevent setup or keep the CLI running indefinitely. -The npm package's existing Reo installation tracking remains independent and -continues to use its own `PACKAGE_TRACKER_ANALYTICS` control. diff --git a/packages/setupWizard/README.md b/packages/setupWizard/README.md index 796483b5c..a27941cfd 100644 --- a/packages/setupWizard/README.md +++ b/packages/setupWizard/README.md @@ -20,24 +20,6 @@ The wizard walks you through: - Node.js 24+ - Docker and Docker Compose -## Setup analytics - -The wizard sends high-level setup progress to Sourcebot's PostHog project: selected -code-host/provider types, counts, coarse system properties, and setup outcomes. -It does not send access tokens, repository/model names, email addresses, URLs, -hostnames, local paths, or raw errors. GeoIP enrichment is disabled. - -Each invocation creates a random UUID in memory. New deployments receive that -same UUID as `SOURCEBOT_INSTALL_ID` in the existing `.env` file; a valid existing -ID is preserved. No telemetry state, identifier file, or disk queue is created. -Analytics failures do not prevent setup. The package's existing Reo installation -tracking remains separate; its `PACKAGE_TRACKER_ANALYTICS` setting and the -deployment's `SOURCEBOT_TELEMETRY_DISABLED` setting do not control wizard analytics. - -Ctrl+C exits with status 130. Once foreground Docker has spawned, setup is recorded -as completed; interrupting it cleans up the CLI without changing that setup outcome. -Completion means configuration handoff, not that Sourcebot is healthy or ready. - ## Development tests From the repository root, under Node 24: diff --git a/packages/setupWizard/telemetry.html b/packages/setupWizard/telemetry.html deleted file mode 100644 index 011985747..000000000 --- a/packages/setupWizard/telemetry.html +++ /dev/null @@ -1,2106 +0,0 @@ - - - - - - - setup-sourcebot telemetry proposal - - - -
- -
-
- - -
-

setup-sourcebot PostHog telemetry proposal

-

Status: Draft for review · Canonical plan

-

Source of truth: this HTML document is the authoritative proposal. Make future plan changes here first. telemetry.md is a convenience mirror and should only be regenerated from this page when explicitly needed.

-

Implementation status (September 11, 2026): the required repository-root entrypoint.sh change was merged through Sourcebot PR #1648 as commit a0ee2233, with all automated checks passing. Runtime support is therefore landed in the Sourcebot repository; setup-to-deployment continuity becomes available to users once the wizard selects a published Sourcebot image containing that merge. A container-level compatibility test verified first boot, same-version restart, upgrade restart, generated-ID fallback, telemetry-disabled behavior, PostHog-compatible HTTPS payloads, and stable identity propagation using the merged entrypoint. The setup-wizard implementation is in progress in the separate codex/setup-wizard-posthog worktree. It is not release-complete until every mandatory verification gate below has passed.

-

Executive summary

-

Instrument the setup-sourcebot CLI so Sourcebot can measure progression from wizard start through a completed setup, identify where users leave the flow, and understand high-level configuration choices without collecting repository information, credentials, real-world user identifiers, or other sensitive values.

-

The canonical product funnel is:

-
started
-  -> chose_setup_directory
-  -> configured_code_sources
-  -> ai_setup_completed (configured or explicitly skipped)
-  -> configured_hosted_url
-  -> generated_configs
-  -> resolved_compose_file
-  -> validated_docker_state
-  -> completed
-
-

The AI checkpoint must fire whether AI is configured or skipped. The compose and Docker checkpoints must likewise fire with an outcome describing success, an existing resource, an explicit skip, or a failure. This keeps valid branches in the funnel instead of reporting them as abandonment.

-

The recommended implementation upgrades setup-sourcebot to Node 24 LTS and uses the official posthog-node SDK behind a small package-owned privacy wrapper. At process start, the wizard creates one canonical lowercase UUIDv4 setupSessionId with crypto.randomUUID() and uses it as the PostHog distinctId. The normative application format is exactly ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$: 36 ASCII characters, lowercase hexadecimal, canonical hyphens, version 4, and the IETF UUID variant. RFC 9562 is the current UUID standard and supersedes RFC 4122; Node’s current API documentation still describes randomUUID() as generating an “RFC 4122 version 4 UUID.” These descriptions are compatible for this value, but the regex and byte-for-byte reuse rule are authoritative for this plan.

-

For a new generated deployment, that exact, unmodified value is written to .env as SOURCEBOT_INSTALL_ID. The required Sourcebot repository change to the root entrypoint.sh is merged in PR #1648. When built into a published Sourcebot container image, the script preserves the supplied ID on first boot. It does not validate, normalize, regenerate, uppercase, or otherwise transform a non-empty supplied value; the wizard is responsible for supplying the canonical UUIDv4. The setup wizard does not download, patch, or rewrite entrypoint.sh at runtime. Together, these changes allow setup events and resulting deployment telemetry to share the same installation identity without a telemetry-only state file. No email, name, account, or other PII is associated with it.

-

Goals

-
    -
  • Measure conversion from wizard start through completion.
    • -
  • Explain conversion using only coarse product choices, counts, branch outcomes, and runtime compatibility information.
    • -
  • Associate a successful first-time setup with the resulting Sourcebot deployment without storing telemetry-only state or linking the random ID to a real person, organization, repository, or machine.
    • -
  • Make the schema stable enough to build PostHog funnels and breakdowns before implementation ships.
    • -
  • Require a comprehensive isolated E2E suite against the compiled npm artifact—including all setup branches, exact PostHog contracts, behavioral regression checks, Docker identity continuity, and cleanup—before the feature can be marked complete.
    • -
-

Non-goals

-
    -
  • Product analytics after the deployed Sourcebot instance starts. Existing Sourcebot telemetry owns that lifecycle.
    • -
  • Capturing prompt text, user-entered values, configuration objects, errors, logs, or debugging traces.
    • -
  • Proving that Sourcebot became healthy. The v1 completion event records the wizard’s terminal handoff, not application readiness.
    • -
  • Exactly-once delivery. A CLI can be killed before a best-effort request completes.
    • -
  • Building a durable workflow or adding a queue. This is a local interactive CLI whose telemetry must remain non-blocking.
    • -
-

Decisions and rationale

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
DecisionRecommendationRationale
Installation boundaryDo not emit a Sourcebot PostHog event from package installation. Begin the funnel at started.This removes the need to correlate two processes or persist telemetry state. Reo’s existing installation tracker remains independent and unchanged.
Setup identityCreate one canonical lowercase UUIDv4 setupSessionId in memory at wizard startup with crypto.randomUUID(); require the exact plan regex and use the same string as the setup PostHog distinctId.One identifier is sufficient for the complete started -> completed funnel. It requires no telemetry-only persistence and cannot survive or combine separate wizard invocations.
Install-ID formatRequire exactly xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx, where every character is lowercase hexadecimal and y is 8, 9, a, or b. The complete validation expression is ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$.This is the canonical 36-character UUIDv4 representation produced by Node’s crypto.randomUUID() and observed from Sourcebot’s container uuidgen fallback. RFC 9562 is the current standard; it preserves the version-4 and IETF-variant layout formerly specified by RFC 4122. Defining the regex explicitly prevents prefixes, braces, uppercase conversion, hyphen removal, or other transformations.
PostHog person handlingUse setupSessionId as distinctId and keep PostHog’s default person-profile processing. Do not call identify or send $set/$set_once person properties.For a new setup, the same UUID becomes the deployment’s SOURCEBOT_INSTALL_ID. The profile remains pseudonymous and must never be enriched with email, name, organization, repository, credentials, or other PII/sensitive data.
ID storageDo not create any telemetry state outside the selected setup directory. Keep the ID in memory and write it only into the generated .env as SOURCEBOT_INSTALL_ID.The .env is already a required Sourcebot file. Reusing it avoids a second state location while giving the deployment the same identity as its setup session.
Deployment handoffUse a published Sourcebot image containing merged PR #1648, whose repository-root entrypoint.sh preserves a non-empty pre-supplied SOURCEBOT_INSTALL_ID on first boot and generates one with uuidgen only when none was supplied.This is a Sourcebot runtime/container-image change, not a script the wizard modifies. Released images without the PR overwrite the value on first boot and break setup-to-deployment correlation. After first boot, the existing /data/.installedv3 file remains authoritative.
TransportUse the official posthog-node SDK behind a package-owned typed privacy wrapper.The SDK owns delivery mechanics. The wrapper remains responsible for event allowlists, identity, privacy, and failure isolation.
Node runtimeRaise setup-sourcebot from the stale >=18 declaration to >=24.0.0, update @types/node to 24, and run its build, E2E tests, and release job on Node 24.Node 24 is the highest current LTS line. The wizard’s existing dependencies already require newer Node versions than its declared Node 18 floor, and all current runtime dependencies support Node 24. Staying on an LTS line avoids the churn of Node 26 Current while removing the need for a custom PostHog transport.
DeliveryUse posthog-node's asynchronous capture and transport defaults, then perform a bounded SDK shutdown when the process exits.No custom delivery policy is needed. The exit bound is the one CLI-specific choice because the SDK's default shutdown wait can be too long for an interactive command.
Stage timingEmit a checkpoint only when that stage reaches a terminal outcome.Prompt views or button clicks would overstate progress and make branches difficult to compare.
Optional stagesAlways emit the AI, compose, and Docker checkpoints with an explicit outcome.A valid skip or pre-existing resource should remain in the canonical funnel rather than appear as abandonment.
Repository countsSend exact counts for repositories the user selected; bucket only the total discovered by a local filesystem scan.Selected counts answer configuration-depth questions; discovered totals can expose unusually distinctive local environments without adding comparable value.
Completion boundaryEmit completion after Docker’s child process emits spawn, or after manual next steps are printed.Waiting for foreground docker compose up to exit can delay conversion for hours and still does not prove readiness.
ReadinessDefer a separate readiness event until there is a defined health-check requirement.Process launch and application health are different semantics and should not be mixed in one event.
Existing trackerRetain reo-census unchanged alongside the new PostHog telemetry.Reo remains an independent installation tracker with its own endpoint, payload, and PACKAGE_TRACKER_ANALYTICS control. Its events must not be imported into or treated as PostHog funnel events.
PostHog opt-outDo not provide an environment-variable or wizard-level opt-out for the new setup-wizard PostHog telemetry.npm does not require packages to provide a telemetry opt-out. PACKAGE_TRACKER_ANALYTICS remains Reo-only, and SOURCEBOT_TELEMETRY_DISABLED continues to govern deployed Sourcebot telemetry rather than this setup funnel. This policy still requires privacy/legal review and clear documentation before release.
-

Funnel strategy

-

Timestamp ordering: validated during implementation

-

The live dev-project smoke exposed an ordering issue: with immediate independent requests, PostHog adjusts each event by its request's sent_at clock skew, which can reorder closely spaced checkpoints. Set the documented $ignore_sent_at: true control on every setup event. Pass an SDK-envelope timestamp anchored to invocation-start wall time plus monotonic elapsed time, increasing by at least 1 ms per event. This preserves within-run order without delaying setup or serializing network requests. It adds no custom wall-clock application property and does not change transport retry defaults.

-

Tradeoff: absolute dates depend on the invoking machine's clock; durations and within-run order do not. Cross-machine chronological ordering against deployment events is not guaranteed when clocks are badly skewed. Verify stored event order in the live smoke, not only capture-call order. Reference: PostHog timestamp processing.

-

Event semantics

-
    -
  • Event names use snake_case, matching existing Sourcebot PostHog conventions.
    • -
  • Events use the setup_sourcebot_ prefix. The wa_ prefix is not appropriate because these events come from the CLI, not the web app.
    • -
  • Product funnel events fire at most once per setup session, after the corresponding stage reaches a terminal outcome.
    • -
  • Repeated detail events may fire once per configured code source or AI model configuration.
    • -
  • A telemetry failure must never block, delay materially, or fail setup.
    • -
  • Events use PostHog’s default person-profile behavior, keyed by the random setupSessionId. The CLI must never call identify, set custom person properties, or associate the profile with PII or sensitive data.
    • -
-

Feasibility finding

-

This identity handoff fits the existing Sourcebot path with one small deployment-runtime change:

-
    -
  • The generated deployment already loads .env into the Sourcebot container through docker-compose.yml’s env_file, so the wizard can pass SOURCEBOT_INSTALL_ID without changing the compose schema.
    • -
  • Sourcebot images built before PR #1648 replace SOURCEBOT_INSTALL_ID with uuidgen whenever /data/.installedv3 does not exist. The merged repository-root entrypoint.sh now lets a non-empty supplied value win on first boot. The image selected by the wizard must contain the merge before identity continuity is available to users.
    • -
  • Sourcebot’s current runtime and Lighthouse schemas accept a general non-empty string for the ID; they do not impose a stricter UUID validator. This proposal deliberately adopts the canonical format produced by Sourcebot’s own first-boot generator as the setup-wizard contract.
    • -
  • Backend deployment telemetry already uses SOURCEBOT_INSTALL_ID as distinctId, the install_id event property, and the PostHog company group.
    • -
  • Some web telemetry uses a browser- or user-level distinctId, but it still attaches the deployment install_id and company group. Cross-surface deployment analysis should therefore use those deployment dimensions.
    • -
-

Scope boundary: the wizard change ends after writing SOURCEBOT_INSTALL_ID to the generated .env. The root entrypoint.sh change is merged through Sourcebot PR #1648 and ships through the normal Sourcebot container-image build. The wizard never fetches or edits that script.

-

Conclusion: full identity continuity is achievable for deployments newly generated by the wizard. It requires coordinated setup-wizard and Sourcebot container-runtime changes, but does not require a second persisted ID, PostHog aliases, or changes to the deployed telemetry schema.

-

Normative install-ID compatibility contract

-

The wizard creates one canonical lowercase UUIDv4 before emitting started. It remains in memory for that invocation and is reused byte-for-byte as the resulting deployment ID for a new setup.

-

This is a MUST-level implementation requirement: setupSessionId must be the direct string returned by Node’s crypto.randomUUID() and must match ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$. The plan calls this a canonical lowercase UUIDv4 under RFC 9562, compatible with the RFC 4122 UUIDv4 terminology used by Node. The exact regex is normative if terminology differs between documentation versions.

-

The value must remain byte-for-byte identical across this chain:

-
crypto.randomUUID()
-  = setupSessionId
-  = every setup event distinctId
-  = every setup event install_id
-  = every setup event company group key
-  = generated .env SOURCEBOT_INSTALL_ID
-  = first-boot container SOURCEBOT_INSTALL_ID
-  = deployment install event distinct_id
-  = /data/.installedv3 install_id
-  = SOURCEBOT_INSTALL_ID restored on every later boot
-
-

No boundary may add a prefix, braces, quotes, whitespace, or a trailing newline; change letter case; remove hyphens; hash, encode, parse and reformat, or regenerate the value. The generated .env line is exactly SOURCEBOT_INSTALL_ID=<setupSessionId>. Canonical UUID characters require no shell or dotenv escaping.

-

Responsibility boundary: the wizard enforces and tests the UUIDv4 format. PR #1648 intentionally keeps the runtime handoff simple: on first boot, entrypoint.sh preserves any non-empty supplied value and calls uuidgen only when the value is missing or empty. It then safely serializes the selected value with jq. On later boots, /data/.installedv3 wins over the environment. Because the entrypoint does not normalize the supplied value, a conforming wizard value passes through unchanged.

-

Release compatibility gate: the default downloaded Compose file must select a published Sourcebot image containing PR #1648. Existing or user-customized Compose files are outside this compatibility guarantee: reuse them without image inspection, rewriting, warnings, or blocking setup. The wizard still writes the install ID normally, but setup-to-deployment continuity is not guaranteed if a user-selected image replaces it. This exception applies to all image-compatibility and deployment-continuity requirements below.

-

Identity and funnel correlation

- - - - - - - - - - - - - - - - - - - - - - - - - -
IdentifierLifetimePurpose
setupSessionIdOne CLI invocationCanonical lowercase 36-character UUIDv4 created with crypto.randomUUID() before started, conforming to the exact plan regex. It correlates every event in this wizard run.
PostHog distinctIdOne CLI invocation, then deployment identitySet to setupSessionId for every setup event. For a new setup, deployment backend and first-run events later use the same value as SOURCEBOT_INSTALL_ID.
PostHog install_id and company groupOne CLI invocation, then deployment identitySet to setupSessionId on setup events, matching Sourcebot’s existing deployment telemetry convention and enabling group-level continuity even where web-user distinctId values differ.
-

This supports two deliberately simple analyses:

-
    -
  1. Setup funnel: started to completed using setupSessionId as the PostHog person/distinct identity.
    2. -
  2. Setup-to-deployment continuity for a new setup: query the same UUID through setup events and later deployment events/properties/groups.
    3. -
-

Separate wizard invocations intentionally receive different IDs and are not merged. The plan does not attempt eventual conversion across retries because doing so would require persisted telemetry state or identity merging.

-

Recommended PostHog analyses

-
    -
  1. Canonical product funnel: started -> chose_setup_directory -> configured_code_sources -> ai_setup_completed -> configured_hosted_url -> generated_configs -> resolved_compose_file -> validated_docker_state -> completed.
    2. -
  2. Setup-to-deployment continuity: filter or break down setup and deployment events by the shared install_id property or company group.
    3. -
  3. AI branch comparison: break down ai_setup_completed by aiConfigured, then compare downstream completion rather than removing the AI checkpoint.
    4. -
  4. Completion breakdown: break down completed by completionMode, codeHostTypes, aiConfigured, and dockerValidationOutcome.
    5. -
  5. Drop-off diagnosis: compare the next missing checkpoint with cancelled and failed where recoverable: false, broken down only by their fixed stage and reason/category enums. Analyze failed with recoverable: true separately as friction that may precede further progress, completion, cancellation, or a later fatal error.
    6. -
-

The canonical setup funnel can use a standard ordered PostHog funnel because every event in one run has the same distinctId. For a new deployment, backend and container first-run events also use that ID. Browser events may use an anonymous browser or authenticated-user distinctId, but Sourcebot already attaches install_id and the company group, so deployment-level analysis should use those dimensions rather than assuming every web event shares the setup person ID.

-

PostHog event schema

-

Common properties

-

These properties are included on every event unless explicitly noted.

-

The system-derived properties are platform, arch, nodeMajorVersion, packageManager, and isCI. Collect each one independently and best-effort. Failure to read, detect, or parse any system-derived property must never suppress the event, throw into the wizard, change its exit status, or affect setup behavior. Use the fallback defined for that property below while retaining every other successfully collected property.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeDescription
schemaVersion1Version of this event contract.
source"setup-sourcebot-cli"Stable event source.
setupSourcebotVersionstringPublished setup-sourcebot package version.
setupSessionIdcanonical lowercase UUIDv4 stringCanonical lowercase UUIDv4 under RFC 9562, compatible with Node’s RFC 4122 terminology, matching ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$. The exact same string is also used as PostHog distinctId and, for a new generated deployment, SOURCEBOT_INSTALL_ID.
install_idstringSet to setupSessionId, matching Sourcebot deployment telemetry’s existing property name.
platform"darwin" | "linux" | "win32" | "other"Coarse operating-system family. Use "other" when unavailable or outside the allowlist.
arch"arm64" | "x64" | "other"Coarse processor architecture. Use "other" when unavailable or outside the allowlist.
nodeMajorVersionnumber | nullNode.js major version only. Use null if the value cannot be read or parsed; do not substitute 0 or a string.
packageManager"npm" | "yarn" | "pnpm" | "bun" | "unknown"Detected package manager. Use "unknown" when it cannot be detected or is outside the allowlist.
isCIboolean | nullWhether a recognized CI environment is active. Use null when detection cannot determine the answer; do not treat detection failure as false. Do not include CI vendor names or environment values.
elapsedMsnumberMilliseconds since the CLI invocation began.
$ignore_sent_attruePreserve ordered SDK-envelope timestamps instead of per-request clock-skew adjustment. Fixed ingestion control, never a wizard answer.
$geoip_disabletruePrevent IP-based geolocation enrichment.
-

Every capture also sets PostHog distinctId: setupSessionId and groups: { company: setupSessionId }. This mirrors Sourcebot’s deployment-level grouping without adding another identifier.

-

Do not add working directory, OS release, CPU count, hostname, IP-derived location, Git configuration, npm username, or arbitrary environment variables as common properties.

-

Product funnel event catalog

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Funnel conceptProposed eventFires when
startedsetup_sourcebot_startedThe CLI process starts, before the first prompt.
chose setup directorysetup_sourcebot_chose_setup_directoryThe directory is created or the user agrees to reuse an existing directory.
configured code sourcessetup_sourcebot_configured_code_sourcesThe user finishes the one-or-more code-source loop.
optional AI setupsetup_sourcebot_ai_setup_completedThe user finishes AI configuration or explicitly skips it.
configured hosted URLsetup_sourcebot_configured_hosted_urlA valid hosted URL is accepted.
generated configssetup_sourcebot_generated_configsAll required configuration files are written successfully.
downloaded compose filesetup_sourcebot_resolved_compose_fileThe compose-file stage ends with a download, existing file, explicit decline, or failure.
Docker state validatedsetup_sourcebot_validated_docker_stateDocker validation and optional cleanup finish, or validation is skipped for a known reason.
complete setupsetup_sourcebot_completedThe wizard hands off to Docker or prints actionable manual next steps.
-

setup_sourcebot_resolved_compose_file is intentionally named “resolved” rather than “downloaded.” A pre-existing compose file is a successful outcome, while a declined or failed download still allows the wizard to finish with manual next steps.

-

Event definitions

-

setup_sourcebot_started

-

Purpose: establish the start of the setup funnel and its in-memory session/deployment identity.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
invocationMethod"npx" | "global_binary" | "local_binary" | "workspace" | "unknown"yesHow the CLI appears to have been launched.
isInteractivebooleanyesWhether stdin and stdout are interactive terminals.
-

setup_sourcebot_chose_setup_directory

-

Purpose: measure progression through the first prompt and whether setup is new or overwriting an existing directory.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
usedDefaultDirectorybooleanyesWhether the default sourcebot directory was accepted.
directoryExistedbooleanyesWhether the selected directory already existed.
directoryAction"created" | "existing_directory_accepted"yesThe resulting directory path branch.
-

Never include the entered path, its basename, its parent, or any information about files already in the directory.

-

setup_sourcebot_configured_code_source

-

This is a repeated diagnostic event, emitted once after each code-source configuration is accepted. It is not a required step in the product funnel, but it makes the aggregate checkpoint explainable.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
configurationIndexnumberyesOne-based position in the code-source configuration loop.
codeHost"github" | "gitlab" | "bitbucket" | "gitea" | "azure_devops" | "gerrit" | "local_git" | "remote_git"yesConfigured source type.
deploymentType"cloud" | "self_hosted" | "local" | "remote" | "unknown"yesCoarse hosting mode derived locally from the source branch and existing wizard answers. Use "unknown" when the available answers cannot distinguish the mode.
credentialMode"none" | "personal_access_token" | "api_token" | "access_token" | "app_password" | "http_access_token"yesCredential mechanism, never the credential value.
scopeTypesfixed enum arrayyesSubset of all, repositories, organizations, users, groups, projects, and workspaces.
indexAllbooleanyesWhether every repository visible to the credential was selected.
repositoryCountnumberyesExplicit remote repositories or selected local repositories; use 0 when not applicable.
organizationCountnumberyesSelected organizations; use 0 when not applicable.
userCountnumberyesSelected users; use 0 when not applicable.
groupCountnumberyesSelected groups; use 0 when not applicable.
projectCountnumberyesSelected projects; use 0 when not applicable.
workspaceCountnumberyesSelected Bitbucket workspaces; use 0 when not applicable.
generatedConnectionCountnumberyesNumber of config connections generated by this selection.
localDiscoveredRepoCountBucket"1" | "2-5" | "6-20" | "21-100" | "101+" | nullyesBucketed number of discovered local repositories. null for non-local sources.
-

Rules by source:

-
    -
  • For URL-based classification, trim surrounding whitespace. If the value has no URI scheme, prepend https:// for classification only; this does not rewrite the user’s configuration. Accept only http: or https:, parse with Node’s URL, lowercase URL.hostname, remove trailing DNS dots, and remove one leading www.. URL.hostname naturally excludes the protocol, credentials, port, path, query, and fragment, so inputs such as github.com, www.github.com, https://github.com, and https://www.github.com:443/path all classify from the same normalized hostname. Do not perform DNS, HTTP, IP, or ownership lookups. Never retain or send the hostname, URL, path, port, or parse error.
    • -
  • Match provider domains only by exact normalized hostname or a dot-boundary suffix that includes the leading dot, such as .ghe.com. Never use substring matching: a hostname such as notgithub.com must not match github.com.
    • -
  • GitHub is cloud when the normalized hostname is exactly github.com or ends in .ghe.com; other valid hostnames are self_hosted. The .ghe.com rule covers GitHub Enterprise Cloud with data residency.
    • -
  • GitLab is cloud when the normalized hostname is exactly gitlab.com, ends in .gitlab-dedicated.com, or ends in .gitlab-dedicated.systems. Other valid hostnames are unknown, because GitLab Dedicated supports arbitrary custom domains that cannot be distinguished locally from GitLab Self-Managed without adding a new prompt or network lookup.
    • -
  • Bitbucket uses the existing explicit Cloud versus Data Center wizard answer: Cloud maps to cloud and Data Center maps to self_hosted.
    • -
  • Azure DevOps uses the existing explicit Cloud versus Server wizard answer: Cloud maps to cloud and Server maps to self_hosted. Do not report organization names, server URLs, or whether /tfs appears in a real URL.
    • -
  • Gitea is cloud when the normalized hostname is exactly gitea.com; other valid hostnames are self_hosted.
    • -
  • Gerrit always maps to self_hosted.
    • -
  • The Local Git source branch always maps to local. Report selected repository counts and a bucketed discovered count, never paths or repository names.
    • -
  • The arbitrary Remote Git source branch always maps to remote. Report one repository, never its clone URL.
    • -
  • An unparseable URL, missing collector answer, unsupported source type, or unexpected value maps to unknown. The event must still be captured.
    • -
-

setup_sourcebot_configured_code_sources

-

Purpose: product-funnel checkpoint after all code sources have been configured.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
codeSourceConfigurationCountnumberyesNumber of passes through the code-source loop.
generatedConnectionCountnumberyesTotal generated config.json connections. This can exceed the loop count for local repositories.
uniqueCodeHostCountnumberyesNumber of unique code-host types.
codeHostTypesfixed enum arrayyesDeduplicated codeHost values, sorted for stable payloads.
credentialedCodeSourceCountnumberyesConfigured sources with a credential present.
cloudCodeSourceCountnumberyesSources configured against known cloud services.
selfHostedCodeSourceCountnumberyesSources configured against self-hosted services.
localCodeSourceCountnumberyesLocal directory selections.
indexAllCodeSourceCountnumberyesSources configured to index everything visible.
repositoryCountnumberyesTotal explicit remote and selected local repositories.
organizationCountnumberyesTotal selected organizations.
userCountnumberyesTotal selected users.
groupCountnumberyesTotal selected groups.
projectCountnumberyesTotal selected projects.
workspaceCountnumberyesTotal selected Bitbucket workspaces.
-

Do not serialize or spread connections into telemetry. Construct this event from an explicit allowlisted summary object.

-

setup_sourcebot_configured_ai_provider

-

This is a repeated diagnostic event, emitted once per configured model. A user can configure multiple models for the same provider.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
configurationIndexnumberyesOne-based position in the model configuration loop.
providerfixed enumyesOne of anthropic, openai, openai-compatible, amazon-bedrock, google-generative-ai, google-vertex, google-vertex-anthropic, azure, deepseek, mistral, openrouter, or xai.
modelSelectionMethod"catalog" | "custom_entry" | "manual_fallback"yesWhether the model came from the catalog, a custom value entered through catalog search, or manual input after catalog failure/unavailability.
credentialMode"api_key" | "aws_default_chain" | "aws_explicit_keys" | "google_application_default_credentials" | "google_credentials_file"yesCoarse credential strategy.
usesCustomEndpointbooleanyesTrue only for an OpenAI-compatible custom endpoint.
hasDisplayNamebooleanyesWhether the optional display-name field was populated.
-

Never include model names or IDs, display names, API keys, resource names, base URLs, cloud project IDs, regions, API versions, access-key IDs, or credential-file paths.

-

setup_sourcebot_ai_setup_completed

-

Purpose: preserve a linear funnel while distinguishing configured and intentionally skipped AI setup.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
aiConfiguredbooleanyesWhether at least one model was configured.
aiConfigurationCountnumberyesNumber of configured models; 0 when skipped.
uniqueProviderCountnumberyesNumber of unique provider types; 0 when skipped.
providerTypesfixed enum arrayyesDeduplicated provider values, sorted; empty when skipped.
usesCustomEndpointbooleanyesWhether any OpenAI-compatible endpoint was configured.
credentialModesfixed enum arrayyesDeduplicated credential strategies, sorted; empty when skipped.
modelSelectionMethodsfixed enum arrayyesDeduplicated selection methods, sorted; empty when skipped.
-

setup_sourcebot_configured_hosted_url

-

Purpose: measure progression through deployment URL configuration without collecting the URL.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
usedDefaultUrlbooleanyesWhether http://localhost:3000 was accepted unchanged.
protocol"http" | "https"yesURL protocol only.
hostCategory"localhost" | "address" | "unknown"yesLocal structural classification that distinguishes confirmed loopback hosts from every other successfully parsed address.
-

Determine hostCategory only from the already-entered hosted URL. Parse it with Node’s URL, lowercase the hostname, remove trailing DNS dots, and remove IPv6 URL brackets before applying these rules:

-
    -
  • localhost: the parsed hostname is exactly localhost, ends in .localhost, is an IPv4 address in 127.0.0.0/8, or is the IPv6 loopback address ::1.
    • -
  • address: the URL parses successfully and has any other non-empty hostname or IP address. This value makes no claim about whether the address is public, private, reachable, or resolvable.
    • -
  • unknown: the URL or hostname is unavailable, cannot be parsed, or the classifier fails unexpectedly. The event must still be captured.
    • -
-

This classifier performs no DNS, HTTP, socket, IP ownership, or reachability lookup. It must discard the parsed value after producing the fixed category.

-

Never include the URL, hostname, domain, path, query, fragment, or port.

-

setup_sourcebot_generated_configs

-

Purpose: record that the configuration-writing stage completed successfully.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
filesWrittenfixed enum arrayyesSubset of config_json, env, and compose_override.
overwroteExistingFilesfixed enum arrayyesExisting fixed file types the user agreed to overwrite. Empty for a new setup.
wroteComposeOverridebooleanyesWhether local-repository mounts required an override file.
localMountCountnumberyesNumber of local root directories mounted.
generatedConnectionCountnumberyesNumber of connections written to config.json.
aiConfigurationCountnumberyesNumber of model configurations written.
credentialVariableCountnumberyesCount of credential environment variables written, never their names or values.
deploymentIdentityAction"created_from_setup_session" | "preserved_existing"yesWhether the generated deployment received this run’s setupSessionId or retained a valid ID from an existing .env. The existing ID itself is never copied into setup telemetry.
-

The event fires only after every required write succeeds. It must not include file paths, file contents, environment-variable names, or generated secrets.

-

setup_sourcebot_resolved_compose_file

-

Purpose: represent every terminal branch of the compose-file stage.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
outcome"downloaded" | "already_present" | "declined" | "download_failed"yesResult of the compose-file stage.
composeAvailablebooleanyesWhether a compose file is available for later validation and startup.
downloadPromptShownbooleanyesWhether the download prompt was shown.
downloadAttemptedbooleanyesWhether an HTTP download was attempted.
failureCategory"network" | "http_4xx" | "http_5xx" | "filesystem" | "timeout" | "unknown" | nullyesCoarse failure reason. null unless outcome is download_failed.
-

Never include the download URL, HTTP body, destination path, or raw error message.

-

setup_sourcebot_validated_docker_state

-

Purpose: measure whether the generated setup can be started immediately and how often existing Docker state requires intervention.

-

This event must fire even when the phase is skipped because no compose file is available or because an existing deployment is intentionally left running.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
outcome"passed" | "passed_after_cleanup" | "unresolved_conflicts" | "skipped_no_compose" | "skipped_existing_deployment_running" | "validation_failed"yesOverall result of the Docker validation phase.
dockerStatus"available" | "unavailable" | "error" | "not_checked"yesWhether Docker commands could be executed successfully.
composeContainerState"none" | "running" | "stopped" | "mixed" | "unknown"yesAggregate state, never container names.
runningComposeContainerCountnumber | nullyesCount of running containers belonging to the compose project.
stoppedComposeContainerCountnumber | nullyesCount of stopped containers belonging to the compose project.
existingVolumeCountnumber | nullyesCount of matching existing volumes.
initialPortConflictCountnumber | nullyesConflicts found before remediation. Do not include port numbers.
remainingPortConflictCountnumber | nullyesConflicts remaining after remediation.
portConflictSource"none" | "docker" | "non_docker" | "mixed" | "unknown"yesCoarse owner category.
existingDeploymentAction"none" | "stopped" | "left_running" | "stop_failed"yesOutcome of the running-deployment prompt.
stoppedContainerAction"none" | "removed" | "kept" | "remove_failed"yesOutcome of stopped-container cleanup.
volumeAction"none" | "removed" | "kept" | "remove_failed"yesOutcome of existing-volume cleanup.
portConflictAction"none" | "containers_stopped" | "kept" | "stop_failed"yesOutcome of port-conflict cleanup.
leftExistingDeploymentRunningbooleanyesWhether an existing deployment was intentionally or unsuccessfully left running.
-

Unknown and skipped measurements: all five Docker count properties remain required, but use null when their check was skipped, failed, or could not produce a complete result. Use 0 only after a successful check confirmed zero matching resources/conflicts (including successfully parsing a Compose file with no published ports). Never coerce unknown values to zero in completion summaries or PostHog analyses. Preserve independent successful measurements when another check fails.

-
    -
  • Container counts and composeContainerState describe the initial successful Compose inventory, before cleanup. Volume count describes the inventory before volume cleanup. An unavailable inventory produces null counts and, for containers, unknown state.
    • -
  • initialPortConflictCount and portConflictSource describe the first complete port check, after deployment/volume cleanup and before port-conflict remediation. If the necessary Docker-owner or socket checks fail, use null and unknown rather than claiming a complete inventory. remainingPortConflictCount describes the recheck after attempted port remediation; use null if that recheck fails. If no port remediation is attempted, reuse the initial count, including null.
    • -
  • With no Compose file, emit skipped_no_compose, dockerStatus: "not_checked", all five counts as null, container state and conflict source as unknown, actions as none, and leftExistingDeploymentRunning: false.
    • -
  • When an existing deployment is left running, retain the measured initial container inventory; skipped volume and port checks stay null and conflict source stays unknown. Set leftExistingDeploymentRunning: true and retain the actual deployment action, including stop_failed when applicable.
    • -
  • Action none means no action was offered or attempted, including skipped checks; it does not assert that no resources existed. kept and left_running mean the user explicitly declined the corresponding action.
    • -
  • dockerStatus is not_checked if no Docker check ran, unavailable if a check established a missing/inaccessible executable, Compose plugin, daemon, or socket, error if attempted checks cannot establish availability, and available when availability was confirmed. An individual cleanup command failure does not by itself mean Docker is unavailable.
    • -
-

Overall outcome precedence: first use skipped_no_compose when applicable. Otherwise, any required inspection or attempted cleanup failure yields validation_failed, even when later checks succeed or conflicts also exist. With no such failure, use skipped_existing_deployment_running for a deliberately retained running deployment, then unresolved_conflicts for remaining measured port conflicts, then passed_after_cleanup if cleanup occurred successfully, otherwise passed. Action fields and counts retain the additional detail. This classification does not alter prompts, Docker actions, or continuation behavior; operation failures also emit the nonterminal diagnostics defined below.

-

Never include container names, service names, volume names, Docker project names, port numbers, process information, compose contents, or raw command output.

-

setup_sourcebot_completed

-

Purpose: final funnel conversion event. “Completed” means the wizard finished its work and either handed off to Docker or printed sufficient manual next steps; it does not claim that Sourcebot became healthy.

-

Additional properties:

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
completionMode"sourcebot_start_spawned" | "sourcebot_start_failed" | "existing_deployment_left_running" | "manual_start_required"yesHow the wizard ended.
sourcebotStartOfferedbooleanyesWhether the final start prompt was shown.
sourcebotStartRequestedbooleanyesWhether the user chose to run docker compose up.
sourcebotStartOutcome"spawned" | "declined" | "not_offered" | "spawn_failed"yesResult of the start handoff. This describes process launch, not application readiness.
composeAvailablebooleanyesWhether the compose file was available at completion.
dockerValidationOutcomeenum from validation eventyesFinal Docker validation result.
remainingPortConflictCountnumber | nullyesCopy the Docker validation value unchanged, including null when unmeasured. Do not replace unknown with zero.
generatedConnectionCountnumberyesTotal generated code connections.
codeHostTypesfixed enum arrayyesDeduplicated configured code-host types.
repositoryCountnumberyesTotal explicit remote and selected local repositories.
aiConfiguredbooleanyesWhether any AI model was configured.
aiConfigurationCountnumberyesNumber of configured models.
providerTypesfixed enum arrayyesDeduplicated AI provider types.
deploymentIdentityAction"created_from_setup_session" | "preserved_existing"yesSafe summary of whether this run established deployment identity continuity. This repeats the generated-config value so completion can be filtered directly.
totalDurationMsnumberyesDuration from CLI start to this terminal handoff.
-

When the user chooses to start Sourcebot, emit this event after the child process successfully emits spawn, not after foreground docker compose up exits. Waiting for process exit can delay the event until the user stops Sourcebot hours later or presses Ctrl+C.

-

If actual readiness is important, add a separate setup_sourcebot_became_ready event when the local readiness poll succeeds. Do not overload setup_sourcebot_completed with a health claim.

-

Drop-off diagnostic events

-

These events are not funnel checkpoints, but they are necessary to explain missing next-step events.

-

setup_sourcebot_cancelled

- - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
stagefixed enumyesOne of setup_directory, code_sources, ai_setup, hosted_url, config_overwrite, compose_file, docker_validation, or start.
reason"keyboard_interrupt" | "existing_directory_declined" | "config_overwrite_declined"yesAllowlisted cancellation reason.
-

The config-overwrite case may additionally include fileType: "config_json" | "env" | "compose_override". Never include a path or filename supplied by the user.

-

setup_sourcebot_failed

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
PropertyTypeRequiredDescription
stagesame stage enum as cancellationyesStage active when the failure occurred.
failureCategoryfixed enumyesOne of validation, network, filesystem, docker_unavailable, docker_command, process_spawn, or unknown.
recoverablebooleanyestrue when the wizard handles the error and continues to further setup work or manual next steps; false only when the error causes the wizard to terminate. This describes actual control flow, not whether the underlying problem could theoretically be fixed.
-

Failure-event lifecycle: setup_sourcebot_failed with recoverable: true is a nonterminal diagnostic event. Emit it once per distinct handled operation failure; multiple such events may occur in one stage or session, including with identical stage/category values. Keep the same session ID and the SDK active so later checkpoints and diagnostics can be captured. A stage outcome may summarize the same failure without emitting a second failure event for that operation. Only failed with recoverable: false ends the setup telemetry chain due to an error.

-

These categories intentionally stay coarse and cover the wizard’s current fatal and recoverable failure paths:

-
    -
  • validation: malformed or unsupported user/configuration data that escapes normal prompt validation, including local parsing or invariant failures. Ordinary rejected prompt input is not a failure event because the prompt remains open.
    • -
  • network: a required fetch, HTTP response, timeout, or response-decoding operation fails. Best-effort autocomplete/model-catalog failures that successfully fall back do not emit setup_sourcebot_failed.
    • -
  • filesystem: directory creation/change, local repository inspection, configuration or compose-file read/write, permissions, missing files, or storage-capacity operations fail.
    • -
  • docker_unavailable: the Docker executable, Compose plugin, daemon, or daemon socket is unavailable or inaccessible.
    • -
  • docker_command: Docker is available, but a Docker/Compose inspect, stop, remove, volume, or startup command fails or returns unusable output.
    • -
  • process_spawn: a required child process fails to spawn and the failure is not more specifically docker_unavailable. A missing Docker executable therefore remains docker_unavailable.
    • -
  • unknown: any remaining unexpected runtime, prompt-library, cryptography, serialization, dependency, or programming failure. This catch-all ensures telemetry classification never replaces or masks the original wizard behavior.
    • -
-

Classify from the operation that failed and fixed error metadata such as a known operation result or error code; never inspect or transmit free-form exception messages, command output, paths, URLs, or user input. When more than one category appears applicable, prefer the most specific category above rather than emitting multiple failure events.

-

Do not send exception messages, stack traces, command lines, stderr, HTTP bodies, or arbitrary error codes. If more detail is needed, add a reviewed fixed enum rather than forwarding runtime text.

-

Privacy and data minimization

-

Explicitly allowed

-
    -
  • Fixed enums describing product choices.
    • -
  • Booleans describing whether optional capabilities or credentials were configured.
    • -
  • Counts of selected configuration entities.
    • -
  • Bucketed counts where the exact value is not needed, such as repositories discovered during a local filesystem scan.
    • -
  • Coarse runtime compatibility fields: OS family, architecture family, Node major version, and package manager.
    • -
  • Random UUIDs generated solely for pseudonymous telemetry correlation and PostHog’s default Person profile.
    • -
-

Explicitly prohibited

-
    -
  • Access tokens, API keys, passwords, generated secrets, credential contents, or arbitrary environment variables.
    • -
  • Repository, organization, group, project, workspace, or user names.
    • -
  • Search/autocomplete input.
    • -
  • Git clone URLs, code-host URLs, hosted Sourcebot URLs, custom AI endpoint URLs, domains, hostnames, IP addresses, or port numbers.
    • -
  • Setup directories, working directories, local repository paths, credential-file paths, filenames derived from user input, or file contents.
    • -
  • Email addresses or domains, Git usernames, npm usernames, OS usernames, or machine hostnames.
    • -
  • Model names/IDs, display names, cloud project/resource names, regions, or API versions.
    • -
  • Container, service, volume, process, or Docker project names.
    • -
  • Raw errors, stack traces, command output, or HTTP response bodies.
    • -
  • Sourcebot configuration objects or environment maps, even after attempted redaction.
    • -
  • PostHog identify, alias, $set, $set_once, or any custom Person property that could enrich or link the random session/deployment profile to a real person, organization, repository, or machine.
    • -
-

Every event must be built from an event-specific allowlisted object. Code must never spread prompt results, connection configs, model configs, environment maps, process.env, errors, or Docker command results into event properties.

-

The SDK wrapper does not deliberately send an IP address, hostname, or user agent as an event property, and disableGeoip: true prevents PostHog GeoIP enrichment. As with any direct HTTPS request, the receiving endpoint can observe the source IP at the transport layer. If policy requires that PostHog never receive the client IP at all, direct client-side capture is insufficient and the design must instead use a reviewed first-party relay. That stricter requirement is not assumed by this proposal.

-

Telemetry controls and delivery

-
    -
  • The new setup-wizard PostHog telemetry has no package-level opt-out. It does not inspect SOURCEBOT_TELEMETRY_DISABLED or PACKAGE_TRACKER_ANALYTICS before creating its random ID or sending events.
    • -
  • PACKAGE_TRACKER_ANALYTICS remains owned and interpreted only by reo-census. Setting it to false disables Reo’s tracking but does not disable PostHog setup-wizard telemetry.
    • -
  • SOURCEBOT_TELEMETRY_DISABLED remains applicable to telemetry from the deployed Sourcebot product. It does not disable the setup-wizard PostHog funnel proposed here.
    • -
  • Use the SDK's non-blocking capture queue during the wizard and a bounded SDK shutdown at terminal events.
    • -
  • Do not add a custom telemetry transport around posthog-node.
    • -
  • Suppress telemetry transport errors unless verbose debugging is explicitly enabled; never print payloads because future payload changes could expose data in terminal logs.
    • -
  • Document the always-on setup-wizard PostHog telemetry, its collected data categories, Person-profile behavior, in-memory session identity, deployment-ID handoff, and separation from Reo before release.
    • -
-

Existing reo-census tracker

-

The package currently depends on reo-census, whose install hook sends data to Reo rather than PostHog. Its default payload includes the working directory, Git username, email domain, npm username when available, detailed OS information, and CPU count. Full-data mode can include a full email address and dependency lists.

-

Reo remains installed and continues its existing behavior, including its own PACKAGE_TRACKER_ANALYTICS variable. The PostHog allowlist in this proposal governs only the new Sourcebot-owned PostHog events; it does not modify or make claims about Reo’s separate payload. The two systems must use separate event definitions and reporting so Reo installation data is not mistaken for the Sourcebot PostHog funnel.

-

Implementation proposal

-

Architecture

-

Keep telemetry isolated from prompt and configuration objects:

-
setup-sourcebot CLI
-        |
-        +--- create in-memory UUIDv4 setupSessionId
-        |
-        +--- typed, allowlisted PostHog events
-        |          distinctId = setupSessionId
-        |          install_id/company = setupSessionId
-        |
-        +--- generated .env
-                   SOURCEBOT_INSTALL_ID = setupSessionId
-                              |
-                              +--- Sourcebot first boot preserves the ID
-                              +--- deployed telemetry continues on that ID
-
-

The implementation does not need a postinstall hook, telemetry state file, background daemon, local queue, or server-side proxy. Delivery is best-effort from the short-lived wizard process. The only persisted identifier is SOURCEBOT_INSTALL_ID in the generated deployment configuration, where it is operational Sourcebot state rather than setup-wizard telemetry state.

-

Proposed files and responsibilities

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
FileChangeResponsibility
src/telemetry.tsaddCreate the in-memory setupSessionId, create common properties, initialize posthog-node, send allowlisted events, and perform bounded shutdown.
src/telemetryEvents.tsaddDefine event-name-to-property TypeScript types, fixed enums, schema version, and pure event payload builders.
bin.cjsaddPerform a dependency-free Node 24 runtime check before dynamically importing dist/index.js, ensuring unsupported runtimes fail with a clear message before prompts, telemetry, file writes, or other side effects.
src/index.tsmodifyCreate the setup session, track the active stage, emit aggregate checkpoints, write or preserve SOURCEBOT_INSTALL_ID in .env, and route terminal paths through bounded flush helpers.
src/models.ts and code-host collectorsmodifyReturn non-sensitive telemetry summaries beside generated config, built from already-known branch choices and counts.
Docker helper module(s)modifyReturn structured outcomes instead of swallowing command failures that need to be distinguished from an empty Docker state. Never expose stderr to telemetry.
Repository-root entrypoint.shmergedPR #1648 implements the Sourcebot runtime change built into subsequent published container images: on first boot, preserve a non-empty pre-supplied SOURCEBOT_INSTALL_ID; generate a UUID only when one was not supplied. It safely serializes the selected ID into /data/.installedv3 and telemetry JSON, propagates uuidgen failures, and includes restart regression coverage for JSON-special characters. This file is not owned, downloaded, or modified by the setup wizard at runtime.
package.jsonmodifySet engines.node to >=24.0.0, point bin at bin.cjs, update @types/node to 24, add posthog-node, publish the runtime bootstrap, and retain reo-census and its existing hook unchanged.
packages/setupWizard/tests/e2e/addOwn the compiled-artifact builder, PTY driver, scenario manifest, prompt fixtures, PostHog-compatible TLS capture service, fake Docker executable/state machine, filesystem fixtures, payload oracle, privacy scanner, cleanup audit, and redacted completion reporter.
packages/setupWizard/tests/integration/addExercise the real posthog-node client and package-owned wrapper against local capture fixtures, including transport degradation and exact envelope/property validation. These tests complement but do not replace packed CLI E2E runs.
.github/workflows/setup-wizard-e2e.ymlmodifyAdd the required setup-wizard-e2e jobs: exhaustive Linux/Node 24 packed-artifact and Docker coverage, macOS/Windows Node 24 smoke coverage, and intentional Node 22.22 rejection. Upload only redacted failure diagnostics and cleanup reports.
.github/workflows/release-setup-sourcebot.ymlmodifyChange the setup-sourcebot release runtime from Node 20 to Node 24, build one candidate tarball, and require packed-artifact smoke, identity-continuity, PostHog contract, and cleanup gates against that artifact before publishing.
package README and docs/docs/misc/telemetry.mdxmodifyDescribe the events at a category level, pseudonymous ID/profile behavior, deployment-ID handoff, always-on setup-wizard PostHog policy, and the independent Reo/deployed-product controls.
-

The build and files configuration must include bin.cjs and the compiled wizard. No Sourcebot-owned postinstall entrypoint is added. npm pack --dry-run should verify the executable package contents before release.

-

Node 24 migration and PostHog SDK

-

Use Node 24 LTS as the package’s minimum and CI/release runtime. This is the highest production LTS line that does not require an architectural migration for this CLI: the code already targets ES2022, TypeScript is already configured with Node typings, native APIs used by the wizard remain available, and the current runtime dependencies accept Node 24. Node 26 is a Current release rather than LTS and is therefore not selected for a published setup tool.

-

The existing >=18 declaration is already inaccurate: @inquirer/prompts 8.4.3 requires at least Node 20.12 on the Node 20 line, and ora 9.4.0 requires Node 20. Raising the floor makes the supported runtime truthful. Updating @types/node from 22 to 24 keeps compile-time APIs aligned with the declared runtime; no application rewrite is expected.

-

Initialize the official SDK once per process:

-
const posthog = new PostHog(projectToken, {
-    host: "https://us.i.posthog.com",
-    flushAt: 1,
-    flushInterval: 0,
-    disableGeoip: true,
-    isServer: false,
-});
-
-

PostHog's Node guide recommends flushAt: 1 and flushInterval: 0 when a short-lived runtime should send queued events immediately. Leave the SDK's request and retry options unspecified so its maintained defaults apply. disableGeoip: true prevents location enrichment, and isServer: false tells PostHog this is a CLI-like runtime. Use the same public project token already configured by Sourcebot, defined as a setup-package constant with a comment linking it to the canonical shared default. Do not source the token or host from a user’s POSTHOG_* environment variables.

-

The package-owned wrapper should:

-
    -
  1. Initialize independently of Reo and deployed-Sourcebot telemetry environment variables.
    2. -
  2. Accept only a typed event name and the matching allowlisted property type.
    3. -
  3. Add common properties and use PostHog’s default person-profile processing without sending $set, $set_once, or other person properties.
    4. -
  4. Call posthog.capture() without forwarding configuration objects, prompt values, exceptions, or arbitrary environment data.
    5. -
  5. Swallow SDK capture and shutdown errors without printing payloads or changing setup behavior.
    6. -
-

At normal completion and handled-exit boundaries, call await posthog.shutdown(1_000). PostHog recommends awaiting shutdown() in short-lived runtimes so queued events are sent; supplying 1,000 ms instead of accepting its longer default caps the effect on CLI exit. Validate that budget in E2E tests. No additional delivery machinery is required.

-

setupSessionId creation and deployment handoff

-
    -
  1. At the start of main(), before setup_sourcebot_started, create setupSessionId directly with Node’s crypto.randomUUID(). Do not use a custom random-byte encoder or derive it from any user, repository, path, machine, or environment value.
    2. -
  2. Assert in tests that the generated value matches the exact canonical contract: ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$. This enforces 36 ASCII characters, lowercase hexadecimal, standard hyphen positions, UUID version 4, and the IETF variant defined by RFC 9562 (historically called the RFC 4122 variant).
    3. -
  3. Keep the value only in process memory while the wizard runs. Every setup event uses the same string, without normalization or transformation, as distinctId, setupSessionId, the install_id property, and the PostHog company group key.
    4. -
  4. When generating a new .env, write the same string byte-for-byte as SOURCEBOT_INSTALL_ID=<setupSessionId> alongside the other generated Sourcebot settings. Do not uppercase it, add braces or a prefix, remove hyphens, hash it, or encode it again. Docker Compose already loads this file through env_file, so no compose-file change is required.
    5. -
  5. Require a published Sourcebot image containing merged PR #1648. In that image, first boot uses a non-empty supplied SOURCEBOT_INSTALL_ID when present and calls uuidgen only as a fallback. The entrypoint intentionally does not reformat or regenerate a supplied value; the wizard guarantees that it already matches the canonical UUIDv4 contract. The existing deployment install event and /data/.installedv3 record then use that same ID. The wizard itself never modifies this script.
    6. -
  6. On subsequent container boots, keep the current behavior: the ID persisted in /data/.installedv3 is authoritative. This prevents an edited environment file from silently changing the identity of an existing deployment.
    7. -
-

Recommended implementation:

-
import { randomUUID } from "node:crypto";
-
-const setupSessionId = randomUUID();
-
-

No conversion step is needed or allowed between generation and use. randomUUID() already returns the canonical lowercase hyphenated UUIDv4 string required by this plan. A runtime reformatter would add risk without adding validation; enforce the invariant at the generation boundary and in tests instead.

-

No telemetry-only file or registry entry is created. If the wizard exits before configuration generation, the ID disappears with the process. Once configuration is generated, the value exists only as the deployment’s required SOURCEBOT_INSTALL_ID in .env and later in Sourcebot’s existing .installedv3 deployment record.

-

Existing setup directories: before overwriting an existing .env, read only the exact SOURCEBOT_INSTALL_ID key and preserve it if it matches the canonical UUIDv4 expression above. Do not send that existing value in setup telemetry or change the current session’s distinctId. Record only a safe enum such as deploymentIdentityAction: "preserved_existing". Therefore end-to-end identity continuity is guaranteed for newly generated deployments; a rerun against an existing deployment remains a separate setup session and is not aliased or merged.

-

Tracker-specific controls

- - - - - - - - - -
TrackerControlEffect
Setup-wizard PostHogNoneAlways attempts capture when the relevant wizard checkpoint executes.
ReoPACKAGE_TRACKER_ANALYTICS=falseInterpreted only by reo-census; it has no effect on PostHog.
Deployed SourcebotSOURCEBOT_TELEMETRY_DISABLED=trueApplies to product telemetry after Sourcebot is deployed; it has no effect on setup-wizard PostHog events.
-

No Sourcebot code should intercept, override, remove, or reinterpret Reo’s variable. Conversely, the setup-wizard PostHog wrapper must not use either variable as an early return. A setup event can still be absent because the wizard never reached that checkpoint, the process was terminated, the network was blocked, or best-effort delivery failed.

-

Collector result contract

-

Collectors currently return generated configuration. Extend their result type with an explicitly constructed summary:

-
type CollectResult<TConfig, TSummary> = {
-    config: TConfig;
-    env: Record<string, string>;
-    telemetry: TSummary;
-};
-
-

telemetry must be built from booleans, fixed enums, and counts at the point where the collector already knows the user’s branch. It must never be produced by serializing, cloning, redacting, or spreading config, env, a prompt result, or an exception. The aggregate code-source and AI summaries should be computed from these safe summaries, not reconstructed from final configuration files.

-

Use exhaustive TypeScript unions for codeHost, providerType, stage, outcomes, credential modes, and failure categories. A new provider or branch should fail type checking until its safe telemetry mapping is explicitly selected.

-

Instrumentation points

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
EventExact implementation point
setup_sourcebot_startedAt the beginning of main(), after telemetry/session initialization and before the banner or first prompt.
setup_sourcebot_chose_setup_directoryAfter directory creation succeeds or reuse is confirmed. A declined existing-directory prompt emits cancelled instead.
setup_sourcebot_configured_code_sourceImmediately after each collector returns an accepted safe summary.
setup_sourcebot_configured_code_sourcesOnce after the add-another-source loop ends, using aggregates of per-source safe summaries.
setup_sourcebot_configured_ai_providerAfter each selected model/provider configuration is accepted.
setup_sourcebot_ai_setup_completedOnce when the AI loop returns, including the zero-configuration skip branch.
setup_sourcebot_configured_hosted_urlAfter URL validation succeeds; derive only fixed booleans such as HTTPS/default acceptance, never preserve the URL.
setup_sourcebot_generated_configsAfter all required writes complete, including writing this session’s ID for a new setup or preserving a valid existing deployment ID. If any write fails, emit failed at config_overwrite or generation stage and do not emit this checkpoint.
setup_sourcebot_resolved_compose_fileOnce after the existing/download/decline/failure branch resolves. Record only the fixed outcome and coarse failure category.
setup_sourcebot_validated_docker_stateOnce after validation/cleanup, or immediately with an explicit skip outcome when compose is unavailable or an existing deployment is left running.
setup_sourcebot_completedAfter manual instructions are printed, after an existing deployment is intentionally retained, after a Docker child emits spawn, or after a spawn failure is converted to manual instructions. Begin bounded SDK shutdown without waiting for the foreground child to exit; the spawned child may already be running and producing output.
setup_sourcebot_cancelledIn centralized handling for Ctrl+C and known declined terminal prompts, using the current fixed stage.
setup_sourcebot_failedAt the handling site for each recoverable infrastructure operation failure, with recoverable: true, even if later recovery succeeds. Emit recoverable: false at centralized fatal-error handling only when the error terminates the wizard. Do not recapture the same handled failure at the stage boundary.
-

Maintain a currentStage enum in main() and update it immediately before each stage begins. This makes Ctrl+C and top-level failures classifiable without inspecting prompt text or exception messages.

-

Docker and process handling

-

Docker helpers must distinguish command succeeded and found zero resources from command failed. Return internal structured results such as { ok, value, failureCategory }; keep stdout/stderr and command details local. This is necessary for truthful dockerStatus, cleanup actions, and validation_failed outcomes.

-

For docker compose up, attach spawn and error listeners before deciding the completion event:

-
    -
  • spawn: commit completion with sourcebotStartOutcome: "spawned" and begin bounded SDK shutdown while the foreground process continues. Keep child exit/error listeners and process cleanup active.
    • -
  • error: capture failed with recoverable: true and the applicable failure category, print existing/manual recovery guidance, and emit completed with sourcebotStartOutcome: "spawn_failed" and completionMode: "sourcebot_start_failed".
    • -
  • Later child exit: do not change the setup completion event. Runtime uptime and health belong to deployed Sourcebot observability.
    • -
-

Terminal paths and error policy

-

Use one lifecycle coordinator with separate setup-outcome and process-shutdown state. The mutually exclusive setup terminal events are completed, cancelled, and failed with recoverable: false. Commit the outcome synchronously before awaiting SDK shutdown so competing callbacks cannot emit another terminal event. Process cleanup must still run on Ctrl+C even when the setup outcome is already committed. Replace immediate exits with this coordinator; after main has returned and bounded SDK shutdown has finished, explicitly exit to prevent SDK retry sockets/timers from keeping the process alive. Never perform this final exit at the foreground Docker handoff; continue supervising Docker until it exits or Ctrl+C is handled.

-

Recoverable failure capture must bypass the terminal helper: it does not mark the session finished, call shutdown(), or change the wizard’s existing continuation or exit behavior. For example, a failed stopped-container removal followed by a failed volume removal emits two failed events with recoverable: true; the run can then emit validated_docker_state and completed, or end with cancelled or a later fatal failed. Capture each failure only once for that operation; do not deduplicate distinct failures just because their stage/category values match.

-

Telemetry code is outside the setup success/failure contract:

-
    -
  • Session-ID generation failure drops telemetry initialization and setup continues; no fallback file is created.
    • -
  • Failure to collect one or more system-derived properties uses the schema-defined fallback values and the event is still captured.
    • -
  • Payload construction failure drops that event.
    • -
  • Network, HTTP, timeout, and flush failure are swallowed.
    • -
  • Telemetry never changes generated files, Docker decisions, wizard exit status, or normal terminal messaging.
    • -
  • Debug logs, if added for development, report only event name and fixed failure category and are disabled in published normal operation.
    • -
-

Completion, Ctrl+C, and process cleanup

-

Code findings: src/index.ts currently catches Inquirer ExitPromptError and exits immediately, starts openBrowserWhenReady() before Docker emits spawn, and leaves its fetch/sleep loop active for up to 120 seconds. Docker helpers create child processes, port checks create temporary servers, and GitHub/GitLab autocomplete fetches have no shared cancellation signal. The custom tabCheckbox uses Inquirer core; prompt interruption can arrive through readline rather than the process-level signal listener. These resources need explicit ownership and cancellation.

- - - - - - - - - - -
SituationTelemetry decisionProcess behavior
Manual instructions printed, or existing deployment retainedCommit and capture completed once.Finish bounded SDK shutdown and release wizard resources; exit normally.
Docker emits spawn while setup is activeCommit and capture completed immediately, then begin the one bounded SDK shutdown.Keep waiting for foreground Docker. Begin cancellable readiness polling only after spawn; do not wait for readiness to count completion.
Ctrl+C before any setup terminal outcomeCommit and capture cancelled with reason: "keyboard_interrupt" and the active stage, once.Abort setup work immediately, begin resource cleanup and bounded telemetry shutdown in parallel, and exit within the interrupt deadline.
Ctrl+C after completion, including during its flushKeep completed; do not emit cancelled or reopen the SDK. Reuse the pending shutdown promise if present.Stop the foreground CLI and readiness work within the same interrupt deadline. An already-finished setup is not cancelled by stopping its foreground runtime.
Spawn error before completionCapture a recoverable failure, print manual instructions, then commit completed with the existing spawn-failed properties.Do not start readiness polling; clean up and exit. If cancellation already won the race, suppress this recovery/completion path.
Foreground Docker exits after completionNo new setup event.Cancel readiness polling and release handles immediately. Preserve the current normal child-exit behavior; do not wait out the readiness timeout.
-

Implementation: introduce a small package-owned lifecycle helper (for example src/lifecycle.ts) used by main(), prompts, collectors, and Docker helpers. Track an initially unset terminal outcome, one shared SDK-shutdown promise, an interrupt flag, an AbortController for setup work, and a registry of owned child processes, probe servers, and timers. Check cancellation after awaited operations and before starting another prompt, writing files, spawning Docker, opening a browser, or emitting a checkpoint. A late callback must not resume setup after cancellation. Keep this lifecycle functioning when SDK initialization or capture fails.

-
    -
  • Install the process SIGINT listener before the first prompt. Route it and typed Inquirer ExitPromptError into the same idempotent interrupt handler. Pass the shared abort signal through the prompt context, including the custom checkbox and search prompts where supported. Treat a prompt abort caused by this controller as cancellation already in progress, not another failure. Do not add a competing raw-stdin reader. Ensure prompt cleanup restores terminal mode/cursor and releases readline; test third-party prompt behavior through the PTY.
    • -
  • Abort pending fetches in autocomplete, model catalog, Compose download, and readiness checks. Combine existing request timeouts with lifecycle cancellation; never let a catch/fallback restart work after the lifecycle signal is aborted. Make readiness sleeps abortable and cancel them on interruption, spawn error, or child exit. Stop local repository traversal at asynchronous boundaries, close probe servers, and stop spinners. Synchronous file writes already underway cannot be interrupted mid-call; do not start another write after interruption is handled, and do not delete user files as cancellation cleanup.
    • -
  • Use a maximum 3,000 ms process-shutdown budget from the first handled Ctrl+C. Start SDK shutdown (at most 1,000 ms) and resource cleanup concurrently. Signal only live subprocesses owned by this invocation; allow up to 2,000 ms for graceful termination, then force termination of those owned processes/descendants within the overall budget. Use a tested platform-specific child cleanup adapter: POSIX signal behavior and Windows process-tree termination differ. Do not assume child.killed proves exit; observe exit/close. Handle both terminal-delivered interruption and a signal sent only to the parent. Do not signal unrelated processes or run volume deletion, broad Docker cleanup, or extra destructive Compose commands.
    • -
  • After cleanup, release stdin and owned handles and exit. A deadline watchdog must force the CLI to exit if a dependency leaves handles open; it must not rely only on process.exitCode. A second Ctrl+C escalates immediately without capturing another event or restarting any deadline. Resource cleanup proceeds even if telemetry throws or times out. Killing a Docker client does not prove its daemon-side operation or containers have stopped; retain Docker's normal interrupt behavior and do not promise deployment shutdown as part of the telemetry contract.
    • -
  • Standardize Ctrl+C exit status to 130 before and after completion, including forced exit. This is an intentional change from the current prompt-cancellation status 0; list it in the behavioral regression allowlist. Explicit decline prompts and ordinary completion keep status 0, and fatal setup errors keep status 1. An interrupt during an already-committed fatal/completion flush changes the process exit reason, not the recorded setup outcome.
    • -
-

Race and delivery contract: whichever handler first commits a setup outcome wins. If cancellation commits before Docker's spawn callback, clean up any subsequently spawned child and emit no completion; if spawn commits completion first, Ctrl+C performs process cleanup without cancellation telemetry. Capture always precedes the shutdown attempt. A healthy collector must receive the selected terminal event in tests; network failure, immediate repeated interruption, or an uncatchable kill can prevent delivery. Never extend the exit deadline or claim guaranteed receipt to compensate.

-

These are lifecycle changes required to satisfy prompt cancellation and cleanup; they do not add application-readiness telemetry. Node's signal documentation explains why installing a signal handler replaces default exit behavior; child-process documentation describes platform differences and why sending a signal alone does not establish process termination. Inquirer documents prompt cancellation through AbortSignal.

-

Reo coexistence

-

Keep reo-census in runtime dependencies without modifying its code, install hook, or PACKAGE_TRACKER_ANALYTICS behavior. The new Sourcebot PostHog telemetry starts only when the interactive wizard runs; no Sourcebot-owned PostHog postinstall hook is added.

-

The Sourcebot PostHog wrapper must not read Reo’s variable, reuse Reo payloads, or attempt to combine the two trackers. Reo installation tracking remains a separate data source and is not a stage in the PostHog setup funnel.

-

The release notes and telemetry documentation should state that setup-wizard PostHog telemetry is always on when its code executes, list the categories collected, explain that PostHog creates a Person profile keyed only by a random pseudonymous session/deployment ID, explain the new-deployment ID handoff, and distinguish PostHog from Reo’s separate tracking and variable. They must not describe the project ingestion token as a secret.

-

Testing plan

-

Implementation verification status

-

Live deployment suite: tests/e2e/liveDeployment.mjs builds and installs the npm tarball, drives the real interactive CLI, and starts the released Sourcebot v5.1.13 image with real PostgreSQL, Redis, migrations, backend, and web processes. It exercises public GitHub repository/organization/user scopes, GitLab projects, Gitea repositories, Gerrit projects, generic Git URLs, three local-clone layouts, automatic startup/interruption, and AI configuration. Each deployment uses an isolated Compose project and loopback port; only the test Compose networking/image settings and deployment telemetry opt-out are changed. Generated configuration, environment values, local mounts, and install-ID handoff remain intact. See tests/e2e/liveDeployment.md for reproduction and limits. The final clean rerun passed all thirteen scenarios against implementation/test commit 7f491c19 and package SHA-256 c4c6157ceacf224e023bf8233dc79830b6683d05fcd614493071037f464eddab. This does not replace the still-required full scenario/branch-coverage accounting.

-
-Live E2E evidence — thirteen scenarios passed; no Ask requests -
    -
  • GitHub: explicit repository, organization (all 16 repositories), and user (all 8 repositories). GitLab project, Gitea repository, Gerrit project, and generic remote Git each indexed successfully.
    • -
  • Local repositories: a single root, two sibling clones represented by a wildcard, and two nested clones represented by separate connections. All expected repositories were indexed; distinct origins avoid Sourcebot's existing clone deduplication.
    • -
  • Every scenario passed owner onboarding, authenticated search API and visible browser search results, HTTP readiness and working search after restart, and persistent install-ID equality with the wizard session.
    • -
  • Automatic docker compose up launched by the CLI reached readiness; Ctrl+C exited with 130, retained exactly one completed event, and emitted no cancelled event after completion.
    • -
  • All twelve AI providers were configured together using synthetic credentials, with exact generated/mounted model configuration and environment propagation verified. A separate Anthropic run verified the available development key without printing it. No Ask or inference calls were made.
    • -
  • The real SDK's captured payloads passed the strict transport/schema/privacy contract, expected checkpoint order, source/provider counts, and identity assertions. This suite captures locally; the separate already-verified dev PostHog smoke remains ingestion/query evidence.
    • -
  • All task-owned containers, volumes, networks, temporary installations, and credentials were removed. Redacted reports/screenshots are outside the repository. The task-downloaded Sourcebot image is retained for the separately requested Vertex-fix task to reuse and clean afterward.
    • -
-

Harness corrections, not production regressions: wait for live autocomplete results; use distinct repository origins in multi-clone fixtures; assert the actual search summary instead of assuming a README result; and supply a credential-free Docker plugin/context configuration when isolating HOME. Cross-platform CI fixes also dispose Windows ConPTY resources, permit initial lockfile creation only in disposable Yarn test projects, and separate Node 24 CLI tests from the application's Node 20 job.

-

Remaining limits: no live authenticated Azure DevOps/Bitbucket or private/self-hosted-host access was available; fixture coverage is not equivalent to live integration verification. AI credential validity, inference, default credential discovery, and service-account-file access are not implied by startup. The pre-existing Google Vertex host-file/mount limitation is assigned to a separate user-requested task, which must validate its fix E2E and open a separate PR with a changelog edit. Full manifest/branch-coverage accounting and final CI evidence still gate overall feature completion; this PR remains draft.

-
-

Live AI validation scope (user clarification): verify that the packaged wizard writes the selected provider/model configuration and credential references correctly, that the generated environment values reach the resulting Sourcebot container unchanged, and that Sourcebot starts successfully. Do not invoke Ask or make model-inference requests. A Pro license is not required for this test scope. Use available development credentials without printing them; where credentials are unavailable, distinguish configuration/startup coverage with synthetic values from actual credential validation. Continue real repository discovery, indexing, search, browser access, and restart tests independently.

-

The feature is not release-complete until the full completion gate below is satisfied. Executable tests cover packed-package collectors, terminal paths, real SDK payloads, Docker branches, and actual container identity continuity. CI definitions are not evidence that Windows or other remote jobs have run.

-

Implementation-discovered changes for the behavioral baseline allowlist: status 130 for Ctrl+C; cancellation-aware readiness cleanup; explicit final process exit after bounded telemetry shutdown; recoverable autocomplete fallback on network/decode failure with an 8-second timeout; and fixed, credential-free SDK shutdown-timeout diagnostics where PostHog itself emits them. Do not patch SDK internals or silence the global console. Empty search/catalog results are not failures; actual infrastructure errors can emit recoverable diagnostics while retaining manual fallback.

-

Docker availability failures can trigger fixed docker info/docker compose version probes for classification. These probes classify the original failed operation and do not create extra failure events. The dedicated workflow is .github/workflows/setup-wizard-e2e.yml.

-
-Local implementation evidence — September 11, 2026; release gate still incomplete -

Implementation is isolated in the msukkari/setup-wizard-telemetry-SOU-2211 branch and dedicated worktree. The original planning checkout is unchanged. The verified release-shaped package has SHA-256 c4c6157ceacf224e023bf8233dc79830b6683d05fcd614493071037f464eddab. Test infrastructure, this plan, certificates, transcripts, and temporary installations are excluded from the tarball.

- - - - - - - - - - - -
CheckObserved result
macOS, Node 24.21106 unit/integration/packed-artifact tests passed, zero failures/skips. The subsequently expanded platform suite passed all four tests, adding foreground Docker spawn/interrupt coverage to the three existing platform cases.
Isolated Linux, Node 2491 packed-CLI cases passed, zero failures/skips, including all code-host collectors, all AI providers, recoverable failures, fatal writes, Ctrl+C races, stubborn subprocesses, and rejected/stalled/reset telemetry transport.
Package-manager launchersnpm 12.0.2, Yarn 4.7.0, pnpm 12.4.1, and Bun 1.4.2 passed on macOS and Linux using the same installed tarball. Bun uses bunx --no-install, which launches the Node CLI; direct execution with the Bun runtime is not this supported launcher test. pnpm's own store/state is allowed only in its known test-owned directories; direct-binary tests still require an empty per-user home.
Unsupported NodeNode 22.22 exits before importing the wizard, with the Node 24 requirement and no generated files.
Baseline differentialTwo representative manual/downloaded-Compose flows match base 31734dc2 for generated configuration and Docker commands, normalizing generated secrets and the intentional install-ID addition. The baseline uses current resolved dependencies and Node 24, not a reconstructed historical dependency/runtime environment.
Real runtime identityEight disposable Sourcebot containers passed first boot, conflicting/missing env on restart, upgrade, generated-ID fallback, and deployment telemetry opt-out. The actual repository entrypoint and image's curl/jq/uuidgen run; database migration and supervisor are fixture stubs. This verifies entrypoint identity/telemetry, not full application health.
Live dev PostHogTwelve events were accepted and queried back in project 323169 for synthetic ID 519ed47f-27e2-45dc-a672-0960af758639: all ten minimal setup checkpoints in order, followed by the real entrypoint's install and upgrade under the same distinct ID. The test relay changes only the project token after local payload assertions; normal tests do not enable this relay.
-

Still required before declaring completion: execute the Windows CI job and resolve any platform failures; finish the complete scenario-manifest/enum/emission-site accounting and compiled branch-coverage review; close the remaining exhaustive fault-injection and behavioral-differential gaps against the full matrix below; and attach passing CI evidence for the final commit. The independent tests/approvedSchema.json snapshot prevents silent field/enum changes, but is not proof of 100% scenario coverage. No claim of full completion or 100% coverage is made by these local results.

-

Test-harness caveat: an exploratory direct-Bun-runtime launch did not produce local collector evidence and is excluded from the passing matrix. Its outbound delivery was not verified, so it must not be used as evidence of network isolation; it may have bypassed the Node-only transport shim. Use the verified bunx Node launcher, and require container-level egress enforcement before repeating unsupported-runtime experiments. Bun's launcher documentation describes respecting the executable's Node shebang.

-
-

Unit tests

-
    -
  • Inject filesystem, environment, clock, UUID generator, package metadata, and transport dependencies; never contact real PostHog in tests.
    • -
  • Verify one ID is created before started, matches ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$, remains stable for all events in that invocation, and is never written outside the selected setup directory.
    • -
  • Verify that setting either SOURCEBOT_TELEMETRY_DISABLED=true or PACKAGE_TRACKER_ANALYTICS=false does not prevent setup-wizard SDK initialization or event capture.
    • -
  • Verify a new .env receives SOURCEBOT_INSTALL_ID=setupSessionId; an existing valid value is preserved; a missing or invalid existing value is replaced with the current session ID; and telemetry records only the approved action enum.
    • -
  • Verify first-boot entrypoint identity selection: preserve a non-empty supplied ID, generate when missing or empty, persist the chosen value in .installedv3, and prefer .installedv3 on subsequent boots. Format enforcement for wizard-generated values belongs to the wizard tests.
    • -
  • Snapshot the exact property keys for every event. Seed collectors with token-, URL-, email-, repository-, model-, path-, hostname-, port-, and raw-error-shaped values and assert none occur anywhere in serialized payloads.
    • -
  • Verify each system-property collector independently handles missing values, unsupported values, malformed values, and thrown errors; the event must still be captured with "other", "unknown", or null as defined in the schema, while successfully collected properties remain intact.
    • -
  • Verify deployment classification with and without an HTTP(S) protocol, optional leading www., mixed hostname case, trailing DNS dots, ports, paths, and surrounding whitespace. Cover exact public-service domains, dot-boundary managed-service suffixes, custom GitLab domains, malformed URLs, and lookalike domains such as notgithub.com. Assert only the fixed deploymentType value is captured and the input URL and normalized hostname are absent.
    • -
  • Verify hosted-URL classification maps localhost, *.localhost, every IPv4 127.0.0.0/8 loopback address, and IPv6 ::1 to localhost; maps every other successfully parsed non-empty host to address; and maps unavailable, malformed, or unexpectedly failing inputs to unknown without suppressing the event. Assert no URL or hostname is captured.
    • -
  • Verify every other fixed enum rejects or maps unknown runtime input to its schema-defined fallback rather than forwarding it.
    • -
  • Inject representative failures from the current wizard operations and verify the coarse mapping: prompt/config parsing to validation; required fetch/HTTP/decode failures to network; directory and file operations to filesystem; missing/inaccessible Docker or Compose to docker_unavailable; non-zero or unusable Docker results to docker_command; other child-process spawn errors to process_spawn; and uncategorized runtime errors to unknown. Assert only one failure event is emitted and no raw error data is captured.
    • -
  • Verify application duration properties use an injected monotonic clock. Standard SDK envelope timestamps are permitted and validated separately; do not add custom wall-clock properties to the application schema.
    • -
  • Verify the exact posthog-node options, use of SDK transport defaults, absence of $process_person_profile: false, absence of custom person properties, and bounded shutdown behavior.
    • -
-

Mandatory end-to-end completion gate

-

The E2E suite is part of the feature implementation, not deferred follow-up work. Implement and run it after the telemetry and Node changes are code-complete but before the feature is marked complete, merged, or released. The feature is incomplete while any required scenario is missing, skipped, quarantined, allowed to fail, or failing. Unit tests and source-level integration tests cannot substitute for this gate.

-

Every required E2E scenario must execute the newly compiled and packed npm artifact through its published binary entrypoint. Tests must not import src/, run tsx, or execute a repository source entrypoint as a substitute. Build @sourcebot/schemas, build setup-sourcebot, create the same .tgz shape used by the release workflow, record its SHA-256 digest, install that tarball with lifecycle scripts enabled into a clean temporary project, and use that exact digest for every scenario in the run.

-

A required CI check named setup-wizard-e2e must gate the feature PR. The release workflow must rebuild and rerun the packed-artifact smoke and identity-continuity tests before publishing. A release must stop before npm publish if the artifact, E2E suite, cleanup audit, or PostHog contract validation fails.

- -

E2E harness and isolation architecture

-
    -
  1. Artifact builder: compile dependencies and the wizard, pack the npm tarball using the release workflow’s Yarn command, inspect the archive, and install it into a new temporary project with npm lifecycle scripts enabled. Assert the published binary resolves and starts without repository-only dependencies.
    2. -
  2. PTY driver: launch node_modules/.bin/setup-sourcebot in a pseudoterminal, wait for each real prompt, submit answers or signals, enforce a per-prompt and per-scenario timeout, and retain a redacted transcript only as an ephemeral CI artifact on failure.
    3. -
  3. Scenario fixture: define each run as declarative prompt answers, filesystem seed state, fixture-server responses, Docker behavior, expected exit result, expected files, and expected ordered telemetry. Generate scenario names from fixed identifiers rather than user data.
    4. -
  4. PostHog capture service: run a local HTTPS endpoint that accepts the actual posthog-node request paths and records raw request bodies. In the isolated test network, resolve the fixed production ingestion hostname to this service and trust a test-only CA through the test environment. Do not add a production environment-variable host override or replace the SDK with a mock in packed-artifact tests.
    5. -
  5. Code-host and catalog fixtures: serve deterministic GitHub-, GitLab-, models.dev-, and compose-download-compatible responses locally. Deny all other outbound traffic during execution so a missed fixture fails the test rather than contacting a real service.
    6. -
  6. Deterministic Docker layer: place a stateful fake docker executable first on PATH for the exhaustive branch matrix. It must emulate command exit codes and stdout/stderr for Compose state, cleanup, volumes, and startup while recording only fixed test metadata.
    7. -
  7. Real Docker layer: separately run the generated files and repository-root entrypoint.sh in disposable Docker containers with unique Compose project names, networks, images, and data volumes. The fake Docker layer does not replace these container-level identity and restart tests.
    8. -
  8. Filesystem sandbox: give every scenario a fresh temporary home, working directory, setup directory, npm cache, and XDG state/config/cache directories. Afterward, assert that no telemetry file or unexpected setup artifact exists inside or outside the selected setup directory.
    9. -
  9. Secret canaries: use unique synthetic token-, email-, repository-, organization-, URL-, model-, path-, hostname-, and error-shaped values for every sensitive prompt and fixture response. Recursively scan captured PostHog bodies, terminal diagnostics, and retained test artifacts to prove none were transmitted through telemetry.
    10. -
  10. Cleanup controller: use try/finally cleanup for processes, PTYs, servers, temporary installations, Docker resources, and certificates. Cleanup runs after success, assertion failure, timeout, or signal and fails the suite if a labeled resource or temporary artifact remains.
    11. -
- -

Required isolated environment matrix

- - - - - - - - - -
EnvironmentRequired coveragePurpose
Linux container, Node 24Full scenario matrix, fake Docker matrix, real Docker suite, PostHog capture, packaging, and cleanup auditPrimary deterministic release gate.
Linux package-manager smoke, Node 24Install/invoke the same packed tarball through npm/npx, Yarn, pnpm, and Bun’s package launcherVerify the reported packageManager/invocationMethod, lifecycle behavior, binary resolution, and graceful unknown fallback without duplicating the full branch matrix.
macOS, Node 24Packed install, minimal happy path, local-repository path handling, cancellation, system properties, and spawn behaviorProtect Darwin-specific paths, process behavior, and package execution.
Windows, Node 24Packed install, minimal happy path, local-repository path handling, cancellation, system properties, and spawn behaviorProtect Win32 paths, executable resolution, signals available on Windows, and package execution.
Linux container, Node 22.22Unsupported-runtime rejection onlyProve the Node 24 boundary fails early without prompts, telemetry, files, or Docker work.
-

The Linux run is authoritative for exhaustive branch coverage. Platform smoke jobs must use the same tarball build recipe and must not be silently omitted from the required check; if CI infrastructure cannot support a platform temporarily, the feature remains incomplete until an equivalent isolated runner is available and passing.

- -

Required setup-flow scenario matrix

- - - - - - - - - - - - - - - - - - - - - - -
AreaScenarios that must execute through the packed CLIPrimary assertions
Installation and invocationFresh tarball install with lifecycle scripts; direct package binary; npm-exec/npx-style invocation; package path containing spaces; read-only package installation after installNo setup PostHog event during installation, no module-resolution failure, correct packageManager/invocationMethod classification or fallback, and no writes into the installed package.
Setup directoryNew relative path, new absolute path, existing directory accepted, existing directory declined, nested creation, spaces/Unicode, creation failure, and chdir failureCorrect checkpoint/cancellation/failure event, unchanged exit semantics, no path leakage, and no writes outside the chosen sandbox.
GitHubDefault cloud and custom/GHE hosts; no token and token; repository, organization, and user scopes; autocomplete success, empty result, authentication/rate-limit response, malformed response, timeout, and connection failureCorrect safe summary and deployment classification, exact counts, successful literal fallback where supported, and no URL, token, search text, owner, or repository leakage.
GitLabGitLab.com, known Dedicated domain, ambiguous custom domain, and self-managed-shaped input; all, group, project, and user scopes; token/no-token; autocomplete success and every fallback/failure classCorrect cloud/unknown rule, scope/count properties, no hostname or selected-name leakage, and continued setup for supported fallback paths.
BitbucketCloud API token, access token, and app password; workspace and repository scopes; Data Center index-all and selected-project paths; cleanup of all credential-shaped canariesCorrect deployment and credential enums, counts, and complete exclusion of emails, usernames, hosts, and credentials.
Azure DevOpsCloud and Server, optional TFS path, organization/project/repository scopes, and multiple selectionsCorrect deployment/scope/count properties without organization, collection, server, URL, or token values.
Gitea and GerritGitea.com and custom Gitea; token/no-token; organization/repository/user selections; Gerrit index-all and selected projectsCorrect fixed deployment type, credential mode, counts, and no host/project leakage.
Local and remote GitDirectory that is itself a repository; all depth-one repositories collapsed to a wildcard; subset and nested repositories generating multiple connections; unreadable directory; no repositories; arbitrary remote Git URLCorrect repository count, bucket, generatedConnectionCount, local/remote deployment type, mount file behavior, and no local path, basename, or clone URL leakage.
Multiple code sourcesOne source, repeated provider, mixed providers, and three-or-more-source loop with both add-another choicesOne repeated event per accepted source, one aggregate checkpoint, one-based indexes, deduplicated host types, exact totals, and stable ordering.
AI setupSkip; every provider offered by the wizard; one and multiple models; OpenAI-compatible endpoint; AWS default chain and explicit keys; Vertex ADC and credential-file path; catalog success, empty, malformed, HTTP failure, timeout, and manual-model fallbackProvider events and aggregate checkpoint are correct, skip remains in the funnel, counts match generated models, and no model, endpoint, region, resource, project, credential path, key, or display name is captured.
Hosted URLDefault URL; localhost, subdomain of localhost, IPv4 loopback range, IPv6 loopback, other hostname/IP address, invalid input followed by valid input, and classifier failure injectionCorrect usedDefaultUrl, protocol, and hostCategory; invalid prompt input does not create a failure event; URL, hostname, port, path, query, and fragment remain absent.
Configuration generationNew files; each existing file accepted for overwrite; each overwrite declined; valid/missing/invalid existing install ID; write failure for each output; generated-secret failureExisting behavior and exit codes remain stable, files parse, only approved files are listed, UUID handoff follows policy, no partial-success checkpoint is emitted, and failures use coarse categories without content leakage.
Compose-file resolutionExisting file; download accepted and successful; download declined; HTTP 4xx/5xx; timeout/disconnect; malformed body where relevant; destination write failureEvery terminal branch emits the compose checkpoint with the right outcome/category, manual instructions remain correct, and setup continues only where it did before instrumentation.
Docker validationDocker executable missing; Compose missing; daemon/socket unavailable; empty state; running, stopped, and mixed containers; cleanup accepted/declined/success/failed; volumes absent/present/remove success/failure; no-compose skipSuccess is not confused with command failure, outcomes and failure categories match, no container/volume names or stderr enter telemetry, and existing user-facing recovery behavior is preserved.
Port conflictsNo published ports; free ports; Docker-owned conflict; non-Docker conflict; mixed conflict; stop accepted/declined/success/failed; conflict remains after cleanupCorrect aggregate conflict fields and completion mode, no port numbers/process/container names in telemetry, and temporary listening sockets are closed.
Completion and startupManual completion, compose unavailable, existing deployment retained, start declined, docker compose up spawn success, spawn error, and later child exitExactly one completion/failure terminal decision, completion fires after spawn rather than child exit, and Sourcebot-start outcome is truthful.
Cancellation and fatal errorsSIGINT at every named stage and during shutdown; repeated signals; every explicit decline; one injected unexpected failure at each stage boundary; cancellation or fatal error after multiple recoverable failuresAt most one terminal event across completion, cancellation, and fatal failure; recoverable diagnostics remain nonterminal. Correct stage/reason/category and exit semantics, bounded shutdown, no hanging PTY, and no corrupted partial files.
Telemetry degradationSDK initialization/capture/shutdown throw; collector returns 4xx/5xx; connection refusal; response timeout; connection reset; malformed response; system-property collector failureThe same prompts, files, Docker actions, terminal output, completion mode, and exit code as the matching telemetry-healthy control; system properties use approved fallbacks and no telemetry state file is created.
- -

Packed-artifact, container, and identity-continuity suite

-

Add a dedicated E2E suite that runs from the packed npm artifact under Node 24, rather than importing TypeScript source. This is the regression gate for the runtime upgrade, telemetry integration, generated files, Docker orchestration, and setup-to-deployment identity handoff.

-
    -
  1. Build and pack setup-sourcebot, install the tarball into an isolated temporary project with PACKAGE_TRACKER_ANALYTICS=false, and assert installation succeeds under Node 24 without warnings or module-resolution errors. Assert no Sourcebot PostHog request occurs during package installation. Using Reo’s opt-out keeps the test from contacting Reo and does not affect wizard telemetry.
    2. -
  2. Redirect the fixed PostHog ingestion hostname at the isolated test-network boundary to the local TLS capture service, leaving the packed production code and official SDK unchanged. Launch the wizard with both PACKAGE_TRACKER_ANALYTICS=false and SOURCEBOT_TELEMETRY_DISABLED=true; assert started is still captured, contains no $process_person_profile: false, PII, sensitive values, or custom person properties, and uses bounded shutdown. Production code must continue using the fixed Sourcebot host, and the E2E design must not add a user-accessible host override.
    3. -
  3. Launch the packed binary through a PTY under Node 24 and drive a deterministic minimal happy path: choose a temporary setup directory, configure a fixture code source, skip AI, accept the hosted URL, generate configuration, resolve the compose branch, skip or stub Docker validation, and decline foreground startup.
    4. -
  4. Assert the wizard exits successfully, generated files parse correctly, the expected ordered started -> completed events arrive, each event contains only its approved keys, and secret-shaped fixture values are absent from serialized requests.
    5. -
  5. Assert the generated .env value for SOURCEBOT_INSTALL_ID matches the canonical UUIDv4 expression and exactly equals, byte-for-byte, the setup events’ distinctId, setupSessionId, install_id, and company group key.
    6. -
  6. Run the actual repository-root entrypoint.sh containing PR #1648 in an isolated Sourcebot container with an empty data volume and the wizard-produced UUID. Stub only external dependencies and long-running processes after identity initialization. Capture the HTTPS request at a local PostHog-compatible endpoint and assert the deployment install event’s distinct_id, the process environment, and .installedv3.install_id all equal the wizard value byte-for-byte.
    7. -
  7. Recreate the container against the same data volume through the complete identity matrix: same-version restart with a conflicting environment value, upgrade restart with a conflicting value, another same-version restart with the environment value absent, first boot with the ID absent so uuidgen is exercised, and telemetry-disabled first boot/restart. Assert the persisted value always wins after first boot, upgrade telemetry retains the same distinct_id, same-version restarts do not duplicate install/upgrade events, generated fallback IDs match the canonical UUIDv4 expression, and telemetry-disabled runs emit no events.
    8. -
  8. Run the wizard over a fixture existing .env with a valid deployment ID. Assert the ID is preserved, setup events remain keyed to the new session UUID, and deploymentIdentityAction is preserved_existing without transmitting the existing ID.
    9. -
  9. Run a second packed invocation that sends SIGINT during a prompt. With a healthy collector, assert exactly one allowlisted cancelled event, bounded SDK shutdown, exit status 130 under the lifecycle policy, and no partial configuration corruption. This protects signal and process behavior that can differ across Node majors.
    10. -
  10. Run the packed binary under Node 22.22 in a separate CI container and assert bin.cjs fails immediately with a clear Node 24 requirement before prompts, UUID generation, telemetry, file writes, or Docker commands. This verifies the migration boundary is intentional rather than a late dependency/runtime crash.
    11. -
- -

PostHog E2E contract assertions

-

The packed-artifact tests must use the real posthog-node dependency shipped in the tarball. Mocking the package-owned typed wrapper is appropriate for unit tests but does not satisfy E2E completion.

-
    -
  1. Decode every request accepted by the local PostHog-compatible capture service, including SDK batching or compression, and validate the actual ingestion envelope rather than only the arguments passed to posthog.capture().
    2. -
  2. For each scenario, compare the received logical event sequence with an explicit oracle. Assert no missing or unexpected event, correct repeated-event count, checkpoint order, and at most one terminal event across completed, cancelled, and failed with recoverable: false. Recoverable failure events do not count toward this limit. Assert one logical failure capture per failed operation, allowing distinct operation failures with identical stage/category values. Transport retries caused by an induced ambiguous network failure are not treated as duplicate instrumentation.
    3. -
  3. Run packed-CLI scenarios with both stopped-container removal and volume removal failing in the same session, followed respectively by manual completion, cancellation at a later prompt, and a later fatal error. Assert both recoverable events arrive under the same setup ID, subsequent reached checkpoints are still captured, and exactly one appropriate terminal event arrives. Add a spawn-error-to-manual-completion scenario. Verify the SDK remains active after recoverable failures and shuts down only at the terminal boundary; repeated signals or callbacks must not add a terminal event after that boundary.
    4. -
  4. Verify Docker measurement semantics for no Compose file, deliberately retained deployment, unavailable Docker, partial inspection failure, confirmed empty inventories, and failed port rechecks. Assert skipped/failed counts are null, confirmed empty counts are 0, independent measurements survive other failures, and completion copies the remaining-conflict value exactly. Combine cleanup failure with remaining port conflicts and with a left-running deployment to verify validation_failed takes precedence while action fields preserve the details.
    5. -
  5. Validate setup event names against the 13-event allowlist, and validate application property keys/types against the common and event-specific schema. Separately validate SDK-generated properties and transport envelopes using the versioned allowlist below. Existing deployment install/upgrade events in the identity suite use their own runtime contract, not the setup-event allowlist.
    6. -
  6. Assert one canonical UUIDv4 is reused for distinct_id, setupSessionId, install_id, and the company group throughout a new setup. Assert separate invocations receive different setup IDs and are not accidentally merged.
    7. -
  7. Assert schemaVersion, package version, platform, architecture, Node major, package manager, CI state, elapsed timing, GeoIP disablement, and source attribution are correct for the test environment. Inject each permitted system-property failure and verify the documented fallback without dropping the event.
    8. -
  8. Assert PostHog’s default Person behavior is preserved: no $process_person_profile: false, identify, alias, $set, or $set_once mutation is emitted, while the random setup identity and company group remain present.
    9. -
  9. Recursively inspect both keys and values in the serialized requests. Reject exact canaries, substrings, URL-encoded forms, JSON-escaped forms, and accidental nested configuration objects containing credentials, repositories, emails, URLs, paths, model identifiers, hostnames, ports, command output, errors, or environment data.
    10. -
  10. Exercise a successful collector, HTTP rejection, connection refusal/reset, and non-responsive collector. Assert capture remains non-blocking during prompts and the explicit shutdown deadline bounds normal completion, cancellation, and failure exits without changing setup correctness.
    11. -
  11. Assert package installation itself produces no Sourcebot PostHog request, and assert the wizard creates no telemetry-only file, durable queue, identifier, cache entry, or state directory.
    12. -
- -

SDK metadata and transport-envelope allowlist

-

Keep two explicit contracts: the application schema above, and a transport contract for the exact posthog-node/@posthog/core versions resolved in the tested artifact. Inspect those resolved versions during implementation and record the enabled wire format in the fixtures. The inspected Node SDK 5.52.1/core 1.53.2 adds library metadata, event timestamps, event UUIDs, and grouping metadata. These fields must pass tests when correctly generated; they are not additional wizard answers.

- - - - - - - -
LayerPermitted metadata and assertions
SDK event properties$lib equals posthog-node; $lib_version equals the resolved SDK version; $groups contains exactly { company: setupSessionId }; $geoip_disable and $ignore_sent_at are true. Group and GeoIP assertions complement the application's existing contract. With the planned isServer: false, $is_server is absent.
Event envelopeValidate the observed SDK fields such as event, distinct_id, properties, timestamp, and uuid. Timestamps must parse, increase strictly within the invocation, and fall within the test's allowed time window. The SDK's event UUID is separate from the setup UUID: the inspected core generates UUIDv7 event IDs, so do not require these to equal setupSessionId or match the setup UUIDv4 regex.
Batch and HTTP transportAllow the actual resolved format's project token field, batch array, send/creation timestamp, and content-type/compression/SDK-identification headers. For a capture-v1 format, validate its options object and SDK metadata in PostHog-Sdk-Info rather than requiring $lib/$lib_version inside every event. Only enable these alternatives when the tested artifact actually uses that format; do not accept arbitrary extra envelope keys.
-

The wire allowlist is implemented in test fixtures, not by stripping SDK-generated fields in production. SDK upgrades must update the fixture contract after inspecting their actual output. Do not blanket-allow $* properties: unexpected user identifiers, person mutations, feature-flag/session context, location data, raw errors, or arbitrary SDK enrichment must still fail. Scan both permitted metadata and application properties for sensitive canaries. The live PostHog smoke may observe additional server-generated ingestion metadata; distinguish that from outbound capture data and review unexpected enrichment rather than comparing a stored event object directly to the wire schema.

-

Behavioral regression comparison

-

Required lifecycle regressions: through the packed CLI, exercise real PTY Ctrl+C and parent-directed SIGINT at prompts, stalled autocomplete/catalog/Compose fetches, repository scanning, Docker commands, port checks, immediately before/after spawn, readiness fetch/sleep, and pending terminal shutdown. Cover multiple recoverable errors followed by interruption, spawn failure, immediate child exit, and repeated Ctrl+C. For each interrupted case, assert no subsequent setup work, CLI exit within the 3-second budget plus a small documented CI scheduling tolerance, expected exit code, released terminal state, and no surviving owned test subprocesses/timers/listeners. With a healthy collector, assert exactly one cancelled event before completion, or exactly one completed event and no cancellation after completion. Repeat with a stalled collector and stubborn child to verify the bounded fallback. Verify Windows console/PTY interruption separately using supported platform mechanisms. Add the intentional status-130 and readiness-cleanup differences to the baseline comparison allowlist.

-

Run representative non-telemetry setup scenarios against both the feature artifact and an artifact built from the PR base revision. Normalize only nondeterministic values such as temporary paths, ANSI timing, generated secrets, and the intentionally added SOURCEBOT_INSTALL_ID. Maintain a small reviewed allowlist of intentional differences; an unlisted difference fails the gate.

-
    -
  • Compare prompt order, prompt defaults, validation behavior, cancellation points, terminal success/failure messages, and process exit codes.
    • -
  • Compare generated config.json, .env, and compose override semantics after redacting generated secrets. Validate JSON against the Sourcebot schema, parse compose YAML, and confirm the only telemetry-related persisted change is the install-ID entry in the existing .env.
    • -
  • Compare fake-Docker command order and arguments, cleanup choices, port-conflict decisions, and whether foreground startup occurs.
    • -
  • Run every happy-path scenario twice—once with a healthy local collector and once with telemetry transport forced to fail—and require identical wizard files, Docker actions, user-facing behavior, and exit status.
    • -
  • Exercise the Node-migration-sensitive APIs used by the package, including ESM loading, global fetch, AbortSignal.timeout, cryptography, child-process events, filesystem/path behavior, and signal handling through the compiled package.
    • -
- -

Coverage accounting and scenario completeness

-

Store a reviewed, machine-readable scenario manifest with the E2E tests. It must map every wizard stage, prompt branch, code-host collector, AI-provider branch, compose outcome, Docker outcome, terminal path, event name, event enum value, and failure category to at least one scenario ID. CI must fail when an implementation adds or changes a branch/event enum without updating the manifest and its assertions.

-
    -
  • Require 100% coverage of the manifest, all telemetry emission sites, all terminal-event paths, all event names, and every documented outcome enum before completion.
    • -
  • Collect JavaScript branch coverage from the compiled dist execution as supporting evidence and review uncovered setup logic. Numeric source coverage alone is not a substitute for manifest coverage or behavioral assertions.
    • -
  • Generate a compact machine-readable report containing the tarball digest, Node/OS matrix, scenario totals, pass/fail/skip counts, event names observed, enum values observed, cleanup result, and duration. Do not include prompt answers, request bodies, paths, credentials, or other canary values.
    • -
  • Use zero test-level retries. A failed scenario must remain a failure until its cause is understood and fixed; rerunning an entire CI job for diagnosed infrastructure failure does not waive the original result.
    • -
- -

Execution order, cleanup, and completion evidence

-
    -
  1. After implementation is ready, run formatting, type checking, unit tests, and source-level integration tests.
    2. -
  2. Build and pack one candidate tarball, record its digest, and run the full isolated Linux E2E matrix against that artifact.
    3. -
  3. Run the required macOS, Windows, unsupported-Node, fake-Docker, real-Docker, entrypoint identity, and telemetry-degradation jobs.
    4. -
  4. Run one final PostHog ingestion smoke in the supplied dev project using the external relay described below and synthetic, non-sensitive fixture data. Query the resulting events by the known generated setup/deployment UUID and verify event visibility, ordering, property types, Person behavior, company grouping, and setup-to-deployment identity continuity. Delete or expire test data according to the test project’s retention policy; never send fixture secrets.
    5. -
  5. Run the cleanup audit and repository-dirtiness check after every job. No temporary tarball, certificate, npm cache, setup directory, telemetry file, process, listener, Docker container, image, network, or volume may remain.
    6. -
  6. Attach the redacted completion report and required CI links to the feature PR. Mark the feature complete only when every required scenario and environment passes against the final commit and artifact, the manifest reports no uncovered entry, cleanup passes, and the PostHog smoke is verified.
    7. -
-

All E2E infrastructure and tests land with the feature changes. None of these checks may be converted into a post-merge task merely to unblock completion.

-

Dev-project PostHog smoke routing

-

Use the user-supplied dev-project ingestion token phc_EJR6BsaBbvIKhM4t4zp1boYC92Tpp5Fgb9Csa9Us5aw for live test ingestion. Keep the production package token unchanged. Run the same packed artifact and digest used by the local E2E suite; the existing external HTTPS capture service acts as a forwarding relay only for the final live smoke. It substitutes the dev ingestion token in the SDK request’s project-authentication field, forwards to the dev project’s verified PostHog ingestion host, and preserves event names, UUIDs, timestamps, distinct IDs, groups, and properties. Apply the same routing to container install/upgrade telemetry so both surfaces reach the dev project.

-

The relay must validate the original and forwarded envelopes and assert that only project-authentication metadata changes. Support the shipped SDK’s batching/compression and the entrypoint’s capture payload. Resolve the upstream host outside the local hostname override to avoid a forwarding loop. Confirm the dev project’s region/ingestion host before running; do not infer it from the token. The ingestion token does not grant event-query access: use the authenticated PostHog connection or a separately configured read credential to verify stored events. Record the observed results; request acceptance alone does not satisfy ingestion verification.

-

Only the final synthetic live smoke permits this relay to reach PostHog. Exhaustive canary and failure tests remain on the local collector with external traffic denied. Never embed the dev token or relay controls into the published CLI; test routing is entirely external. The live smoke, including event-query verification, remains part of the mandatory development completion gate.

- -

Packaging and integration tests

-
    -
  • Use a local HTTP capture server with the real posthog-node client to verify the final JSON envelope, SDK options, default profile behavior, and absence of PII, sensitive fields, and custom person-property mutations.
    • -
  • Add explicit package scripts for unit, integration, packed-artifact E2E, platform smoke, and cleanup-audit jobs so local development, pull-request CI, and release CI invoke the same commands.
    • -
  • Run the packed CLI against deterministic local service fixtures and the fake Docker executable for exhaustive branches, then run the separate real-Docker identity suite; neither layer may be represented as covering the other.
    • -
  • Run npm pack --dry-run and assert bin.cjs and the compiled wizard are included and executable through the package entrypoint, with no new Sourcebot-owned postinstall entrypoint.
    • -
  • Inspect the actual release-shaped tarball and assert test harnesses, fixture secrets, certificates, transcripts, coverage output, and temporary files are excluded from the published package.
    • -
  • Assert reo-census remains in the package manifest and packed dependency graph, and that Sourcebot code neither removes nor rewrites PACKAGE_TRACKER_ANALYTICS.
    • -
  • Assert no test-only PostHog host/token override, fake transport, deterministic UUID hook, or failure-injection switch is reachable through the published CLI. Test redirection belongs to the external harness or internal dependency injection exercised outside the production entrypoint.
    • -
  • Verify formatting, Node 24 type checking/build, the setup-wizard unit suite, integration suite, full packed-package E2E matrix, platform/package-manager smoke jobs, and cleanup audit in CI.
    • -
-

Rollout and operational validation

-
    -
  1. PR #1648 is merged for the repository-root entrypoint.sh first-boot behavior. Land the setup-wizard telemetry and .env handoff in packages/setupWizard.
    2. -
  2. Publish a Sourcebot container image containing the updated root entrypoint.sh before or alongside the telemetry-enabled wizard. Older images will overwrite the wizard-supplied ID and therefore cannot provide setup-to-deployment continuity.
    3. -
  3. Include the Node 24 runtime floor, Node typings, runtime bootstrap, posthog-node wrapper, typed schema, in-memory session identity, instrumentation, E2E tests, documentation, and explicit Reo coexistence behavior in the setup-wizard change.
    4. -
  4. Run the packed-package E2E suite under Node 24 and its intentional Node 22.22 rejection case before testing production ingestion.
    5. -
  5. After every required local packed-artifact scenario passes, run the final synthetic smoke in a non-production PostHog project and verify ingestion, Person/group behavior, and identity continuity by the generated UUID. Keep exhaustive sensitive-canary and failure testing on the isolated local collector.
    6. -
  6. Point the production build at the existing Sourcebot PostHog project and release the Node requirement as a clearly documented breaking package version rather than a silent minor runtime change.
    7. -
  7. Build the ordered started -> completed funnel and setup-to-deployment analysis from the names and shared identity fields in this document; save breakdowns only on allowlisted properties.
    8. -
  8. During the first release, monitor event counts, property-key cardinality, unknown enum rates, and impossible orderings such as completed without started in the same session. Do not add payload logging to diagnose delivery.
    9. -
  9. If telemetry causes a setup regression, ship a patch that repairs or removes the faulty integration path; setup behavior and exit status must remain independent of telemetry delivery success.
    10. -
-

Schema changes after release require incrementing schemaVersion when meaning, type, or enum interpretation changes. Additive event properties still require privacy review and exact-key tests. Renaming an event or changing checkpoint timing should use a new event/schema version rather than silently changing the existing funnel.

-

Acceptance criteria for implementation

-
    -
  • Every product-funnel checkpoint fires exactly once per setup session after its stage resolves.
    • -
  • Skipping AI emits setup_sourcebot_ai_setup_completed with zero counts and aiConfigured: false.
    • -
  • Existing, downloaded, declined, and failed compose-file branches all emit setup_sourcebot_resolved_compose_file with distinct outcomes.
    • -
  • Docker validation emits an outcome even when skipped for a known reason. Skipped or failed measurements use required nullable counts, never fabricated zeroes; initial/final measurements, status values, action semantics, and outcome precedence follow the Docker schema and are covered by E2E tests.
    • -
  • Completion is captured at the defined handoff without waiting for foreground Docker to exit. Ctrl+C before completion captures cancellation; Ctrl+C after completion cleans up and exits without changing the successful setup outcome. The lifecycle coordinator cancels owned background work, restores terminal state, and enforces the 3-second interrupt exit budget independently of telemetry success.
    • -
  • Cancelling with Ctrl+C or declining an overwrite emits only fixed stage/reason values.
    • -
  • Telemetry failure cannot alter wizard output files, Docker actions, exit status, or user-visible success.
    • -
  • Every reported setup failure maps to exactly one coarse failureCategory; expected prompt rejection, cancellation, and best-effort autocomplete/model-catalog fallback are not misreported as failures. Handled infrastructure errors remain reportable as recoverable failures even when the wizard later completes successfully.
    • -
  • Each handled infrastructure operation failure emits one nonterminal failed event with recoverable: true and leaves the telemetry session active. Multiple recoverable failures may precede further checkpoints. Only completion, cancellation, or an error that actually terminates the wizard ends the chain; at most one terminal event is emitted and SDK shutdown occurs only at that boundary.
    • -
  • Automated tests assert separate exact application and resolved-SDK/envelope allowlists for every setup event and seed inputs with token-, URL-, email-, repository-, model-, path-, and error-shaped values to prove none enter captured payloads. Legitimate SDK-generated library metadata, timestamps, event UUIDs, and grouping fields pass the versioned transport contract.
    • -
  • Deployment classification normalizes common URL formatting variants, uses exact or dot-boundary hostname matches, emits unknown when the mode is ambiguous, and never sends the entered or normalized hostname.
    • -
  • Hosted-URL classification reports only confirmed loopback, another parsed address, or unknown; it performs no network lookup and makes no public/private or reachability claim.
    • -
  • reo-census remains installed and retains sole ownership of PACKAGE_TRACKER_ANALYTICS.
    • -
  • A UUIDv4 setupSessionId is created in memory at wizard startup regardless of Reo or deployed-product telemetry variables; no telemetry-only state is persisted.
    • -
  • The generated setupSessionId is exactly 36 ASCII characters and matches ^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$; no code path transforms it before use. This is the plan’s normative UUIDv4 contract under current RFC 9562 and is compatible with Node’s RFC 4122 terminology.
    • -
  • For a new setup, SOURCEBOT_INSTALL_ID in .env, every setup event identity field, the deployment’s first-run event, and .installedv3 all use the same UUID.
    • -
  • The identity-continuity E2E test runs against Sourcebot runtime code containing the committed repository-root entrypoint.sh change; the setup wizard never edits or substitutes that script.
    • -
  • Existing valid deployment IDs are preserved without being sent in setup telemetry; .installedv3 remains authoritative after first boot.
    • -
  • package.json declares Node >=24.0.0, Node typings use major 24, and setup-sourcebot CI/release jobs run on Node 24.
    • -
  • The packed-package E2E suite passes on Node 24 and verifies a clear, side-effect-free failure on Node 22.22.
    • -
  • The required setup-wizard-e2e check executes the final compiled npm tarball across the complete scenario manifest and required OS/package-manager environments with no skipped, quarantined, retried, or allowed-failure scenario.
    • -
  • The E2E completion report proves exact event/property contracts, privacy canary exclusion, baseline behavioral equivalence, telemetry-failure isolation, real Docker identity continuity, non-production PostHog ingestion, and complete resource cleanup for the final feature commit.
    • -
  • The official posthog-node SDK is configured with immediate flush, SDK-maintained transport defaults, a bounded shutdown timeout, GeoIP disabled, and CLI runtime attribution.
    • -
  • PostHog uses default Person profiles keyed by random setupSessionId values, with GeoIP disabled and no identify, $set, or $set_once calls.
    • -
  • Documentation explains collection categories, ephemeral setup identity, new-deployment ID handoff, always-on setup-wizard PostHog telemetry, Reo’s independent variable, and the deployed product’s separate telemetry setting.
    • -
-

References

-
    -
  • PostHog Node.js SDK: short-lived process shutdown/flush behavior and current Node runtime guidance.
    • -
  • PostHog Node SDK reference: shutdown(timeoutMs) behavior and its default timeout.
    • -
  • PostHog anonymous vs. identified events: default identified-event profile processing and the distinction between event correlation and person properties.
    • -
  • PostHog capture API: raw event ingestion envelope and endpoint.
    • -
  • GitHub Enterprise Cloud with data residency: managed GitHub Enterprise Cloud instances use dedicated ghe.com subdomains.
    • -
  • GitLab Dedicated: default managed-instance domains and support for arbitrary custom domains, which makes some URL-only classifications ambiguous.
    • -
  • Node.js release schedule: Node 24 LTS status and the production recommendation to use an Active or Maintenance LTS release.
    • -
  • Node.js crypto.randomUUID(): generates a cryptographically random version-4 UUID and returns it as a string.
    • -
  • RFC 9562: the current UUID standard, including the canonical hex-and-dash text representation, IETF variant bits, and UUIDv4 version bits; it obsoletes RFC 4122.
    • -
  • Alpine Linux 3.23 uuidgen package: confirms the Sourcebot image uses the uuidgen implementation from util-linux.
    • -
  • util-linux uuidgen source and UUID formatting source: the random generator emits a standard UUID and the default formatter uses lowercase hexadecimal with canonical hyphens.
    • -
  • Existing Sourcebot identity flow: entrypoint.sh, docker-compose.yml, packages/backend/src/posthog.ts, packages/web/src/lib/posthog.ts, packages/shared/src/env.server.ts, and docs/docs/misc/telemetry.mdx.
    • -
-

Reviewer checklist

-
    -
  • [ ] Approve started -> completed as the canonical and only Sourcebot PostHog setup funnel.
    • -
  • [ ] Approve setupSessionId as the in-memory setup distinctId and, for new deployments, the resulting SOURCEBOT_INSTALL_ID.
    • -
  • [ ] Approve the exact canonical lowercase UUIDv4 regex under RFC 9562, its compatibility with Node’s RFC 4122 terminology, and byte-for-byte reuse across all setup and deployment identity fields.
    • -
  • [ ] Approve each event name, checkpoint timing, property type, and fixed enum in PostHog event schema.
    • -
  • [ ] Approve no telemetry-only persistence and the explicit existing-deployment behavior.
    • -
  • [ ] Approve exact selected-repository counts and bucketed local-discovery totals.
    • -
  • [ ] Approve Node >=24.0.0, Node 24 CI/release execution, and the explicit Node 22.22 rejection test.
    • -
  • [ ] Approve the official posthog-node transport defaults, privacy wrapper, immediate-flush configuration, and bounded shutdown timing.
    • -
  • [ ] Approve PostHog’s default Person profile keyed only by the random setupSessionId, with no identify, aliases, custom person properties, PII, or sensitive data.
    • -
  • [ ] Approve retaining reo-census and leaving PACKAGE_TRACKER_ANALYTICS exclusively under Reo’s control.
    • -
  • [ ] Approve no package-level opt-out for setup-wizard PostHog telemetry, subject to privacy/legal review and clear release documentation.
    • -
  • [ ] Approve completion as successful handoff/manual guidance rather than application readiness.
    • -
  • [ ] Confirm that a separate readiness event is out of scope for v1.
    • -
  • [ ] Approve the mandatory compiled-artifact E2E matrix, scenario-manifest coverage, platform/package-manager jobs, PostHog contract oracle, behavioral differential checks, Docker identity suite, cleanup audit, and no-deferral completion gate.
    • -
-
-
-
- - - From f970c911a5a0496dfd4287cee4ebd0cf5e44d553 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:21:28 -0700 Subject: [PATCH 08/14] Support setup wizard on PostHog's minimum Node runtime --- .github/workflows/release-setup-sourcebot.yml | 2 +- .github/workflows/setup-wizard-e2e.yml | 10 ++++- .github/workflows/test.yml | 2 +- CHANGELOG.md | 2 +- packages/setupWizard/README.md | 6 ++- packages/setupWizard/bin.cjs | 8 +++- packages/setupWizard/package.json | 6 +-- packages/setupWizard/tests/e2e/harness.mjs | 4 +- .../tests/e2e/nodeCompatibility.mjs | 38 +++++++++++++++++++ .../setupWizard/tests/e2e/unsupportedNode.mjs | 19 ---------- .../setupWizard/tests/unit/bootstrap.test.mjs | 29 ++++++++++++++ yarn.lock | 19 +++------- 12 files changed, 100 insertions(+), 45 deletions(-) create mode 100644 packages/setupWizard/tests/e2e/nodeCompatibility.mjs delete mode 100644 packages/setupWizard/tests/e2e/unsupportedNode.mjs create mode 100644 packages/setupWizard/tests/unit/bootstrap.test.mjs diff --git a/.github/workflows/release-setup-sourcebot.yml b/.github/workflows/release-setup-sourcebot.yml index 484fe5926..a56fac51c 100644 --- a/.github/workflows/release-setup-sourcebot.yml +++ b/.github/workflows/release-setup-sourcebot.yml @@ -112,7 +112,7 @@ jobs: docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest yarn workspace setup-sourcebot test yarn workspace setup-sourcebot test:e2e - yarn workspace setup-sourcebot test:node22 + yarn workspace setup-sourcebot test:node-compatibility - name: Upgrade npm for Trusted Publishing working-directory: . diff --git a/.github/workflows/setup-wizard-e2e.yml b/.github/workflows/setup-wizard-e2e.yml index d220ae6d4..7ebbb838f 100644 --- a/.github/workflows/setup-wizard-e2e.yml +++ b/.github/workflows/setup-wizard-e2e.yml @@ -21,6 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + node: ['20.20.0', '22.22.0', '24.x'] runs-on: ${{ matrix.os }} env: PACKAGE_TRACKER_ANALYTICS: 'false' @@ -36,14 +37,19 @@ jobs: - run: yarn rebuild node-pty - run: yarn workspace @sourcebot/schemas build - run: yarn workspace setup-sourcebot build + - name: Select end-user test runtime + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - run: corepack enable - run: yarn workspace setup-sourcebot test - run: yarn workspace setup-sourcebot test:platform - name: Linux packed-artifact and runtime checks - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.node == '24.x' run: | docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest yarn workspace setup-sourcebot test:e2e - yarn workspace setup-sourcebot test:node22 + yarn workspace setup-sourcebot test:node-compatibility yarn workspace setup-sourcebot test:baseline node packages/setupWizard/tests/e2e/packageManagers.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 15faf3056..d624ebd3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,6 +41,6 @@ jobs: run: yarn install --frozen-lockfile - name: Test - # The CLI requires Node 24 and is built/tested by setup-wizard-e2e. + # The CLI has separate packed-artifact and Node-compatibility gates in setup-wizard-e2e. # Keep the application workspaces on their existing runtime here. run: yarn workspaces foreach --all --topological --exclude setup-sourcebot --exclude 'root-workspace-*' run test diff --git a/CHANGELOG.md b/CHANGELOG.md index 06f26598f..e4f9c3cf6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Added privacy-scoped setup wizard funnel telemetry with deployment identity handoff and Node 24 support. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) +- Added privacy-scoped setup wizard funnel telemetry with deployment identity handoff, supporting Node.js from 20.20.0. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) - Added isolated live setup CLI deployment tests covering public code hosts, local clones, AI configuration, search, and restart identity; repaired cross-platform test-runner cleanup. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) ## [5.1.13] - 2026-09-12 diff --git a/packages/setupWizard/README.md b/packages/setupWizard/README.md index a27941cfd..8b17439c9 100644 --- a/packages/setupWizard/README.md +++ b/packages/setupWizard/README.md @@ -17,7 +17,7 @@ The wizard walks you through: ## Requirements -- Node.js 24+ +- Node.js 20.20+ (20.x), 22.22+ (22.x), or 23.5+ (including 24+). Node 24 LTS is recommended; Node 20 is supported for compatibility but is end-of-life. - Docker and Docker Compose ## Development tests @@ -29,11 +29,15 @@ yarn workspace @sourcebot/schemas build yarn workspace setup-sourcebot build yarn workspace setup-sourcebot test yarn workspace setup-sourcebot test:e2e +yarn workspace setup-sourcebot test:node-compatibility ``` The E2E tests compile and pack the package, install it outside the repository, drive its published binary in a PTY, inspect real SDK requests through a local TLS collector, and clean up temporary installations. OpenSSL and Docker are required. +The compatibility suite tests the same tarball under Node 20.20.0 and 22.22.0, +and checks early rejection on Node 18, 20.19, and 22.21. Build/release tooling +continues to use Node 24; that is not the end-user minimum. The runtime suite uses `docker.sourcebot.dev/sourcebot-dev/sourcebot:latest` (override only the test image with `SETUP_TEST_SOURCEBOT_IMAGE`). diff --git a/packages/setupWizard/bin.cjs b/packages/setupWizard/bin.cjs index 9c5cd3791..41e4cf662 100755 --- a/packages/setupWizard/bin.cjs +++ b/packages/setupWizard/bin.cjs @@ -1,6 +1,10 @@ #!/usr/bin/env node -if (Number(process.versions.node.split('.')[0]) < 24) { - console.error('setup-sourcebot requires Node.js 24 or newer. Please upgrade Node.js.'); +// Intersection of posthog-node and @inquirer/prompts runtime requirements. +const [major, minor] = process.versions.node.split('.').map(Number); +const supported = (major === 20 && minor >= 20) || + (major === 22 && minor >= 22) || (major === 23 && minor >= 5) || major >= 24; +if (!supported) { + console.error('setup-sourcebot requires Node.js 20.20+, 22.22+, or 23.5+ (including Node.js 24+). Please upgrade Node.js.'); process.exitCode = 1; } else { import('./dist/index.js').catch(() => { diff --git a/packages/setupWizard/package.json b/packages/setupWizard/package.json index 201a5cc94..06cd5e894 100644 --- a/packages/setupWizard/package.json +++ b/packages/setupWizard/package.json @@ -19,7 +19,7 @@ "test:platform": "node --test tests/e2e/platform.test.mjs", "test:linux": "node tests/e2e/linux.mjs", "test:runtime": "node --test tests/e2e/runtime.test.mjs", - "test:node22": "node tests/e2e/unsupportedNode.mjs", + "test:node-compatibility": "node tests/e2e/nodeCompatibility.mjs", "test:baseline": "node tests/e2e/baseline.mjs", "test:live": "node tests/e2e/liveSmoke.mjs" }, @@ -33,14 +33,14 @@ }, "devDependencies": { "@sourcebot/schemas": "workspace:^", - "@types/node": "^24.0.0", + "@types/node": "^20.19.43", "node-pty": "^1.1.0", "tsx": "^4.21.0", "typescript": "^5.6.2", "undici": "^7" }, "engines": { - "node": ">=24.0.0" + "node": "^20.20.0 || ^22.22.0 || >=23.5.0" }, "files": [ "dist", diff --git a/packages/setupWizard/tests/e2e/harness.mjs b/packages/setupWizard/tests/e2e/harness.mjs index a71b5fa1b..ecc34c92e 100644 --- a/packages/setupWizard/tests/e2e/harness.mjs +++ b/packages/setupWizard/tests/e2e/harness.mjs @@ -44,7 +44,7 @@ export function contract(events, requireTerminal = true) { assert.equal(event.properties.$lib_version, '5.52.1'); assert.equal(event.properties.$geoip_disable, true); assert.equal(event.properties.$ignore_sent_at, true); - assert.equal(event.properties.nodeMajorVersion, 24); + assert.equal(event.properties.nodeMajorVersion, Number(process.versions.node.split('.')[0])); assert.equal(event.properties.source, 'setup-sourcebot-cli'); assert.equal(event.properties.schemaVersion, 1); assert.ok(event.properties.elapsedMs >= elapsed); @@ -63,7 +63,7 @@ export function contract(events, requireTerminal = true) { } export function artifact() { - assert.equal(Number(process.versions.node.split('.')[0]), 24, 'Run packed-artifact tests using Node 24'); + assert.ok(Number(process.versions.node.split('.')[0]) >= 20, 'Run packed-artifact tests on Node 20 or newer'); // node-pty 1.1.0 ships its macOS helper without its executable bit in the npm tarball. if (process.platform === 'darwin') { const helper = join(dirname(require.resolve('node-pty/package.json')), 'prebuilds', `darwin-${process.arch}`, 'spawn-helper'); diff --git a/packages/setupWizard/tests/e2e/nodeCompatibility.mjs b/packages/setupWizard/tests/e2e/nodeCompatibility.mjs new file mode 100644 index 000000000..f9eef92f7 --- /dev/null +++ b/packages/setupWizard/tests/e2e/nodeCompatibility.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdirSync, readdirSync } from 'node:fs'; +import { join, dirname, delimiter } from 'node:path'; +import { artifact } from './harness.mjs'; + +// Build once on the release runtime, then install and exercise that exact tarball +// on each minimum supported LTS runtime. No production telemetry is sent. +const packed = artifact(); +const packageRoot = new URL('../..', import.meta.url); +try { + for (const version of ['20.20.0', '22.22.0', '18.20.8', '20.19.0', '22.21.0']) { + const runtimeRoot = join(packed.root, `node-${version}`); + mkdirSync(runtimeRoot); + execFileSync('npm', ['install', '--prefix', runtimeRoot, '--no-audit', '--no-fund', '--cache', join(packed.root, 'runtime-cache'), `node@${version}`], { stdio: 'pipe', timeout: 120000 }); + const node = join(runtimeRoot, 'node_modules/node/bin/node'); + const env = { ...process.env, PATH: `${dirname(node)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false', SETUP_TEST_TARBALL: packed.tarball }; + if (['20.20.0', '22.22.0'].includes(version)) { + const tests = [ + ...['unit', 'integration'].flatMap(dir => readdirSync(new URL(`../${dir}/`, import.meta.url)).filter(f => f.endsWith('.test.mjs')).map(f => `tests/${dir}/${f}`)), + ...['wizard', 'collectors', 'docker', 'safety', 'platform'].map(f => `tests/e2e/${f}.test.mjs`), + ]; + execFileSync(node, ['--test', '--test-concurrency=1', ...tests], { cwd: packageRoot, env, stdio: 'inherit', timeout: 600000 }); + console.log(`Node ${version}: packed CLI and telemetry regression suites passed; artifact ${packed.digest}`); + } else { + const work = join(runtimeRoot, 'empty-home'); + mkdirSync(work); + const result = spawnSync(node, [join(packed.installed, 'bin.cjs')], { cwd: work, encoding: 'utf8', env: { PATH: '', HOME: work }, timeout: 10000 }); + assert.equal(result.status, 1); + assert.match(result.stderr, /requires Node.js 20\.20\+, 22\.22\+, or 23\.5\+/); + assert.equal(result.stdout, ''); + assert.deepEqual(readdirSync(work), []); + console.log(`Node ${version}: rejected before importing the wizard`); + } + } +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/unsupportedNode.mjs b/packages/setupWizard/tests/e2e/unsupportedNode.mjs deleted file mode 100644 index fe81593cc..000000000 --- a/packages/setupWizard/tests/e2e/unsupportedNode.mjs +++ /dev/null @@ -1,19 +0,0 @@ -import assert from 'node:assert/strict'; -import { execFileSync, spawnSync } from 'node:child_process'; -import { mkdirSync, readdirSync } from 'node:fs'; -import { join } from 'node:path'; -import { artifact } from './harness.mjs'; -const packed = artifact(); -try { - const node22 = process.env.SETUP_TEST_NODE22 ?? execFileSync('npm', ['exec', '--yes', '--package=node@22.22.0', '--', 'node', '-p', 'process.execPath'], { encoding: 'utf8' }).trim(); - const work = join(packed.root, 'node22'); - mkdirSync(work); - const result = spawnSync(node22, [join(packed.installed, 'bin.cjs')], { cwd: work, encoding: 'utf8', env: { PATH: '', HOME: work } }); - assert.equal(result.status, 1); - assert.match(result.stderr, /requires Node.js 24 or newer/); - assert.equal(result.stdout, ''); - assert.deepEqual(readdirSync(work), []); - console.log(`Node 22.22 rejected before loading the wizard; artifact ${packed.digest}`); -} finally { - packed.cleanup(); -} diff --git a/packages/setupWizard/tests/unit/bootstrap.test.mjs b/packages/setupWizard/tests/unit/bootstrap.test.mjs new file mode 100644 index 000000000..9e4f846e7 --- /dev/null +++ b/packages/setupWizard/tests/unit/bootstrap.test.mjs @@ -0,0 +1,29 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, mkdtempSync, rmSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +test('bootstrap admits dependency-compatible versions and rejects unsupported gaps before import', () => { + const root = mkdtempSync(join(tmpdir(), 'setup-bootstrap-')); + const bin = join(root, 'bin.cjs'); + // Deliberately omit dist: entering the supported branch must stop at its + // handled missing-module error, never run the wizard or send telemetry. + copyFileSync(new URL('../../bin.cjs', import.meta.url), bin); + try { + for (const [version, supported] of [ + ['18.20.8', false], ['20.19.9', false], ['20.20.0', true], ['20.21.0', true], + ['21.7.3', false], ['22.21.9', false], ['22.22.0', true], + ['23.4.9', false], ['23.5.0', true], ['24.0.0', true], ['26.0.0', true], + ]) { + const result = spawnSync(process.execPath, ['-e', `Object.defineProperty(process.versions, 'node', { value: ${JSON.stringify(version)} }); require(${JSON.stringify(bin)});`], { cwd: root, encoding: 'utf8', env: { HOME: root, PATH: '' }, timeout: 10000 }); + assert.equal(result.status, 1, version); + assert.equal(result.stdout, '', version); + assert.match(result.stderr, supported ? /Unable to start setup-sourcebot/ : /requires Node.js 20\.20\+/, version); + } + assert.deepEqual(readdirSync(root), ['bin.cjs']); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/yarn.lock b/yarn.lock index 71783e786..16aa3ddd3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10036,12 +10036,12 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^24.0.0": - version: 24.13.4 - resolution: "@types/node@npm:24.13.4" +"@types/node@npm:^20.19.43": + version: 20.19.43 + resolution: "@types/node@npm:20.19.43" dependencies: - undici-types: "npm:~7.18.0" - checksum: 10c0/a12196e984cb09ead549651217b4b395961ea011f3a3791a9f78311d70b5f3be343c6fc935376553c06a34c14d963d4f637ad7829837b0c88cf441919f422893 + undici-types: "npm:~6.21.0" + checksum: 10c0/9bcec3b5295bdd77ff0b44a528a69f7e22028c347507ba2c69be47ec84e30299f45043b222e9c86c510e138c9c53b2419dd5cd34920602a4a5a381c288075318 languageName: node linkType: hard @@ -21298,7 +21298,7 @@ __metadata: dependencies: "@inquirer/prompts": "npm:^8.4.3" "@sourcebot/schemas": "workspace:^" - "@types/node": "npm:^24.0.0" + "@types/node": "npm:^20.19.43" chalk: "npm:^5.6.2" inquirer-select-pro: "npm:^1.0.0-alpha.9" node-pty: "npm:^1.1.0" @@ -23032,13 +23032,6 @@ __metadata: languageName: node linkType: hard -"undici-types@npm:~7.18.0": - version: 7.18.2 - resolution: "undici-types@npm:7.18.2" - checksum: 10c0/85a79189113a238959d7a647368e4f7c5559c3a404ebdb8fc4488145ce9426fcd82252a844a302798dfc0e37e6fb178ff481ed03bc4caf634c5757d9ef43521d - languageName: node - linkType: hard - "undici@npm:^7": version: 7.29.1 resolution: "undici@npm:7.29.1" From ccfd14671eb5646ea4d606ab283203a9163688d7 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:25:43 -0700 Subject: [PATCH 09/14] Consolidate telemetry PR changelog entry --- CHANGELOG.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4f9c3cf6..efdfe6c12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,8 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Added privacy-scoped setup wizard funnel telemetry with deployment identity handoff, supporting Node.js from 20.20.0. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) -- Added isolated live setup CLI deployment tests covering public code hosts, local clones, AI configuration, search, and restart identity; repaired cross-platform test-runner cleanup. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) +- Added privacy-scoped setup wizard funnel telemetry with deployment identity handoff, Node.js 20.20.0 support, and cross-platform end-to-end test coverage. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) ## [5.1.13] - 2026-09-12 From c3a1928e2630b47e8741142e11e68d808be456e9 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:33:18 -0700 Subject: [PATCH 10/14] Remove unrelated formatting changes from telemetry PR --- .github/workflows/release-setup-sourcebot.yml | 1 + packages/setupWizard/src/azuredevops.ts | 2 +- packages/setupWizard/src/bitbucket.ts | 19 ++-- packages/setupWizard/src/github.ts | 11 +-- packages/setupWizard/src/gitlab.ts | 23 ++--- packages/setupWizard/src/index.ts | 99 +++++++------------ packages/setupWizard/src/localRepos.ts | 62 ++++++------ packages/setupWizard/src/models.ts | 83 ++++++++-------- 8 files changed, 140 insertions(+), 160 deletions(-) diff --git a/.github/workflows/release-setup-sourcebot.yml b/.github/workflows/release-setup-sourcebot.yml index a56fac51c..219f62531 100644 --- a/.github/workflows/release-setup-sourcebot.yml +++ b/.github/workflows/release-setup-sourcebot.yml @@ -142,3 +142,4 @@ jobs: git tag -a "setup-sourcebot-v$VERSION" -m "setup-sourcebot v$VERSION" git push origin HEAD:main git push origin "setup-sourcebot-v$VERSION" + diff --git a/packages/setupWizard/src/azuredevops.ts b/packages/setupWizard/src/azuredevops.ts index b60356919..4b9d10b2e 100644 --- a/packages/setupWizard/src/azuredevops.ts +++ b/packages/setupWizard/src/azuredevops.ts @@ -61,7 +61,7 @@ export async function collectAzureDevOpsConfig(connectionName: string): Promise< const token = await password({ message: `Azure DevOps Personal Access Token (stored locally in .env as ${envKey})`, mask: true, - validate: (v) => (!v?.trim() ? 'Token is required' : true), + validate: (v) => !v?.trim() ? 'Token is required' : true, }); env[envKey] = token; config.token = { env: envKey }; diff --git a/packages/setupWizard/src/bitbucket.ts b/packages/setupWizard/src/bitbucket.ts index def0bf02a..1a38a8df0 100644 --- a/packages/setupWizard/src/bitbucket.ts +++ b/packages/setupWizard/src/bitbucket.ts @@ -42,11 +42,14 @@ async function collectBitbucketCloud( }); if (authMethod === 'api-token') { - note('The email you use to sign in to Atlassian (e.g. you@example.com).', 'Atlassian account email'); + note( + 'The email you use to sign in to Atlassian (e.g. you@example.com).', + 'Atlassian account email', + ); const email = await input({ message: 'Atlassian account email', - validate: (v) => (!v?.trim() ? 'Email is required' : true), + validate: (v) => !v?.trim() ? 'Email is required' : true, }); config.user = email; @@ -60,7 +63,7 @@ async function collectBitbucketCloud( const gitUser = await input({ message: 'Bitbucket username', - validate: (v) => (!v?.trim() ? 'Username is required' : true), + validate: (v) => !v?.trim() ? 'Username is required' : true, }); config.gitUser = gitUser; @@ -79,7 +82,7 @@ async function collectBitbucketCloud( const token = await password({ message: `API Token (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => (!v?.trim() ? 'Token is required' : true), + validate: (v) => !v?.trim() ? 'Token is required' : true, }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; @@ -96,7 +99,7 @@ async function collectBitbucketCloud( const token = await password({ message: `Access Token (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => (!v?.trim() ? 'Token is required' : true), + validate: (v) => !v?.trim() ? 'Token is required' : true, }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; @@ -114,7 +117,7 @@ async function collectBitbucketCloud( const username = await input({ message: 'Bitbucket username', - validate: (v) => (!v?.trim() ? 'Username is required' : true), + validate: (v) => !v?.trim() ? 'Username is required' : true, }); config.user = username; @@ -122,7 +125,7 @@ async function collectBitbucketCloud( const token = await password({ message: `Bitbucket App Password (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => (!v?.trim() ? 'App Password is required' : true), + validate: (v) => !v?.trim() ? 'App Password is required' : true, }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; @@ -212,7 +215,7 @@ async function collectBitbucketServer( const token = await password({ message: `Bitbucket HTTP Access Token (stored locally in .env as ${tokenEnvKey})`, mask: true, - validate: (v) => (!v?.trim() ? 'Token is required' : true), + validate: (v) => !v?.trim() ? 'Token is required' : true, }); env[tokenEnvKey] = token; config.token = { env: tokenEnvKey }; diff --git a/packages/setupWizard/src/github.ts b/packages/setupWizard/src/github.ts index ed85c8fed..0d1fbc09d 100644 --- a/packages/setupWizard/src/github.ts +++ b/packages/setupWizard/src/github.ts @@ -41,13 +41,12 @@ async function searchGitHub( 'User-Agent': 'setup-sourcebot', ...(token ? { Authorization: `Bearer ${token}` } : {}), }; - const url = - type === 'repo' - ? `${apiBase}/search/repositories?q=${encodeURIComponent(query)}&per_page=8` - : `${apiBase}/search/users?q=${encodeURIComponent(query)}+type:${type}&per_page=8`; + const url = type === 'repo' + ? `${apiBase}/search/repositories?q=${encodeURIComponent(query)}&per_page=8` + : `${apiBase}/search/users?q=${encodeURIComponent(query)}+type:${type}&per_page=8`; try { const res = await wizardFetch(url, { headers, signal: AbortSignal.timeout(8000) }); - const data = (await res.json()) as { items?: Array<{ login?: string; full_name?: string }> }; + const data = await res.json() as { items?: Array<{ login?: string; full_name?: string }> }; const literalFallback = (): SearchOption | null => { return { name: query, value: query }; @@ -56,7 +55,7 @@ async function searchGitHub( if (!res.ok) { lifecycle.fail('network', true); const warning = - res.status === 403 && res.headers.get('x-ratelimit-remaining') === '0' + (res.status === 403 && res.headers.get('x-ratelimit-remaining') === '0') ? '⚠ Autocomplete disabled — GitHub rate limit exceeded.' : '⚠ Autocomplete disabled — authentication failed, check your PAT.'; const fallback = literalFallback(); diff --git a/packages/setupWizard/src/gitlab.ts b/packages/setupWizard/src/gitlab.ts index cc8c0f30f..fbaac23ac 100644 --- a/packages/setupWizard/src/gitlab.ts +++ b/packages/setupWizard/src/gitlab.ts @@ -54,15 +54,14 @@ async function searchGitLab( if (!res.ok) { lifecycle.fail('network', true); - const warning = - res.status === 401 - ? '⚠ Autocomplete disabled — authentication failed, check your PAT.' - : `⚠ Autocomplete disabled — GitLab API error (${res.status}).`; + const warning = res.status === 401 + ? '⚠ Autocomplete disabled — authentication failed, check your PAT.' + : `⚠ Autocomplete disabled — GitLab API error (${res.status}).`; const fallback = literalFallback(); return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; } - const data = (await res.json()) as Array<{ + const data = await res.json() as Array<{ full_path?: string; path_with_namespace?: string; username?: string; @@ -132,7 +131,11 @@ export async function collectGitLabConfig(connectionName: string): Promise 0 && current.length + 1 + word.length > width) { + if (current.length > 0 && (current.length + 1 + word.length) > width) { lines.push(indent + current); current = word; } else { @@ -72,7 +72,9 @@ function wrapText(text: string, indent: string, width: number): string[] { } function openBrowser(url: string): void { - const cmd = process.platform === 'darwin' ? 'open' : process.platform === 'win32' ? 'cmd' : 'xdg-open'; + const cmd = process.platform === 'darwin' ? 'open' + : process.platform === 'win32' ? 'cmd' + : 'xdg-open'; const args = process.platform === 'win32' ? ['/c', 'start', '""', url] : [url]; lifecycle.check(); const browser = spawn(cmd, args, { stdio: 'ignore', detached: true }); @@ -134,9 +136,9 @@ type PublishedPort = { host: string; port: number }; // undefined for specs with no fixed host port (container-only, ranges, ${VAR}). function parseHostPortSpec(spec: string): PublishedPort | undefined { let s = spec.trim(); - s = s.replace(/\s+#.*$/, '').trim(); // strip inline comment - s = s.replace(/^["']|["']$/g, '').trim(); // strip surrounding quotes - s = s.replace(/\/(tcp|udp|sctp)$/i, ''); // strip protocol suffix + s = s.replace(/\s+#.*$/, '').trim(); // strip inline comment + s = s.replace(/^["']|["']$/g, '').trim(); // strip surrounding quotes + s = s.replace(/\/(tcp|udp|sctp)$/i, ''); // strip protocol suffix const parts = s.split(':'); let host = '0.0.0.0'; let hostPort: string; @@ -209,9 +211,7 @@ function isPortInUse({ host, port }: PublishedPort): Promise { const release = lifecycle.own(() => server.close()); server.once('close', release); server.once('error', (err: NodeJS.ErrnoException) => { - server.close(() => { - /* noop */ - }); + server.close(() => { /* noop */ }); // EADDRINUSE = taken. Other errors (e.g. EACCES on privileged ports) aren't // a "someone else has it" conflict we can meaningfully report, so treat as free. if (err.code !== 'EADDRINUSE') { @@ -234,6 +234,9 @@ function isPortInUse({ host, port }: PublishedPort): Promise { const docker = new Docker((category) => lifecycle.fail(category, true)); let portInspectionFailed = false; + +// Mirrors Docker Compose's project-name normalization for the default case +// where the project name is derived from the working directory basename. function dockerComposeProjectName(): string { return basename(process.cwd()) .toLowerCase() @@ -307,7 +310,9 @@ async function main() { const allEnv: EnvVars = {}; const localRepoIndex = new Map(); - note('Code is cloned and indexed locally on this machine. No code is ever transmitted to Sourcebot.'); + note( + 'Code is cloned and indexed locally on this machine. No code is ever transmitted to Sourcebot.', + ); // eslint-disable-next-line no-constant-condition while (true) { @@ -315,21 +320,9 @@ async function main() { message: 'Which code host do you want to connect?', loop: false, choices: [ - { - value: 'github', - name: 'GitHub', - description: 'github.com, GitHub Enterprise Server, or GitHub Enterprise Cloud', - }, - { - value: 'gitlab', - name: 'GitLab', - description: 'gitlab.com, GitLab Self Managed, or GitLab Dedicated', - }, - { - value: 'local', - name: 'Local git repositories', - description: 'git repositories in a local directory', - }, + { value: 'github', name: 'GitHub', description: 'github.com, GitHub Enterprise Server, or GitHub Enterprise Cloud' }, + { value: 'gitlab', name: 'GitLab', description: 'gitlab.com, GitLab Self Managed, or GitLab Dedicated' }, + { value: 'local', name: 'Local git repositories', description: 'git repositories in a local directory' }, { value: 'git', name: 'Remote git repository', description: 'Arbitrary git URL' }, { value: 'azuredevops', name: 'Azure DevOps', description: 'dev.azure.com or Azure Devops Server' }, { value: 'bitbucket', name: 'Bitbucket', description: 'Bitbucket Cloud or Bitbucket Data Center' }, @@ -380,7 +373,9 @@ async function main() { ...result.telemetry, }); for (const { name, config } of result.connections) { - const finalName = name ? generateConnectionName(name, connections) : connectionName; + const finalName = name + ? generateConnectionName(name, connections) + : connectionName; connections[finalName] = config; } Object.assign(allEnv, result.env); @@ -496,19 +491,10 @@ async function main() { const TOP_LEVEL_ENV_KEYS = ['AUTH_URL']; const connectionEnv = Object.fromEntries( - Object.entries(allEnv).filter( - ([k]) => - !Object.values(PROVIDER_ENV_KEYS).includes(k) && - !['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k) && - !TOP_LEVEL_ENV_KEYS.includes(k), - ), + Object.entries(allEnv).filter(([k]) => !Object.values(PROVIDER_ENV_KEYS).includes(k) && !['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k) && !TOP_LEVEL_ENV_KEYS.includes(k)) ); const aiEnv = Object.fromEntries( - Object.entries(allEnv).filter( - ([k]) => - Object.values(PROVIDER_ENV_KEYS).includes(k) || - ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k), - ), + Object.entries(allEnv).filter(([k]) => Object.values(PROVIDER_ENV_KEYS).includes(k) || ['AWS_ACCESS_KEY_ID', 'AWS_SECRET_ACCESS_KEY'].includes(k)) ); const envLines: string[] = [ @@ -578,20 +564,17 @@ async function main() { releaseWriter(); const fileInfo: Record = { 'config.json': { - description: - 'The Sourcebot configuration file. This controls which repos Sourcebot indexes and which language models it connects to.', + description: 'The Sourcebot configuration file. This controls which repos Sourcebot indexes and which language models it connects to.', docsLabel: 'Configuration file docs', docsUrl: 'https://docs.sourcebot.dev/docs/configuration/config-file', }, '.env': { - description: - 'The environment file your Sourcebot deployment will load. This includes any of the access tokens you provided here, as well as generated secrets required to run Sourcebot.', + description: 'The environment file your Sourcebot deployment will load. This includes any of the access tokens you provided here, as well as generated secrets required to run Sourcebot.', docsLabel: 'Environment variables docs', docsUrl: 'https://docs.sourcebot.dev/docs/configuration/environment-variables', }, 'docker-compose.override.yml': { - description: - 'Mounts your local repositories into the Sourcebot container so they can be indexed. Merged with docker-compose.yml at `docker compose up` time.', + description: 'Mounts your local repositories into the Sourcebot container so they can be indexed. Merged with docker-compose.yml at `docker compose up` time.', }, }; @@ -695,8 +678,7 @@ async function main() { console.log(' ' + chalk.dim('- ') + `${c.Name} ${chalk.dim(`(${c.Service})`)}`); } const stop = await confirm({ - message: - 'Stop and remove the running deployment? (required before any volume changes or restart can apply)', + message: 'Stop and remove the running deployment? (required before any volume changes or restart can apply)', default: true, }); if (stop) { @@ -715,9 +697,7 @@ async function main() { } } else if (stopped.length > 0) { console.log(); - console.log( - chalk.yellow('⚠ ') + 'Stopped containers from a previous run exist and will conflict on next start:', - ); + console.log(chalk.yellow('⚠ ') + 'Stopped containers from a previous run exist and will conflict on next start:'); for (const c of stopped) { console.log(' ' + chalk.dim('- ') + `${c.Name} ${chalk.dim(`(${c.Service})`)}`); } @@ -799,7 +779,7 @@ async function main() { const inUse: PublishedPort[] = []; for (const p of publishedPorts) { const ownedByContainer = (owners.get(p.port)?.length ?? 0) > 0; - if (ownedByContainer || (await isPortInUse(p))) { + if (ownedByContainer || await isPortInUse(p)) { inUse.push(p); } } @@ -825,13 +805,16 @@ async function main() { for (const p of inUse) { const display = p.host === '0.0.0.0' ? `${p.port}` : `${p.host}:${p.port}`; const by = owners.get(p.port); - const suffix = - by && by.length > 0 ? chalk.dim(` (in use by Docker container ${by.join(', ')})`) : ''; + const suffix = by && by.length > 0 + ? chalk.dim(` (in use by Docker container ${by.join(', ')})`) + : ''; console.log(' ' + chalk.dim('- ') + display + suffix); } // Containers we can stop ourselves; ports held by non-Docker processes we can't. - const conflictingContainers = [...new Set(inUse.flatMap((p) => owners.get(p.port) ?? []))]; + const conflictingContainers = [...new Set( + inUse.flatMap((p) => owners.get(p.port) ?? []), + )]; if (conflictingContainers.length > 0) { console.log(); @@ -856,7 +839,7 @@ async function main() { const freshOwners = freshResult.ok ? freshResult.value : new Map(); for (const p of inUse) { const ownedByContainer = (freshOwners.get(p.port)?.length ?? 0) > 0; - if (ownedByContainer || (await isPortInUse(p))) { + if (ownedByContainer || await isPortInUse(p)) { stillInUse.push(p); } } @@ -867,9 +850,7 @@ async function main() { console.log(chalk.green('✓ ') + 'All required ports are now free'); } else { console.log(); - console.log( - chalk.yellow('⚠ ') + 'These ports are still in use (likely a non-Docker process):', - ); + console.log(chalk.yellow('⚠ ') + 'These ports are still in use (likely a non-Docker process):'); for (const p of stillInUse) { const display = p.host === '0.0.0.0' ? `${p.port}` : `${p.host}:${p.port}`; console.log(' ' + chalk.dim('- ') + display); @@ -880,9 +861,7 @@ async function main() { if (hasPortConflicts) { console.log(); - console.log( - chalk.dim(' Free these ports (stop the process or container using them), or change the host'), - ); + console.log(chalk.dim(' Free these ports (stop the process or container using them), or change the host')); console.log(chalk.dim(' port mappings in docker-compose.yml, before starting Sourcebot.')); } } @@ -1001,9 +980,7 @@ async function main() { } if (hasPortConflicts) { - nextSteps.push( - `${step++}. Free the host ports listed above (or change the host port mappings in docker-compose.yml).`, - ); + nextSteps.push(`${step++}. Free the host ports listed above (or change the host port mappings in docker-compose.yml).`); nextSteps.push(''); } diff --git a/packages/setupWizard/src/localRepos.ts b/packages/setupWizard/src/localRepos.ts index 30eefd654..c9be3a041 100644 --- a/packages/setupWizard/src/localRepos.ts +++ b/packages/setupWizard/src/localRepos.ts @@ -13,7 +13,16 @@ import { note } from './utils.js'; const MAX_DEPTH = 5; -const SKIP_DIRS = new Set(['node_modules', 'dist', 'build', 'out', 'target', 'vendor', 'coverage', '__pycache__']); +const SKIP_DIRS = new Set([ + 'node_modules', + 'dist', + 'build', + 'out', + 'target', + 'vendor', + 'coverage', + '__pycache__', +]); function expandHostPath(p: string): string { const trimmed = p.trim(); @@ -59,7 +68,9 @@ async function findGitRepos(root: string, maxDepth: number): Promise { return repos.sort(); } -export async function collectLocalReposConfig(localRepoIndex: Map): Promise { +export async function collectLocalReposConfig( + localRepoIndex: Map, +): Promise { note( [ 'Point at a directory on your machine that contains git repositories.', @@ -120,15 +131,13 @@ export async function collectLocalReposConfig(localRepoIndex: Map !posixRel(p).includes('/')); - const connections = - allSelected && allAtDepthOne - ? [ - { - config: { - type: 'git', - url: `file://${containerRoot}/*`, - } satisfies GenericGitHostConnectionConfig, - }, - ] - : selected.map((repoPath) => { - const config: GenericGitHostConnectionConfig = { - type: 'git', - url: `file://${containerRoot}/${posixRel(repoPath)}`, - }; - return { name: basename(repoPath), config }; - }); + const connections = allSelected && allAtDepthOne + ? [{ + config: { + type: 'git', + url: `file://${containerRoot}/*`, + } satisfies GenericGitHostConnectionConfig, + }] + : selected.map((repoPath) => { + const config: GenericGitHostConnectionConfig = { + type: 'git', + url: `file://${containerRoot}/${posixRel(repoPath)}`, + }; + return { name: basename(repoPath), config }; + }); return { connections, diff --git a/packages/setupWizard/src/models.ts b/packages/setupWizard/src/models.ts index f28069aea..2b5a8410e 100644 --- a/packages/setupWizard/src/models.ts +++ b/packages/setupWizard/src/models.ts @@ -15,15 +15,15 @@ import { INPUT_THEME, note, type EnvVars } from './utils.js'; type Provider = LanguageModel['provider']; export const PROVIDER_ENV_KEYS: Record = { - anthropic: 'ANTHROPIC_API_KEY', - openai: 'OPENAI_API_KEY', + 'anthropic': 'ANTHROPIC_API_KEY', + 'openai': 'OPENAI_API_KEY', 'google-generative-ai': 'GOOGLE_GENERATIVE_AI_API_KEY', - deepseek: 'DEEPSEEK_API_KEY', - mistral: 'MISTRAL_API_KEY', - xai: 'XAI_API_KEY', - openrouter: 'OPENROUTER_API_KEY', + 'deepseek': 'DEEPSEEK_API_KEY', + 'mistral': 'MISTRAL_API_KEY', + 'xai': 'XAI_API_KEY', + 'openrouter': 'OPENROUTER_API_KEY', 'openai-compatible': 'OPENAI_COMPATIBLE_API_KEY', - azure: 'AZURE_OPENAI_API_KEY', + 'azure': 'AZURE_OPENAI_API_KEY', }; // ─── models.dev catalog ──────────────────────────────────────────────────── @@ -132,7 +132,10 @@ async function getModelOptionsForProvider(providerKey: string): Promise { +async function searchModel(options: { + message: string; + models: ModelOption[]; +}): Promise { const choices = options.models.map((m) => ({ name: m.name === m.id ? m.id : `${m.id} · ${m.name}`, value: m.id, @@ -150,8 +153,8 @@ async function searchModel(options: { message: string; models: ModelOption[] }): return choices; } const lowered = trimmed.toLowerCase(); - const filtered = choices.filter( - (c) => c.value.toLowerCase().includes(lowered) || c.name.toLowerCase().includes(lowered), + const filtered = choices.filter((c) => + c.value.toLowerCase().includes(lowered) || c.name.toLowerCase().includes(lowered), ); const hasExact = choices.some((c) => c.value === trimmed); if (!hasExact) { @@ -172,7 +175,7 @@ async function ensureApiKey(provider: Provider, env: EnvVars): Promise { const apiKey = await password({ message: `API key (stored locally in .env as ${envKey})`, mask: true, - validate: (v) => (!v?.trim() ? 'API key is required' : true), + validate: (v) => !v?.trim() ? 'API key is required' : true, }); env[envKey] = apiKey; } @@ -221,13 +224,13 @@ async function collectModelConfig( case 'azure': { const resourceName = await input({ message: 'Azure resource name', - validate: (v) => (!v?.trim() ? 'Resource name is required' : true), + validate: (v) => !v?.trim() ? 'Resource name is required' : true, }); const apiVersion = await input({ message: 'API version', default: '2024-08-01-preview', theme: INPUT_THEME, - validate: (v) => (!v?.trim() ? 'API version is required' : true), + validate: (v) => !v?.trim() ? 'API version is required' : true, }); const envKey = await ensureApiKey(provider, env); const config: AzureLanguageModel = { @@ -252,7 +255,7 @@ async function collectModelConfig( if (!env['AWS_ACCESS_KEY_ID']) { env['AWS_ACCESS_KEY_ID'] = await input({ message: 'AWS Access Key ID (stored locally in .env as AWS_ACCESS_KEY_ID)', - validate: (v) => (!v?.trim() ? 'Access Key ID is required' : true), + validate: (v) => !v?.trim() ? 'Access Key ID is required' : true, }); } config.accessKeyId = { env: 'AWS_ACCESS_KEY_ID' }; @@ -261,7 +264,7 @@ async function collectModelConfig( env['AWS_SECRET_ACCESS_KEY'] = await password({ message: 'AWS Secret Access Key (stored locally in .env as AWS_SECRET_ACCESS_KEY)', mask: true, - validate: (v) => (!v?.trim() ? 'Secret Access Key is required' : true), + validate: (v) => !v?.trim() ? 'Secret Access Key is required' : true, }); } config.accessKeySecret = { env: 'AWS_SECRET_ACCESS_KEY' }; @@ -271,7 +274,7 @@ async function collectModelConfig( message: 'AWS region', default: 'us-east-1', theme: INPUT_THEME, - validate: (v) => (!v?.trim() ? 'Region is required' : true), + validate: (v) => !v?.trim() ? 'Region is required' : true, }); return config; } @@ -280,7 +283,7 @@ async function collectModelConfig( if (!env['GOOGLE_VERTEX_PROJECT']) { env['GOOGLE_VERTEX_PROJECT'] = await input({ message: 'Google Cloud project ID (stored locally in .env as GOOGLE_VERTEX_PROJECT)', - validate: (v) => (!v?.trim() ? 'Project ID is required' : true), + validate: (v) => !v?.trim() ? 'Project ID is required' : true, }); } if (!env['GOOGLE_VERTEX_REGION']) { @@ -288,7 +291,7 @@ async function collectModelConfig( message: 'Google Cloud region (stored locally in .env as GOOGLE_VERTEX_REGION)', default: 'us-central1', theme: INPUT_THEME, - validate: (v) => (!v?.trim() ? 'Region is required' : true), + validate: (v) => !v?.trim() ? 'Region is required' : true, }); } @@ -306,9 +309,8 @@ async function collectModelConfig( if (!useAppDefault) { if (!env['GOOGLE_APPLICATION_CREDENTIALS']) { env['GOOGLE_APPLICATION_CREDENTIALS'] = await input({ - message: - 'Path to service account credentials JSON (stored locally in .env as GOOGLE_APPLICATION_CREDENTIALS)', - validate: (v) => (!v?.trim() ? 'Credentials path is required' : true), + message: 'Path to service account credentials JSON (stored locally in .env as GOOGLE_APPLICATION_CREDENTIALS)', + validate: (v) => !v?.trim() ? 'Credentials path is required' : true, }); } config.credentials = { env: 'GOOGLE_APPLICATION_CREDENTIALS' }; @@ -331,7 +333,7 @@ export async function collectModels( 'in natural language and get answers grounded in your indexed code.', ' https://docs.sourcebot.dev/docs/features/ask/ask-sourcebot', '', - "You'll need an API key from at least one supported provider", + 'You\'ll need an API key from at least one supported provider', '(Anthropic, OpenAI, Google, etc.) to enable these features.', ].join('\n'), 'AI features', @@ -358,11 +360,7 @@ export async function collectModels( { value: 'amazon-bedrock', name: 'Amazon Bedrock' }, { value: 'google-generative-ai', name: 'Google Gemini' }, { value: 'google-vertex', name: 'Google Vertex AI', description: 'Gemini via Vertex' }, - { - value: 'google-vertex-anthropic', - name: 'Google Vertex AI (Anthropic)', - description: 'Claude via Vertex', - }, + { value: 'google-vertex-anthropic', name: 'Google Vertex AI (Anthropic)', description: 'Claude via Vertex' }, { value: 'azure', name: 'Azure OpenAI' }, { value: 'deepseek', name: 'DeepSeek' }, { value: 'mistral', name: 'Mistral' }, @@ -371,28 +369,27 @@ export async function collectModels( ], }); - const modelOptions = provider === 'openai-compatible' ? null : await getModelOptionsForProvider(provider); - const model = - modelOptions && modelOptions.length > 0 - ? await searchModel({ - message: 'Model name', - models: modelOptions, - }) - : await input({ - message: 'Model name', - validate: (v) => (!v?.trim() ? 'Model name is required' : true), - }); + const modelOptions = provider === 'openai-compatible' + ? null + : await getModelOptionsForProvider(provider); + const model = modelOptions && modelOptions.length > 0 + ? await searchModel({ + message: 'Model name', + models: modelOptions, + }) + : await input({ + message: 'Model name', + validate: (v) => !v?.trim() ? 'Model name is required' : true, + }); let credentialMode: AiSummary['credentialMode'] = 'api_key'; const config = await collectModelConfig(provider, model, env, (mode) => { credentialMode = mode; }); - const displayName = ( - await input({ - message: 'Display name (optional, press enter to skip)', - }) - ).trim(); + const displayName = (await input({ + message: 'Display name (optional, press enter to skip)', + })).trim(); if (displayName) { config.displayName = displayName; } From b3e68f927f4302acd534cc038439b3081e9d0bc9 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:36:48 -0700 Subject: [PATCH 11/14] Run setup wizard verification on Node 24 only --- .github/workflows/release-setup-sourcebot.yml | 1 - .github/workflows/setup-wizard-e2e.yml | 9 +-------- .github/workflows/test.yml | 2 +- packages/setupWizard/README.md | 7 +++---- 4 files changed, 5 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release-setup-sourcebot.yml b/.github/workflows/release-setup-sourcebot.yml index 219f62531..03662f10c 100644 --- a/.github/workflows/release-setup-sourcebot.yml +++ b/.github/workflows/release-setup-sourcebot.yml @@ -112,7 +112,6 @@ jobs: docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest yarn workspace setup-sourcebot test yarn workspace setup-sourcebot test:e2e - yarn workspace setup-sourcebot test:node-compatibility - name: Upgrade npm for Trusted Publishing working-directory: . diff --git a/.github/workflows/setup-wizard-e2e.yml b/.github/workflows/setup-wizard-e2e.yml index 7ebbb838f..d2b97d3e1 100644 --- a/.github/workflows/setup-wizard-e2e.yml +++ b/.github/workflows/setup-wizard-e2e.yml @@ -21,7 +21,6 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - node: ['20.20.0', '22.22.0', '24.x'] runs-on: ${{ matrix.os }} env: PACKAGE_TRACKER_ANALYTICS: 'false' @@ -37,19 +36,13 @@ jobs: - run: yarn rebuild node-pty - run: yarn workspace @sourcebot/schemas build - run: yarn workspace setup-sourcebot build - - name: Select end-user test runtime - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node }} - - run: corepack enable - run: yarn workspace setup-sourcebot test - run: yarn workspace setup-sourcebot test:platform - name: Linux packed-artifact and runtime checks - if: runner.os == 'Linux' && matrix.node == '24.x' + if: runner.os == 'Linux' run: | docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest yarn workspace setup-sourcebot test:e2e - yarn workspace setup-sourcebot test:node-compatibility yarn workspace setup-sourcebot test:baseline node packages/setupWizard/tests/e2e/packageManagers.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d624ebd3a..c99a57907 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,6 +41,6 @@ jobs: run: yarn install --frozen-lockfile - name: Test - # The CLI has separate packed-artifact and Node-compatibility gates in setup-wizard-e2e. + # The CLI has separate Node 24 packed-artifact checks in setup-wizard-e2e. # Keep the application workspaces on their existing runtime here. run: yarn workspaces foreach --all --topological --exclude setup-sourcebot --exclude 'root-workspace-*' run test diff --git a/packages/setupWizard/README.md b/packages/setupWizard/README.md index 8b17439c9..9262aacd2 100644 --- a/packages/setupWizard/README.md +++ b/packages/setupWizard/README.md @@ -29,15 +29,14 @@ yarn workspace @sourcebot/schemas build yarn workspace setup-sourcebot build yarn workspace setup-sourcebot test yarn workspace setup-sourcebot test:e2e -yarn workspace setup-sourcebot test:node-compatibility ``` The E2E tests compile and pack the package, install it outside the repository, drive its published binary in a PTY, inspect real SDK requests through a local TLS collector, and clean up temporary installations. OpenSSL and Docker are required. -The compatibility suite tests the same tarball under Node 20.20.0 and 22.22.0, -and checks early rejection on Node 18, 20.19, and 22.21. Build/release tooling -continues to use Node 24; that is not the end-user minimum. +PR and release verification run on Node 24 only; that is not the end-user minimum. +The optional `yarn workspace setup-sourcebot test:node-compatibility` command remains +available for targeted compatibility investigations, but is not part of CI or release verification. The runtime suite uses `docker.sourcebot.dev/sourcebot-dev/sourcebot:latest` (override only the test image with `SETUP_TEST_SOURCEBOT_IMAGE`). From 5cfc6d12024dcd0c07385080fc97453c0218b5c8 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:38:49 -0700 Subject: [PATCH 12/14] Restore Node compatibility coverage in parallel CI jobs --- .github/workflows/release-setup-sourcebot.yml | 1 + .github/workflows/setup-wizard-e2e.yml | 19 ++++++++++++++++++- .github/workflows/test.yml | 2 +- packages/setupWizard/README.md | 12 +++++++++--- 4 files changed, 29 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release-setup-sourcebot.yml b/.github/workflows/release-setup-sourcebot.yml index 03662f10c..219f62531 100644 --- a/.github/workflows/release-setup-sourcebot.yml +++ b/.github/workflows/release-setup-sourcebot.yml @@ -112,6 +112,7 @@ jobs: docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest yarn workspace setup-sourcebot test yarn workspace setup-sourcebot test:e2e + yarn workspace setup-sourcebot test:node-compatibility - name: Upgrade npm for Trusted Publishing working-directory: . diff --git a/.github/workflows/setup-wizard-e2e.yml b/.github/workflows/setup-wizard-e2e.yml index d2b97d3e1..0b4e2e39f 100644 --- a/.github/workflows/setup-wizard-e2e.yml +++ b/.github/workflows/setup-wizard-e2e.yml @@ -21,6 +21,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] + node: ['20.20.0', '22.22.0', '24.x'] runs-on: ${{ matrix.os }} env: PACKAGE_TRACKER_ANALYTICS: 'false' @@ -36,10 +37,26 @@ jobs: - run: yarn rebuild node-pty - run: yarn workspace @sourcebot/schemas build - run: yarn workspace setup-sourcebot build + - name: Pack on the release runtime + shell: bash + env: + SETUP_TEST_TARBALL: ${{ runner.temp }}/setup-sourcebot.tgz + run: | + yarn workspace setup-sourcebot pack --out "$SETUP_TEST_TARBALL" + echo "SETUP_TEST_TARBALL=$SETUP_TEST_TARBALL" >> "$GITHUB_ENV" + - name: Select end-user test runtime + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - run: corepack enable - run: yarn workspace setup-sourcebot test - run: yarn workspace setup-sourcebot test:platform + - name: Linux minimum-runtime regression checks + if: runner.os == 'Linux' && matrix.node != '24.x' + working-directory: packages/setupWizard + run: node --test --test-concurrency=1 tests/e2e/wizard.test.mjs tests/e2e/collectors.test.mjs tests/e2e/docker.test.mjs tests/e2e/safety.test.mjs - name: Linux packed-artifact and runtime checks - if: runner.os == 'Linux' + if: runner.os == 'Linux' && matrix.node == '24.x' run: | docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest yarn workspace setup-sourcebot test:e2e diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c99a57907..d624ebd3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,6 +41,6 @@ jobs: run: yarn install --frozen-lockfile - name: Test - # The CLI has separate Node 24 packed-artifact checks in setup-wizard-e2e. + # The CLI has separate packed-artifact and Node-compatibility gates in setup-wizard-e2e. # Keep the application workspaces on their existing runtime here. run: yarn workspaces foreach --all --topological --exclude setup-sourcebot --exclude 'root-workspace-*' run test diff --git a/packages/setupWizard/README.md b/packages/setupWizard/README.md index 9262aacd2..ac8afb2ef 100644 --- a/packages/setupWizard/README.md +++ b/packages/setupWizard/README.md @@ -29,14 +29,20 @@ yarn workspace @sourcebot/schemas build yarn workspace setup-sourcebot build yarn workspace setup-sourcebot test yarn workspace setup-sourcebot test:e2e +yarn workspace setup-sourcebot test:node-compatibility ``` The E2E tests compile and pack the package, install it outside the repository, drive its published binary in a PTY, inspect real SDK requests through a local TLS collector, and clean up temporary installations. OpenSSL and Docker are required. -PR and release verification run on Node 24 only; that is not the end-user minimum. -The optional `yarn workspace setup-sourcebot test:node-compatibility` command remains -available for targeted compatibility investigations, but is not part of CI or release verification. +PR verification runs nine parallel OS/Node jobs (Linux, macOS, Windows × Node +20.20.0, 22.22.0, 24). Each job builds and packs on Node 24 before selecting its +test runtime. Linux runs the full CLI regression suite on each version; macOS and +Windows run unit/integration and platform smoke tests. Docker identity, baseline, +and package-manager checks run only in the Node 24 Linux job. +Release verification also runs `test:node-compatibility` against the exact publish +tarball; that command checks the older runtimes sequentially, including early +rejection on Node 18, 20.19, and 22.21. Node 24 is not the end-user minimum. The runtime suite uses `docker.sourcebot.dev/sourcebot-dev/sourcebot:latest` (override only the test image with `SETUP_TEST_SOURCEBOT_IMAGE`). From 7a2ff6645f1ee46c004e33fe7cc8fa575b5d242b Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 22:59:43 -0700 Subject: [PATCH 13/14] Capture privacy-scoped Docker startup failures after setup handoff --- CHANGELOG.md | 2 +- .../setupWizard/src/dockerStartFailure.ts | 55 ++++++++++++++++++ packages/setupWizard/src/index.ts | 29 ++++++++-- packages/setupWizard/src/lifecycle.ts | 14 ++++- packages/setupWizard/src/telemetryEvents.ts | 13 +++++ .../setupWizard/tests/approvedSchema.json | 15 +++++ .../setupWizard/tests/e2e/docker.test.mjs | 57 +++++++++++++++++++ packages/setupWizard/tests/e2e/fakeDocker.cjs | 18 +++++- .../setupWizard/tests/e2e/runtime.test.mjs | 49 ++++++++++++++++ .../tests/unit/dockerStartFailure.test.mjs | 42 ++++++++++++++ 10 files changed, 284 insertions(+), 10 deletions(-) create mode 100644 packages/setupWizard/src/dockerStartFailure.ts create mode 100644 packages/setupWizard/tests/unit/dockerStartFailure.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index efdfe6c12..2bc271528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- Added privacy-scoped setup wizard funnel telemetry with deployment identity handoff, Node.js 20.20.0 support, and cross-platform end-to-end test coverage. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) +- Added privacy-scoped setup wizard funnel and Docker startup-failure telemetry with deployment identity handoff, Node.js 20.20.0 support, and cross-platform end-to-end test coverage. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) ## [5.1.13] - 2026-09-12 diff --git a/packages/setupWizard/src/dockerStartFailure.ts b/packages/setupWizard/src/dockerStartFailure.ts new file mode 100644 index 000000000..83d95f2a3 --- /dev/null +++ b/packages/setupWizard/src/dockerStartFailure.ts @@ -0,0 +1,55 @@ +import { stripVTControlCharacters } from 'node:util'; +import type { Events } from './telemetryEvents.js'; + +type Reason = Events['start_failed']['failureReason']; + +// Docker exposes an exit status, not structured error reasons, for compose up. +// Recognize only specific CLI/daemon diagnostics locally. Never serialize output, +// captures, paths, image names, container names or IDs into an event. +export class DockerStartFailure { + private pending = ''; + private detected: Reason = 'unknown'; + + write(chunk: Buffer): void { + // Bound retained output even for a long-running foreground Compose process. + const lines = (this.pending + chunk.toString()).split(/[\r\n]/); + this.pending = (lines.pop() ?? '').slice(-8192); + for (const line of lines) { + this.classify(line.slice(-8192)); + } + } + + reason(): Reason { + this.classify(this.pending); + this.pending = ''; + return this.detected; + } + + private classify(raw: string): void { + if (this.detected !== 'unknown') { + return; + } + const line = stripVTControlCharacters(raw).trim(); + // Attached application logs are not Docker diagnostics. + if (line.includes(' | ')) { + return; + } + const daemon = /^Error response from daemon:/i.test(line); + if (daemon && /container name .+ is already in use by container/i.test(line)) { + this.detected = 'container_name_conflict'; + } else if (daemon && /port is already allocated|address already in use/i.test(line)) { + this.detected = 'port_conflict'; + } else if ( + /^(?:Error response from daemon:|unable to get image|pull access denied|failed to resolve reference)/i.test(line) && + /pull access denied|manifest unknown|manifest for .+ not found|failed to resolve reference|no matching manifest|toomanyrequests|unauthorized: authentication required/i.test(line) + ) { + this.detected = 'image_pull_failed'; + } else if (daemon && /invalid mount config|mounts denied|error while creating mount source path|invalid volume specification/i.test(line)) { + this.detected = 'mount_failed'; + } else if (/^(?:Cannot connect to the Docker daemon|error during connect:|permission denied while trying to connect to the Docker daemon|docker: ['"]?compose['"]? is not a docker command)/i.test(line)) { + this.detected = 'docker_unavailable'; + } else if (/^(?:validating .+:|no configuration file provided:|yaml: line \d+:|services\..+:|service .+ refers to undefined (?:volume|network) .+: invalid compose project)/i.test(line)) { + this.detected = 'compose_configuration'; + } + } +} diff --git a/packages/setupWizard/src/index.ts b/packages/setupWizard/src/index.ts index 355c3e46f..430d06a53 100644 --- a/packages/setupWizard/src/index.ts +++ b/packages/setupWizard/src/index.ts @@ -15,6 +15,7 @@ import { dockerOutcome, } from './telemetrySummary.js'; import { Docker } from './docker.js'; +import { DockerStartFailure } from './dockerStartFailure.js'; import type { CodeSourceSummary, Events } from './telemetryEvents.js'; import net from 'node:net'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'; @@ -891,7 +892,9 @@ async function main() { deploymentIdentityAction: deploymentIdentity.action, totalDurationMs: 0, }; - const complete = () => lifecycle.complete({ ...completion, totalDurationMs: lifecycle.telemetry.elapsed() }); + const complete = (keepTelemetryOpen = false) => lifecycle.complete( + { ...completion, totalDurationMs: lifecycle.telemetry.elapsed() }, keepTelemetryOpen, + ); if (downloadedCompose && !leftDeploymentRunning) { const startNow = await confirm({ message: hasPortConflicts @@ -910,10 +913,13 @@ async function main() { const readiness = new AbortController(); const releaseReadiness = lifecycle.own(() => readiness.abort()); let spawned = false; + const startFailure = new DockerStartFailure(); await new Promise((resolve) => { const child = lifecycle.child( - spawn('docker', ['compose', 'up'], { stdio: 'inherit', detached: process.platform !== 'win32' }), + spawn('docker', ['compose', 'up'], { stdio: ['inherit', 'inherit', 'pipe'], detached: process.platform !== 'win32' }), ); + child.stderr?.pipe(process.stderr, { end: false }); + child.stderr?.on('data', (chunk: Buffer) => startFailure.write(chunk)); child.once('spawn', () => { if (lifecycle.interrupted) { child.kill(); @@ -922,19 +928,34 @@ async function main() { spawned = true; completion.sourcebotStartOutcome = 'spawned'; completion.completionMode = 'sourcebot_start_spawned'; - void complete(); + // Complete the setup handoff now, but keep diagnostics available + // until Compose exits or the user interrupts it. + void complete(true); void openBrowserWhenReady( SOURCEBOT_URL, AbortSignal.any([lifecycle.signal, readiness.signal]), ).catch(() => {}); }); - child.once('close', () => { + child.once('close', (code, signal) => { readiness.abort(); + if (spawned && code !== 0 && signal !== 'SIGINT' && signal !== 'SIGTERM') { + const reason = startFailure.reason(); + lifecycle.startFailed({ + failurePhase: 'compose_exit', + failureCategory: reason === 'docker_unavailable' ? 'docker_unavailable' : 'docker_command', + failureReason: reason, + }); + } resolve(); }); child.once('error', (error: NodeJS.ErrnoException) => { readiness.abort(); if (!lifecycle.interrupted) { + lifecycle.startFailed({ + failurePhase: 'spawn', + failureCategory: error.code === 'ENOENT' || error.code === 'EACCES' ? 'docker_unavailable' : 'process_spawn', + failureReason: error.code === 'ENOENT' || error.code === 'EACCES' ? 'docker_unavailable' : 'unknown', + }); lifecycle.fail( error.code === 'ENOENT' || error.code === 'EACCES' ? 'docker_unavailable' : 'process_spawn', true, diff --git a/packages/setupWizard/src/lifecycle.ts b/packages/setupWizard/src/lifecycle.ts index a15846694..5bec26f20 100644 --- a/packages/setupWizard/src/lifecycle.ts +++ b/packages/setupWizard/src/lifecycle.ts @@ -18,6 +18,7 @@ export class Lifecycle { terminal?: 'completed' | 'cancelled' | 'failed'; interrupted = false; private installed = false; + private startFailureCaptured = false; constructor(readonly telemetry = new Telemetry()) {} get signal(): AbortSignal { return this.controller.signal; @@ -89,13 +90,22 @@ export class Lifecycle { recoverable, }); } - async complete(properties: Events['completed']): Promise { + startFailed(properties: Events['start_failed']): void { + if (this.interrupted || this.startFailureCaptured || (this.terminal && this.terminal !== 'completed')) { + return; + } + this.startFailureCaptured = true; + this.telemetry.capture('start_failed', properties); + } + async complete(properties: Events['completed'], keepTelemetryOpen = false): Promise { if (this.terminal || this.interrupted) { return; } this.terminal = 'completed'; this.telemetry.capture('completed', properties); - await this.telemetry.shutdown(); + if (!keepTelemetryOpen) { + await this.telemetry.shutdown(); + } } async decline(reason: Events['cancelled']['reason']): Promise { if (!this.terminal) { diff --git a/packages/setupWizard/src/telemetryEvents.ts b/packages/setupWizard/src/telemetryEvents.ts index ac198b29b..e6bcb4b1a 100644 --- a/packages/setupWizard/src/telemetryEvents.ts +++ b/packages/setupWizard/src/telemetryEvents.ts @@ -310,6 +310,19 @@ export const eventSchemas = { ), }, failed: { stage, failureCategory: category, recoverable: boolean }, + start_failed: { + failurePhase: choice('spawn', 'compose_exit'), + failureCategory: choice('docker_unavailable', 'process_spawn', 'docker_command'), + failureReason: choice( + 'container_name_conflict', + 'port_conflict', + 'image_pull_failed', + 'mount_failed', + 'compose_configuration', + 'docker_unavailable', + 'unknown', + ), + }, }; export type EventName = keyof typeof eventSchemas; export type Events = { [K in EventName]: Fields<(typeof eventSchemas)[K]> }; diff --git a/packages/setupWizard/tests/approvedSchema.json b/packages/setupWizard/tests/approvedSchema.json index e1c7b9725..29e19acb5 100644 --- a/packages/setupWizard/tests/approvedSchema.json +++ b/packages/setupWizard/tests/approvedSchema.json @@ -568,6 +568,21 @@ ] } }, + "start_failed": { + "failurePhase": { "enum": ["spawn", "compose_exit"] }, + "failureCategory": { "enum": ["docker_unavailable", "process_spawn", "docker_command"] }, + "failureReason": { + "enum": [ + "container_name_conflict", + "port_conflict", + "image_pull_failed", + "mount_failed", + "compose_configuration", + "docker_unavailable", + "unknown" + ] + } + }, "failed": { "stage": { "enum": [ diff --git a/packages/setupWizard/tests/e2e/docker.test.mjs b/packages/setupWizard/tests/e2e/docker.test.mjs index 9d3843bac..86e836c17 100644 --- a/packages/setupWizard/tests/e2e/docker.test.mjs +++ b/packages/setupWizard/tests/e2e/docker.test.mjs @@ -93,9 +93,66 @@ test('failed spawn offers manual steps and completes after recoverable failures' }); assert.equal(p(result, 'completed').sourcebotStartOutcome, 'spawn_failed'); assert.equal(p(result, 'completed').completionMode, 'sourcebot_start_failed'); + assert.equal(result.events.filter(e => e.event === 'setup_sourcebot_start_failed').length, 1); + assert.deepEqual( + ['failurePhase', 'failureCategory', 'failureReason'].map(key => p(result, 'start_failed')[key]), + ['spawn', 'docker_unavailable', 'docker_unavailable'], + ); assert.ok(result.events.filter(e => e.event === 'setup_sourcebot_failed').every(e => e.properties.recoverable)); }); +for (const [reason, diagnostic] of [ + ['container_name_conflict', 'Error response from daemon: Conflict. The container name "/canary-sensitive-container" is already in use by container "canary-sensitive-id".'], + ['port_conflict', 'Error response from daemon: driver failed programming external connectivity: Bind for canary-sensitive-address failed: port is already allocated'], + ['image_pull_failed', 'Error response from daemon: pull access denied for canary-sensitive-image, repository does not exist or may require docker login'], + ['mount_failed', 'Error response from daemon: invalid mount config for type "bind": bind source path does not exist: canary-sensitive-path'], + ['compose_configuration', 'validating canary-sensitive-path: services.sourcebot Additional property canary-sensitive is not allowed'], + ['docker_unavailable', 'Cannot connect to the Docker daemon at canary-sensitive-socket. Is the docker daemon running?'], + ['unknown', 'canary-sensitive-unrecognized-error'], + ['unknown', 'canary-sensitive-app | Error response from daemon: pull access denied for canary-sensitive-image'], +]) { + test(`post-handoff start failure: ${reason} (${diagnostic.slice(0, 24)})`, async () => { + const result = await scenario(packed, { docker: { start: { stderrChunks: [diagnostic.slice(0, 17), diagnostic.slice(17)], delayMs: 1100 } } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + await d.wait(diagnostic); + }); + const failures = result.events.filter(e => e.event === 'setup_sourcebot_start_failed'); + assert.equal(failures.length, 1); + assert.equal(failures[0].properties.failurePhase, 'compose_exit'); + assert.equal(failures[0].properties.failureReason, reason); + assert.equal(failures[0].properties.failureCategory, reason === 'docker_unavailable' ? 'docker_unavailable' : 'docker_command'); + const completed = result.events.find(e => e.event === 'setup_sourcebot_completed'); + assert.ok(result.events.indexOf(completed) < result.events.indexOf(failures[0])); + assert.equal(failures[0].distinct_id, completed.distinct_id); + assert.equal(result.events.filter(e => e.event === 'setup_sourcebot_completed').length, 1); + assert.equal(result.events.some(e => e.event === 'setup_sourcebot_cancelled'), false); + assert.equal(result.exitCode, 0, 'Telemetry must not change existing CLI exit behavior'); + }); +} + +for (const termination of [{ exitCode: 0 }, { signal: 'SIGINT' }, { signal: 'SIGTERM' }]) { + test(`normal Compose termination is not a start failure: ${JSON.stringify(termination)}`, { skip: process.platform === 'win32' && !!termination.signal }, async () => { + const result = await scenario(packed, { docker: { start: { ...termination, stderrChunks: ['Error response from daemon: pull access denied for canary-sensitive-image\n'] } } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + }); + assert.equal(result.events.some(e => e.event === 'setup_sourcebot_start_failed'), false); + assert.equal(p(result, 'completed').sourcebotStartOutcome, 'spawned'); + }); +} + +test('start failure with unavailable PostHog still exits and retains generated files', async () => { + const began = Date.now(); + const result = await scenario(packed, { telemetry: 'stall', docker: { start: { stderrChunks: ['canary-sensitive-error'] } } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + }); + assert.equal(result.exitCode, 0); + assert.ok(Date.now() - began < 7000); + assert.deepEqual(Object.keys(result.files).sort(), ['.env', 'config.json', 'docker-compose.yml']); +}); + for (const stage of ['fetch', 'docker', 'after_failures']) { test(`Ctrl+C during ${stage} cancels outstanding work`, async () => { const result = await scenario(packed, { diff --git a/packages/setupWizard/tests/e2e/fakeDocker.cjs b/packages/setupWizard/tests/e2e/fakeDocker.cjs index b162a3a4d..6df21d395 100644 --- a/packages/setupWizard/tests/e2e/fakeDocker.cjs +++ b/packages/setupWizard/tests/e2e/fakeDocker.cjs @@ -4,11 +4,23 @@ fs.appendFileSync(process.env.TEST_DOCKER_LOG, JSON.stringify(args) + '\n'); const state = JSON.parse(fs.readFileSync(process.env.TEST_DOCKER_STATE, 'utf8')); const command = args.slice(0, 2).join(' '); fs.appendFileSync(process.env.TEST_DOCKER_PIDS, String(process.pid) + '\n'); -if (state.fail?.includes(command)) { +if (command === 'compose up' && state.start) { + let index = 0; + const write = () => { + if (index < (state.start.stderrChunks ?? []).length) { + process.stderr.write(state.start.stderrChunks[index++]); + setTimeout(write, 10); + } else if (state.start.signal) { + process.kill(process.pid, state.start.signal); + } else { + process.exit(state.start.exitCode ?? 1); + } + }; + setTimeout(write, state.start.delayMs ?? 0); +} else if (state.fail?.includes(command)) { console.error('canary-sensitive-error'); process.exit(1); -} -if (state.stall?.includes(command)) { +} else if (state.stall?.includes(command)) { if (state.descendant) { const child = require('node:child_process').spawn(process.execPath, ['-e', "process.on('SIGINT', () => {}); setInterval(() => {}, 1000)"], { stdio: 'ignore' }); fs.appendFileSync(process.env.TEST_DOCKER_PIDS, String(child.pid) + '\n'); diff --git a/packages/setupWizard/tests/e2e/runtime.test.mjs b/packages/setupWizard/tests/e2e/runtime.test.mjs index 1b3e72249..94c1753e7 100644 --- a/packages/setupWizard/tests/e2e/runtime.test.mjs +++ b/packages/setupWizard/tests/e2e/runtime.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import { randomUUID } from 'node:crypto'; +import { homedir } from 'node:os'; import { execFileSync } from 'node:child_process'; import { writeFileSync, mkdirSync } from 'node:fs'; import { join, resolve } from 'node:path'; @@ -8,6 +9,54 @@ import { fileURLToPath } from 'node:url'; import { artifact, scenario, minimal } from './harness.mjs'; import { INSTALL_ID_PATTERN } from '../../dist/telemetry.js'; +test('real Docker name conflict emits a private diagnostic after setup completion', async () => { + const packed = artifact(); + const docker = execFileSync('which', ['docker'], { encoding: 'utf8' }).trim(); + const image = process.env.SETUP_TEST_SOURCEBOT_IMAGE ?? 'docker.sourcebot.dev/sourcebot-dev/sourcebot:latest'; + const name = `setup-start-conflict-${randomUUID()}`; + const project = `setup-start-${randomUUID()}`; + let created = false; + try { + execFileSync(docker, ['create', '--name', name, '--label', `setup-start-test=${project}`, image], { stdio: 'pipe', timeout: 45000 }); + created = true; + const host = execFileSync(docker, ['context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'], { encoding: 'utf8' }).trim(); + const result = await scenario(packed, { + realDocker: docker, + compose: `services:\n sourcebot:\n image: ${image}\n container_name: ${name}\n pull_policy: never\n`, + sensitiveValues: [name, project], + assertLauncherHome(paths) { + // Docker Desktop creates these empty parent directories itself. + // Still reject files or any wizard-owned per-user state. + const dockerDirectories = process.platform === 'darwin' + ? ['Library', 'Library/Containers', 'Library/Containers/com.docker.docker', 'Library/Containers/com.docker.docker/Data'] + : []; + assert.deepEqual(paths.filter(path => !dockerDirectories.includes(path)), []); + }, + environment: { COMPOSE_PROJECT_NAME: project, DOCKER_HOST: host, DOCKER_CONFIG: process.env.DOCKER_CONFIG ?? join(homedir(), '.docker') }, + async cleanupDeployment({ setup }) { + execFileSync(docker, ['compose', '-p', project, 'down'], { cwd: setup, stdio: 'pipe', timeout: 45000 }); + }, + }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + await d.wait('is already in use by container'); + }); + const failure = result.events.filter(e => e.event === 'setup_sourcebot_start_failed'); + assert.equal(failure.length, 1); + assert.equal(failure[0].properties.failureReason, 'container_name_conflict'); + assert.equal(failure[0].properties.failurePhase, 'compose_exit'); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_start_failed'); + assert.ok(result.events.some(e => e.event === 'setup_sourcebot_completed')); + } finally { + try { + if (created) execFileSync(docker, ['rm', '-v', name], { stdio: 'pipe', timeout: 45000 }); + } finally { + packed.cleanup(); + } + } +}); + test('real Sourcebot containers: packed wizard identity survives first boot, restart, upgrade and opt-out', async () => { const packed = artifact(); const image = process.env.SETUP_TEST_SOURCEBOT_IMAGE ?? 'docker.sourcebot.dev/sourcebot-dev/sourcebot:latest'; diff --git a/packages/setupWizard/tests/unit/dockerStartFailure.test.mjs b/packages/setupWizard/tests/unit/dockerStartFailure.test.mjs new file mode 100644 index 000000000..d81264fa5 --- /dev/null +++ b/packages/setupWizard/tests/unit/dockerStartFailure.test.mjs @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { DockerStartFailure } from '../../dist/dockerStartFailure.js'; +import { Lifecycle } from '../../dist/lifecycle.js'; +import { eventSchemas, validateFields } from '../../dist/telemetryEvents.js'; + +test('Docker reasons are bounded, chunk-safe, ANSI-safe and never contain runtime text', () => { + const classifier = new DockerStartFailure(); + const message = '\u001b[31mError response from daemon: Conflict. The container name "/secret" is already in use by container "secret-id".\u001b[0m'; + for (const char of message) classifier.write(Buffer.from(char)); + assert.equal(classifier.reason(), 'container_name_conflict'); + const huge = new DockerStartFailure(); + huge.write(Buffer.from('secret'.repeat(100000))); + assert.ok(huge.pending.length <= 8192); + assert.equal(huge.reason(), 'unknown'); + const props = { failurePhase: 'compose_exit', failureCategory: 'docker_command', failureReason: 'unknown', stderr: message }; + assert.deepEqual(Object.keys(validateFields(eventSchemas.start_failed, props)).sort(), ['failureCategory', 'failurePhase', 'failureReason']); + assert.throws(() => validateFields(eventSchemas.start_failed, { ...props, failureReason: message })); +}); + +test('only one start diagnostic may follow completion; SDK stays open until process cleanup', async () => { + const events = []; + let shutdowns = 0; + const life = new Lifecycle({ capture: (name, props) => events.push({ name, props }), shutdown: async () => { shutdowns++; } }); + await life.complete({}, true); + assert.equal(shutdowns, 0); + assert.equal(life.terminal, 'completed'); + const failure = { failurePhase: 'compose_exit', failureCategory: 'docker_command', failureReason: 'unknown' }; + life.startFailed(failure); + life.startFailed(failure); + life.fail('docker_command', false); + life.capture('started', {}); + assert.deepEqual(events.map(e => e.name), ['completed', 'start_failed']); + await life.telemetry.shutdown(); + assert.equal(shutdowns, 1); + for (const terminal of [undefined, 'completed', 'cancelled', 'failed']) { + const suppressed = new Lifecycle({ capture() { assert.fail('Unexpected event'); } }); + suppressed.terminal = terminal; + suppressed.interrupted = true; + suppressed.startFailed(failure); + } +}); From fe832d5f28517982e1005483a8d213a640b92938 Mon Sep 17 00:00:00 2001 From: msukkari Date: Fri, 11 Sep 2026 23:01:58 -0700 Subject: [PATCH 14/14] docs: restore npm trusted publishing version requirement --- .github/workflows/release-setup-sourcebot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-setup-sourcebot.yml b/.github/workflows/release-setup-sourcebot.yml index 219f62531..677262e20 100644 --- a/.github/workflows/release-setup-sourcebot.yml +++ b/.github/workflows/release-setup-sourcebot.yml @@ -117,7 +117,7 @@ jobs: - name: Upgrade npm for Trusted Publishing working-directory: . run: | - # Keep npm current for OIDC Trusted Publishing. + # OIDC Trusted Publishing requires npm >= 11.5.1. npm install -g npm@latest npm --version