diff --git a/.github/workflows/release-setup-sourcebot.yml b/.github/workflows/release-setup-sourcebot.yml index 657f2b25e..677262e20 100644 --- a/.github/workflows/release-setup-sourcebot.yml +++ b/.github/workflows/release-setup-sourcebot.yml @@ -60,7 +60,7 @@ jobs: - name: Setup Node.js uses: actions/setup-node@v4 with: - node-version: '20.x' + node-version: '24.x' - name: Install dependencies working-directory: . @@ -103,11 +103,21 @@ jobs: # publishing. yarn pack --out /tmp/setup-sourcebot.tgz + - name: Verify the exact publish artifact + working-directory: . + env: + PACKAGE_TRACKER_ANALYTICS: 'false' + SETUP_TEST_TARBALL: /tmp/setup-sourcebot.tgz + run: | + docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest + yarn workspace setup-sourcebot test + yarn workspace setup-sourcebot test:e2e + yarn workspace setup-sourcebot test:node-compatibility + - name: Upgrade npm for Trusted Publishing working-directory: . run: | - # OIDC Trusted Publishing requires npm >= 11.5.1; Node 20 ships an - # older npm. + # OIDC Trusted Publishing requires npm >= 11.5.1. npm install -g npm@latest npm --version diff --git a/.github/workflows/setup-wizard-e2e.yml b/.github/workflows/setup-wizard-e2e.yml new file mode 100644 index 000000000..0b4e2e39f --- /dev/null +++ b/.github/workflows/setup-wizard-e2e.yml @@ -0,0 +1,74 @@ +name: Setup wizard verification + +on: + pull_request: + paths: + - 'packages/setupWizard/**' + - 'packages/schemas/**' + - 'entrypoint.sh' + - 'yarn.lock' + - '.github/workflows/setup-wizard-e2e.yml' + - '.github/workflows/release-setup-sourcebot.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + platform: + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + node: ['20.20.0', '22.22.0', '24.x'] + runs-on: ${{ matrix.os }} + env: + PACKAGE_TRACKER_ANALYTICS: 'false' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-node@v4 + with: + node-version: '24.x' + - run: corepack enable + - run: yarn install --immutable --mode=skip-build + - run: yarn rebuild node-pty + - run: yarn workspace @sourcebot/schemas build + - run: yarn workspace setup-sourcebot build + - name: Pack on the release runtime + shell: bash + env: + SETUP_TEST_TARBALL: ${{ runner.temp }}/setup-sourcebot.tgz + run: | + yarn workspace setup-sourcebot pack --out "$SETUP_TEST_TARBALL" + echo "SETUP_TEST_TARBALL=$SETUP_TEST_TARBALL" >> "$GITHUB_ENV" + - name: Select end-user test runtime + uses: actions/setup-node@v4 + with: + node-version: ${{ matrix.node }} + - run: corepack enable + - run: yarn workspace setup-sourcebot test + - run: yarn workspace setup-sourcebot test:platform + - name: Linux minimum-runtime regression checks + if: runner.os == 'Linux' && matrix.node != '24.x' + working-directory: packages/setupWizard + run: node --test --test-concurrency=1 tests/e2e/wizard.test.mjs tests/e2e/collectors.test.mjs tests/e2e/docker.test.mjs tests/e2e/safety.test.mjs + - name: Linux packed-artifact and runtime checks + if: runner.os == 'Linux' && matrix.node == '24.x' + run: | + docker pull docker.sourcebot.dev/sourcebot-dev/sourcebot:latest + yarn workspace setup-sourcebot test:e2e + yarn workspace setup-sourcebot test:baseline + node packages/setupWizard/tests/e2e/packageManagers.mjs + + setup-wizard-e2e: + if: always() + needs: [platform] + runs-on: ubuntu-latest + steps: + - name: Require every platform job + env: + RESULT: ${{ needs.platform.result }} + run: test "$RESULT" = success diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3ac20e85..d624ebd3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,4 +41,6 @@ jobs: run: yarn install --frozen-lockfile - name: Test - run: yarn test + # The CLI has separate packed-artifact and Node-compatibility gates in setup-wizard-e2e. + # Keep the application workspaces on their existing runtime here. + run: yarn workspaces foreach --all --topological --exclude setup-sourcebot --exclude 'root-workspace-*' run test diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a14d69cd..2bc271528 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- Added privacy-scoped setup wizard funnel and Docker startup-failure telemetry with deployment identity handoff, Node.js 20.20.0 support, and cross-platform end-to-end test coverage. [#1653](https://github.com/sourcebot-dev/sourcebot/pull/1653) + ## [5.1.13] - 2026-09-12 ### Fixed diff --git a/packages/setupWizard/README.md b/packages/setupWizard/README.md index 4224b2fc2..ac8afb2ef 100644 --- a/packages/setupWizard/README.md +++ b/packages/setupWizard/README.md @@ -17,9 +17,35 @@ The wizard walks you through: ## Requirements -- Node.js 18+ +- Node.js 20.20+ (20.x), 22.22+ (22.x), or 23.5+ (including 24+). Node 24 LTS is recommended; Node 20 is supported for compatibility but is end-of-life. - Docker and Docker Compose +## Development tests + +From the repository root, under Node 24: + +```bash +yarn workspace @sourcebot/schemas build +yarn workspace setup-sourcebot build +yarn workspace setup-sourcebot test +yarn workspace setup-sourcebot test:e2e +yarn workspace setup-sourcebot test:node-compatibility +``` + +The E2E tests compile and pack the package, install it outside the repository, +drive its published binary in a PTY, inspect real SDK requests through a local TLS +collector, and clean up temporary installations. OpenSSL and Docker are required. +PR verification runs nine parallel OS/Node jobs (Linux, macOS, Windows × Node +20.20.0, 22.22.0, 24). Each job builds and packs on Node 24 before selecting its +test runtime. Linux runs the full CLI regression suite on each version; macOS and +Windows run unit/integration and platform smoke tests. Docker identity, baseline, +and package-manager checks run only in the Node 24 Linux job. +Release verification also runs `test:node-compatibility` against the exact publish +tarball; that command checks the older runtimes sequentially, including early +rejection on Node 18, 20.19, and 22.21. Node 24 is not the end-user minimum. +The runtime suite uses `docker.sourcebot.dev/sourcebot-dev/sourcebot:latest` +(override only the test image with `SETUP_TEST_SOURCEBOT_IMAGE`). + ## Docs Full deployment guide: [docs.sourcebot.dev/docs/deployment/docker-compose](https://docs.sourcebot.dev/docs/deployment/docker-compose) diff --git a/packages/setupWizard/bin.cjs b/packages/setupWizard/bin.cjs new file mode 100755 index 000000000..41e4cf662 --- /dev/null +++ b/packages/setupWizard/bin.cjs @@ -0,0 +1,14 @@ +#!/usr/bin/env node +// Intersection of posthog-node and @inquirer/prompts runtime requirements. +const [major, minor] = process.versions.node.split('.').map(Number); +const supported = (major === 20 && minor >= 20) || + (major === 22 && minor >= 22) || (major === 23 && minor >= 5) || major >= 24; +if (!supported) { + console.error('setup-sourcebot requires Node.js 20.20+, 22.22+, or 23.5+ (including Node.js 24+). Please upgrade Node.js.'); + process.exitCode = 1; +} else { + import('./dist/index.js').catch(() => { + console.error('Unable to start setup-sourcebot. Please reinstall the package.'); + process.exitCode = 1; + }); +} diff --git a/packages/setupWizard/package.json b/packages/setupWizard/package.json index 464d21b53..06cd5e894 100644 --- a/packages/setupWizard/package.json +++ b/packages/setupWizard/package.json @@ -8,31 +8,43 @@ "url": "git+https://github.com/sourcebot-dev/sourcebot.git", "directory": "packages/setupWizard" }, - "bin": "./dist/index.js", + "bin": "./bin.cjs", "scripts": { "build": "tsc", "watch": "tsc --watch", "dev": "tsx src/index.ts", - "prepublishOnly": "yarn build" + "prepublishOnly": "yarn build", + "test": "node --test tests/unit/*.test.mjs tests/integration/*.test.mjs", + "test:e2e": "node --test --test-concurrency=1 tests/e2e/*.test.mjs", + "test:platform": "node --test tests/e2e/platform.test.mjs", + "test:linux": "node tests/e2e/linux.mjs", + "test:runtime": "node --test tests/e2e/runtime.test.mjs", + "test:node-compatibility": "node tests/e2e/nodeCompatibility.mjs", + "test:baseline": "node tests/e2e/baseline.mjs", + "test:live": "node tests/e2e/liveSmoke.mjs" }, "dependencies": { "@inquirer/prompts": "^8.4.3", "chalk": "^5.6.2", "inquirer-select-pro": "^1.0.0-alpha.9", "ora": "^9.4.0", + "posthog-node": "5.52.1", "reo-census": "^1.2.10" }, "devDependencies": { "@sourcebot/schemas": "workspace:^", - "@types/node": "^22.7.5", + "@types/node": "^20.19.43", + "node-pty": "^1.1.0", "tsx": "^4.21.0", - "typescript": "^5.6.2" + "typescript": "^5.6.2", + "undici": "^7" }, "engines": { - "node": ">=18" + "node": "^20.20.0 || ^22.22.0 || >=23.5.0" }, "files": [ "dist", + "bin.cjs", "README.md" ] } diff --git a/packages/setupWizard/src/azuredevops.ts b/packages/setupWizard/src/azuredevops.ts index fead965c9..4b9d10b2e 100644 --- a/packages/setupWizard/src/azuredevops.ts +++ b/packages/setupWizard/src/azuredevops.ts @@ -1,5 +1,6 @@ -import { confirm, input, password, select } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { sourceSummary } from './telemetrySummary.js'; +import { confirm, input, password, select } from './prompts.js'; +import { checkbox } from './prompts.js'; import type { AzureDevOpsConnectionConfig } from '@sourcebot/schemas/v3/azuredevops.type'; import type { CollectResult, EnvVars } from './utils.js'; import { multiInput, note, toEnvKey } from './utils.js'; @@ -96,5 +97,20 @@ export async function collectAzureDevOpsConfig(connectionName: string): Promise< }); } - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('azure_devops', { + deploymentType: deploymentType === 'cloud' ? 'cloud' : 'self_hosted', + credentialMode: 'personal_access_token', + scopeTypes: [ + ...(targets.includes('orgs') ? ['organizations' as const] : []), + ...(targets.includes('projects') ? ['projects' as const] : []), + ...(targets.includes('repos') ? ['repositories' as const] : []), + ], + organizationCount: config.orgs?.length ?? 0, + projectCount: config.projects?.length ?? 0, + repositoryCount: config.repos?.length ?? 0, + }), + }; } diff --git a/packages/setupWizard/src/bitbucket.ts b/packages/setupWizard/src/bitbucket.ts index 4e547fc83..1a38a8df0 100644 --- a/packages/setupWizard/src/bitbucket.ts +++ b/packages/setupWizard/src/bitbucket.ts @@ -1,5 +1,6 @@ -import { confirm, input, password, select } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { sourceSummary } from './telemetrySummary.js'; +import { confirm, input, password, select } from './prompts.js'; +import { checkbox } from './prompts.js'; import type { BitbucketConnectionConfig } from '@sourcebot/schemas/v3/bitbucket.type'; import type { CollectResult, EnvVars } from './utils.js'; import { multiInput, note, toEnvKey } from './utils.js'; @@ -151,7 +152,25 @@ async function collectBitbucketCloud( }); } - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('bitbucket', { + deploymentType: 'cloud', + credentialMode: + authMethod === 'api-token' + ? 'api_token' + : authMethod === 'access-token' + ? 'access_token' + : 'app_password', + scopeTypes: [ + ...(targets.includes('workspaces') ? ['workspaces' as const] : []), + ...(targets.includes('repos') ? ['repositories' as const] : []), + ], + workspaceCount: config.workspaces?.length ?? 0, + repositoryCount: config.repos?.length ?? 0, + }), + }; } async function collectBitbucketServer( @@ -208,7 +227,16 @@ async function collectBitbucketServer( if (indexAll) { config.all = true; - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('bitbucket', { + deploymentType: 'self_hosted', + credentialMode: 'http_access_token', + indexAll: true, + scopeTypes: ['all'], + }), + }; } const targets = await checkbox({ @@ -232,5 +260,18 @@ async function collectBitbucketServer( }); } - return { connections: [{ config }], env }; + return { + connections: [{ config }], + env, + telemetry: sourceSummary('bitbucket', { + deploymentType: 'self_hosted', + credentialMode: 'http_access_token', + scopeTypes: [ + ...(targets.includes('projects') ? ['projects' as const] : []), + ...(targets.includes('repos') ? ['repositories' as const] : []), + ], + projectCount: config.projects?.length ?? 0, + repositoryCount: config.repos?.length ?? 0, + }), + }; } diff --git a/packages/setupWizard/src/docker.ts b/packages/setupWizard/src/docker.ts new file mode 100644 index 000000000..4b41b6961 --- /dev/null +++ b/packages/setupWizard/src/docker.ts @@ -0,0 +1,196 @@ +import { spawn } from 'node:child_process'; +import { lifecycle } from './lifecycle.js'; +import type { FailureCategory } from './telemetryEvents.js'; + +export type DockerResult = + | { ok: true; value: T } + | { ok: false; failureCategory: FailureCategory }; +export type ComposeContainer = { Name: string; Service: string; State: string }; +export class Docker { + status: 'available' | 'unavailable' | 'error' | 'not_checked' = + 'not_checked'; + failed = false; + constructor( + private readonly onFailure: (category: FailureCategory) => void, + ) {} + private failure(failureCategory: FailureCategory): DockerResult { + lifecycle.check(); + this.failed = true; + this.onFailure(failureCategory); + return { ok: false, failureCategory }; + } + private async execute(args: string[]) { + lifecycle.check(); + const result = await new Promise<{ + code: number | null; + stdout: string; + missing: boolean; + stderr: string; + }>((resolve) => { + const child = lifecycle.child( + spawn('docker', args, { + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + }), + ); + let stdout = ''; + let stderr = ''; + child.stdout?.on('data', (chunk) => { + stdout += chunk.toString(); + }); + child.stderr?.on('data', (chunk) => { + stderr += chunk.toString(); + }); + child.once('error', (error: NodeJS.ErrnoException) => + resolve({ + code: null, + stdout: '', + stderr: '', + missing: ['ENOENT', 'EACCES'].includes(error.code ?? ''), + }), + ); + child.once('close', (code) => + resolve({ code, stdout, stderr, missing: false }), + ); + }); + lifecycle.check(); + return result; + } + async run(args: string[]): Promise> { + const result = await this.execute(args); + if (result.code !== 0) { + if (result.missing) { + this.status = 'unavailable'; + } else if ( + this.status !== 'available' && + this.status !== 'unavailable' + ) { + if (result.code === null) { + this.status = 'error'; + } else { + // Diagnose availability using fixed commands, never by parsing user stderr. + // These probes classify this single failed operation; they do not emit more failures. + const engine = await this.execute([ + 'info', + '--format', + '{{.ServerVersion}}', + ]); + const compose = + args[0] === 'compose' && engine.code === 0 + ? await this.execute([ + 'compose', + 'version', + '--short', + ]) + : undefined; + this.status = + engine.code === 0 && (!compose || compose.code === 0) + ? 'available' + : 'unavailable'; + } + } + // Human diagnostics stay local; never use stderr for analytics classification. + if ( + ['stop', 'rm'].includes(args[0]) || + ['down', 'rm'].includes(args[1]) + ) { + if (result.stderr.trim()) { + console.error(result.stderr.trim()); + } + } + return this.failure( + this.status === 'unavailable' + ? 'docker_unavailable' + : 'docker_command', + ); + } + // A working volume command does not repair a known missing Compose plugin. + if (this.status !== 'unavailable') { + this.status = 'available'; + } + return { ok: true, value: result.stdout }; + } + async containers(): Promise> { + const result = await this.run([ + 'compose', + 'ps', + '-a', + '--format', + 'json', + ]); + if (!result.ok) { + return result; + } + try { + const text = result.value.trim(); + const items: unknown = !text + ? [] + : text.startsWith('[') + ? JSON.parse(text) + : text + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)); + if ( + !Array.isArray(items) || + items.some( + (c) => + !c || + typeof c.Name !== 'string' || + typeof c.Service !== 'string' || + typeof c.State !== 'string', + ) + ) { + throw new Error('Invalid container inventory'); + } + return { ok: true, value: items }; + } catch { + return this.failure('docker_command'); + } + } + async volumes(expected: string[]): Promise> { + if (!expected.length) { + return { ok: true, value: [] }; + } + const result = await this.run([ + 'volume', + 'ls', + '--format', + '{{.Name}}', + ]); + if (!result.ok) { + return result; + } + const names = new Set( + result.value + .split('\n') + .map((s) => s.trim()) + .filter(Boolean), + ); + return { ok: true, value: expected.filter((name) => names.has(name)) }; + } + async portOwners(): Promise>> { + const result = await this.run([ + 'ps', + '--format', + '{{.Names}}\t{{.Ports}}', + ]); + if (!result.ok) { + return result; + } + const owners = new Map(); + for (const line of result.value.split('\n').filter(Boolean)) { + const [name, ports] = line.split('\t'); + if (ports === undefined) { + return this.failure('docker_command'); + } + for (const match of ports.matchAll(/(\d+)->/g)) { + const port = Number(match[1]); + owners.set(port, [ + ...new Set([...(owners.get(port) ?? []), name]), + ]); + } + } + return { ok: true, value: owners }; + } +} diff --git a/packages/setupWizard/src/dockerStartFailure.ts b/packages/setupWizard/src/dockerStartFailure.ts new file mode 100644 index 000000000..83d95f2a3 --- /dev/null +++ b/packages/setupWizard/src/dockerStartFailure.ts @@ -0,0 +1,55 @@ +import { stripVTControlCharacters } from 'node:util'; +import type { Events } from './telemetryEvents.js'; + +type Reason = Events['start_failed']['failureReason']; + +// Docker exposes an exit status, not structured error reasons, for compose up. +// Recognize only specific CLI/daemon diagnostics locally. Never serialize output, +// captures, paths, image names, container names or IDs into an event. +export class DockerStartFailure { + private pending = ''; + private detected: Reason = 'unknown'; + + write(chunk: Buffer): void { + // Bound retained output even for a long-running foreground Compose process. + const lines = (this.pending + chunk.toString()).split(/[\r\n]/); + this.pending = (lines.pop() ?? '').slice(-8192); + for (const line of lines) { + this.classify(line.slice(-8192)); + } + } + + reason(): Reason { + this.classify(this.pending); + this.pending = ''; + return this.detected; + } + + private classify(raw: string): void { + if (this.detected !== 'unknown') { + return; + } + const line = stripVTControlCharacters(raw).trim(); + // Attached application logs are not Docker diagnostics. + if (line.includes(' | ')) { + return; + } + const daemon = /^Error response from daemon:/i.test(line); + if (daemon && /container name .+ is already in use by container/i.test(line)) { + this.detected = 'container_name_conflict'; + } else if (daemon && /port is already allocated|address already in use/i.test(line)) { + this.detected = 'port_conflict'; + } else if ( + /^(?:Error response from daemon:|unable to get image|pull access denied|failed to resolve reference)/i.test(line) && + /pull access denied|manifest unknown|manifest for .+ not found|failed to resolve reference|no matching manifest|toomanyrequests|unauthorized: authentication required/i.test(line) + ) { + this.detected = 'image_pull_failed'; + } else if (daemon && /invalid mount config|mounts denied|error while creating mount source path|invalid volume specification/i.test(line)) { + this.detected = 'mount_failed'; + } else if (/^(?:Cannot connect to the Docker daemon|error during connect:|permission denied while trying to connect to the Docker daemon|docker: ['"]?compose['"]? is not a docker command)/i.test(line)) { + this.detected = 'docker_unavailable'; + } else if (/^(?:validating .+:|no configuration file provided:|yaml: line \d+:|services\..+:|service .+ refers to undefined (?:volume|network) .+: invalid compose project)/i.test(line)) { + this.detected = 'compose_configuration'; + } + } +} diff --git a/packages/setupWizard/src/genericGit.ts b/packages/setupWizard/src/genericGit.ts index 5f636220e..f5deabe01 100644 --- a/packages/setupWizard/src/genericGit.ts +++ b/packages/setupWizard/src/genericGit.ts @@ -1,4 +1,5 @@ -import { input } from '@inquirer/prompts'; +import { sourceSummary } from './telemetrySummary.js'; +import { input } from './prompts.js'; import type { GenericGitHostConnectionConfig } from '@sourcebot/schemas/v3/genericGitHost.type'; import type { CollectResult } from './utils.js'; @@ -21,5 +22,13 @@ export async function collectGenericGitConfig(): Promise { url, }; - return { connections: [{ config }], env: {} }; + return { + connections: [{ config }], + env: {}, + telemetry: sourceSummary('remote_git', { + deploymentType: 'remote', + repositoryCount: 1, + scopeTypes: ['repositories'], + }), + }; } diff --git a/packages/setupWizard/src/gerrit.ts b/packages/setupWizard/src/gerrit.ts index 51ccd97e0..873fb5ed6 100644 --- a/packages/setupWizard/src/gerrit.ts +++ b/packages/setupWizard/src/gerrit.ts @@ -1,4 +1,5 @@ -import { confirm, input } from '@inquirer/prompts'; +import { sourceSummary } from './telemetrySummary.js'; +import { confirm, input } from './prompts.js'; import type { GerritConnectionConfig } from '@sourcebot/schemas/v3/gerrit.type'; import type { CollectResult } from './utils.js'; import { multiInput } from './utils.js'; @@ -33,5 +34,14 @@ export async function collectGerritConfig(): Promise { }); } - return { connections: [{ config }], env: {} }; + return { + connections: [{ config }], + env: {}, + telemetry: sourceSummary('gerrit', { + deploymentType: 'self_hosted', + indexAll, + scopeTypes: indexAll ? ['all'] : ['projects'], + projectCount: config.projects?.length ?? 0, + }), + }; } diff --git a/packages/setupWizard/src/gitea.ts b/packages/setupWizard/src/gitea.ts index 050a55e51..c2f56f267 100644 --- a/packages/setupWizard/src/gitea.ts +++ b/packages/setupWizard/src/gitea.ts @@ -1,5 +1,6 @@ -import { input, password } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { sourceSummary, deployment } from './telemetrySummary.js'; +import { input, password } from './prompts.js'; +import { checkbox } from './prompts.js'; import type { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type'; import type { CollectResult, EnvVars } from './utils.js'; import { INPUT_THEME, multiInput, toEnvKey } from './utils.js'; @@ -64,5 +65,20 @@ export async function collectGiteaConfig(connectionName: string): Promise }; + try { + const res = await wizardFetch(url, { headers, signal: AbortSignal.timeout(8000) }); + const data = await res.json() as { items?: Array<{ login?: string; full_name?: string }> }; - const literalFallback = (): SearchOption | null => { - return { name: query, value: query }; - }; + const literalFallback = (): SearchOption | null => { + return { name: query, value: query }; + }; - if (!res.ok) { - const warning = - (res.status === 403 && res.headers.get('x-ratelimit-remaining') === '0') - ? '⚠ Autocomplete disabled — GitHub rate limit exceeded.' - : '⚠ Autocomplete disabled — authentication failed, check your PAT.'; - const fallback = literalFallback(); - return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; - } + if (!res.ok) { + lifecycle.fail('network', true); + const warning = + (res.status === 403 && res.headers.get('x-ratelimit-remaining') === '0') + ? '⚠ Autocomplete disabled — GitHub rate limit exceeded.' + : '⚠ Autocomplete disabled — authentication failed, check your PAT.'; + const fallback = literalFallback(); + return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; + } - const results: SearchOption[] = (data.items ?? []).map((item) => { - const value = type === 'repo' ? item.full_name! : item.login!; - return { name: value, value }; - }); - if (results.length === 0) { - const fallback = literalFallback(); - return fallback ? [fallback] : []; + if ( + !data || + (data.items !== undefined && + (!Array.isArray(data.items) || + data.items.some((item) => typeof (type === 'repo' ? item?.full_name : item?.login) !== 'string'))) + ) { + throw new Error('Invalid search response'); + } + const results: SearchOption[] = (data.items ?? []).map((item) => { + const value = type === 'repo' ? item.full_name! : item.login!; + return { name: value, value }; + }); + if (results.length === 0) { + const fallback = literalFallback(); + return fallback ? [fallback] : []; + } + githubSearchCache.set(cacheKey, results); + return results; + } catch { + lifecycle.check(); + lifecycle.fail('network', true); + const fallback = true; + const warning = new Separator('⚠ Autocomplete unavailable — enter a value manually.'); + return fallback ? [{ name: query, value: query }, warning] : [warning]; } - githubSearchCache.set(cacheKey, results); - return results; } export async function collectGitHubConfig(connectionName: string): Promise { @@ -218,5 +238,20 @@ export async function collectGitHubConfig(connectionName: string): Promise { + if (type === 'project') { + return PROJECT_PATTERN.test(query) ? { name: query, value: query } : null; + } + return { name: query, value: query }; + }; - const literalFallback = (): SearchOption | null => { - if (type === 'project') { - return PROJECT_PATTERN.test(query) ? { name: query, value: query } : null; + if (!res.ok) { + lifecycle.fail('network', true); + const warning = res.status === 401 + ? '⚠ Autocomplete disabled — authentication failed, check your PAT.' + : `⚠ Autocomplete disabled — GitLab API error (${res.status}).`; + const fallback = literalFallback(); + return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; } - return { name: query, value: query }; - }; - if (!res.ok) { - const warning = res.status === 401 - ? '⚠ Autocomplete disabled — authentication failed, check your PAT.' - : `⚠ Autocomplete disabled — GitLab API error (${res.status}).`; - const fallback = literalFallback(); - return fallback ? [fallback, new Separator(warning)] : [new Separator(warning)]; - } + const data = await res.json() as Array<{ + full_path?: string; + path_with_namespace?: string; + username?: string; + }>; - const data = await res.json() as Array<{ - full_path?: string; - path_with_namespace?: string; - username?: string; - }>; - - const results: SearchOption[] = data.map((item) => { - let value: string; - if (type === 'group') { - value = item.full_path!; - } else if (type === 'project') { - value = item.path_with_namespace!; - } else { - value = item.username!; + if ( + !Array.isArray(data) || + data.some( + (item) => + typeof (type === 'group' + ? item?.full_path + : type === 'project' + ? item?.path_with_namespace + : item?.username) !== 'string', + ) + ) { + throw new Error('Invalid search response'); } - return { name: value, value }; - }); + const results: SearchOption[] = data.map((item) => { + let value: string; + if (type === 'group') { + value = item.full_path!; + } else if (type === 'project') { + value = item.path_with_namespace!; + } else { + value = item.username!; + } + return { name: value, value }; + }); - if (results.length === 0) { - const fallback = literalFallback(); - return fallback ? [fallback] : []; - } + if (results.length === 0) { + const fallback = literalFallback(); + return fallback ? [fallback] : []; + } - gitlabSearchCache.set(cacheKey, results); - return results; + gitlabSearchCache.set(cacheKey, results); + return results; + } catch { + lifecycle.check(); + lifecycle.fail('network', true); + const fallback = type !== 'project' || PROJECT_PATTERN.test(query); + const warning = new Separator('⚠ Autocomplete unavailable — enter a value manually.'); + return fallback ? [{ name: query, value: query }, warning] : [warning]; + } } export async function collectGitLabConfig(connectionName: string): Promise { @@ -235,5 +260,22 @@ export async function collectGitLabConfig(connectionName: string): Promise {}); + browser.unref(); } -async function openBrowserWhenReady(url: string, timeoutMs = 120_000): Promise { +async function openBrowserWhenReady(url: string, signal: AbortSignal, timeoutMs = 120_000): Promise { const start = Date.now(); while (Date.now() - start < timeoutMs) { try { - const res = await fetch(url, { signal: AbortSignal.timeout(2000) }); + const res = await wizardFetch(url, { signal: AbortSignal.any([signal, AbortSignal.timeout(2000)]) }); if (res.status < 500) { + signal.throwIfAborted(); openBrowser(url); return; } } catch { // not yet ready } - await new Promise((r) => setTimeout(r, 2000)); + await sleep(2000, undefined, { signal }); } } @@ -189,11 +207,19 @@ function parsePublishedHostPorts(composeYaml: string): PublishedPort[] { // 5432), which is the actual failure mode `docker compose up` hits. function isPortInUse({ host, port }: PublishedPort): Promise { return new Promise((resolve) => { + lifecycle.check(); const server = net.createServer(); + const release = lifecycle.own(() => server.close()); + server.once('close', release); server.once('error', (err: NodeJS.ErrnoException) => { server.close(() => { /* noop */ }); // EADDRINUSE = taken. Other errors (e.g. EACCES on privileged ports) aren't // a "someone else has it" conflict we can meaningfully report, so treat as free. + if (err.code !== 'EADDRINUSE') { + portInspectionFailed = true; + docker.failed = true; + lifecycle.fail('validation', true); + } resolve(err.code === 'EADDRINUSE'); }); server.once('listening', () => { @@ -207,65 +233,8 @@ function isPortInUse({ host, port }: PublishedPort): Promise { }); } -// Best-effort: maps a host port to the running Docker container(s) publishing it, so -// a conflict can name the offender. Returns an empty map if Docker isn't available. -async function getDockerPublishedPortOwners(): Promise> { - return new Promise>((resolve) => { - const child = spawn('docker', ['ps', '--format', '{{.Names}}\t{{.Ports}}'], { - stdio: ['ignore', 'pipe', 'ignore'], - }); - let out = ''; - child.stdout?.on('data', (chunk: Buffer) => { - out += chunk.toString(); - }); - child.on('exit', (code) => { - const map = new Map(); - if (code !== 0) { - resolve(map); - return; - } - for (const line of out.split('\n')) { - const [name, portsStr] = line.split('\t'); - if (!name || !portsStr) { - continue; - } - // e.g. "0.0.0.0:5432->5432/tcp, :::5432->5432/tcp" — the number before - // each "->" is the published host port. - for (const m of portsStr.matchAll(/(\d+)->/g)) { - const port = Number(m[1]); - const list = map.get(port) ?? []; - if (!list.includes(name)) { - list.push(name); - } - map.set(port, list); - } - } - resolve(map); - }); - child.on('error', () => resolve(new Map())); - }); -} - -// Stops the given containers (by name). Returns true only if all stopped cleanly. -async function stopDockerContainers(names: string[]): Promise { - if (names.length === 0) { - return true; - } - return new Promise((resolve) => { - const child = spawn('docker', ['stop', ...names], { stdio: ['ignore', 'ignore', 'pipe'] }); - let err = ''; - child.stderr?.on('data', (chunk: Buffer) => { - err += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0 && err.trim()) { - console.error(chalk.red('✗ ') + err.trim()); - } - resolve(code === 0); - }); - child.on('error', () => resolve(false)); - }); -} +const docker = new Docker((category) => lifecycle.fail(category, true)); +let portInspectionFailed = false; // Mirrors Docker Compose's project-name normalization for the default case // where the project name is derived from the working directory basename. @@ -275,115 +244,6 @@ function dockerComposeProjectName(): string { .replace(/[^a-z0-9_-]/g, ''); } -async function listExistingDockerVolumes(expectedNames: string[]): Promise { - if (expectedNames.length === 0) { - return []; - } - return new Promise((resolve) => { - const child = spawn('docker', ['volume', 'ls', '--format', '{{.Name}}'], { - stdio: ['ignore', 'pipe', 'ignore'], - }); - let out = ''; - child.stdout?.on('data', (chunk: Buffer) => { - out += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0) { - resolve([]); - return; - } - const existing = new Set(out.split('\n').map((l) => l.trim()).filter(Boolean)); - resolve(expectedNames.filter((name) => existing.has(name))); - }); - child.on('error', () => resolve([])); - }); -} - -async function removeDockerVolumes(volumes: string[]): Promise { - if (volumes.length === 0) { - return true; - } - return new Promise((resolve) => { - const child = spawn('docker', ['volume', 'rm', ...volumes], { stdio: ['ignore', 'ignore', 'pipe'] }); - let err = ''; - child.stderr?.on('data', (chunk: Buffer) => { - err += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0 && err.trim()) { - console.error(chalk.red('✗ ') + err.trim()); - } - resolve(code === 0); - }); - child.on('error', () => resolve(false)); - }); -} - -type ComposeContainer = { Name: string; Service: string; State: string }; - -function parseComposePsOutput(output: string): ComposeContainer[] { - const trimmed = output.trim(); - if (!trimmed) { - return []; - } - if (trimmed.startsWith('[')) { - try { - return JSON.parse(trimmed) as ComposeContainer[]; - } catch { - // fall through to line-based parse - } - } - const containers: ComposeContainer[] = []; - for (const line of trimmed.split('\n')) { - if (!line.trim()) { - continue; - } - try { - containers.push(JSON.parse(line) as ComposeContainer); - } catch { - // skip unparseable line - } - } - return containers; -} - -async function listComposeContainers(): Promise { - return new Promise((resolve) => { - const child = spawn('docker', ['compose', 'ps', '-a', '--format', 'json'], { - stdio: ['ignore', 'pipe', 'ignore'], - }); - let out = ''; - child.stdout?.on('data', (chunk: Buffer) => { - out += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0) { - resolve([]); - return; - } - resolve(parseComposePsOutput(out)); - }); - child.on('error', () => resolve([])); - }); -} - -async function runComposeCommand(args: string[], label: string): Promise { - return new Promise((resolve) => { - const child = spawn('docker', ['compose', ...args], { stdio: ['ignore', 'ignore', 'pipe'] }); - let err = ''; - child.stderr?.on('data', (chunk: Buffer) => { - err += chunk.toString(); - }); - child.on('exit', (code) => { - if (code !== 0 && err.trim()) { - console.error(chalk.red('✗ ') + `${label}: ` + err.trim()); - } - resolve(code === 0); - }); - child.on('error', () => resolve(false)); - }); -} - const PLATFORM_LABELS: Record = { github: 'GitHub', gitlab: 'GitLab', @@ -396,6 +256,9 @@ const PLATFORM_LABELS: Record = { }; async function main() { + lifecycle.install(); + lifecycle.capture('started', { invocationMethod: invocationMethod(), isInteractive: !!process.stdin.isTTY }); + lifecycle.failureCategory = 'filesystem'; console.log(String.raw` ███████╗ ██████╗ ██╗ ██╗██████╗ ██████╗███████╗██████╗ ██████╗ ████████╗ ██╔════╝██╔═══██╗██║ ██║██╔══██╗██╔════╝██╔════╝██╔══██╗██╔═══██╗╚══██╔══╝ @@ -417,7 +280,8 @@ async function main() { }, }); - if (existsSync(setupDir)) { + const selectedDirectoryExisted = existsSync(setupDir); + if (selectedDirectoryExisted) { const overwrite = await confirm({ message: `Directory '${setupDir}' already exists. Do you want to overwrite it?`, default: false, @@ -425,13 +289,23 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + 'Setup cancelled.'); - process.exit(0); + await lifecycle.decline('existing_directory_declined'); + return; } } else { mkdirSync(setupDir, { recursive: true }); } + lifecycle.check(); process.chdir(setupDir); + lifecycle.capture('chose_setup_directory', { + usedDefaultDirectory: setupDir === 'sourcebot', + directoryExisted: selectedDirectoryExisted, + directoryAction: selectedDirectoryExisted ? 'existing_directory_accepted' : 'created', + }); + lifecycle.stage = 'code_sources'; + lifecycle.failureCategory = 'unknown'; + const sourceSummaries: CodeSourceSummary[] = []; const connections: Record = {}; const allEnv: EnvVars = {}; @@ -493,6 +367,12 @@ async function main() { continue; } + lifecycle.check(); + sourceSummaries.push(result.telemetry); + lifecycle.capture('configured_code_source', { + configurationIndex: sourceSummaries.length, + ...result.telemetry, + }); for (const { name, config } of result.connections) { const finalName = name ? generateConnectionName(name, connections) @@ -511,7 +391,20 @@ async function main() { } } - const { models, env: modelEnv } = await collectModels(); + const sourceSummary = aggregateSources(sourceSummaries); + lifecycle.capture('configured_code_sources', sourceSummary); + lifecycle.stage = 'ai_setup'; + let modelIndex = 0; + const { + models, + env: modelEnv, + telemetry: modelSummaries, + } = await collectModels((summary) => { + lifecycle.capture('configured_ai_provider', { configurationIndex: ++modelIndex, ...summary }); + }); + const aiSummary = aggregateAi(modelSummaries); + lifecycle.capture('ai_setup_completed', aiSummary); + lifecycle.stage = 'hosted_url'; Object.assign(allEnv, modelEnv); const authUrl = await input({ @@ -529,8 +422,17 @@ async function main() { }, }); allEnv.AUTH_URL = authUrl; + lifecycle.capture('configured_hosted_url', { + usedDefaultUrl: authUrl === SOURCEBOT_URL, + protocol: authUrl.startsWith('https:') ? 'https' : 'http', + hostCategory: hostCategory(authUrl), + }); + lifecycle.stage = 'config_overwrite'; + lifecycle.failureCategory = 'filesystem'; + const overwritten: Events['generated_configs']['overwroteExistingFiles'] = []; if (existsSync('config.json')) { + overwritten.push('config_json'); const overwrite = await confirm({ message: 'config.json already exists. Overwrite?', default: true, @@ -538,11 +440,13 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + 'config.json was not overwritten.'); - process.exit(0); + await lifecycle.decline('config_overwrite_declined'); + return; } } if (existsSync('.env')) { + overwritten.push('env'); const overwrite = await confirm({ message: '.env already exists. Overwrite?', default: true, @@ -550,11 +454,13 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + '.env was not overwritten.'); - process.exit(0); + await lifecycle.decline('config_overwrite_declined'); + return; } } if (localRepoIndex.size > 0 && existsSync('docker-compose.override.yml')) { + overwritten.push('compose_override'); const overwrite = await confirm({ message: 'docker-compose.override.yml already exists. Overwrite?', default: true, @@ -562,11 +468,18 @@ async function main() { if (!overwrite) { console.log(); console.log(chalk.red('✗ ') + 'docker-compose.override.yml was not overwritten.'); - process.exit(0); + await lifecycle.decline('config_overwrite_declined'); + return; } } - const s = ora('Writing configuration files...').start(); + lifecycle.check(); + const deploymentIdentity = selectInstallId( + existsSync('.env') ? readFileSync('.env', 'utf8') : '', + lifecycle.telemetry.setupSessionId, + ); + const s = spinner('Writing configuration files...'); + const releaseWriter = lifecycle.own(() => s.stop()); const configOutput: Record = { $schema: 'https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/schemas/v3/index.json', @@ -596,6 +509,10 @@ async function main() { `AUTH_URL=${allEnv.AUTH_URL}`, ]; + if (deploymentIdentity.id) { + envLines.push('', '# Deployment identifier', `SOURCEBOT_INSTALL_ID=${deploymentIdentity.id}`); + } + if (Object.keys(connectionEnv).length > 0) { envLines.push('', '# Code host credentials'); for (const [key, value] of Object.entries(connectionEnv)) { @@ -610,6 +527,7 @@ async function main() { } } + lifecycle.check(); writeFileSync('config.json', configJson + '\n'); writeFileSync('.env', envLines.join('\n') + '\n'); @@ -632,6 +550,19 @@ async function main() { writtenFiles.push('docker-compose.override.yml'); } + lifecycle.capture('generated_configs', { + filesWritten: ['config_json', 'env', ...(localRepoIndex.size > 0 ? ['compose_override' as const] : [])], + overwroteExistingFiles: overwritten, + wroteComposeOverride: localRepoIndex.size > 0, + localMountCount: localRepoIndex.size, + generatedConnectionCount: sourceSummary.generatedConnectionCount, + aiConfigurationCount: aiSummary.aiConfigurationCount, + credentialVariableCount: + Object.keys(connectionEnv).filter((k) => !['GOOGLE_VERTEX_PROJECT', 'GOOGLE_VERTEX_REGION'].includes(k)) + .length + Object.keys(aiEnv).length, + deploymentIdentityAction: deploymentIdentity.action, + }); + releaseWriter(); const fileInfo: Record = { 'config.json': { description: 'The Sourcebot configuration file. This controls which repos Sourcebot indexes and which language models it connects to.', @@ -669,25 +600,48 @@ async function main() { s.succeed(chalk.bold('Wrote the following files:')); console.log(['', ...fileLines].join('\n')); + lifecycle.stage = 'compose_file'; let downloadedCompose = false; + const compose: Events['resolved_compose_file'] = { + outcome: 'already_present', + composeAvailable: true, + downloadPromptShown: false, + downloadAttempted: false, + failureCategory: null, + }; if (!existsSync('docker-compose.yml')) { + compose.downloadPromptShown = true; + compose.outcome = 'declined'; + compose.composeAvailable = false; const download = await confirm({ message: 'Download docker-compose.yml?', default: true, }); if (download) { - const ds = ora('Downloading docker-compose.yml...').start(); + compose.downloadAttempted = true; + let downloadFailure: NonNullable = 'network'; + const ds = spinner('Downloading docker-compose.yml...'); try { - const res = await fetch(DOCKER_COMPOSE_URL); + const res = await wizardFetch(DOCKER_COMPOSE_URL); + downloadFailure = res.status >= 500 ? 'http_5xx' : res.status >= 400 ? 'http_4xx' : 'network'; if (!res.ok) { throw new Error(`HTTP ${res.status}`); } - await writeFile('docker-compose.yml', await res.text()); + const body = await res.text(); + lifecycle.check(); + downloadFailure = 'filesystem'; + await writeFile('docker-compose.yml', body); ds.succeed('Downloaded docker-compose.yml'); downloadedCompose = true; + compose.outcome = 'downloaded'; + compose.composeAvailable = true; } catch { + lifecycle.check(); + compose.outcome = 'download_failed'; + compose.failureCategory = downloadFailure; + lifecycle.fail(downloadFailure === 'filesystem' ? 'filesystem' : 'network', true); ds.fail('Download failed — you can get it manually (see next steps)'); } } @@ -695,12 +649,28 @@ async function main() { downloadedCompose = true; } + lifecycle.capture('resolved_compose_file', compose); + lifecycle.stage = 'docker_validation'; + lifecycle.failureCategory = 'docker_command'; + const dockerSummary = emptyDockerSummary(); let leftDeploymentRunning = false; if (downloadedCompose) { - const containers = await listComposeContainers(); + const containerResult = await docker.containers(); + const containers = containerResult.ok ? containerResult.value : []; const running = containers.filter((c) => c.State === 'running'); const stopped = containers.filter((c) => c.State !== 'running'); + if (containerResult.ok) { + dockerSummary.runningComposeContainerCount = running.length; + dockerSummary.stoppedComposeContainerCount = stopped.length; + dockerSummary.composeContainerState = running.length + ? stopped.length + ? 'mixed' + : 'running' + : stopped.length + ? 'stopped' + : 'none'; + } if (running.length > 0) { console.log(); @@ -713,8 +683,9 @@ async function main() { default: true, }); if (stop) { - const ds = ora('Stopping deployment...').start(); - const ok = await runComposeCommand(['down'], 'docker compose down'); + const ds = spinner('Stopping deployment...'); + const ok = (await docker.run(['compose', 'down'])).ok; + dockerSummary.existingDeploymentAction = ok ? 'stopped' : 'stop_failed'; if (ok) { ds.succeed('Stopped deployment'); } else { @@ -722,6 +693,7 @@ async function main() { leftDeploymentRunning = true; } } else { + dockerSummary.existingDeploymentAction = 'left_running'; leftDeploymentRunning = true; } } else if (stopped.length > 0) { @@ -734,9 +706,11 @@ async function main() { message: 'Remove them now to prevent name conflicts when Sourcebot starts?', default: true, }); + dockerSummary.stoppedContainerAction = 'kept'; if (remove) { - const rs = ora('Removing containers...').start(); - const ok = await runComposeCommand(['rm', '-f'], 'docker compose rm'); + const rs = spinner('Removing containers...'); + const ok = (await docker.run(['compose', 'rm', '-f'])).ok; + dockerSummary.stoppedContainerAction = ok ? 'removed' : 'remove_failed'; if (ok) { rs.succeed('Removed containers'); } else { @@ -751,7 +725,9 @@ async function main() { const declaredVolumes = parseTopLevelVolumes(readFileSync('docker-compose.yml', 'utf-8')); const project = dockerComposeProjectName(); const expectedNames = declaredVolumes.map((v) => `${project}_${v}`); - const existing = await listExistingDockerVolumes(expectedNames); + const volumeResult = await docker.volumes(expectedNames); + const existing = volumeResult.ok ? volumeResult.value : []; + dockerSummary.existingVolumeCount = volumeResult.ok ? existing.length : null; if (existing.length > 0) { console.log(); @@ -763,9 +739,11 @@ async function main() { message: 'Wipe these volumes? This will permanently delete any existing Sourcebot data in them.', default: false, }); + dockerSummary.volumeAction = 'kept'; if (wipe) { - const ws = ora('Removing volumes...').start(); - const ok = await removeDockerVolumes(existing); + const ws = spinner('Removing volumes...'); + const ok = (await docker.run(['volume', 'rm', ...existing])).ok; + dockerSummary.volumeAction = ok ? 'removed' : 'remove_failed'; if (ok) { ws.succeed(`Removed ${existing.length} volume${existing.length === 1 ? '' : 's'}`); } else { @@ -786,13 +764,19 @@ async function main() { } const publishedPorts = parsePublishedHostPorts(composeYaml); + if (publishedPorts.length === 0) { + dockerSummary.initialPortConflictCount = 0; + dockerSummary.remainingPortConflictCount = 0; + dockerSummary.portConflictSource = 'none'; + } if (publishedPorts.length > 0) { - const ps = ora('Checking for port conflicts...').start(); + const ps = spinner('Checking for port conflicts...'); // Detect via two complementary sources: `docker ps` (authoritative for ports // published by other containers — a plain socket bind can't see those reliably, // e.g. Docker Desktop on macOS lets us bind a port it already forwards), and a // socket bind (catches non-Docker processes like a local Postgres/Redis). - const owners = await getDockerPublishedPortOwners(); + const ownersResult = await docker.portOwners(); + const owners = ownersResult.ok ? ownersResult.value : new Map(); const inUse: PublishedPort[] = []; for (const p of publishedPorts) { const ownedByContainer = (owners.get(p.port)?.length ?? 0) > 0; @@ -800,6 +784,18 @@ async function main() { inUse.push(p); } } + if (ownersResult.ok && !portInspectionFailed) { + dockerSummary.initialPortConflictCount = inUse.length; + dockerSummary.remainingPortConflictCount = inUse.length; + const dockerOwned = inUse.filter((p) => owners.has(p.port)).length; + dockerSummary.portConflictSource = !inUse.length + ? 'none' + : dockerOwned === inUse.length + ? 'docker' + : dockerOwned === 0 + ? 'non_docker' + : 'mixed'; + } if (inUse.length === 0) { ps.succeed('No port conflicts detected'); } else { @@ -827,9 +823,11 @@ async function main() { message: `Stop ${conflictingContainers.length === 1 ? 'this container' : 'these containers'} (${conflictingContainers.join(', ')}) to free the ports?`, default: true, }); + dockerSummary.portConflictAction = 'kept'; if (stop) { - const ss = ora('Stopping containers...').start(); - const ok = await stopDockerContainers(conflictingContainers); + const ss = spinner('Stopping containers...'); + const ok = (await docker.run(['stop', ...conflictingContainers])).ok; + dockerSummary.portConflictAction = ok ? 'containers_stopped' : 'stop_failed'; if (ok) { ss.succeed(`Stopped ${conflictingContainers.join(', ')}`); } else { @@ -837,13 +835,17 @@ async function main() { } // Re-check the conflicting ports now that the containers are stopped. const stillInUse: PublishedPort[] = []; - const freshOwners = await getDockerPublishedPortOwners(); + portInspectionFailed = false; + const freshResult = await docker.portOwners(); + const freshOwners = freshResult.ok ? freshResult.value : new Map(); for (const p of inUse) { const ownedByContainer = (freshOwners.get(p.port)?.length ?? 0) > 0; if (ownedByContainer || await isPortInUse(p)) { stillInUse.push(p); } } + dockerSummary.remainingPortConflictCount = + freshResult.ok && !portInspectionFailed ? stillInUse.length : null; if (stillInUse.length === 0) { hasPortConflicts = false; console.log(chalk.green('✓ ') + 'All required ports are now free'); @@ -867,6 +869,32 @@ async function main() { } } + dockerSummary.dockerStatus = docker.status; + dockerSummary.leftExistingDeploymentRunning = leftDeploymentRunning; + dockerSummary.outcome = dockerOutcome(dockerSummary, downloadedCompose, docker.failed); + lifecycle.capture('validated_docker_state', dockerSummary); + lifecycle.stage = 'start'; + lifecycle.failureCategory = 'process_spawn'; + const completion: Events['completed'] = { + completionMode: leftDeploymentRunning ? 'existing_deployment_left_running' : 'manual_start_required', + sourcebotStartOffered: downloadedCompose && !leftDeploymentRunning, + sourcebotStartRequested: false, + sourcebotStartOutcome: 'not_offered', + composeAvailable: downloadedCompose, + dockerValidationOutcome: dockerSummary.outcome, + remainingPortConflictCount: dockerSummary.remainingPortConflictCount, + generatedConnectionCount: sourceSummary.generatedConnectionCount, + codeHostTypes: sourceSummary.codeHostTypes, + repositoryCount: sourceSummary.repositoryCount, + aiConfigured: aiSummary.aiConfigured, + aiConfigurationCount: aiSummary.aiConfigurationCount, + providerTypes: aiSummary.providerTypes, + deploymentIdentityAction: deploymentIdentity.action, + totalDurationMs: 0, + }; + const complete = (keepTelemetryOpen = false) => lifecycle.complete( + { ...completion, totalDurationMs: lifecycle.telemetry.elapsed() }, keepTelemetryOpen, + ); if (downloadedCompose && !leftDeploymentRunning) { const startNow = await confirm({ message: hasPortConflicts @@ -880,16 +908,74 @@ async function main() { `Sourcebot will open at ${SOURCEBOT_URL} once it's ready.\nPress Ctrl+C to stop.`, 'Starting Sourcebot', ); - void openBrowserWhenReady(SOURCEBOT_URL).catch(() => { /* best effort */ }); + lifecycle.check(); + completion.sourcebotStartRequested = true; + const readiness = new AbortController(); + const releaseReadiness = lifecycle.own(() => readiness.abort()); + let spawned = false; + const startFailure = new DockerStartFailure(); await new Promise((resolve) => { - const child = spawn('docker', ['compose', 'up'], { stdio: 'inherit' }); - child.on('exit', () => resolve()); - child.on('error', (err) => { - console.error(chalk.red('✗ ') + 'Failed to run `docker compose up`: ' + (err instanceof Error ? err.message : String(err))); + const child = lifecycle.child( + spawn('docker', ['compose', 'up'], { stdio: ['inherit', 'inherit', 'pipe'], detached: process.platform !== 'win32' }), + ); + child.stderr?.pipe(process.stderr, { end: false }); + child.stderr?.on('data', (chunk: Buffer) => startFailure.write(chunk)); + child.once('spawn', () => { + if (lifecycle.interrupted) { + child.kill(); + return; + } + spawned = true; + completion.sourcebotStartOutcome = 'spawned'; + completion.completionMode = 'sourcebot_start_spawned'; + // Complete the setup handoff now, but keep diagnostics available + // until Compose exits or the user interrupts it. + void complete(true); + void openBrowserWhenReady( + SOURCEBOT_URL, + AbortSignal.any([lifecycle.signal, readiness.signal]), + ).catch(() => {}); + }); + child.once('close', (code, signal) => { + readiness.abort(); + if (spawned && code !== 0 && signal !== 'SIGINT' && signal !== 'SIGTERM') { + const reason = startFailure.reason(); + lifecycle.startFailed({ + failurePhase: 'compose_exit', + failureCategory: reason === 'docker_unavailable' ? 'docker_unavailable' : 'docker_command', + failureReason: reason, + }); + } + resolve(); + }); + child.once('error', (error: NodeJS.ErrnoException) => { + readiness.abort(); + if (!lifecycle.interrupted) { + lifecycle.startFailed({ + failurePhase: 'spawn', + failureCategory: error.code === 'ENOENT' || error.code === 'EACCES' ? 'docker_unavailable' : 'process_spawn', + failureReason: error.code === 'ENOENT' || error.code === 'EACCES' ? 'docker_unavailable' : 'unknown', + }); + lifecycle.fail( + error.code === 'ENOENT' || error.code === 'EACCES' ? 'docker_unavailable' : 'process_spawn', + true, + ); + completion.sourcebotStartOutcome = 'spawn_failed'; + completion.completionMode = 'sourcebot_start_failed'; + console.error(chalk.red('✗ ') + 'Failed to run docker compose up.'); + } resolve(); }); }); - return; + readiness.abort(); + releaseReadiness(); + lifecycle.check(); + if (spawned) { + await lifecycle.telemetry.shutdown(); + return; + } + } else { + completion.sourcebotStartOutcome = 'declined'; } } @@ -904,6 +990,7 @@ async function main() { nextSteps.push(`${step++}. To apply your new configuration, restart Sourcebot:`); nextSteps.push(' docker compose down && docker compose up'); note(nextSteps.join('\n'), 'Sourcebot is already running'); + await complete(); return; } @@ -924,16 +1011,31 @@ async function main() { nextSteps.push(`${step}. Open ${SOURCEBOT_URL}`); note(nextSteps.join('\n'), 'Next steps'); + await complete(); } -main().catch(err => { - const isExitPrompt = err instanceof Error - && (err.name === 'ExitPromptError' || err.message?.startsWith('User force closed the prompt')); - if (isExitPrompt) { - console.log(); - console.log(chalk.red('✗ ') + 'Setup cancelled.'); - process.exit(0); - } - console.error(err); - process.exit(1); -}); +main() + .catch(async (error) => { + if (lifecycle.interrupted) { + return; + } + if (error instanceof Error && error.name === 'ExitPromptError') { + lifecycle.interrupt(); + return; + } + const code = error && typeof error === 'object' ? error.code : undefined; + const category = ['ENOENT', 'ENOTDIR', 'EISDIR', 'EROFS', 'ENOSPC', 'EACCES', 'EPERM'].includes(code) + ? 'filesystem' + : error instanceof Error && error.name === 'ValidationError' + ? 'validation' + : lifecycle.failureCategory; + lifecycle.fail(category, false); + console.error(error); + await lifecycle.telemetry.shutdown(); + process.exitCode = 1; + }) + .finally(() => { + if (!lifecycle.interrupted) { + lifecycle.exit(Number(process.exitCode ?? 0)); + } + }); diff --git a/packages/setupWizard/src/lifecycle.ts b/packages/setupWizard/src/lifecycle.ts new file mode 100644 index 000000000..5bec26f20 --- /dev/null +++ b/packages/setupWizard/src/lifecycle.ts @@ -0,0 +1,221 @@ +import { spawn, type ChildProcess } from 'node:child_process'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { join } from 'node:path'; +import { Telemetry } from './telemetry.js'; +import type { + Events, + EventName, + Stage, + FailureCategory, +} from './telemetryEvents.js'; + +export class Lifecycle { + readonly controller = new AbortController(); + readonly children = new Set(); + private readonly cleanups = new Set<() => void>(); + stage: Stage = 'setup_directory'; + failureCategory: FailureCategory = 'unknown'; + terminal?: 'completed' | 'cancelled' | 'failed'; + interrupted = false; + private installed = false; + private startFailureCaptured = false; + constructor(readonly telemetry = new Telemetry()) {} + get signal(): AbortSignal { + return this.controller.signal; + } + check(): void { + this.signal.throwIfAborted(); + } + install(): void { + if (!this.installed) { + this.installed = true; + process.on('SIGINT', this.interrupt); + } + } + dispose(): void { + process.off('SIGINT', this.interrupt); + this.installed = false; + } + exit(code: number): never { + this.dispose(); + this.controller.abort(); + for (const cleanup of this.cleanups) { + try { + cleanup(); + } catch { + /* Cleanup is independent of telemetry. */ + } + } + this.killChildren(true); + // SDK retry sockets/timers can outlive its bounded shutdown promise. + // Only call after main returns, never at the foreground Docker handoff. + process.exit(code); + } + own(cleanup: () => void): () => void { + this.cleanups.add(cleanup); + return () => this.cleanups.delete(cleanup); + } + child(child: ChildProcess): ChildProcess { + this.children.add(child); + child.once('close', () => { + // A client can exit before descendants that ignored its interrupt. + // Tear down its dedicated group before relinquishing ownership. + if (process.platform !== 'win32' && child.pid) { + try { + process.kill(-child.pid, 'SIGKILL'); + } catch { /* The group normally no longer exists. */ } + } + this.children.delete(child); + }); + if (this.interrupted) { + child.kill(); + } + return child; + } + capture(name: K, props: Events[K]): void { + if (!this.terminal && !this.interrupted) { + this.telemetry.capture(name, props); + } + } + fail(category: FailureCategory, recoverable: boolean): void { + if (this.interrupted || this.terminal) { + return; + } + if (!recoverable) { + this.terminal = 'failed'; + } + this.telemetry.capture('failed', { + stage: this.stage, + failureCategory: category, + recoverable, + }); + } + startFailed(properties: Events['start_failed']): void { + if (this.interrupted || this.startFailureCaptured || (this.terminal && this.terminal !== 'completed')) { + return; + } + this.startFailureCaptured = true; + this.telemetry.capture('start_failed', properties); + } + async complete(properties: Events['completed'], keepTelemetryOpen = false): Promise { + if (this.terminal || this.interrupted) { + return; + } + this.terminal = 'completed'; + this.telemetry.capture('completed', properties); + if (!keepTelemetryOpen) { + await this.telemetry.shutdown(); + } + } + async decline(reason: Events['cancelled']['reason']): Promise { + if (!this.terminal) { + this.terminal = 'cancelled'; + this.telemetry.capture('cancelled', { stage: this.stage, reason }); + } + await this.telemetry.shutdown(); + } + private killChildren(force: boolean): void { + for (const child of this.children) { + if ( + child.exitCode !== null || + child.signalCode !== null || + !child.pid + ) { + continue; + } + try { + if (process.platform === 'win32') { + const taskkill = join( + process.env.SystemRoot ?? 'C:\\Windows', + 'System32', + 'taskkill.exe', + ); + const killer = spawn( + taskkill, + [ + '/pid', + String(child.pid), + '/T', + ...(force ? ['/F'] : []), + ], + { stdio: 'ignore' }, + ); + killer.on('error', () => {}); + killer.unref(); + } else { + // Owned Docker clients run in their own POSIX group, never the user's shell group. + process.kill(-child.pid, force ? 'SIGKILL' : 'SIGINT'); + } + } catch { + /* Child may have exited concurrently. */ + } + } + } + interrupt = (): void => { + if (this.interrupted) { + this.killChildren(true); + process.exit(130); + } + this.interrupted = true; + if (!this.terminal) { + this.terminal = 'cancelled'; + this.telemetry.capture('cancelled', { + stage: this.stage, + reason: 'keyboard_interrupt', + }); + } + this.controller.abort(); + for (const cleanup of this.cleanups) { + try { + cleanup(); + } catch { + /* Independent cleanup must proceed. */ + } + } + this.killChildren(false); + const force = setTimeout(() => this.killChildren(true), 2000); + const deadline = setTimeout(() => { + this.killChildren(true); + process.exit(130); + }, 3000); + void (async () => { + await this.telemetry.shutdown(); + while ( + [...this.children].some( + (c) => c.exitCode === null && c.signalCode === null, + ) + ) { + await sleep(20); + } + clearTimeout(force); + clearTimeout(deadline); + if (process.stdin.isTTY) { + process.stdin.setRawMode(false); + } + process.stdin.pause(); + process.stdout.write('\u001b[?25h\n'); + process.exit(130); + })().catch(() => { + /* Deadline still forces exit. */ + }); + }; +} +export const lifecycle = new Lifecycle(); +export async function wizardFetch( + input: string | URL, + init: RequestInit = {}, +): Promise { + lifecycle.check(); + const signal = init.signal + ? AbortSignal.any([lifecycle.signal, init.signal]) + : lifecycle.signal; + try { + const result = await fetch(input, { ...init, signal }); + lifecycle.check(); + return result; + } catch (error) { + lifecycle.check(); + lifecycle.failureCategory = 'network'; + throw error; + } +} diff --git a/packages/setupWizard/src/localRepos.ts b/packages/setupWizard/src/localRepos.ts index f22c17155..c9be3a041 100644 --- a/packages/setupWizard/src/localRepos.ts +++ b/packages/setupWizard/src/localRepos.ts @@ -1,5 +1,7 @@ -import { input } from '@inquirer/prompts'; -import { tabCheckbox as checkbox } from './tabCheckbox.js'; +import { lifecycle } from './lifecycle.js'; +import { sourceSummary, discoveredBucket } from './telemetrySummary.js'; +import { input } from './prompts.js'; +import { checkbox } from './prompts.js'; import { existsSync, statSync } from 'fs'; import { readdir } from 'fs/promises'; import { homedir } from 'os'; @@ -34,6 +36,7 @@ async function findGitRepos(root: string, maxDepth: number): Promise { const repos: string[] = []; async function walk(dir: string, depth: number): Promise { + lifecycle.check(); if (existsSync(join(dir, '.git'))) { repos.push(dir); return; @@ -102,7 +105,14 @@ export async function collectLocalReposConfig( hostPath = expandHostPath(rawPath); const spinner = ora(`Scanning ${hostPath} for git repositories...`).start(); - repos = await findGitRepos(hostPath, MAX_DEPTH); + const releaseSpinner = lifecycle.own(() => spinner.stop()); + try { + repos = await findGitRepos(hostPath, MAX_DEPTH); + lifecycle.check(); + } finally { + spinner.stop(); + releaseSpinner(); + } if (repos.length === 0) { spinner.fail(`No git repositories found under ${hostPath}`); continue; @@ -130,6 +140,12 @@ export async function collectLocalReposConfig( }], env: {}, localRepoHostPath: hostPath, + telemetry: sourceSummary('local_git', { + deploymentType: 'local', + scopeTypes: ['repositories'], + repositoryCount: 1, + localDiscoveredRepoCountBucket: '1', + }), }; } @@ -167,5 +183,16 @@ export async function collectLocalReposConfig( return { name: basename(repoPath), config }; }); - return { connections, env: {}, localRepoHostPath: hostPath }; + return { + connections, + env: {}, + localRepoHostPath: hostPath, + telemetry: sourceSummary('local_git', { + deploymentType: 'local', + scopeTypes: ['repositories'], + repositoryCount: selected.length, + generatedConnectionCount: connections.length, + localDiscoveredRepoCountBucket: discoveredBucket(repos.length), + }), + }; } diff --git a/packages/setupWizard/src/models.ts b/packages/setupWizard/src/models.ts index 50faea65f..2b5a8410e 100644 --- a/packages/setupWizard/src/models.ts +++ b/packages/setupWizard/src/models.ts @@ -1,5 +1,7 @@ -import { confirm, input, password, select } from '@inquirer/prompts'; -import { select as searchSelect } from 'inquirer-select-pro'; +import { wizardFetch, lifecycle } from './lifecycle.js'; +import type { AiSummary } from './telemetryEvents.js'; +import { confirm, input, password, select } from './prompts.js'; +import { searchSelect } from './prompts.js'; import type { AmazonBedrockLanguageModel, AzureLanguageModel, @@ -59,14 +61,21 @@ async function loadCatalog(): Promise { if (!catalogPromise) { catalogPromise = (async () => { try { - const response = await fetch(MODELS_DEV_API_URL, { + const response = await wizardFetch(MODELS_DEV_API_URL, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS), }); if (!response.ok) { + lifecycle.fail('network', true); return null; } - return await response.json() as ModelsDevCatalog; + const catalog = await response.json(); + if (!catalog || typeof catalog !== 'object' || Array.isArray(catalog)) { + throw new Error('Invalid model catalog'); + } + return catalog as ModelsDevCatalog; } catch { + lifecycle.check(); + lifecycle.fail('network', true); return null; } })(); @@ -88,6 +97,19 @@ async function getModelOptionsForProvider(providerKey: string): Promise + !m || + typeof m.id !== 'string' || + !m.id || + (m.name !== undefined && typeof m.name !== 'string') || + (m.release_date !== undefined && typeof m.release_date !== 'string'), + ) + ) { + lifecycle.fail('network', true); + return null; + } return models .map((m) => ({ id: m.id, @@ -164,6 +186,7 @@ async function collectModelConfig( provider: Provider, model: string, env: EnvVars, + setCredentialMode: (mode: AiSummary['credentialMode']) => void, ): Promise { switch (provider) { case 'anthropic': @@ -224,6 +247,7 @@ async function collectModelConfig( message: 'Use the default AWS credential chain? (No to provide Access Key ID and Secret explicitly)', default: true, }); + setCredentialMode(useDefaultChain ? 'aws_default_chain' : 'aws_explicit_keys'); const config: AmazonBedrockLanguageModel = { provider, model }; @@ -275,6 +299,7 @@ async function collectModelConfig( message: 'Use Application Default Credentials? (No to provide a service account credentials file path)', default: true, }); + setCredentialMode(useAppDefault ? 'google_application_default_credentials' : 'google_credentials_file'); const config: GoogleVertexLanguageModel | GoogleVertexAnthropicLanguageModel = { provider, @@ -295,9 +320,12 @@ async function collectModelConfig( } } -export async function collectModels(): Promise<{ models: LanguageModel[]; env: EnvVars }> { +export async function collectModels( + onAccepted: (summary: AiSummary) => void = () => {}, +): Promise<{ models: LanguageModel[]; env: EnvVars; telemetry: AiSummary[] }> { const models: LanguageModel[] = []; const env: EnvVars = {}; + const telemetry: AiSummary[] = []; note( [ @@ -317,7 +345,7 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E }); if (!wantsAI) { - return { models, env }; + return { models, env, telemetry }; } // eslint-disable-next-line no-constant-condition @@ -354,7 +382,10 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E validate: (v) => !v?.trim() ? 'Model name is required' : true, }); - const config = await collectModelConfig(provider, model, env); + let credentialMode: AiSummary['credentialMode'] = 'api_key'; + const config = await collectModelConfig(provider, model, env, (mode) => { + credentialMode = mode; + }); const displayName = (await input({ message: 'Display name (optional, press enter to skip)', @@ -363,6 +394,19 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E config.displayName = displayName; } models.push(config); + const summary: AiSummary = { + provider, + credentialMode, + usesCustomEndpoint: provider === 'openai-compatible', + hasDisplayName: !!displayName, + modelSelectionMethod: modelOptions?.length + ? modelOptions.some((option) => option.id === model) + ? 'catalog' + : 'custom_entry' + : 'manual_fallback', + }; + telemetry.push(summary); + onAccepted(summary); const addAnother = await confirm({ message: 'Add another model?', @@ -374,5 +418,5 @@ export async function collectModels(): Promise<{ models: LanguageModel[]; env: E } } - return { models, env }; + return { models, env, telemetry }; } diff --git a/packages/setupWizard/src/prompts.ts b/packages/setupWizard/src/prompts.ts new file mode 100644 index 000000000..fcb190aeb --- /dev/null +++ b/packages/setupWizard/src/prompts.ts @@ -0,0 +1,37 @@ +import { + confirm as rawConfirm, + input as rawInput, + password as rawPassword, + select as rawSelect, +} from '@inquirer/prompts'; +import { select as rawSearchSelect } from 'inquirer-select-pro'; +import { tabCheckbox } from './tabCheckbox.js'; +import { lifecycle } from './lifecycle.js'; + +// Preserve the library's generic call signatures, including multi-select return types. +function cancellable(prompt: T): T { + return (async (config: unknown, context: Record = {}) => { + lifecycle.check(); + try { + const result = await ( + prompt as ( + config: unknown, + context: unknown, + ) => Promise + )(config, { ...context, signal: lifecycle.signal }); + lifecycle.check(); + return result; + } catch (error) { + if (error instanceof Error && error.name === 'ExitPromptError') { + lifecycle.interrupt(); + } + throw error; + } + }) as T; +} +export const confirm = cancellable(rawConfirm); +export const input = cancellable(rawInput); +export const password = cancellable(rawPassword); +export const select = cancellable(rawSelect); +export const searchSelect = cancellable(rawSearchSelect); +export const checkbox = cancellable(tabCheckbox); diff --git a/packages/setupWizard/src/spinner.ts b/packages/setupWizard/src/spinner.ts new file mode 100644 index 000000000..fa807f435 --- /dev/null +++ b/packages/setupWizard/src/spinner.ts @@ -0,0 +1,9 @@ +import ora from 'ora'; +import { lifecycle } from './lifecycle.js'; + +export function spinner(text: string) { + lifecycle.check(); + const result = ora(text).start(); + lifecycle.own(() => result.stop()); + return result; +} diff --git a/packages/setupWizard/src/telemetry.ts b/packages/setupWizard/src/telemetry.ts new file mode 100644 index 000000000..f10325101 --- /dev/null +++ b/packages/setupWizard/src/telemetry.ts @@ -0,0 +1,182 @@ +import { randomUUID } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { PostHog } from 'posthog-node'; +import { + eventSchemas, + validateFields, + type Events, + type EventName, +} from './telemetryEvents.js'; + +export const INSTALL_ID_PATTERN = + /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +// Same public ingestion token as packages/shared/src/env.server.ts. Never read user overrides. +export const POSTHOG_PROJECT_TOKEN = + 'phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW'; +export const POSTHOG_OPTIONS = { + host: 'https://us.i.posthog.com', + flushAt: 1, + flushInterval: 0, + disableGeoip: true, + isServer: false, +} as const; +function bestEffort(read: () => T, fallback: T): T { + try { + return read(); + } catch { + return fallback; + } +} +export function systemProperties(runtime = process) { + return { + platform: bestEffort( + () => + ['darwin', 'linux', 'win32'].includes(runtime.platform) + ? runtime.platform + : 'other', + 'other', + ), + arch: bestEffort( + () => + ['arm64', 'x64'].includes(runtime.arch) + ? runtime.arch + : 'other', + 'other', + ), + nodeMajorVersion: bestEffort(() => { + const v = Number(runtime.versions.node.split('.')[0]); + return Number.isSafeInteger(v) && v > 0 ? v : null; + }, null), + packageManager: bestEffort( + () => + /^(npm|yarn|pnpm|bun)\//.exec( + runtime.env.npm_config_user_agent ?? '', + )?.[1] ?? 'unknown', + 'unknown', + ), + isCI: bestEffort( + () => + ['CI', 'CONTINUOUS_INTEGRATION', 'BUILD_NUMBER', 'RUN_ID'].some( + (k) => !!runtime.env[k] && runtime.env[k] !== 'false', + ), + null, + ), + }; +} +export function invocationMethod(): Events['started']['invocationMethod'] { + return bestEffort(() => { + if ( + process.env.npm_command === 'exec' || + process.argv[1]?.includes('_npx') + ) { + return 'npx'; + } + if (process.env.npm_lifecycle_event) { + return 'workspace'; + } + return process.argv[1]?.includes('node_modules') + ? 'local_binary' + : 'unknown'; + }, 'unknown'); +} +type Client = Pick; +export class Telemetry { + readonly setupSessionId: string | undefined; + private client?: Client; + private readonly began: number; + private shutdownPromise?: Promise; + private closed = false; + private readonly wallClockStart = Date.now(); + private lastTimestamp = 0; + constructor( + createClient: () => Client = () => + new PostHog(POSTHOG_PROJECT_TOKEN, POSTHOG_OPTIONS), + uuid: () => string = randomUUID, + private readonly clock = () => performance.now(), + ) { + this.began = clock(); + this.setupSessionId = bestEffort(() => { + const id = uuid(); + return INSTALL_ID_PATTERN.test(id) ? id : undefined; + }, undefined); + if (this.setupSessionId) { + this.client = bestEffort(() => createClient(), undefined); + bestEffort(() => this.client?.on('error', () => {}), undefined); + } + } + elapsed(): number { + return Math.max(0, Math.round(this.clock() - this.began)); + } + capture(event: K, values: Events[K]): void { + if (this.closed || !this.client || !this.setupSessionId) { + return; + } + try { + const properties = validateFields( + eventSchemas[event], + values as never, + ); + const version: unknown = bestEffort( + () => + JSON.parse( + readFileSync( + new URL('../package.json', import.meta.url), + 'utf8', + ), + ).version, + 'unknown', + ); + // Immediate independent requests can be ingested out of order. Preserve + // session order even across wall-clock adjustments and same-ms checkpoints. + this.lastTimestamp = Math.max( + this.lastTimestamp + 1, + this.wallClockStart + this.elapsed(), + ); + this.client.capture({ + distinctId: this.setupSessionId, + event: `setup_sourcebot_${event}`, + groups: { company: this.setupSessionId }, + timestamp: new Date(this.lastTimestamp), + properties: { + ...properties, + ...systemProperties(), + schemaVersion: 1, + source: 'setup-sourcebot-cli', + setupSourcebotVersion: + typeof version === 'string' && + /^\d+\.\d+\.\d+(?:[-+][\w.-]+)?$/.test(version) + ? version + : 'unknown', + setupSessionId: this.setupSessionId, + install_id: this.setupSessionId, + elapsedMs: this.elapsed(), + $geoip_disable: true, + $ignore_sent_at: true, + }, + }); + } catch { + /* Telemetry never participates in setup success. */ + } + } + shutdown(): Promise { + if (!this.shutdownPromise) { + this.closed = true; + this.shutdownPromise = (async () => { + let timer: ReturnType | undefined; + try { + await Promise.race([ + this.client?.shutdown(1000), + new Promise((resolve) => { + timer = setTimeout(resolve, 1000); + }), + ]); + } catch { + /* Best effort. */ + } finally { + clearTimeout(timer); + } + })(); + } + return this.shutdownPromise; + } +} diff --git a/packages/setupWizard/src/telemetryEvents.ts b/packages/setupWizard/src/telemetryEvents.ts new file mode 100644 index 000000000..e6bcb4b1a --- /dev/null +++ b/packages/setupWizard/src/telemetryEvents.ts @@ -0,0 +1,345 @@ +// Runtime allowlists are also the TypeScript event contract. Never accept arbitrary maps. +type Rule = { parse: (value: unknown) => T }; +type Value = R extends Rule ? T : never; +type Shape = Record>; +type Fields = { [K in keyof S]: Value }; +function rule(check: (v: unknown) => boolean): Rule { + return { + parse(value) { + if (!check(value)) { + throw new Error('Invalid telemetry property'); + } + return value as T; + }, + }; +} +export function choice( + ...values: T +): Rule { + return rule((v) => typeof v === 'string' && values.includes(v)); +} +const boolean = rule((v) => typeof v === 'boolean'); +const count = rule( + (v) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 0, +); +const position = rule( + (v) => typeof v === 'number' && Number.isSafeInteger(v) && v >= 1, +); +const nullable = (inner: Rule): Rule => ({ + parse: (v) => (v === null ? null : inner.parse(v)), +}); +const array = (inner: Rule): Rule => ({ + parse(v) { + if (!Array.isArray(v)) { + throw new Error('Invalid telemetry array'); + } + return v.map((item) => inner.parse(item)); + }, +}); +export const codeHost = choice( + 'github', + 'gitlab', + 'bitbucket', + 'gitea', + 'azure_devops', + 'gerrit', + 'local_git', + 'remote_git', +); +export const provider = choice( + 'anthropic', + 'openai', + 'openai-compatible', + 'amazon-bedrock', + 'google-generative-ai', + 'google-vertex', + 'google-vertex-anthropic', + 'azure', + 'deepseek', + 'mistral', + 'openrouter', + 'xai', +); +export const deploymentType = choice( + 'cloud', + 'self_hosted', + 'local', + 'remote', + 'unknown', +); +const sourceCredential = choice( + 'none', + 'personal_access_token', + 'api_token', + 'access_token', + 'app_password', + 'http_access_token', +); +const aiCredential = choice( + 'api_key', + 'aws_default_chain', + 'aws_explicit_keys', + 'google_application_default_credentials', + 'google_credentials_file', +); +const modelSelection = choice('catalog', 'custom_entry', 'manual_fallback'); +const scope = choice( + 'all', + 'repositories', + 'organizations', + 'users', + 'groups', + 'projects', + 'workspaces', +); +const file = choice('config_json', 'env', 'compose_override'); +const identity = choice('created_from_setup_session', 'preserved_existing'); +const stage = choice( + 'setup_directory', + 'code_sources', + 'ai_setup', + 'hosted_url', + 'config_overwrite', + 'compose_file', + 'docker_validation', + 'start', +); +const category = choice( + 'validation', + 'network', + 'filesystem', + 'docker_unavailable', + 'docker_command', + 'process_spawn', + 'unknown', +); +const dockerOutcome = choice( + 'passed', + 'passed_after_cleanup', + 'unresolved_conflicts', + 'skipped_no_compose', + 'skipped_existing_deployment_running', + 'validation_failed', +); +const entityCounts = { + repositoryCount: count, + organizationCount: count, + userCount: count, + groupCount: count, + projectCount: count, + workspaceCount: count, +}; +export const codeSourceSchema = { + codeHost, + deploymentType, + credentialMode: sourceCredential, + scopeTypes: array(scope), + indexAll: boolean, + ...entityCounts, + generatedConnectionCount: count, + localDiscoveredRepoCountBucket: nullable( + choice('1', '2-5', '6-20', '21-100', '101+'), + ), +}; +export const aiSchema = { + provider, + modelSelectionMethod: modelSelection, + credentialMode: aiCredential, + usesCustomEndpoint: boolean, + hasDisplayName: boolean, +}; +export const eventSchemas = { + started: { + invocationMethod: choice( + 'npx', + 'global_binary', + 'local_binary', + 'workspace', + 'unknown', + ), + isInteractive: boolean, + }, + chose_setup_directory: { + usedDefaultDirectory: boolean, + directoryExisted: boolean, + directoryAction: choice('created', 'existing_directory_accepted'), + }, + configured_code_source: { + configurationIndex: position, + ...codeSourceSchema, + }, + configured_code_sources: { + codeSourceConfigurationCount: count, + generatedConnectionCount: count, + uniqueCodeHostCount: count, + codeHostTypes: array(codeHost), + credentialedCodeSourceCount: count, + cloudCodeSourceCount: count, + selfHostedCodeSourceCount: count, + localCodeSourceCount: count, + indexAllCodeSourceCount: count, + ...entityCounts, + }, + configured_ai_provider: { configurationIndex: position, ...aiSchema }, + ai_setup_completed: { + aiConfigured: boolean, + aiConfigurationCount: count, + uniqueProviderCount: count, + providerTypes: array(provider), + usesCustomEndpoint: boolean, + credentialModes: array(aiCredential), + modelSelectionMethods: array(modelSelection), + }, + configured_hosted_url: { + usedDefaultUrl: boolean, + protocol: choice('http', 'https'), + hostCategory: choice('localhost', 'address', 'unknown'), + }, + generated_configs: { + filesWritten: array(file), + overwroteExistingFiles: array(file), + wroteComposeOverride: boolean, + localMountCount: count, + generatedConnectionCount: count, + aiConfigurationCount: count, + credentialVariableCount: count, + deploymentIdentityAction: identity, + }, + resolved_compose_file: { + outcome: choice( + 'downloaded', + 'already_present', + 'declined', + 'download_failed', + ), + composeAvailable: boolean, + downloadPromptShown: boolean, + downloadAttempted: boolean, + failureCategory: nullable( + choice( + 'network', + 'http_4xx', + 'http_5xx', + 'filesystem', + 'timeout', + 'unknown', + ), + ), + }, + validated_docker_state: { + outcome: dockerOutcome, + dockerStatus: choice( + 'available', + 'unavailable', + 'error', + 'not_checked', + ), + composeContainerState: choice( + 'none', + 'running', + 'stopped', + 'mixed', + 'unknown', + ), + runningComposeContainerCount: nullable(count), + stoppedComposeContainerCount: nullable(count), + existingVolumeCount: nullable(count), + initialPortConflictCount: nullable(count), + remainingPortConflictCount: nullable(count), + portConflictSource: choice( + 'none', + 'docker', + 'non_docker', + 'mixed', + 'unknown', + ), + existingDeploymentAction: choice( + 'none', + 'stopped', + 'left_running', + 'stop_failed', + ), + stoppedContainerAction: choice( + 'none', + 'removed', + 'kept', + 'remove_failed', + ), + volumeAction: choice('none', 'removed', 'kept', 'remove_failed'), + portConflictAction: choice( + 'none', + 'containers_stopped', + 'kept', + 'stop_failed', + ), + leftExistingDeploymentRunning: boolean, + }, + completed: { + completionMode: choice( + 'sourcebot_start_spawned', + 'sourcebot_start_failed', + 'existing_deployment_left_running', + 'manual_start_required', + ), + sourcebotStartOffered: boolean, + sourcebotStartRequested: boolean, + sourcebotStartOutcome: choice( + 'spawned', + 'declined', + 'not_offered', + 'spawn_failed', + ), + composeAvailable: boolean, + dockerValidationOutcome: dockerOutcome, + remainingPortConflictCount: nullable(count), + generatedConnectionCount: count, + codeHostTypes: array(codeHost), + repositoryCount: count, + aiConfigured: boolean, + aiConfigurationCount: count, + providerTypes: array(provider), + deploymentIdentityAction: identity, + totalDurationMs: count, + }, + cancelled: { + stage, + reason: choice( + 'keyboard_interrupt', + 'existing_directory_declined', + 'config_overwrite_declined', + ), + }, + failed: { stage, failureCategory: category, recoverable: boolean }, + start_failed: { + failurePhase: choice('spawn', 'compose_exit'), + failureCategory: choice('docker_unavailable', 'process_spawn', 'docker_command'), + failureReason: choice( + 'container_name_conflict', + 'port_conflict', + 'image_pull_failed', + 'mount_failed', + 'compose_configuration', + 'docker_unavailable', + 'unknown', + ), + }, +}; +export type EventName = keyof typeof eventSchemas; +export type Events = { [K in EventName]: Fields<(typeof eventSchemas)[K]> }; +export type CodeSourceSummary = Fields; +export type AiSummary = Fields; +export type Stage = Value; +export type FailureCategory = Value; +export type DockerSummary = Events['validated_docker_state']; +export function validateFields( + schema: S, + input: Fields, +): Fields { + // Read only explicitly approved fields; the source object is never spread or serialized. + return Object.fromEntries( + Object.entries(schema).map(([key, validator]) => [ + key, + validator.parse(input[key]), + ]), + ) as Fields; +} diff --git a/packages/setupWizard/src/telemetrySummary.ts b/packages/setupWizard/src/telemetrySummary.ts new file mode 100644 index 000000000..764ee8d06 --- /dev/null +++ b/packages/setupWizard/src/telemetrySummary.ts @@ -0,0 +1,228 @@ +import { isIP } from 'node:net'; +import { INSTALL_ID_PATTERN } from './telemetry.js'; +import type { + CodeSourceSummary, + AiSummary, + Events, + DockerSummary, +} from './telemetryEvents.js'; + +export function sourceSummary( + codeHost: CodeSourceSummary['codeHost'], + fields: Partial> = {}, +): CodeSourceSummary { + return { + codeHost, + deploymentType: 'unknown', + credentialMode: 'none', + scopeTypes: [], + indexAll: false, + repositoryCount: 0, + organizationCount: 0, + userCount: 0, + groupCount: 0, + projectCount: 0, + workspaceCount: 0, + generatedConnectionCount: 1, + localDiscoveredRepoCountBucket: null, + ...fields, + }; +} +export function normalizeHost(value: string): string | undefined { + try { + const input = value.trim(); + if (!input) { + return undefined; + } + const url = new URL( + /^[a-z][a-z\d+.-]*:\/\//i.test(input) ? input : `https://${input}`, + ); + if (!['http:', 'https:'].includes(url.protocol)) { + return undefined; + } + return url.hostname + .toLowerCase() + .replace(/\.+$/, '') + .replace(/^www\./, ''); + } catch { + return undefined; + } +} +export function deployment( + host: 'github' | 'gitlab' | 'gitea', + url: string, +): CodeSourceSummary['deploymentType'] { + const name = normalizeHost(url); + if (!name) { + return 'unknown'; + } + if (host === 'github') { + return name === 'github.com' || name.endsWith('.ghe.com') + ? 'cloud' + : 'self_hosted'; + } + if (host === 'gitlab') { + return name === 'gitlab.com' || + name.endsWith('.gitlab-dedicated.com') || + name.endsWith('.gitlab-dedicated.systems') + ? 'cloud' + : 'unknown'; + } + return name === 'gitea.com' ? 'cloud' : 'self_hosted'; +} +export function hostCategory( + value: string, +): Events['configured_hosted_url']['hostCategory'] { + try { + const url = new URL(value); + const host = url.hostname + .toLowerCase() + .replace(/\.+$/, '') + .replace(/^\[|\]$/g, ''); + if (!host) { + return 'unknown'; + } + return host === 'localhost' || + host.endsWith('.localhost') || + host === '::1' || + (isIP(host) === 4 && host.startsWith('127.')) + ? 'localhost' + : 'address'; + } catch { + return 'unknown'; + } +} +export function discoveredBucket( + n: number, +): CodeSourceSummary['localDiscoveredRepoCountBucket'] { + return n <= 1 + ? '1' + : n <= 5 + ? '2-5' + : n <= 20 + ? '6-20' + : n <= 100 + ? '21-100' + : '101+'; +} +const unique = (items: T[]): T[] => + [...new Set(items)].sort(); +export function aggregateSources( + sources: CodeSourceSummary[], +): Events['configured_code_sources'] { + const sum = ( + key: + | 'generatedConnectionCount' + | 'repositoryCount' + | 'organizationCount' + | 'userCount' + | 'groupCount' + | 'projectCount' + | 'workspaceCount', + ) => sources.reduce((n, s) => n + s[key], 0); + const codeHostTypes = unique(sources.map((s) => s.codeHost)); + return { + codeSourceConfigurationCount: sources.length, + generatedConnectionCount: sum('generatedConnectionCount'), + uniqueCodeHostCount: codeHostTypes.length, + codeHostTypes, + credentialedCodeSourceCount: sources.filter( + (s) => s.credentialMode !== 'none', + ).length, + cloudCodeSourceCount: sources.filter( + (s) => s.deploymentType === 'cloud', + ).length, + selfHostedCodeSourceCount: sources.filter( + (s) => s.deploymentType === 'self_hosted', + ).length, + localCodeSourceCount: sources.filter( + (s) => s.deploymentType === 'local', + ).length, + indexAllCodeSourceCount: sources.filter((s) => s.indexAll).length, + repositoryCount: sum('repositoryCount'), + organizationCount: sum('organizationCount'), + userCount: sum('userCount'), + groupCount: sum('groupCount'), + projectCount: sum('projectCount'), + workspaceCount: sum('workspaceCount'), + }; +} +export function aggregateAi(models: AiSummary[]): Events['ai_setup_completed'] { + const providerTypes = unique(models.map((m) => m.provider)); + return { + aiConfigured: models.length > 0, + aiConfigurationCount: models.length, + uniqueProviderCount: providerTypes.length, + providerTypes, + usesCustomEndpoint: models.some((m) => m.usesCustomEndpoint), + credentialModes: unique(models.map((m) => m.credentialMode)), + modelSelectionMethods: unique( + models.map((m) => m.modelSelectionMethod), + ), + }; +} +export function selectInstallId( + contents: string, + sessionId: string | undefined, +) { + // Parse only this setting. Accept normal dotenv quoting; never evaluate or expand it. + const lines = contents + .split(/\r?\n/) + .filter((l) => /^\s*(?:export\s+)?SOURCEBOT_INSTALL_ID\s*=/.test(l)); + const raw = lines + .at(-1) + ?.replace(/^\s*(?:export\s+)?SOURCEBOT_INSTALL_ID\s*=\s*/, '') + .trim(); + const existing = raw + ?.match(/^(?:"([0-9a-f-]+)"|'([0-9a-f-]+)'|([0-9a-f-]+))\s*(?:#.*)?$/) + ?.slice(1) + .find(Boolean); + return existing && INSTALL_ID_PATTERN.test(existing) + ? { id: existing, action: 'preserved_existing' as const } + : { id: sessionId, action: 'created_from_setup_session' as const }; +} +export function emptyDockerSummary(): DockerSummary { + return { + outcome: 'skipped_no_compose', + dockerStatus: 'not_checked', + composeContainerState: 'unknown', + runningComposeContainerCount: null, + stoppedComposeContainerCount: null, + existingVolumeCount: null, + initialPortConflictCount: null, + remainingPortConflictCount: null, + portConflictSource: 'unknown', + existingDeploymentAction: 'none', + stoppedContainerAction: 'none', + volumeAction: 'none', + portConflictAction: 'none', + leftExistingDeploymentRunning: false, + }; +} +export function dockerOutcome( + summary: DockerSummary, + composeAvailable: boolean, + failed: boolean, +): DockerSummary['outcome'] { + if (!composeAvailable) { + return 'skipped_no_compose'; + } + if (failed) { + return 'validation_failed'; + } + if (summary.leftExistingDeploymentRunning) { + return 'skipped_existing_deployment_running'; + } + if ( + (summary.remainingPortConflictCount ?? 0) > 0 || + summary.stoppedContainerAction === 'kept' + ) { + return 'unresolved_conflicts'; + } + return summary.existingDeploymentAction === 'stopped' || + summary.stoppedContainerAction === 'removed' || + summary.volumeAction === 'removed' || + summary.portConflictAction === 'containers_stopped' + ? 'passed_after_cleanup' + : 'passed'; +} diff --git a/packages/setupWizard/src/utils.ts b/packages/setupWizard/src/utils.ts index 8a7b08f1f..415ab0740 100644 --- a/packages/setupWizard/src/utils.ts +++ b/packages/setupWizard/src/utils.ts @@ -1,6 +1,7 @@ +import type { CodeSourceSummary } from './telemetryEvents.js'; import chalk from 'chalk'; import { randomBytes } from 'crypto'; -import { select as searchSelect } from 'inquirer-select-pro'; +import { searchSelect } from './prompts.js'; import type { ConnectionConfig } from '@sourcebot/schemas/v3/index.type'; export type { ConnectionConfig }; @@ -13,6 +14,7 @@ export type CollectResult = { */ connections: Array<{ name?: string; config: ConnectionConfig }>; env: EnvVars; + telemetry: CodeSourceSummary; /** * Optional host path that needs to be mounted into the Sourcebot container. * Surfaced in the wizard's next-steps so users get the matching volume mount line. diff --git a/packages/setupWizard/tests/approvedSchema.json b/packages/setupWizard/tests/approvedSchema.json new file mode 100644 index 000000000..29e19acb5 --- /dev/null +++ b/packages/setupWizard/tests/approvedSchema.json @@ -0,0 +1,614 @@ +{ + "started": { + "invocationMethod": { + "enum": [ + "npx", + "global_binary", + "local_binary", + "workspace", + "unknown" + ] + }, + "isInteractive": { + "type": "boolean" + } + }, + "chose_setup_directory": { + "usedDefaultDirectory": { + "type": "boolean" + }, + "directoryExisted": { + "type": "boolean" + }, + "directoryAction": { + "enum": [ + "created", + "existing_directory_accepted" + ] + } + }, + "configured_code_source": { + "configurationIndex": { + "type": "number" + }, + "codeHost": { + "enum": [ + "github", + "gitlab", + "bitbucket", + "gitea", + "azure_devops", + "gerrit", + "local_git", + "remote_git" + ] + }, + "deploymentType": { + "enum": [ + "cloud", + "self_hosted", + "local", + "remote", + "unknown" + ] + }, + "credentialMode": { + "enum": [ + "none", + "personal_access_token", + "api_token", + "access_token", + "app_password", + "http_access_token" + ] + }, + "scopeTypes": { + "array": { + "enum": [ + "all", + "repositories", + "organizations", + "users", + "groups", + "projects", + "workspaces" + ] + } + }, + "indexAll": { + "type": "boolean" + }, + "repositoryCount": { + "type": "number" + }, + "organizationCount": { + "type": "number" + }, + "userCount": { + "type": "number" + }, + "groupCount": { + "type": "number" + }, + "projectCount": { + "type": "number" + }, + "workspaceCount": { + "type": "number" + }, + "generatedConnectionCount": { + "type": "number" + }, + "localDiscoveredRepoCountBucket": { + "nullable": { + "enum": [ + "1", + "2-5", + "6-20", + "21-100", + "101+" + ] + } + } + }, + "configured_code_sources": { + "codeSourceConfigurationCount": { + "type": "number" + }, + "generatedConnectionCount": { + "type": "number" + }, + "uniqueCodeHostCount": { + "type": "number" + }, + "codeHostTypes": { + "array": { + "enum": [ + "github", + "gitlab", + "bitbucket", + "gitea", + "azure_devops", + "gerrit", + "local_git", + "remote_git" + ] + } + }, + "credentialedCodeSourceCount": { + "type": "number" + }, + "cloudCodeSourceCount": { + "type": "number" + }, + "selfHostedCodeSourceCount": { + "type": "number" + }, + "localCodeSourceCount": { + "type": "number" + }, + "indexAllCodeSourceCount": { + "type": "number" + }, + "repositoryCount": { + "type": "number" + }, + "organizationCount": { + "type": "number" + }, + "userCount": { + "type": "number" + }, + "groupCount": { + "type": "number" + }, + "projectCount": { + "type": "number" + }, + "workspaceCount": { + "type": "number" + } + }, + "configured_ai_provider": { + "configurationIndex": { + "type": "number" + }, + "provider": { + "enum": [ + "anthropic", + "openai", + "openai-compatible", + "amazon-bedrock", + "google-generative-ai", + "google-vertex", + "google-vertex-anthropic", + "azure", + "deepseek", + "mistral", + "openrouter", + "xai" + ] + }, + "modelSelectionMethod": { + "enum": [ + "catalog", + "custom_entry", + "manual_fallback" + ] + }, + "credentialMode": { + "enum": [ + "api_key", + "aws_default_chain", + "aws_explicit_keys", + "google_application_default_credentials", + "google_credentials_file" + ] + }, + "usesCustomEndpoint": { + "type": "boolean" + }, + "hasDisplayName": { + "type": "boolean" + } + }, + "ai_setup_completed": { + "aiConfigured": { + "type": "boolean" + }, + "aiConfigurationCount": { + "type": "number" + }, + "uniqueProviderCount": { + "type": "number" + }, + "providerTypes": { + "array": { + "enum": [ + "anthropic", + "openai", + "openai-compatible", + "amazon-bedrock", + "google-generative-ai", + "google-vertex", + "google-vertex-anthropic", + "azure", + "deepseek", + "mistral", + "openrouter", + "xai" + ] + } + }, + "usesCustomEndpoint": { + "type": "boolean" + }, + "credentialModes": { + "array": { + "enum": [ + "api_key", + "aws_default_chain", + "aws_explicit_keys", + "google_application_default_credentials", + "google_credentials_file" + ] + } + }, + "modelSelectionMethods": { + "array": { + "enum": [ + "catalog", + "custom_entry", + "manual_fallback" + ] + } + } + }, + "configured_hosted_url": { + "usedDefaultUrl": { + "type": "boolean" + }, + "protocol": { + "enum": [ + "http", + "https" + ] + }, + "hostCategory": { + "enum": [ + "localhost", + "address", + "unknown" + ] + } + }, + "generated_configs": { + "filesWritten": { + "array": { + "enum": [ + "config_json", + "env", + "compose_override" + ] + } + }, + "overwroteExistingFiles": { + "array": { + "enum": [ + "config_json", + "env", + "compose_override" + ] + } + }, + "wroteComposeOverride": { + "type": "boolean" + }, + "localMountCount": { + "type": "number" + }, + "generatedConnectionCount": { + "type": "number" + }, + "aiConfigurationCount": { + "type": "number" + }, + "credentialVariableCount": { + "type": "number" + }, + "deploymentIdentityAction": { + "enum": [ + "created_from_setup_session", + "preserved_existing" + ] + } + }, + "resolved_compose_file": { + "outcome": { + "enum": [ + "downloaded", + "already_present", + "declined", + "download_failed" + ] + }, + "composeAvailable": { + "type": "boolean" + }, + "downloadPromptShown": { + "type": "boolean" + }, + "downloadAttempted": { + "type": "boolean" + }, + "failureCategory": { + "nullable": { + "enum": [ + "network", + "http_4xx", + "http_5xx", + "filesystem", + "timeout", + "unknown" + ] + } + } + }, + "validated_docker_state": { + "outcome": { + "enum": [ + "passed", + "passed_after_cleanup", + "unresolved_conflicts", + "skipped_no_compose", + "skipped_existing_deployment_running", + "validation_failed" + ] + }, + "dockerStatus": { + "enum": [ + "available", + "unavailable", + "error", + "not_checked" + ] + }, + "composeContainerState": { + "enum": [ + "none", + "running", + "stopped", + "mixed", + "unknown" + ] + }, + "runningComposeContainerCount": { + "nullable": { + "type": "number" + } + }, + "stoppedComposeContainerCount": { + "nullable": { + "type": "number" + } + }, + "existingVolumeCount": { + "nullable": { + "type": "number" + } + }, + "initialPortConflictCount": { + "nullable": { + "type": "number" + } + }, + "remainingPortConflictCount": { + "nullable": { + "type": "number" + } + }, + "portConflictSource": { + "enum": [ + "none", + "docker", + "non_docker", + "mixed", + "unknown" + ] + }, + "existingDeploymentAction": { + "enum": [ + "none", + "stopped", + "left_running", + "stop_failed" + ] + }, + "stoppedContainerAction": { + "enum": [ + "none", + "removed", + "kept", + "remove_failed" + ] + }, + "volumeAction": { + "enum": [ + "none", + "removed", + "kept", + "remove_failed" + ] + }, + "portConflictAction": { + "enum": [ + "none", + "containers_stopped", + "kept", + "stop_failed" + ] + }, + "leftExistingDeploymentRunning": { + "type": "boolean" + } + }, + "completed": { + "completionMode": { + "enum": [ + "sourcebot_start_spawned", + "sourcebot_start_failed", + "existing_deployment_left_running", + "manual_start_required" + ] + }, + "sourcebotStartOffered": { + "type": "boolean" + }, + "sourcebotStartRequested": { + "type": "boolean" + }, + "sourcebotStartOutcome": { + "enum": [ + "spawned", + "declined", + "not_offered", + "spawn_failed" + ] + }, + "composeAvailable": { + "type": "boolean" + }, + "dockerValidationOutcome": { + "enum": [ + "passed", + "passed_after_cleanup", + "unresolved_conflicts", + "skipped_no_compose", + "skipped_existing_deployment_running", + "validation_failed" + ] + }, + "remainingPortConflictCount": { + "nullable": { + "type": "number" + } + }, + "generatedConnectionCount": { + "type": "number" + }, + "codeHostTypes": { + "array": { + "enum": [ + "github", + "gitlab", + "bitbucket", + "gitea", + "azure_devops", + "gerrit", + "local_git", + "remote_git" + ] + } + }, + "repositoryCount": { + "type": "number" + }, + "aiConfigured": { + "type": "boolean" + }, + "aiConfigurationCount": { + "type": "number" + }, + "providerTypes": { + "array": { + "enum": [ + "anthropic", + "openai", + "openai-compatible", + "amazon-bedrock", + "google-generative-ai", + "google-vertex", + "google-vertex-anthropic", + "azure", + "deepseek", + "mistral", + "openrouter", + "xai" + ] + } + }, + "deploymentIdentityAction": { + "enum": [ + "created_from_setup_session", + "preserved_existing" + ] + }, + "totalDurationMs": { + "type": "number" + } + }, + "cancelled": { + "stage": { + "enum": [ + "setup_directory", + "code_sources", + "ai_setup", + "hosted_url", + "config_overwrite", + "compose_file", + "docker_validation", + "start" + ] + }, + "reason": { + "enum": [ + "keyboard_interrupt", + "existing_directory_declined", + "config_overwrite_declined" + ] + } + }, + "start_failed": { + "failurePhase": { "enum": ["spawn", "compose_exit"] }, + "failureCategory": { "enum": ["docker_unavailable", "process_spawn", "docker_command"] }, + "failureReason": { + "enum": [ + "container_name_conflict", + "port_conflict", + "image_pull_failed", + "mount_failed", + "compose_configuration", + "docker_unavailable", + "unknown" + ] + } + }, + "failed": { + "stage": { + "enum": [ + "setup_directory", + "code_sources", + "ai_setup", + "hosted_url", + "config_overwrite", + "compose_file", + "docker_validation", + "start" + ] + }, + "failureCategory": { + "enum": [ + "validation", + "network", + "filesystem", + "docker_unavailable", + "docker_command", + "process_spawn", + "unknown" + ] + }, + "recoverable": { + "type": "boolean" + } + } +} diff --git a/packages/setupWizard/tests/e2e/baseline.mjs b/packages/setupWizard/tests/e2e/baseline.mjs new file mode 100644 index 000000000..426f9224f --- /dev/null +++ b/packages/setupWizard/tests/e2e/baseline.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, writeFileSync, readFileSync, symlinkSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { join, dirname, resolve, delimiter } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { artifact, scenario, minimal, defaultCompose } from './harness.mjs'; +const require = createRequire(import.meta.url); +const repo = resolve(dirname(fileURLToPath(import.meta.url)), '../../../..'); +const packed = artifact(); +const base = process.env.SETUP_TEST_BASE_REF ?? '31734dc2'; +try { + const checkout = join(packed.root, 'baseline'); + mkdirSync(checkout); + const archive = execFileSync('git', ['archive', base, 'packages/setupWizard'], { cwd: repo }); + execFileSync('tar', ['-x', '-C', checkout], { input: archive }); + const pkg = join(checkout, 'packages/setupWizard'); + symlinkSync(join(repo, 'node_modules'), join(checkout, 'node_modules'), 'junction'); + symlinkSync(join(repo, 'packages/setupWizard/node_modules'), join(pkg, 'node_modules'), 'junction'); + execFileSync(process.execPath, [require.resolve('typescript/bin/tsc'), '-p', pkg], { stdio: 'inherit' }); + const manifest = JSON.parse(readFileSync(join(pkg, 'package.json'), 'utf8')); + // Yarn pack rewrites this dev-only workspace reference; mirror that transformation. + manifest.devDependencies['@sourcebot/schemas'] = JSON.parse(readFileSync(join(repo, 'packages/schemas/package.json'), 'utf8')).version; + writeFileSync(join(pkg, 'package.json'), JSON.stringify(manifest)); + const env = { ...process.env, PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false' }; + const tgz = execFileSync('npm', ['pack', '--ignore-scripts', '--silent'], { cwd: pkg, env, encoding: 'utf8' }).trim(); + const install = join(packed.root, 'baseline-installed'); + mkdirSync(install); + writeFileSync(join(install, 'package.json'), '{"private":true,"allowScripts":{"reo-census":true}}'); + execFileSync('npm', ['install', '--no-audit', '--no-fund', join(pkg, tgz)], { cwd: install, env }); + const prior = { ...packed, bin: join(install, 'node_modules/.bin/setup-sourcebot'), installed: join(install, 'node_modules/setup-sourcebot') }; + const normalize = files => Object.fromEntries(Object.entries(files).map(([name, contents]) => [name, name === '.env' + ? contents.replace(/^SOURCEBOT_INSTALL_ID=.*\n/gm, '').replace(/^# Deployment identifier\n/gm, '').replace(/^(AUTH_SECRET|SOURCEBOT_ENCRYPTION_KEY)=.*$/gm, '$1=').replace(/\n{3,}/g, '\n\n').trim() + : contents])); + for (const download of [false, true]) { + const run = (target, baseline) => scenario(target, { baseline }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', download ? 'y' : 'n'); + if (download) { + await d.answer('Start Sourcebot now?', 'n'); + } + }); + const original = await run(prior, true); + const feature = await run(packed, false); + assert.deepEqual(original.events, []); + assert.deepEqual(normalize(feature.files), normalize(original.files)); + assert.deepEqual(feature.dockerCalls, original.dockerCalls); + assert.equal(feature.exitCode, original.exitCode); + } + console.log(`Baseline ${base}: generated config/env and Docker operations match (manual and downloaded Compose); only UUID persistence and generated secrets normalized.`); +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/collectors.test.mjs b/packages/setupWizard/tests/e2e/collectors.test.mjs new file mode 100644 index 000000000..e6de1202c --- /dev/null +++ b/packages/setupWizard/tests/e2e/collectors.test.mjs @@ -0,0 +1,223 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { artifact, scenario, minimal, canary } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Collector artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const props = (r, name) => r.events.filter(e => e.event === `setup_sourcebot_${name}`).map(e => e.properties); +const end = async d => { + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); +}; + +for (const [host, index] of [['github', 0], ['gitlab', 1], ['gitea', 6]]) { + for (const token of [false, true]) { + test(`${host}: all explicit scopes, ${token ? 'credentialed' : 'public'}`, async () => { + const label = host === 'github' ? 'GitHub' : host === 'gitlab' ? 'GitLab' : 'Gitea'; + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', index); + await d.answer(`${label} URL`); + await d.answer(`${label} ${host === 'gitea' ? 'Access Token' : 'Personal Access Token'}`, token ? `${canary}-token` : ''); + await d.check('What do you want to index?', [0, 1, 2]); + if (host === 'github') { + await d.multi('Repositories to index'); + await d.multi('Organizations to index', canary); + await d.multi('GitHub users to index', canary); + } else if (host === 'gitlab') { + await d.multi('Groups to index', canary); + await d.multi('Projects to index'); + await d.multi('Users to index', canary); + } else { + await d.multi('Organizations to index', canary); + await d.multi('Repositories to index'); + await d.multi('Users to index', canary); + } + await end(d); + }); + const summary = props(result, 'configured_code_source')[0]; + assert.equal(summary.codeHost, host); + assert.equal(summary.deploymentType, 'cloud'); + assert.equal(summary.scopeTypes.length, 3); + assert.equal(summary.userCount, 1); + assert.equal(summary.credentialMode === 'none', !token); + assert.equal(summary.repositoryCount, host === 'gitlab' ? 0 : 1); + assert.equal(summary.projectCount, host === 'gitlab' ? 1 : 0); + }); + } +} + +for (const all of [true, false]) { + test(`Gerrit: ${all ? 'all' : 'projects'}`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 7); + await d.answer('Gerrit URL', `https://${canary}.example.invalid`); + await d.answer('Index all projects?', all ? 'y' : 'n'); + if (!all) { + await d.multi('Projects to index', canary); + } + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.indexAll, all); + assert.equal(s.projectCount, all ? 0 : 1); + assert.equal(s.deploymentType, 'self_hosted'); + }); +} + +for (const server of [false, true]) { + test(`Azure DevOps ${server ? 'Server/TFS' : 'Cloud'}: all scopes`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 4); + await d.select('Which Azure DevOps deployment?', server ? 1 : 0); + if (server) { + await d.answer('Azure DevOps Server URL', `https://${canary}.example.invalid`); + await d.answer('Use legacy TFS path format', 'y'); + } + await d.answer('Azure DevOps Personal Access Token', `${canary}-token`); + await d.check('What do you want to index?', [0, 1, 2]); + await d.multi(server ? 'Collections to index' : 'Organizations to index', canary); + await d.multi('Projects to index'); + await d.multi('Repositories to index', `${canary}/project/repo`); + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.deploymentType, server ? 'self_hosted' : 'cloud'); + assert.equal(s.organizationCount, 1); + assert.equal(s.projectCount, 1); + assert.equal(s.repositoryCount, 1); + }); +} + +for (const auth of [0, 1, 2]) { + test(`Bitbucket Cloud auth ${auth}: workspaces and repositories`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 5); + await d.select('Which Bitbucket deployment?'); + await d.select('How will you authenticate?', auth); + if (auth === 0) { + await d.answer('Atlassian account email', `${canary}@example.invalid`); + await d.answer('Bitbucket username', canary); + await d.answer('API Token (', `${canary}-token`); + } else if (auth === 1) { + await d.answer('Access Token (', `${canary}-token`); + } else { + await d.answer('Bitbucket username', canary); + await d.answer('Bitbucket App Password', `${canary}-password`); + } + await d.check('What do you want to index?', [0, 1]); + await d.multi('Workspaces to index', canary); + await d.multi('Repositories to index'); + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.credentialMode, ['api_token', 'access_token', 'app_password'][auth]); + assert.equal(s.workspaceCount, 1); + assert.equal(s.repositoryCount, 1); + }); +} + +for (const all of [false, true]) { + test(`Bitbucket Data Center: ${all ? 'all' : 'selected'}`, async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 5); + await d.select('Which Bitbucket deployment?', 1); + await d.answer('Bitbucket Data Center URL', `https://${canary}.example.invalid`); + await d.answer('Bitbucket username'); + await d.answer('Bitbucket HTTP Access Token', `${canary}-token`); + await d.answer('Index every repository visible to the token?', all ? 'y' : 'n'); + if (!all) { + await d.check('What do you want to index?', [0, 1]); + await d.multi('Project keys to index', canary); + await d.multi('Repositories to index'); + } + await end(d); + }); + assert.equal(props(result, 'configured_code_source')[0].indexAll, all); + assert.equal(props(result, 'configured_code_source')[0].deploymentType, 'self_hosted'); + }); +} + +for (const shape of ['root', 'wildcard', 'nested']) { + test(`Local repositories: ${shape}`, async () => { + const result = await scenario(packed, { prepare({ cwd }) { + const root = join(cwd, canary); + if (shape === 'root') { + mkdirSync(join(root, '.git'), { recursive: true }); + } else { + mkdirSync(join(root, 'one/.git'), { recursive: true }); + mkdirSync(join(root, shape === 'wildcard' ? 'two/.git' : 'nested/two/.git'), { recursive: true }); + } + } }, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 2); + await d.answer('Path to your repos directory', join(d.cwd, canary)); + if (shape !== 'root') { + await d.answer('Which repositories should be indexed?'); + } + await end(d); + }); + const s = props(result, 'configured_code_source')[0]; + assert.equal(s.repositoryCount, shape === 'root' ? 1 : 2); + assert.equal(s.generatedConnectionCount, shape === 'nested' ? 2 : 1); + assert.equal(props(result, 'generated_configs')[0].wroteComposeOverride, true); + assert.ok(result.files['docker-compose.override.yml']); + }); +} + +const providers = ['anthropic', 'openai', 'openai-compatible', 'amazon-bedrock', 'google-generative-ai', 'google-vertex', 'google-vertex-anthropic', 'azure', 'deepseek', 'mistral', 'openrouter', 'xai']; +for (const provider of providers) { + for (const explicit of (provider === 'amazon-bedrock' || provider.startsWith('google-vertex') ? [false, true] : [false])) { + test(`AI ${provider}${explicit ? ' explicit credentials' : ''}`, async () => { + const result = await scenario(packed, {}, async d => { + await minimal(d, { ai: true }); + await d.select('Which AI provider?', providers.indexOf(provider)); + await d.answer('Model name', `${canary}-model`); + if (provider === 'openai-compatible') { + await d.answer('Base URL', `https://${canary}.example.invalid/v1`); + } + if (provider === 'azure') { + await d.answer('Azure resource name', canary); + await d.answer('API version'); + } + if (provider === 'amazon-bedrock') { + await d.answer('Use the default AWS credential chain?', explicit ? 'n' : 'y'); + if (explicit) { + await d.answer('AWS Access Key ID', canary); + await d.answer('AWS Secret Access Key', `${canary}-secret`); + } + await d.answer('AWS region'); + } else if (provider.startsWith('google-vertex')) { + await d.answer('Google Cloud project ID', canary); + await d.answer('Google Cloud region'); + await d.answer('Use Application Default Credentials?', explicit ? 'n' : 'y'); + if (explicit) { + await d.answer('Path to service account credentials JSON', `/${canary}/credentials.json`); + } + } else { + await d.answer('API key (', `${canary}-api-key`); + } + await d.answer('Display name', `${canary}-display`); + await d.answer('Add another model?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + const s = props(result, 'configured_ai_provider')[0]; + assert.equal(s.provider, provider); + assert.equal(s.hasDisplayName, true); + assert.equal(s.usesCustomEndpoint, provider === 'openai-compatible'); + assert.equal(s.modelSelectionMethod, 'manual_fallback'); + assert.deepEqual(props(result, 'ai_setup_completed')[0].providerTypes, [provider]); + assert.equal(JSON.parse(result.files['config.json']).models[0].provider, provider); + }); + } +} diff --git a/packages/setupWizard/tests/e2e/docker.test.mjs b/packages/setupWizard/tests/e2e/docker.test.mjs new file mode 100644 index 000000000..86e836c17 --- /dev/null +++ b/packages/setupWizard/tests/e2e/docker.test.mjs @@ -0,0 +1,199 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { artifact, scenario, minimal, defaultCompose } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Docker artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const p = (r, name) => r.events.find(e => e.event === `setup_sourcebot_${name}`)?.properties; +const running = { Name: 'canary-sensitive-running', Service: 'sourcebot', State: 'running' }; +const stopped = { Name: 'canary-sensitive-stopped', Service: 'sourcebot', State: 'exited' }; +const initial = async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'y'); }; + +for (const action of ['stop', 'keep', 'fail']) { + test(`running deployment: ${action}`, async () => { + const result = await scenario(packed, { docker: { containers: [running], fail: action === 'fail' ? ['compose down'] : [] } }, async d => { + await initial(d); + await d.answer('Stop and remove the running deployment?', action === 'keep' ? 'n' : 'y'); + if (action === 'stop') { + await d.answer('Start Sourcebot now?', 'n'); + } + }); + assert.equal(p(result, 'validated_docker_state').existingDeploymentAction, { stop: 'stopped', keep: 'left_running', fail: 'stop_failed' }[action]); + assert.equal(p(result, 'validated_docker_state').outcome, { stop: 'passed_after_cleanup', keep: 'skipped_existing_deployment_running', fail: 'validation_failed' }[action]); + assert.equal(p(result, 'completed').sourcebotStartOffered, action === 'stop'); + if (action !== 'stop') { + assert.equal(p(result, 'validated_docker_state').existingVolumeCount, null); + assert.equal(p(result, 'validated_docker_state').remainingPortConflictCount, null); + } + }); +} + +for (const action of ['remove', 'keep', 'fail']) { + test(`stopped containers and volumes: ${action}`, async () => { + const result = await scenario(packed, { docker: { containers: [stopped], volumes: ['sourcebot_cache'], fail: action === 'fail' ? ['compose rm', 'volume rm'] : [] } }, async d => { + await initial(d); + await d.answer('Remove them now', action === 'keep' ? 'n' : 'y'); + await d.answer('Wipe these volumes?', action === 'keep' ? 'n' : 'y'); + await d.answer('Start Sourcebot now?', 'n'); + }); + const state = p(result, 'validated_docker_state'); + assert.equal(state.stoppedComposeContainerCount, 1); + assert.equal(state.existingVolumeCount, 1); + assert.equal(state.stoppedContainerAction, { remove: 'removed', keep: 'kept', fail: 'remove_failed' }[action]); + assert.equal(state.volumeAction, { remove: 'removed', keep: 'kept', fail: 'remove_failed' }[action]); + const failures = result.events.filter(e => e.event === 'setup_sourcebot_failed'); + assert.equal(failures.length, action === 'fail' ? 2 : 0); + assert.ok(failures.every(e => e.properties.recoverable)); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_completed'); + }); +} + +for (const failure of ['missing', 'inventory', 'malformed', 'volumes']) { + test(`Docker failed measurement: ${failure}`, async () => { + const result = await scenario(packed, { dockerMissing: failure === 'missing', docker: { malformed: failure === 'malformed', fail: failure === 'inventory' ? ['compose ps'] : failure === 'volumes' ? ['volume ls'] : [] } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'n'); + }); + const state = p(result, 'validated_docker_state'); + assert.equal(state.outcome, 'validation_failed'); + if (failure !== 'volumes') { + assert.equal(state.runningComposeContainerCount, null); + assert.equal(state.composeContainerState, 'unknown'); + } else { + assert.equal(state.runningComposeContainerCount, 0); + } + if (failure === 'volumes' || failure === 'missing') { + assert.equal(state.existingVolumeCount, null); + } + assert.equal(state.dockerStatus, failure === 'missing' ? 'unavailable' : 'available'); + }); +} + +for (const action of ['keep', 'stop', 'fail', 'remain']) { + test(`Docker port conflict: ${action}`, async () => { + const result = await scenario(packed, { compose: 'services:\n sourcebot:\n ports:\n - "43187:3000"\n', docker: { ports: 'canary-sensitive-container\t0.0.0.0:43187->3000/tcp', fail: action === 'fail' ? ['stop canary-sensitive-container'] : [], keepPorts: action === 'remain' } }, async d => { + await initial(d); + await d.answer('Stop this container', action === 'keep' ? 'n' : 'y'); + await d.answer(action === 'stop' ? 'Start Sourcebot now?' : 'Start Sourcebot anyway?', 'n'); + }); + const state = p(result, 'validated_docker_state'); + assert.equal(state.initialPortConflictCount, 1); + assert.equal(state.remainingPortConflictCount, action === 'stop' ? 0 : 1); + assert.equal(state.portConflictSource, 'docker'); + assert.equal(state.outcome, action === 'fail' ? 'validation_failed' : action === 'stop' ? 'passed_after_cleanup' : 'unresolved_conflicts'); + }); +} + +test('failed spawn offers manual steps and completes after recoverable failures', async () => { + const result = await scenario(packed, { dockerMissing: true }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + }); + assert.equal(p(result, 'completed').sourcebotStartOutcome, 'spawn_failed'); + assert.equal(p(result, 'completed').completionMode, 'sourcebot_start_failed'); + assert.equal(result.events.filter(e => e.event === 'setup_sourcebot_start_failed').length, 1); + assert.deepEqual( + ['failurePhase', 'failureCategory', 'failureReason'].map(key => p(result, 'start_failed')[key]), + ['spawn', 'docker_unavailable', 'docker_unavailable'], + ); + assert.ok(result.events.filter(e => e.event === 'setup_sourcebot_failed').every(e => e.properties.recoverable)); +}); + +for (const [reason, diagnostic] of [ + ['container_name_conflict', 'Error response from daemon: Conflict. The container name "/canary-sensitive-container" is already in use by container "canary-sensitive-id".'], + ['port_conflict', 'Error response from daemon: driver failed programming external connectivity: Bind for canary-sensitive-address failed: port is already allocated'], + ['image_pull_failed', 'Error response from daemon: pull access denied for canary-sensitive-image, repository does not exist or may require docker login'], + ['mount_failed', 'Error response from daemon: invalid mount config for type "bind": bind source path does not exist: canary-sensitive-path'], + ['compose_configuration', 'validating canary-sensitive-path: services.sourcebot Additional property canary-sensitive is not allowed'], + ['docker_unavailable', 'Cannot connect to the Docker daemon at canary-sensitive-socket. Is the docker daemon running?'], + ['unknown', 'canary-sensitive-unrecognized-error'], + ['unknown', 'canary-sensitive-app | Error response from daemon: pull access denied for canary-sensitive-image'], +]) { + test(`post-handoff start failure: ${reason} (${diagnostic.slice(0, 24)})`, async () => { + const result = await scenario(packed, { docker: { start: { stderrChunks: [diagnostic.slice(0, 17), diagnostic.slice(17)], delayMs: 1100 } } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + await d.wait(diagnostic); + }); + const failures = result.events.filter(e => e.event === 'setup_sourcebot_start_failed'); + assert.equal(failures.length, 1); + assert.equal(failures[0].properties.failurePhase, 'compose_exit'); + assert.equal(failures[0].properties.failureReason, reason); + assert.equal(failures[0].properties.failureCategory, reason === 'docker_unavailable' ? 'docker_unavailable' : 'docker_command'); + const completed = result.events.find(e => e.event === 'setup_sourcebot_completed'); + assert.ok(result.events.indexOf(completed) < result.events.indexOf(failures[0])); + assert.equal(failures[0].distinct_id, completed.distinct_id); + assert.equal(result.events.filter(e => e.event === 'setup_sourcebot_completed').length, 1); + assert.equal(result.events.some(e => e.event === 'setup_sourcebot_cancelled'), false); + assert.equal(result.exitCode, 0, 'Telemetry must not change existing CLI exit behavior'); + }); +} + +for (const termination of [{ exitCode: 0 }, { signal: 'SIGINT' }, { signal: 'SIGTERM' }]) { + test(`normal Compose termination is not a start failure: ${JSON.stringify(termination)}`, { skip: process.platform === 'win32' && !!termination.signal }, async () => { + const result = await scenario(packed, { docker: { start: { ...termination, stderrChunks: ['Error response from daemon: pull access denied for canary-sensitive-image\n'] } } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + }); + assert.equal(result.events.some(e => e.event === 'setup_sourcebot_start_failed'), false); + assert.equal(p(result, 'completed').sourcebotStartOutcome, 'spawned'); + }); +} + +test('start failure with unavailable PostHog still exits and retains generated files', async () => { + const began = Date.now(); + const result = await scenario(packed, { telemetry: 'stall', docker: { start: { stderrChunks: ['canary-sensitive-error'] } } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'y'); + }); + assert.equal(result.exitCode, 0); + assert.ok(Date.now() - began < 7000); + assert.deepEqual(Object.keys(result.files).sort(), ['.env', 'config.json', 'docker-compose.yml']); +}); + +for (const stage of ['fetch', 'docker', 'after_failures']) { + test(`Ctrl+C during ${stage} cancels outstanding work`, async () => { + const result = await scenario(packed, { + composeStatus: stage === 'fetch' ? 'stall' : undefined, + docker: stage === 'docker' ? { stall: ['compose ps'], stubborn: true } : stage === 'after_failures' ? { containers: [stopped], volumes: ['sourcebot_cache'], fail: ['compose rm', 'volume rm'] } : {}, + }, async d => { + await initial(d); + if (stage === 'after_failures') { + await d.answer('Remove them now', 'y'); + await d.answer('Wipe these volumes?', 'y'); + await d.wait('Start Sourcebot now?'); + } else { + await sleep(300); + } + const began = Date.now(); + d.interrupt(); + await d.finish(130); + assert.ok(Date.now() - began < 3500); + }); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_cancelled'); + assert.equal(p(result, 'cancelled').stage, { fetch: 'compose_file', docker: 'docker_validation', after_failures: 'start' }[stage]); + assert.equal(result.events.some(e => e.event === 'setup_sourcebot_completed'), false); + }); +} + +test('Ctrl+C cleans stubborn descendants even if their parent exits first', async () => { + await scenario(packed, { docker: { stall: ['compose ps'], descendant: true } }, async d => { + await initial(d); + await sleep(350); + d.interrupt(); + await d.finish(130); + }); +}); + +for (const missing of ['info', 'compose version']) { + test(`availability probe distinguishes unavailable ${missing}`, async () => { + const result = await scenario(packed, { docker: { fail: ['compose ps', missing === 'info' ? 'info --format' : missing] } }, async d => { + await initial(d); + await d.answer('Start Sourcebot now?', 'n'); + }); + assert.equal(p(result, 'validated_docker_state').dockerStatus, 'unavailable'); + assert.equal(p(result, 'failed').failureCategory, 'docker_unavailable'); + }); +} diff --git a/packages/setupWizard/tests/e2e/fakeDocker.cjs b/packages/setupWizard/tests/e2e/fakeDocker.cjs new file mode 100644 index 000000000..6df21d395 --- /dev/null +++ b/packages/setupWizard/tests/e2e/fakeDocker.cjs @@ -0,0 +1,43 @@ +const fs = require('node:fs'); +const args = process.argv.slice(2); +fs.appendFileSync(process.env.TEST_DOCKER_LOG, JSON.stringify(args) + '\n'); +const state = JSON.parse(fs.readFileSync(process.env.TEST_DOCKER_STATE, 'utf8')); +const command = args.slice(0, 2).join(' '); +fs.appendFileSync(process.env.TEST_DOCKER_PIDS, String(process.pid) + '\n'); +if (command === 'compose up' && state.start) { + let index = 0; + const write = () => { + if (index < (state.start.stderrChunks ?? []).length) { + process.stderr.write(state.start.stderrChunks[index++]); + setTimeout(write, 10); + } else if (state.start.signal) { + process.kill(process.pid, state.start.signal); + } else { + process.exit(state.start.exitCode ?? 1); + } + }; + setTimeout(write, state.start.delayMs ?? 0); +} else if (state.fail?.includes(command)) { + console.error('canary-sensitive-error'); + process.exit(1); +} else if (state.stall?.includes(command)) { + if (state.descendant) { + const child = require('node:child_process').spawn(process.execPath, ['-e', "process.on('SIGINT', () => {}); setInterval(() => {}, 1000)"], { stdio: 'ignore' }); + fs.appendFileSync(process.env.TEST_DOCKER_PIDS, String(child.pid) + '\n'); + } + if (state.stubborn) { + process.on('SIGINT', () => {}); + } + setInterval(() => {}, 1000); +} else if (command === 'compose ps') { + console.log(state.malformed ? 'malformed' : JSON.stringify(state.containers ?? [])); +} else if (command === 'volume ls') { + console.log((state.volumes ?? []).join('\n')); +} else if (args[0] === 'ps') { + console.log(state.ports ?? ''); +} else if (args[0] === 'stop') { + if (!state.keepPorts) { + state.ports = ''; + fs.writeFileSync(process.env.TEST_DOCKER_STATE, JSON.stringify(state)); + } +} diff --git a/packages/setupWizard/tests/e2e/harness.mjs b/packages/setupWizard/tests/e2e/harness.mjs new file mode 100644 index 000000000..ecc34c92e --- /dev/null +++ b/packages/setupWizard/tests/e2e/harness.mjs @@ -0,0 +1,347 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { createServer } from 'node:https'; +import { once } from 'node:events'; +import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync, chmodSync, existsSync, readdirSync, copyFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, delimiter, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { stripVTControlCharacters } from 'node:util'; +import { gunzipSync } from 'node:zlib'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { createRequire } from 'node:module'; +import pty from 'node-pty'; +import { eventSchemas, validateFields } from '../../dist/telemetryEvents.js'; +import { INSTALL_ID_PATTERN } from '../../dist/telemetry.js'; + +const require = createRequire(import.meta.url); +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..'); +const commandEnv = { ...process.env, PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false' }; +export const down = '\x1b[B'; +export const canary = 'canary-sensitive'; +export const defaultCompose = 'services:\n sourcebot:\n image: sourcebot-test\nvolumes:\n cache:\n'; +const commonKeys = ['platform', 'arch', 'nodeMajorVersion', 'packageManager', 'isCI', 'schemaVersion', 'source', 'setupSourcebotVersion', 'setupSessionId', 'install_id', 'elapsedMs', '$geoip_disable', '$ignore_sent_at', '$groups', '$lib', '$lib_version']; +const approvedSchema = JSON.parse(readFileSync(new URL('../approvedSchema.json', import.meta.url), 'utf8')); + +export function contract(events, requireTerminal = true) { + assert.ok(events.length > 0); + const id = events[0].distinct_id; + assert.match(id, INSTALL_ID_PATTERN); + let elapsed = -1; + let timestamp = -Infinity; + for (const event of events) { + const name = event.event.replace(/^setup_sourcebot_/, ''); + assert.ok(approvedSchema[name], `Unexpected event ${name}`); + assert.deepEqual(Object.keys(event).sort(), ['distinct_id', 'event', 'properties', 'timestamp', 'uuid'].sort()); + assert.deepEqual(Object.keys(event.properties).sort(), [...commonKeys, ...Object.keys(approvedSchema[name])].sort()); + validateFields(eventSchemas[name], event.properties); + assert.equal(event.distinct_id, id); + assert.equal(event.properties.setupSessionId, id); + assert.equal(event.properties.install_id, id); + assert.deepEqual(event.properties.$groups, { company: id }); + assert.equal(event.properties.$lib, 'posthog-node'); + assert.equal(event.properties.$lib_version, '5.52.1'); + assert.equal(event.properties.$geoip_disable, true); + assert.equal(event.properties.$ignore_sent_at, true); + assert.equal(event.properties.nodeMajorVersion, Number(process.versions.node.split('.')[0])); + assert.equal(event.properties.source, 'setup-sourcebot-cli'); + assert.equal(event.properties.schemaVersion, 1); + assert.ok(event.properties.elapsedMs >= elapsed); + elapsed = event.properties.elapsedMs; + assert.ok(Number.isFinite(Date.parse(event.timestamp))); + assert.ok(Date.parse(event.timestamp) > timestamp, 'Event timestamps must preserve funnel order'); + timestamp = Date.parse(event.timestamp); + assert.match(event.uuid, /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + } + assert.equal(JSON.stringify(events).includes(canary), false, 'Sensitive canary reached telemetry'); + const terminal = events.filter(e => ['setup_sourcebot_completed', 'setup_sourcebot_cancelled'].includes(e.event) || (e.event === 'setup_sourcebot_failed' && !e.properties.recoverable)); + if (requireTerminal) { + assert.equal(terminal.length, 1); + } + return id; +} + +export function artifact() { + assert.ok(Number(process.versions.node.split('.')[0]) >= 20, 'Run packed-artifact tests on Node 20 or newer'); + // node-pty 1.1.0 ships its macOS helper without its executable bit in the npm tarball. + if (process.platform === 'darwin') { + const helper = join(dirname(require.resolve('node-pty/package.json')), 'prebuilds', `darwin-${process.arch}`, 'spawn-helper'); + if (existsSync(helper)) { + chmodSync(helper, 0o755); + } + } + const root = mkdtempSync(join(tmpdir(), 'sourcebot-e2e-')); + // Windows cannot exec .cmd shims directly. Invoke the bundled JS entry points + // without a shell so temporary paths containing spaces remain literal. + const packageCommand = (name, args, options) => { + if (process.platform !== 'win32') { + return execFileSync(name, args, options); + } + const script = join(dirname(process.execPath), 'node_modules', name === 'npm' ? 'npm/bin/npm-cli.js' : 'corepack/dist/yarn.js'); + assert.ok(existsSync(script), `Missing package-manager entry point: ${script}`); + return execFileSync(process.execPath, [script, ...args], options); + }; + try { + const tarball = process.env.SETUP_TEST_TARBALL || join(root, 'setup-sourcebot.tgz'); + if (!process.env.SETUP_TEST_TARBALL) { + packageCommand('yarn', ['workspace', '@sourcebot/schemas', 'build'], { cwd: packageRoot, env: commandEnv }); + packageCommand('yarn', ['build'], { cwd: packageRoot, env: commandEnv }); + packageCommand('yarn', ['pack', '--out', tarball], { cwd: packageRoot, env: commandEnv }); + } + const digest = createHash('sha256').update(readFileSync(tarball)).digest('hex'); + const installation = join(root, 'packed install with spaces'); + mkdirSync(installation); + writeFileSync(join(installation, 'package.json'), '{"name":"isolated-setup-test","private":true,"allowScripts":{"reo-census":true}}'); + packageCommand('npm', ['install', '--no-audit', '--no-fund', '--cache', join(root, 'npm-cache'), tarball], { cwd: installation, env: commandEnv, timeout: 120000 }); + const installed = join(installation, 'node_modules/setup-sourcebot'); + assert.deepEqual(readdirSync(installed).sort(), ['README.md', 'bin.cjs', 'dist', 'package.json'].sort()); + const cert = join(root, 'cert.pem'); + const key = join(root, 'key.pem'); + const openssl = process.platform === 'win32' + ? [join(process.env.ProgramFiles ?? 'C:\\Program Files', 'Git', 'usr', 'bin', 'openssl.exe'), join(process.env.ProgramFiles ?? 'C:\\Program Files', 'Git', 'mingw64', 'bin', 'openssl.exe')].find(existsSync) ?? 'openssl' + : 'openssl'; + execFileSync(openssl, ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-keyout', key, '-out', cert, '-subj', '/CN=setup-fixture', '-addext', 'subjectAltName=DNS:us.i.posthog.com,DNS:raw.githubusercontent.com,DNS:models.dev,DNS:api.github.com,DNS:gitlab.com,DNS:*.example.invalid'], { stdio: 'ignore' }); + return { root, installed, cert, key, digest, tarball, bin: join(installation, 'node_modules/.bin/setup-sourcebot'), cleanup() { rmSync(root, { recursive: true, force: true }); assert.equal(existsSync(root), false); } }; + } catch (error) { + rmSync(root, { recursive: true, force: true }); + throw error; + } +} + +export async function scenario(artifact, options, drive) { + const root = mkdtempSync(join(artifact.root, 'scenario-')); + const events = []; + const requests = []; + const forwarding = []; + const cwd = join(root, 'work'); + const setup = join(cwd, options.setupName ?? 'sourcebot'); + const fakeBin = join(root, 'bin'); + const home = join(root, 'home'); + for (const dir of [cwd, fakeBin, home]) { + mkdirSync(dir); + } + if (options.files) { + mkdirSync(setup); + for (const [name, data] of Object.entries(options.files)) { + writeFileSync(join(setup, name), data); + } + } + options.prepare?.({ cwd, setup, root }); + writeFileSync(join(root, 'docker-state.json'), JSON.stringify(options.docker ?? {})); + if (options.realDocker) { + writeFileSync(join(fakeBin, 'docker'), `#!${process.execPath}\n` + + `const {spawn}=require('node:child_process'); const args=process.argv.slice(2);\n` + + `if(!['compose','volume','ps','info'].includes(args[0])){throw Error('Live test Docker command not permitted');}\n` + + `if(args[0]==='volume' && args[1]!=='ls'){throw Error('Live test volume mutation not permitted');}\n` + + `const p=spawn(${JSON.stringify(options.realDocker)},args,{stdio:'inherit'});\n` + + `process.on('SIGINT',()=>{});p.on('error',()=>process.exit(1));p.on('close',c=>process.exit(c??130));\n`, { mode: 0o755 }); + } else if (!options.dockerMissing) { + if (process.platform === 'win32') { + // A real executable is needed: Windows spawn(shell:false) cannot + // execute Unix shebangs or .cmd wrappers. The external preload + // dispatches this dedicated Node copy to the Docker fixture. + copyFileSync(process.execPath, join(fakeBin, 'docker.exe')); + } else { + writeFileSync(join(fakeBin, 'docker'), `#!${process.execPath}\n` + readFileSync(new URL('./fakeDocker.cjs', import.meta.url), 'utf8'), { mode: 0o755 }); + } + } + const server = createServer({ key: readFileSync(artifact.key), cert: readFileSync(artifact.cert) }, async (req, res) => { + if (req.headers.host === 'us.i.posthog.com') { + const parts = []; + for await (const chunk of req) { + parts.push(chunk); + } + const bytes = Buffer.concat(parts); + const body = JSON.parse((req.headers['content-encoding'] === 'gzip' ? gunzipSync(bytes) : bytes).toString()); + requests.push(body); + if (options.live) { + // Only a separately invoked dev smoke enables network forwarding. + contract(body.batch, false); + assert.equal(body.api_key, 'phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW'); + assert.ok(process.env.SETUP_TEST_DEV_TOKEN, 'Dev project token required'); + const forwarded = { ...body, api_key: process.env.SETUP_TEST_DEV_TOKEN }; + assert.deepEqual({ ...forwarded, api_key: body.api_key }, body); + assert.equal(req.url, '/batch/', 'Unexpected PostHog ingestion path'); + const request = fetch('https://us.i.posthog.com/batch/', { + method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(forwarded), signal: AbortSignal.timeout(15000), + }).then(async response => { + assert.ok(response.ok, `Dev ingestion rejected: ${response.status}`); + await response.text(); + }); + forwarding.push(request); + request.catch(() => {}); + } + if (options.telemetry === 'stall') { + return; + } + if (options.telemetry === 'reset') { + req.socket.destroy(); + return; + } + if (options.telemetry === 'reject') { + res.writeHead(503).end('{}'); + return; + } + events.push(...body.batch); + res.writeHead(200, { 'content-type': 'application/json' }).end('{"status":1}'); + } else if (req.headers.host === 'raw.githubusercontent.com') { + if (options.composeStatus === 'stall') { + return; + } + res.writeHead(options.composeStatus ?? 200).end(options.compose ?? defaultCompose); + } else if (req.headers.host === 'models.dev') { + if (options.catalog === 'stall') { + return; + } + res.writeHead(options.catalogStatus ?? 200).end(JSON.stringify(options.catalog ?? {})); + } else if (req.headers.host === 'api.github.com') { + if (options.searchStatus === 'stall') { + return; + } + if (options.searchStatus === 'reset') { + req.socket.destroy(); + return; + } + res.writeHead(options.searchStatus ?? 200).end(JSON.stringify(options.search ?? { items: [] })); + } else if (req.headers.host === 'gitlab.com' || req.headers.host?.endsWith('.example.invalid')) { + res.writeHead(options.searchStatus ?? 200).end(JSON.stringify(options.search ?? [])); + } else { + res.writeHead(403).end('Unexpected test egress denied'); + } + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + let child; + let transcript = ''; + let ended; + let cursor = 0; + const env = { + PATH: `${fakeBin}${delimiter}${dirname(process.execPath)}`, + HOME: home, USERPROFILE: home, TMPDIR: root, TEMP: root, TMP: root, + XDG_STATE_HOME: join(home, 'state'), XDG_CONFIG_HOME: join(home, 'config'), XDG_CACHE_HOME: join(home, 'cache'), + SYSTEMROOT: process.env.SYSTEMROOT ?? '', LANG: 'en_US.UTF-8', TERM: 'xterm-256color', + NODE_OPTIONS: `--import=${new URL(options.networkModule ?? './network.mjs', import.meta.url).href}`, + NODE_EXTRA_CA_CERTS: artifact.cert, TEST_CAPTURE_PORT: String(server.address().port), + TEST_DOCKER_STATE: join(root, 'docker-state.json'), TEST_DOCKER_LOG: join(root, 'docker.log'), + TEST_DOCKER_PIDS: join(root, 'docker-pids'), + PACKAGE_TRACKER_ANALYTICS: 'false', SOURCEBOT_TELEMETRY_DISABLED: 'true', + ...options.environment, + }; + try { + child = options.launcher + ? pty.spawn(options.launcher.file, options.launcher.args, { name: 'xterm-256color', cols: 180, rows: 60, cwd, env }) + : process.platform === 'win32' + ? pty.spawn(process.execPath, [join(artifact.installed, 'bin.cjs')], { name: 'xterm-256color', cols: 180, rows: 60, cwd, env }) + : pty.spawn(artifact.bin, [], { name: 'xterm-256color', cols: 180, rows: 60, cwd, env }); + child.onData(data => { transcript += stripVTControlCharacters(data); }); + child.onExit(result => { ended = result; }); + const wait = async text => { + const started = Date.now(); + while (!transcript.slice(cursor).includes(text)) { + assert.equal(ended, undefined, `CLI exited before ${text}: ${transcript.slice(-1500)}`); + assert.ok(Date.now() - started < 12000, `Prompt timeout ${text}: ${transcript.slice(-1500)}`); + await sleep(10); + } + cursor = transcript.indexOf(text, cursor) + text.length; + await sleep(25); + }; + const answer = async (text, value = '') => { await wait(text); child.write(value + '\r'); }; + const select = async (text, index = 0) => { + await wait(text); + for (let n = 0; n < index; n++) { + child.write(down); + await sleep(15); + } + child.write('\r'); + }; + const multi = async (text, value = `${canary}/repository`) => { + await wait(text); + // select-pro initializes its asynchronous option loader after a debounce. + await sleep(300); + child.write(value); + await sleep(550); + child.write('\t'); + await sleep(50); + child.write('\r'); + }; + const check = async (text, indexes = [0]) => { + await wait(text); + let position = 0; + for (const index of indexes) { + while (position < index) { + child.write(down); + position++; + await sleep(15); + } + child.write('\t'); + await sleep(20); + } + child.write('\r'); + }; + const finish = async (code = 0) => { + const start = Date.now(); + while (!ended) { + assert.ok(Date.now() - start < 7000, `CLI did not exit: ${transcript.slice(-1000)}`); + await sleep(20); + } + assert.equal(ended.exitCode, code, transcript.slice(-1000)); + }; + await drive({ answer, select, multi, check, wait, write: data => child.write(data), interrupt: () => process.kill(child.pid, 'SIGINT'), finish, events, setup, root, cwd, home, get transcript() { return transcript; } }); + if (!ended) { + await finish(); + } + await Promise.all(forwarding); + if (!options.telemetry && !options.baseline) { + contract(events); + } + assert.equal(JSON.stringify(requests).includes(canary), false); + for (const sensitive of options.sensitiveValues ?? []) { + assert.equal(JSON.stringify(requests).includes(sensitive), false, 'Live credential reached telemetry'); + } + if (options.assertLauncherHome) { + options.assertLauncherHome(readdirSync(home, { recursive: true })); + } else { + assert.deepEqual(readdirSync(home), [], 'Wizard wrote unexpected per-user state'); + } + const files = existsSync(setup) ? Object.fromEntries(readdirSync(setup).filter(f => !options.ignoreFiles?.includes(f)).map(f => [f, readFileSync(join(setup, f), 'utf8')])) : {}; + const dockerCalls = existsSync(join(root, 'docker.log')) ? readFileSync(join(root, 'docker.log'), 'utf8').trim().split('\n').map(JSON.parse) : []; + if (existsSync(join(root, 'docker-pids'))) { + for (const pid of readFileSync(join(root, 'docker-pids'), 'utf8').trim().split('\n').map(Number)) { + assert.throws(() => process.kill(pid, 0), { code: 'ESRCH' }, 'Owned Docker fixture process survived CLI exit'); + } + } + const result = { events, files, dockerCalls, requests, exitCode: ended.exitCode }; + await options.verifyDeployment?.({ ...result, setup, root }); + return result; + } finally { + if (child && process.platform === 'win32') { + // node-pty's ConPTY worker survives a natural child exit unless + // the terminal is disposed; Windows kill() accepts no signal. + child.kill(); + } else if (child && !ended) { + child.kill('SIGKILL'); + } + try { + await options.cleanupDeployment?.({ setup, root }); + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + rmSync(root, { recursive: true, force: true }); + assert.equal(existsSync(root), false); + } + } +} + +export async function minimal(driver, { existing = false, ai = false, pauseBeforeHosted = false } = {}) { + await driver.answer('What directory would you like'); + if (existing) { + await driver.answer('Do you want to overwrite it?', 'y'); + } + await driver.select('Which code host', 3); + await driver.answer('Git clone URL', `https://${canary}.example.invalid/repository`); + await driver.answer('Add another code host?', 'n'); + await driver.answer('Would you like to configure AI features?', ai ? 'y' : 'n'); + if (!ai && !pauseBeforeHosted) { + await driver.answer('What URL will Sourcebot be hosted at?'); + } +} diff --git a/packages/setupWizard/tests/e2e/linux.mjs b/packages/setupWizard/tests/e2e/linux.mjs new file mode 100644 index 000000000..40b2546f3 --- /dev/null +++ b/packages/setupWizard/tests/e2e/linux.mjs @@ -0,0 +1,27 @@ +import assert from 'node:assert/strict'; +import { cpSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { randomUUID } from 'node:crypto'; +import { execFileSync } from 'node:child_process'; +import { artifact } from './harness.mjs'; + +const packed = artifact(); +const name = `sourcebot-linux-e2e-${randomUUID()}`; +const packageRoot = resolve(fileURLToPath(new URL('.', import.meta.url)), '../..'); +try { + const snapshot = join(packed.root, 'linux-package'); + mkdirSync(snapshot); + // Never mount the changing worktree into a running verification job. + for (const path of ['dist', 'tests', 'package.json']) { + cpSync(join(packageRoot, path), join(snapshot, path), { recursive: true }); + } + execFileSync('docker', ['run', '--rm', '--name', name, '-e', 'PACKAGE_TRACKER_ANALYTICS=false', '-e', 'SETUP_TEST_TARBALL=/fixture/setup-sourcebot.tgz', + '-v', `${snapshot}:/work/packages/setupWizard:ro`, '-v', `${packed.tarball}:/fixture/setup-sourcebot.tgz:ro`, '-w', '/work', 'node:24-bookworm', 'sh', '-c', + 'npm install --no-audit --no-fund posthog-node@5.52.1 node-pty@1.1.0 undici@7.29.1 && node --test --test-concurrency=1 packages/setupWizard/tests/e2e/wizard.test.mjs packages/setupWizard/tests/e2e/collectors.test.mjs packages/setupWizard/tests/e2e/docker.test.mjs packages/setupWizard/tests/e2e/safety.test.mjs && node packages/setupWizard/tests/e2e/packageManagers.mjs'], { stdio: 'inherit', timeout: 600000 }); + console.log(`Linux packed-artifact matrix passed: ${packed.digest}`); +} finally { + try { execFileSync('docker', ['rm', '-f', name], { stdio: 'ignore' }); } catch { /* --rm already handled completion. */ } + assert.equal(execFileSync('docker', ['ps', '-aq', '--filter', `name=^/${name}$`], { encoding: 'utf8' }).trim(), ''); + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/liveDeployment.md b/packages/setupWizard/tests/e2e/liveDeployment.md new file mode 100644 index 000000000..9218c2650 --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveDeployment.md @@ -0,0 +1,52 @@ +# Live deployment verification + +Run `liveDeployment.mjs` with Node 24 on macOS or Linux with Docker available. +It builds and installs the actual npm tarball in a disposable directory, drives +the interactive CLI through a PTY, starts the real Sourcebot/Postgres/Redis stack, +onboards a synthetic owner through Chromium, verifies indexing and authenticated +search, then verifies health, search, and install-ID continuity after restart. + +Pull `docker.sourcebot.dev/sourcebot-dev/sourcebot:v5.1.13`, `postgres:16`, and +`redis:8` first. The default scenario is `github_repo`. For the public/configuration +matrix, set: + +```sh +SETUP_TEST_LIVE_CASES=github_repo,github_org,github_user,gitlab_project,gitea_repo,gerrit_project,remote_git,local_root,local_wildcard,local_nested,auto_start,ai_all \ +SETUP_TEST_LIVE_OUTPUT=/absolute/path/outside/the/repository \ +node packages/setupWizard/tests/e2e/liveDeployment.mjs +``` + +The optional `anthropic` scenario reads `ANTHROPIC_API_KEY` from development env +files under `SETUP_TEST_CREDENTIAL_DIR`. Never pass the credential as a command +argument. `ai_all` configures all twelve providers with synthetic values. Neither +scenario calls Ask or an inference endpoint; neither proves credential validity +or requires a Pro license. The tests check generated model configuration, +credential references, container environment propagation, and successful startup. + +Every deployment uses a unique Compose project, random loopback HTTP port, and +unpublished database/Redis ports. The Compose download is intercepted only to +apply these isolation settings, pin the image, and disable deployment telemetry. +The wizard's generated config, `.env`, identity, and local-repository override +remain intact. Wizard PostHog requests use the real SDK and are routed to the +local TLS collector; live public autocomplete/model-catalog requests remain real. +This suite does not forward to production or dev PostHog. The separate dev smoke +test provides actual ingestion/query evidence. + +Public fixtures are bounded: one explicit repository/project per host, the +small `chalk` organization, and the `octocat` user. Organization/user tests obtain +the expected repository count before starting and require all repositories to +be discovered and indexed. Multi-local tests clone two distinct origins because +Sourcebot deduplicates multiple local clones of the same origin. + +Each scenario records success/failure and removes its containers, volumes, +network, generated secrets, and temporary files in cleanup. Only redacted reports +and browser screenshots remain when an output directory is supplied. Task-owned +image downloads should be removed by the operator afterward without removing +images that existed before the run. Do not commit generated artifacts. + +This live matrix supplements, not replaces, the deterministic collector, fault, +transport, lifecycle, and platform suites. It does not establish live access to +private/GHE/GitLab self-managed, Azure DevOps, or Bitbucket deployments, and does +not claim every possible code-host scope or credential mode has live coverage. +Unavailable integrations must remain explicitly unverified in the completion +report rather than being inferred from fixture-based coverage. diff --git a/packages/setupWizard/tests/e2e/liveDeployment.mjs b/packages/setupWizard/tests/e2e/liveDeployment.mjs new file mode 100644 index 000000000..90fcc252f --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveDeployment.mjs @@ -0,0 +1,392 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { randomUUID, createHash } from 'node:crypto'; +import { createServer } from 'node:net'; +import { once } from 'node:events'; +import { mkdirSync, readFileSync, writeFileSync, existsSync, chmodSync, realpathSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { artifact, scenario } from './harness.mjs'; + +const packed = artifact(); +const docker = execFileSync('which', ['docker'], { encoding: 'utf8' }).trim(); +const image = 'docker.sourcebot.dev/sourcebot-dev/sourcebot:v5.1.13'; +const reports = []; +const providers = ['anthropic', 'openai', 'openai-compatible', 'amazon-bedrock', 'google-generative-ai', 'google-vertex', 'google-vertex-anthropic', 'azure', 'deepseek', 'mistral', 'openrouter', 'xai']; +const output = process.env.SETUP_TEST_LIVE_OUTPUT; +if (output) { + mkdirSync(output, { recursive: true }); +} +let browser; +let parse, stringify, dotenv; +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +const liveMulti = async (driver, prompt, value) => { + await driver.wait(prompt); + await pause(350); + driver.write(value); + await driver.wait(`[ ] ${value}`); + driver.write('\t'); + await pause(100); + driver.write('\r'); +}; +const run = (args, options = {}) => execFileSync(docker, args, { encoding: 'utf8', timeout: 45000, stdio: ['ignore', 'pipe', 'pipe'], ...options }); +const freePort = async () => { + const server = createServer().listen(0, '127.0.0.1'); + await once(server, 'listening'); + const port = server.address().port; + await new Promise(resolve => server.close(resolve)); + return port; +}; +function credentials() { + assert.ok(process.env.SETUP_TEST_CREDENTIAL_DIR, 'Set SETUP_TEST_CREDENTIAL_DIR to the development env-file directory'); + for (const file of ['.env.local', '.env.development.local', '.env.local.development']) { + const path = join(process.env.SETUP_TEST_CREDENTIAL_DIR, file); + if (existsSync(path)) { + const values = dotenv(readFileSync(path)); + if (values.ANTHROPIC_API_KEY) { + return values.ANTHROPIC_API_KEY; + } + } + } +} + +try { + const dockerConfig = join(packed.root, 'docker-client'); + mkdirSync(dockerConfig); + const pluginDirectory = resolve(dirname(realpathSync(docker)), '../cli-plugins'); + writeFileSync(join(dockerConfig, 'config.json'), JSON.stringify({ cliPluginsExtraDirs: existsSync(pluginDirectory) ? [pluginDirectory] : [] })); + const dockerHost = run(['context', 'inspect', '--format', '{{.Endpoints.docker.Host}}']).trim(); + const tools = join(packed.root, 'browser-tools'); + mkdirSync(tools); + writeFileSync(join(tools, 'package.json'), '{"private":true}'); + process.env.PLAYWRIGHT_BROWSERS_PATH = join(packed.root, 'browsers'); + execFileSync('npm', ['install', '--no-audit', '--no-fund', 'playwright@1.63.0', 'yaml@2.8.3', 'dotenv@16.4.7'], { cwd: tools, timeout: 120000, stdio: 'ignore' }); + ({ parse, stringify } = await import(pathToFileURL(join(tools, 'node_modules/yaml/dist/index.js')).href)); + ({ parse: dotenv } = await import(pathToFileURL(join(tools, 'node_modules/dotenv/lib/main.js')).href)); + execFileSync(process.execPath, [join(tools, 'node_modules/playwright/cli.js'), 'install', 'chromium'], { timeout: 180000, stdio: 'ignore' }); + const { chromium } = await import(pathToFileURL(join(tools, 'node_modules/playwright/index.mjs')).href); + browser = await chromium.launch({ headless: true }); + const upstream = await fetch('https://raw.githubusercontent.com/sourcebot-dev/sourcebot/main/docker-compose.yml'); + assert.ok(upstream.ok); + const composeSource = await upstream.text(); + const cases = (process.env.SETUP_TEST_LIVE_CASES ?? 'github_repo').split(','); + const knownCases = new Set(['github_repo', 'github_org', 'github_user', 'gitlab_project', 'remote_git', 'local_root', 'local_wildcard', 'local_nested', 'anthropic', 'gitea_repo', 'gerrit_project', 'auto_start', 'ai_all']); + assert.ok(cases.every(name => knownCases.has(name)), 'Unknown live deployment scenario'); + for (const name of cases) { + const project = `sb-e2e-${randomUUID().slice(0, 8)}`; + const port = await freePort(); + const url = `http://localhost:${port}`; + let expectedRepositories = name === 'local_wildcard' || name === 'local_nested' ? 2 : 1; + if (name === 'github_org' || name === 'github_user') { + const endpoint = name === 'github_org' ? 'orgs/chalk' : 'users/octocat'; + const response = await fetch(`https://api.github.com/${endpoint}/repos?per_page=100`); + assert.ok(response.ok, 'Could not establish expected public repository count'); + const repositories = await response.json(); + assert.ok(repositories.length > 0 && repositories.length < 100, 'Public fixture must fit a single bounded page'); + expectedRepositories = repositories.length; + } + const compose = parse(composeSource); + // Isolation only: leave the generated config, secrets and identity intact. + compose.services.sourcebot.image = image; + compose.services.sourcebot.pull_policy = 'never'; + compose.services.sourcebot.container_name = `${project}-sourcebot`; + compose.services.sourcebot.ports = [`127.0.0.1:${port}:3000`]; + compose.services.sourcebot.environment.push('SOURCEBOT_TELEMETRY_DISABLED=true'); + for (const [service, config] of Object.entries(compose.services)) { + config.restart = 'no'; + if (service !== 'sourcebot') { + config.ports = []; + } + } + const report = { scenario: name, project, url, artifactSha256: packed.digest, composeSourceSha256: createHash('sha256').update(composeSource).digest('hex'), status: 'running', image }; + reports.push(report); + console.log(`Starting live scenario ${name} at ${url}`); + const aiKey = name === 'anthropic' ? credentials() : undefined; + const allAi = name === 'ai_all'; + const aiEnabled = Boolean(aiKey) || allAi; + const fixtureKey = 'fixture-ai-credential-not-real'; + if (name === 'anthropic') { + assert.ok(aiKey, 'Anthropic development credential missing'); + } + let context; + try { + await scenario(packed, { + setupName: project, + realDocker: docker, + compose: stringify(compose), + networkModule: './liveNetwork.mjs', + sensitiveValues: aiKey ? [aiKey] : allAi ? [fixtureKey] : [], + environment: { COMPOSE_PROJECT_NAME: project, DOCKER_CONFIG: dockerConfig, DOCKER_HOST: dockerHost }, + prepare({ cwd }) { + if (name.startsWith('local_')) { + const target = join(cwd, 'cloned repos'); + mkdirSync(target); + if (name === 'local_root') { + execFileSync('git', ['clone', '--depth=1', 'https://github.com/octocat/Hello-World.git', join(target, 'hello')], { stdio: 'ignore', timeout: 45000 }); + } else { + for (const [index, relative] of (name === 'local_nested' ? ['one', 'nested/two'] : ['one', 'two']).entries()) { + // Sourcebot deduplicates clones sharing the same + // origin; use two genuinely different repositories. + const remote = index === 0 ? 'Hello-World' : 'git-consortium'; + execFileSync('git', ['clone', '--depth=1', `https://github.com/octocat/${remote}.git`, join(target, relative)], { stdio: 'ignore', timeout: 45000 }); + } + } + } + }, + async verifyDeployment({ setup, files, events }) { + const props = event => events.find(item => item.event === `setup_sourcebot_${event}`)?.properties; + assert.equal(props('completed').sourcebotStartOutcome, name === 'auto_start' ? 'spawned' : 'declined'); + const source = props('configured_code_source'); + assert.equal(source.codeHost, name.startsWith('local_') ? 'local_git' : name === 'remote_git' ? 'remote_git' : name === 'gitlab_project' ? 'gitlab' : name === 'gitea_repo' ? 'gitea' : name === 'gerrit_project' ? 'gerrit' : 'github'); + assert.equal(source.credentialMode, 'none'); + assert.equal(source.configurationIndex, 1); + assert.equal(source.repositoryCount, ['github_org', 'github_user', 'gitlab_project', 'gerrit_project'].includes(name) ? 0 : name.startsWith('local_') ? expectedRepositories : 1); + assert.equal(source.organizationCount, name === 'github_org' ? 1 : 0); + assert.equal(source.userCount, name === 'github_user' ? 1 : 0); + assert.equal(source.projectCount, ['gitlab_project', 'gerrit_project'].includes(name) ? 1 : 0); + assert.equal(source.generatedConnectionCount, name === 'local_nested' ? 2 : 1); + assert.equal(props('ai_setup_completed').aiConfigured, aiEnabled); + const aiCount = allAi ? providers.length : aiEnabled ? 1 : 0; + assert.equal(props('ai_setup_completed').aiConfigurationCount, aiCount); + assert.deepEqual(events.filter(event => event.event !== 'setup_sourcebot_failed').map(event => event.event.replace('setup_sourcebot_', '')), [ + 'started', 'chose_setup_directory', 'configured_code_source', 'configured_code_sources', + ...Array(aiCount).fill('configured_ai_provider'), 'ai_setup_completed', 'configured_hosted_url', + 'generated_configs', 'resolved_compose_file', 'validated_docker_state', 'completed', + ]); + report.telemetryContract = 'passed'; + chmodSync(join(setup, '.env'), 0o600); + const command = ['compose', '-p', project, '--project-directory', setup]; + run([...command, 'up', '-d', '--pull', 'never']); + const began = Date.now(); + while (true) { + try { + const response = await fetch(`${url}/onboard`, { signal: AbortSignal.timeout(2000) }); + if (response.ok) { + break; + } + } catch { /* Startup is asynchronous. */ } + assert.ok(Date.now() - began < 120000, 'Sourcebot did not become HTTP-ready within two minutes'); + await pause(1000); + } + report.httpReady = true; + report.stage = 'identity'; + const persisted = JSON.parse(run(['exec', `${project}-sourcebot`, 'cat', '/data/.sourcebot/.installedv3'])); + assert.equal(persisted.install_id, events[0].distinct_id); + report.installIdPreserved = true; + context = await browser.newContext(); + const page = await context.newPage(); + report.stage = 'onboarding'; + await page.goto(`${url}/onboard`); + await page.getByRole('link', { name: /Get Started/ }).click(); + await page.getByLabel('Email', { exact: true }).fill(`owner-${project}@example.invalid`); + await page.getByLabel('Password', { exact: true }).fill(`Test-${randomUUID()}!`); + await page.getByRole('button', { name: 'Sign up with credentials' }).click(); + await page.getByRole('link', { name: /Continue/ }).click({ timeout: 30000 }); + await page.getByRole('button', { name: /Skip for now|Continue to Sourcebot/ }).click({ timeout: 30000 }); + await page.waitForURL(current => !current.pathname.startsWith('/onboard')); + report.onboarding = 'passed'; + report.stage = 'indexing'; + const database = run([...command, 'ps', '-q', 'postgres']).trim(); + const started = Date.now(); + let counts; + while (true) { + counts = JSON.parse(run(['exec', database, 'psql', '-U', 'postgres', '-d', 'postgres', '-Atc', `SELECT json_build_object('discovered',count(*),'indexed',count("indexedAt")) FROM "Repo";`])); + if (counts.indexed === expectedRepositories && counts.discovered === expectedRepositories) { + break; + } + assert.ok(Date.now() - started < 180000, `Repository indexing did not complete: discovered=${counts.discovered}, indexed=${counts.indexed}`); + await pause(1000); + } + report.repositories = counts; + report.expectedRepositories = expectedRepositories; + report.stage = 'search'; + const searchQuery = name === 'github_org' ? 'color' : name === 'gitlab_project' ? 'test' : name === 'gitea_repo' ? 'gitea' : name === 'gerrit_project' ? 'readonly' : 'hello'; + const response = await page.request.post(`${url}/api/search`, { data: { query: searchQuery, matches: 10 } }); + assert.equal(response.status(), 200, 'Authenticated search failed'); + const search = await response.json(); + assert.ok(search.stats?.fileCount > 0, `No search matches; response fields: ${Object.keys(search).join(',')}`); + report.searchMatches = search.stats.fileCount; + await page.goto(`${url}/search?query=${encodeURIComponent(searchQuery)}`); + await page.getByText(/Found \d+ match/).first().waitFor({ timeout: 20000 }); + report.searchUi = 'passed'; + if (output) { + mkdirSync(output, { recursive: true }); + await page.screenshot({ path: join(output, `${name}.png`), fullPage: true }); + } + if (name === 'anthropic') { + const config = JSON.parse(files['config.json']); + assert.equal(config.models[0].provider, 'anthropic'); + const expectedEnv = dotenv(files['.env']); + assert.ok(expectedEnv.ANTHROPIC_API_KEY === aiKey, 'Generated AI credential differs from the supplied credential'); + const actual = run(['exec', `${project}-sourcebot`, 'node', '-e', 'process.stdout.write(process.env.ANTHROPIC_API_KEY || "")']); + assert.ok(actual === aiKey, 'Generated AI credential did not reach the container'); + report.ai = { configured: true, environmentVerified: true, askRequest: 'not_requested' }; + } + if (allAi) { + const config = JSON.parse(files['config.json']); + assert.deepEqual(config.models.map(model => model.provider), providers); + assert.deepEqual(events.filter(event => event.event === 'setup_sourcebot_configured_ai_provider').map(event => event.properties.provider), providers); + const expectedEnv = dotenv(files['.env']); + const mountedConfig = JSON.parse(run(['exec', `${project}-sourcebot`, 'cat', '/data/config.json'])); + assert.deepEqual(mountedConfig, config); + const actualEnv = JSON.parse(run(['exec', `${project}-sourcebot`, 'node', '-e', 'process.stdout.write(JSON.stringify(process.env))'])); + for (const key of Object.keys(expectedEnv)) { + assert.ok(actualEnv[key] === expectedEnv[key], `Generated environment mismatch for ${key}`); + } + for (const model of config.models) { + assert.equal(model.model, `fixture-model-${model.provider}`); + if (model.token) { + assert.ok(expectedEnv[model.token.env] === fixtureKey, 'Model credential reference mismatch'); + } + } + report.ai = { providersConfigured: providers.length, environmentVerified: true, credentials: 'synthetic_not_validated', askRequest: 'not_requested' }; + } + run([...command, 'restart', 'sourcebot']); + const restartBegan = Date.now(); + while (true) { + try { + if ((await fetch(`${url}/api/health`, { signal: AbortSignal.timeout(2000) })).ok) { + break; + } + } catch { /* Wait for the restarted application. */ } + assert.ok(Date.now() - restartBegan < 120000, 'Restart did not become HTTP-ready'); + await pause(1000); + } + const afterRestart = JSON.parse(run(['exec', `${project}-sourcebot`, 'cat', '/data/.sourcebot/.installedv3'])); + assert.equal(afterRestart.install_id, persisted.install_id); + const restartedSearch = await page.request.post(`${url}/api/search`, { data: { query: searchQuery, matches: 10 } }); + assert.equal(restartedSearch.status(), 200); + assert.ok((await restartedSearch.json()).stats?.fileCount > 0, 'Search stopped working after restart'); + report.restartIdentity = 'passed'; + report.status = 'passed'; + }, + async cleanupDeployment({ setup }) { + if (output && context && report.status !== 'passed') { + await context.pages()[0]?.screenshot({ path: join(output, `${name}-failure.png`), fullPage: true }).catch(() => {}); + } + await context?.close(); + if (existsSync(join(setup, 'docker-compose.yml'))) { + run(['compose', '-p', project, '--project-directory', setup, 'down', '--volumes', '--remove-orphans'], { timeout: 45000 }); + } + assert.equal(run(['ps', '-aq', '--filter', `label=com.docker.compose.project=${project}`]).trim(), ''); + assert.equal(run(['volume', 'ls', '-q', '--filter', `label=com.docker.compose.project=${project}`]).trim(), ''); + report.cleaned = true; + }, + }, async d => { + await d.answer('What directory would you like', project); + if (name.startsWith('local_')) { + await d.select('Which code host', 2); + await d.answer('Path to your repos directory', join(d.cwd, 'cloned repos', ...(name === 'local_root' ? ['hello'] : []))); + if (name !== 'local_root') { + await d.answer('Which repositories should be indexed?'); + } + } else if (name === 'remote_git') { + await d.select('Which code host', 3); + await d.answer('Git clone URL', 'https://github.com/octocat/Hello-World.git'); + } else if (name === 'gitea_repo') { + await d.select('Which code host', 6); + await d.answer('Gitea URL'); + await d.answer('Gitea Access Token'); + await d.check('What do you want to index?', [1]); + await liveMulti(d, 'Repositories to index', 'gitea/go-sdk'); + } else if (name === 'gerrit_project') { + await d.select('Which code host', 7); + await d.answer('Gerrit URL', 'https://gerrit-review.googlesource.com'); + await d.answer('Index all projects?', 'n'); + await liveMulti(d, 'Projects to index', 'plugins/readonly'); + } else if (name === 'gitlab_project') { + await d.select('Which code host', 1); + await d.answer('GitLab URL'); + await d.answer('GitLab Personal Access Token'); + await d.check('What do you want to index?', [1]); + await liveMulti(d, 'Projects to index', 'gitlab-org/gitlab-test'); + } else { + await d.select('Which code host'); + await d.answer('GitHub URL'); + await d.answer('GitHub Personal Access Token'); + await d.check('What do you want to index?', [name === 'github_org' ? 1 : name === 'github_user' ? 2 : 0]); + await liveMulti(d, name === 'github_org' ? 'Organizations to index' : name === 'github_user' ? 'GitHub users to index' : 'Repositories to index', name === 'github_org' ? 'chalk' : name === 'github_user' ? 'octocat' : 'octocat/Hello-World'); + } + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', aiEnabled ? 'y' : 'n'); + if (allAi) { + for (const [index, provider] of providers.entries()) { + await d.select('Which AI provider?', index); + await d.wait('Model name'); + await pause(500); + d.write(`fixture-model-${provider}`); + await pause(700); + d.write('\r'); + if (provider === 'openai-compatible') { + await d.answer('Base URL', 'https://fixture-ai.example.invalid/v1'); + } + if (provider === 'azure') { + await d.answer('Azure resource name', 'fixture-resource'); + await d.answer('API version'); + } + if (provider === 'amazon-bedrock') { + await d.answer('Use the default AWS credential chain?', 'n'); + await d.answer('AWS Access Key ID', 'fixture-access-id'); + await d.answer('AWS Secret Access Key', fixtureKey); + await d.answer('AWS region'); + } else if (provider.startsWith('google-vertex')) { + if (provider === 'google-vertex') { + await d.answer('Google Cloud project ID', 'fixture-project'); + await d.answer('Google Cloud region'); + } + await d.answer('Use Application Default Credentials?', 'y'); + } else { + await d.answer('API key (', fixtureKey); + } + await d.answer('Display name'); + await d.answer('Add another model?', index === providers.length - 1 ? 'n' : 'y'); + } + } + if (aiKey) { + await d.select('Which AI provider?'); + await d.wait('Model name'); + await pause(500); + d.write('claude-haiku-4-5'); + await pause(500); + d.write('\r'); + await d.answer('API key (', aiKey); + await d.answer('Display name'); + await d.answer('Add another model?', 'n'); + } + await d.answer('What URL will Sourcebot be hosted at?', url); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', name === 'auto_start' ? 'y' : 'n'); + if (name === 'auto_start') { + const began = Date.now(); + while (true) { + try { + if ((await fetch(`${url}/api/health`, { signal: AbortSignal.timeout(2000) })).ok) { + break; + } + } catch { /* Compose is starting the actual application. */ } + assert.ok(Date.now() - began < 120000, 'CLI-started deployment did not become healthy'); + await pause(1000); + } + report.cliStartedDeployment = true; + d.write('\x03'); + await d.finish(130); + assert.equal(d.events.filter(event => event.event === 'setup_sourcebot_completed').length, 1); + assert.equal(d.events.some(event => event.event === 'setup_sourcebot_cancelled'), false); + } + }); + } catch (error) { + report.status = 'failed'; + report.failure = (aiKey ? error.message.replaceAll(aiKey, '[redacted]') : error.message).slice(0, 500); + } + console.log(JSON.stringify(report)); + if (output) { + const path = join(output, 'live-results.json'); + const previous = existsSync(path) ? JSON.parse(readFileSync(path, 'utf8')) : []; + writeFileSync(path, JSON.stringify([...previous.filter(item => !reports.some(report => report.scenario === item.scenario)), ...reports], null, 2)); + } + } + process.exitCode = reports.some(report => report.status !== 'passed') ? 1 : 0; +} finally { + await browser?.close(); + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/liveNetwork.mjs b/packages/setupWizard/tests/e2e/liveNetwork.mjs new file mode 100644 index 000000000..385cc8cbc --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveNetwork.mjs @@ -0,0 +1,21 @@ +// External test routing only. Production PostHog is always captured locally; +// live public code hosts and model catalogs retain their real network behavior. +import tls from 'node:tls'; +import { Agent, buildConnector, setGlobalDispatcher } from 'undici'; + +if (process.versions.bun) { + throw Error('Live tests require the actual Node runtime'); +} +const connect = buildConnector({}); +const permitted = new Set(['api.github.com', 'github.com', 'gitlab.com', 'gitea.com', 'codeberg.org', 'models.dev', 'gerrit-review.googlesource.com']); +setGlobalDispatcher(new Agent({ connect(options, callback) { + if (options.hostname === 'us.i.posthog.com' || options.hostname === 'raw.githubusercontent.com') { + const socket = tls.connect({ host: '127.0.0.1', port: Number(process.env.TEST_CAPTURE_PORT), servername: options.hostname }); + socket.once('secureConnect', () => callback(null, socket)); + socket.once('error', error => callback(error, null)); + } else if (options.protocol === 'https:' && permitted.has(options.hostname)) { + connect(options, callback); + } else { + callback(new Error('Live wizard egress outside the approved code hosts denied'), null); + } +} })); diff --git a/packages/setupWizard/tests/e2e/liveSmoke.mjs b/packages/setupWizard/tests/e2e/liveSmoke.mjs new file mode 100644 index 000000000..1e27759f3 --- /dev/null +++ b/packages/setupWizard/tests/e2e/liveSmoke.mjs @@ -0,0 +1,14 @@ +import assert from 'node:assert/strict'; +import { artifact, scenario, minimal } from './harness.mjs'; +assert.ok(process.env.SETUP_TEST_DEV_TOKEN, 'Provide the designated dev project token; never use the production project.'); +assert.notEqual(process.env.SETUP_TEST_DEV_TOKEN, 'phc_lLPuFFi5LH6c94eFJcqvYVFwiJffVcV6HD8U4a1OnRW'); +const packed = artifact(); +try { + const result = await scenario(packed, { live: true }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'n'); + }); + console.log(JSON.stringify({ artifactSha256: packed.digest, distinctId: result.events[0].distinct_id, eventsForwarded: result.events.length, ingestion: 'accepted; query verification still required' })); +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/network.mjs b/packages/setupWizard/tests/e2e/network.mjs new file mode 100644 index 000000000..778c87e90 --- /dev/null +++ b/packages/setupWizard/tests/e2e/network.mjs @@ -0,0 +1,22 @@ +// Test-only network boundary. This file is outside the installed tarball. It changes +// TCP routing, not the SDK, URL, headers, event payload, or production configuration. +import tls from 'node:tls'; +import { Agent, setGlobalDispatcher } from 'undici'; +import { basename } from 'node:path'; + +if (process.platform === 'win32' && basename(process.execPath).toLowerCase() === 'docker.exe') { + process.argv.splice(1, 0, 'fakeDocker.cjs'); + await import('./fakeDocker.cjs'); + // The fixture owns its exit/timers; never execute "compose" as a JS file. + await new Promise(() => {}); +} + +setGlobalDispatcher(new Agent({ connect(options, callback) { + if (options.protocol !== 'https:') { + callback(new Error('Non-HTTPS test egress denied'), null); + return; + } + const socket = tls.connect({ host: '127.0.0.1', port: Number(process.env.TEST_CAPTURE_PORT), servername: options.hostname }); + socket.once('secureConnect', () => callback(null, socket)); + socket.once('error', error => callback(error, null)); +} })); diff --git a/packages/setupWizard/tests/e2e/nodeCompatibility.mjs b/packages/setupWizard/tests/e2e/nodeCompatibility.mjs new file mode 100644 index 000000000..f9eef92f7 --- /dev/null +++ b/packages/setupWizard/tests/e2e/nodeCompatibility.mjs @@ -0,0 +1,38 @@ +import assert from 'node:assert/strict'; +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdirSync, readdirSync } from 'node:fs'; +import { join, dirname, delimiter } from 'node:path'; +import { artifact } from './harness.mjs'; + +// Build once on the release runtime, then install and exercise that exact tarball +// on each minimum supported LTS runtime. No production telemetry is sent. +const packed = artifact(); +const packageRoot = new URL('../..', import.meta.url); +try { + for (const version of ['20.20.0', '22.22.0', '18.20.8', '20.19.0', '22.21.0']) { + const runtimeRoot = join(packed.root, `node-${version}`); + mkdirSync(runtimeRoot); + execFileSync('npm', ['install', '--prefix', runtimeRoot, '--no-audit', '--no-fund', '--cache', join(packed.root, 'runtime-cache'), `node@${version}`], { stdio: 'pipe', timeout: 120000 }); + const node = join(runtimeRoot, 'node_modules/node/bin/node'); + const env = { ...process.env, PATH: `${dirname(node)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false', SETUP_TEST_TARBALL: packed.tarball }; + if (['20.20.0', '22.22.0'].includes(version)) { + const tests = [ + ...['unit', 'integration'].flatMap(dir => readdirSync(new URL(`../${dir}/`, import.meta.url)).filter(f => f.endsWith('.test.mjs')).map(f => `tests/${dir}/${f}`)), + ...['wizard', 'collectors', 'docker', 'safety', 'platform'].map(f => `tests/e2e/${f}.test.mjs`), + ]; + execFileSync(node, ['--test', '--test-concurrency=1', ...tests], { cwd: packageRoot, env, stdio: 'inherit', timeout: 600000 }); + console.log(`Node ${version}: packed CLI and telemetry regression suites passed; artifact ${packed.digest}`); + } else { + const work = join(runtimeRoot, 'empty-home'); + mkdirSync(work); + const result = spawnSync(node, [join(packed.installed, 'bin.cjs')], { cwd: work, encoding: 'utf8', env: { PATH: '', HOME: work }, timeout: 10000 }); + assert.equal(result.status, 1); + assert.match(result.stderr, /requires Node.js 20\.20\+, 22\.22\+, or 23\.5\+/); + assert.equal(result.stdout, ''); + assert.deepEqual(readdirSync(work), []); + console.log(`Node ${version}: rejected before importing the wizard`); + } + } +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/packageManagers.mjs b/packages/setupWizard/tests/e2e/packageManagers.mjs new file mode 100644 index 000000000..e9fd2d69d --- /dev/null +++ b/packages/setupWizard/tests/e2e/packageManagers.mjs @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, writeFileSync, readFileSync, symlinkSync } from 'node:fs'; +import { join, dirname, delimiter } from 'node:path'; +import { artifact, scenario, minimal } from './harness.mjs'; + +// Install launchers into this test's disposable root, never globally. Their +// executable scripts and the wizard are both run with the required Node 24. +const packed = artifact(); +try { + const managerRoot = join(packed.root, 'managers'); + mkdirSync(managerRoot); + writeFileSync(join(managerRoot, 'package.json'), JSON.stringify({ private: true, allowScripts: { bun: true, pnpm: true } })); + execFileSync('npm', ['install', '--no-audit', '--no-fund', '--cache', join(packed.root, 'manager-cache'), 'npm@12.0.2', '@yarnpkg/cli-dist@4.7.0', 'pnpm@12.4.1', 'bun@1.4.2'], { + cwd: managerRoot, env: { ...process.env, PATH: `${dirname(process.execPath)}${delimiter}${process.env.PATH}`, PACKAGE_TRACKER_ANALYTICS: 'false' }, timeout: 120000, + }); + for (const [name, packageName, prefix] of [ + ['npm', 'npm', ['exec', '--offline', '--']], + ['yarn', '@yarnpkg/cli-dist', ['exec']], + ['pnpm', 'pnpm', ['exec']], + ['bun', 'bun', ['x', '--no-install']], + ]) { + const directory = join(managerRoot, 'node_modules', packageName); + const manifest = JSON.parse(readFileSync(join(directory, 'package.json'), 'utf8')); + const binary = join(directory, typeof manifest.bin === 'string' ? manifest.bin : manifest.bin[name]); + const managerEnv = { npm_config_cache: join(packed.root, 'invocation-cache'), npm_config_script_shell: '/bin/sh', YARN_ENABLE_TELEMETRY: '0', YARN_GLOBAL_FOLDER: join(packed.root, 'yarn-global'), YARN_CACHE_FOLDER: join(packed.root, 'yarn-cache') }; + const result = await scenario(packed, { + launcher: { file: binary, args: [...prefix, name === 'yarn' ? JSON.stringify(packed.bin) : name === 'bun' ? 'setup-sourcebot' : packed.bin] }, + environment: managerEnv, + assertLauncherHome(paths) { + // Package managers may write their own state before launching + // the wizard. Direct-binary suites still require an empty home. + const permitted = /^(Library(?:\/(?:Caches\/)?pnpm(?:\/.*)?)?|state(?:\/pnpm(?:\/.*)?)?|cache(?:\/pnpm(?:\/.*)?)?|\.local(?:\/share(?:\/pnpm(?:\/.*)?)?)?)$/; + const unexpected = paths.filter(path => name !== 'pnpm' || !permitted.test(path)); + assert.deepEqual(unexpected, [], 'Unexpected launcher state outside the known pnpm store/state directories'); + }, + prepare({ cwd }) { + writeFileSync(join(cwd, 'package.json'), JSON.stringify({ private: true, name: 'isolated-launcher', ...(name === 'yarn' ? { packageManager: 'yarn@4.7.0' } : {}) })); + writeFileSync(join(cwd, 'yarn.lock'), ''); + writeFileSync(join(cwd, '.yarnrc.yml'), 'nodeLinker: node-modules\n'); + if (name === 'yarn') { + // This new disposable project intentionally has no lockfile + // yet; CI's immutable-install default cannot apply to it. + execFileSync(process.execPath, [binary, 'install', '--mode=skip-build'], { cwd, env: { ...process.env, ...managerEnv, YARN_ENABLE_IMMUTABLE_INSTALLS: 'false' }, timeout: 30000 }); + } + if (name === 'bun') { + symlinkSync(dirname(packed.installed), join(cwd, 'node_modules'), 'junction'); + } + }, + }, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + const started = result.events[0].properties; + assert.equal(started.packageManager, name); + assert.ok(['npx', 'workspace', 'local_binary', 'unknown'].includes(started.invocationMethod)); + console.log(`${name}@${manifest.version}: packed binary completed; packageManager=${started.packageManager}; invocationMethod=${started.invocationMethod}`); + } + console.log(`Package-manager launcher matrix passed: ${packed.digest}`); +} finally { + packed.cleanup(); +} diff --git a/packages/setupWizard/tests/e2e/platform.test.mjs b/packages/setupWizard/tests/e2e/platform.test.mjs new file mode 100644 index 000000000..cab90a2e5 --- /dev/null +++ b/packages/setupWizard/tests/e2e/platform.test.mjs @@ -0,0 +1,53 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { mkdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { artifact, scenario, minimal, canary } from './harness.mjs'; +let packed; +before(() => { packed = artifact(); }); +after(() => packed?.cleanup()); + +test('platform: published bootstrap and minimal happy path', async () => { + const result = await scenario(packed, {}, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + assert.equal(result.events[0].properties.platform, process.platform); + assert.equal(result.events[0].properties.arch, process.arch); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_completed'); +}); +test('platform: real terminal Ctrl+C exits with cancellation', async () => { + const result = await scenario(packed, {}, async d => { + await d.wait('What directory would you like'); + d.write('\x03'); + await d.finish(130); + }); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_cancelled'); +}); +test('platform: local repository in a path with spaces and Unicode', async () => { + const result = await scenario(packed, { prepare({ cwd }) { mkdirSync(join(cwd, `${canary} space-é`, '.git'), { recursive: true }); } }, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host', 2); + await d.answer('Path to your repos directory', join(d.cwd, `${canary} space-é`)); + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.ok(result.files['docker-compose.override.yml']); + assert.equal(result.events.find(e => e.event === 'setup_sourcebot_configured_code_source').properties.codeHost, 'local_git'); +}); + +test('platform: foreground Docker spawn, completion and Ctrl+C cleanup', async () => { + const result = await scenario(packed, { docker: { stall: ['compose up'], stubborn: true } }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + const started = Date.now(); + while (!d.events.some(event => event.event === 'setup_sourcebot_completed')) { + assert.ok(Date.now() - started < 3000); + await new Promise(resolve => setTimeout(resolve, 20)); + } + d.write('\x03'); + await d.finish(130); + }); + assert.equal(result.events.filter(event => event.event === 'setup_sourcebot_completed').length, 1); + assert.equal(result.events.some(event => event.event === 'setup_sourcebot_cancelled'), false); +}); diff --git a/packages/setupWizard/tests/e2e/runtime.test.mjs b/packages/setupWizard/tests/e2e/runtime.test.mjs new file mode 100644 index 000000000..94c1753e7 --- /dev/null +++ b/packages/setupWizard/tests/e2e/runtime.test.mjs @@ -0,0 +1,141 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { randomUUID } from 'node:crypto'; +import { homedir } from 'node:os'; +import { execFileSync } from 'node:child_process'; +import { writeFileSync, mkdirSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { artifact, scenario, minimal } from './harness.mjs'; +import { INSTALL_ID_PATTERN } from '../../dist/telemetry.js'; + +test('real Docker name conflict emits a private diagnostic after setup completion', async () => { + const packed = artifact(); + const docker = execFileSync('which', ['docker'], { encoding: 'utf8' }).trim(); + const image = process.env.SETUP_TEST_SOURCEBOT_IMAGE ?? 'docker.sourcebot.dev/sourcebot-dev/sourcebot:latest'; + const name = `setup-start-conflict-${randomUUID()}`; + const project = `setup-start-${randomUUID()}`; + let created = false; + try { + execFileSync(docker, ['create', '--name', name, '--label', `setup-start-test=${project}`, image], { stdio: 'pipe', timeout: 45000 }); + created = true; + const host = execFileSync(docker, ['context', 'inspect', '--format', '{{.Endpoints.docker.Host}}'], { encoding: 'utf8' }).trim(); + const result = await scenario(packed, { + realDocker: docker, + compose: `services:\n sourcebot:\n image: ${image}\n container_name: ${name}\n pull_policy: never\n`, + sensitiveValues: [name, project], + assertLauncherHome(paths) { + // Docker Desktop creates these empty parent directories itself. + // Still reject files or any wizard-owned per-user state. + const dockerDirectories = process.platform === 'darwin' + ? ['Library', 'Library/Containers', 'Library/Containers/com.docker.docker', 'Library/Containers/com.docker.docker/Data'] + : []; + assert.deepEqual(paths.filter(path => !dockerDirectories.includes(path)), []); + }, + environment: { COMPOSE_PROJECT_NAME: project, DOCKER_HOST: host, DOCKER_CONFIG: process.env.DOCKER_CONFIG ?? join(homedir(), '.docker') }, + async cleanupDeployment({ setup }) { + execFileSync(docker, ['compose', '-p', project, 'down'], { cwd: setup, stdio: 'pipe', timeout: 45000 }); + }, + }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + await d.wait('is already in use by container'); + }); + const failure = result.events.filter(e => e.event === 'setup_sourcebot_start_failed'); + assert.equal(failure.length, 1); + assert.equal(failure[0].properties.failureReason, 'container_name_conflict'); + assert.equal(failure[0].properties.failurePhase, 'compose_exit'); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_start_failed'); + assert.ok(result.events.some(e => e.event === 'setup_sourcebot_completed')); + } finally { + try { + if (created) execFileSync(docker, ['rm', '-v', name], { stdio: 'pipe', timeout: 45000 }); + } finally { + packed.cleanup(); + } + } +}); + +test('real Sourcebot containers: packed wizard identity survives first boot, restart, upgrade and opt-out', async () => { + const packed = artifact(); + const image = process.env.SETUP_TEST_SOURCEBOT_IMAGE ?? 'docker.sourcebot.dev/sourcebot-dev/sourcebot:latest'; + const label = `setup-wizard-e2e-${randomUUID()}`; + const fixtures = fileURLToPath(new URL('.', import.meta.url)); + const entrypoint = resolve(fixtures, '../../../../entrypoint.sh'); + const containerNames = []; + const live = process.env.SETUP_TEST_LIVE === 'true'; + if (live) { + assert.equal(process.env.SETUP_TEST_DEV_TOKEN, 'phc_EJR6BsaBbvIKhM4t4zp1boYC92Tpp5Fgb9Csa9Us5aw', 'Live runtime verification is restricted to the approved dev project'); + } + try { + const setup = await scenario(packed, { live }, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + const setupId = setup.events[0].distinct_id; + const envFile = join(packed.root, 'deployment.env'); + writeFileSync(envFile, setup.files['.env']); + const versionFile = join(packed.root, 'version.ts'); + const boot = (data, version, options = {}) => { + mkdirSync(join(packed.root, data), { recursive: true }); + writeFileSync(versionFile, `export const SOURCEBOT_VERSION = "${version}";\n`); + const name = `${label}-${containerNames.length}`; + containerNames.push(name); + const output = execFileSync('docker', ['run', '--rm', '--name', name, '--label', label, '--network', 'none', '--add-host', 'us.i.posthog.com:127.0.0.1', + '--env-file', envFile, '-e', `SOURCEBOT_TELEMETRY_DISABLED=${options.disabled ? 'true' : 'false'}`, '-e', 'POSTHOG_PAPIK=phc_runtime_fixture', + ...(options.id !== undefined ? ['-e', `SOURCEBOT_INSTALL_ID=${options.id}`] : []), + ...(options.omit ? ['-e', 'TEST_OMIT_INSTALL_ID=true'] : []), + '-v', `${join(packed.root, data)}:/data`, '-v', `${join(fixtures, 'runtimeFixture.mjs')}:/fixture/runtimeFixture.mjs:ro`, + '-v', `${entrypoint}:/fixture/entrypoint.sh:ro`, '-v', `${packed.cert}:/fixture/cert.pem:ro`, '-v', `${packed.key}:/fixture/key.pem:ro`, + '-v', `${versionFile}:/app/packages/shared/src/version.ts:ro`, '--entrypoint', 'node', image, '/fixture/runtimeFixture.mjs'], { timeout: 45000, encoding: 'utf8' }); + return JSON.parse(output.trim()); + }; + const first = boot('supplied', 'v5.1.3'); + assert.equal(first.installId, setupId); + assert.equal(first.persisted.install_id, setupId); + assert.deepEqual(first.events.map(e => [e.event, e.distinct_id]), [['install', setupId]]); + const restart = boot('supplied', 'v5.1.3', { id: randomUUID() }); + assert.equal(restart.installId, setupId); + assert.deepEqual(restart.events, []); + const upgrade = boot('supplied', 'v5.1.4', { id: randomUUID() }); + assert.equal(upgrade.installId, setupId); + assert.deepEqual(upgrade.events.map(e => [e.event, e.distinct_id]), [['upgrade', setupId]]); + const omittedRestart = boot('supplied', 'v5.1.4', { omit: true }); + assert.equal(omittedRestart.installId, setupId); + assert.deepEqual(omittedRestart.events, []); + const generated = boot('generated', 'v5.1.3', { omit: true }); + assert.match(generated.installId, INSTALL_ID_PATTERN); + assert.notEqual(generated.installId, setupId); + assert.equal(generated.events[0].distinct_id, generated.installId); + const generatedRestart = boot('generated', 'v5.1.3', { omit: true }); + assert.equal(generatedRestart.installId, generated.installId); + const disabled = boot('disabled', 'v5.1.3', { disabled: true }); + assert.equal(disabled.installId, setupId); + assert.deepEqual(disabled.events, []); + const disabledRestart = boot('disabled', 'v5.1.4', { disabled: true, id: randomUUID() }); + assert.equal(disabledRestart.installId, setupId); + assert.deepEqual(disabledRestart.events, []); + if (live) { + // Forward only the two already-validated synthetic payloads captured + // from actual curl. The container itself never has external egress. + for (const payload of [first.events[0], upgrade.events[0]]) { + assert.equal(payload.distinct_id, setupId); + assert.equal(JSON.stringify(payload).includes('canary-sensitive'), false); + const forwarded = { ...payload, api_key: process.env.SETUP_TEST_DEV_TOKEN }; + assert.deepEqual({ ...forwarded, api_key: payload.api_key }, payload); + const response = await fetch('https://us.i.posthog.com/capture/', { + method: 'POST', headers: { 'content-type': 'application/json' }, + body: JSON.stringify(forwarded), signal: AbortSignal.timeout(15000), + }); + assert.ok(response.ok, `Dev runtime ingestion rejected: ${response.status}`); + await response.text(); + } + console.log(JSON.stringify({ artifactSha256: packed.digest, distinctId: setupId, eventsForwarded: setup.events.length + 2, ingestion: 'accepted; query verification still required' })); + } + console.log(`Identity continuity verified across 8 containers; artifact SHA-256 ${packed.digest}`); + } finally { + for (const name of containerNames) { + try { execFileSync('docker', ['rm', '-f', name], { stdio: 'ignore' }); } catch { /* --rm already cleaned completed containers. */ } + } + assert.equal(execFileSync('docker', ['ps', '-aq', '--filter', `label=${label}`], { encoding: 'utf8' }).trim(), ''); + packed.cleanup(); + } +}); diff --git a/packages/setupWizard/tests/e2e/runtimeFixture.mjs b/packages/setupWizard/tests/e2e/runtimeFixture.mjs new file mode 100644 index 000000000..633458ae7 --- /dev/null +++ b/packages/setupWizard/tests/e2e/runtimeFixture.mjs @@ -0,0 +1,37 @@ +// Executed inside a disposable Sourcebot image. The repository entrypoint, real +// curl, jq and uuidgen are unchanged; only DB migration/supervisor are replaced. +import { createServer } from 'node:https'; +import { once } from 'node:events'; +import { spawn } from 'node:child_process'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +const root = mkdtempSync('/tmp/entrypoint-fixture-'); +writeFileSync(`${root}/yarn`, '#!/bin/sh\nexit 0\n', { mode: 0o755 }); +writeFileSync(`${root}/supervisord`, '#!/bin/sh\nprintf "%s" "$SOURCEBOT_INSTALL_ID" > "$TEST_ID_RESULT"\n', { mode: 0o755 }); +const events = []; +const server = createServer({ key: readFileSync('/fixture/key.pem'), cert: readFileSync('/fixture/cert.pem') }, async (req, res) => { + const parts = []; + for await (const part of req) { + parts.push(part); + } + events.push(JSON.parse(Buffer.concat(parts).toString())); + res.writeHead(200).end('{"status":1}'); +}).listen(443, '127.0.0.1'); +await once(server, 'listening'); +try { + const env = { ...process.env, PATH: `${root}:${process.env.PATH}`, CONFIG_PATH: '', DATA_CACHE_DIR: '/data', DATABASE_URL: 'postgresql://fixture', REDIS_URL: 'redis://fixture', CURL_CA_BUNDLE: '/fixture/cert.pem', TEST_ID_RESULT: `${root}/id` }; + if (env.TEST_OMIT_INSTALL_ID === 'true') { + delete env.SOURCEBOT_INSTALL_ID; + } + const child = spawn('/bin/sh', ['/fixture/entrypoint.sh'], { env, stdio: ['ignore', 'pipe', 'pipe'] }); + let diagnostics = ''; + child.stdout.on('data', b => { diagnostics += b; }); + child.stderr.on('data', b => { diagnostics += b; }); + const [code] = await once(child, 'close'); + if (code !== 0) { + throw Error(`Entrypoint failed (${code}): ${diagnostics}`); + } + console.log(JSON.stringify({ events, installId: readFileSync(`${root}/id`, 'utf8'), persisted: JSON.parse(readFileSync('/data/.installedv3', 'utf8')) })); +} finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); +} diff --git a/packages/setupWizard/tests/e2e/safety.test.mjs b/packages/setupWizard/tests/e2e/safety.test.mjs new file mode 100644 index 000000000..4a9e39b31 --- /dev/null +++ b/packages/setupWizard/tests/e2e/safety.test.mjs @@ -0,0 +1,149 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { setTimeout as sleep } from 'node:timers/promises'; +import { artifact, scenario, minimal, canary } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Safety artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const prop = (r, n) => r.events.find(e => e.event === `setup_sourcebot_${n}`)?.properties; + +for (const stage of ['directory', 'config', 'env']) { + test(`explicit decline: ${stage}`, async () => { + const files = stage === 'directory' ? {} : stage === 'config' ? { 'config.json': 'original' } : { '.env': 'original' }; + const result = await scenario(packed, { files }, async d => { + if (stage === 'directory') { + await d.answer('What directory would you like'); + await d.answer('Do you want to overwrite it?', 'n'); + } else { + await minimal(d, { existing: true }); + await d.answer(stage === 'config' ? 'config.json already exists. Overwrite?' : '.env already exists. Overwrite?', 'n'); + } + }); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_cancelled'); + assert.deepEqual(result.files, files); + assert.equal(prop(result, 'cancelled').reason, stage === 'directory' ? 'existing_directory_declined' : 'config_overwrite_declined'); + }); +} + +test('fatal directory creation failure: failed without checkpoint or files', async () => { + const result = await scenario(packed, { prepare({ cwd }) { writeFileSync(join(cwd, 'blocked'), 'fixture'); } }, async d => { + await d.answer('What directory would you like', 'blocked/child'); + await d.finish(1); + }); + assert.deepEqual(result.events.map(e => e.event), ['setup_sourcebot_started', 'setup_sourcebot_failed']); + assert.equal(prop(result, 'failed').failureCategory, 'filesystem'); + assert.equal(prop(result, 'failed').recoverable, false); +}); + +test('fatal config write failure does not emit generated_configs', async () => { + const result = await scenario(packed, { files: {}, ignoreFiles: ['config.json'], prepare({ setup }) { mkdirSync(join(setup, 'config.json')); } }, async d => { + await minimal(d, { existing: true }); + await d.answer('config.json already exists. Overwrite?', 'y'); + await d.finish(1); + }); + assert.equal(prop(result, 'generated_configs'), undefined); + assert.equal(prop(result, 'failed').failureCategory, 'filesystem'); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_failed'); +}); + +test('multiple code-source loop emits sequential indexes and exact aggregate', async () => { + const result = await scenario(packed, {}, async d => { + await d.answer('What directory would you like'); + for (let i = 0; i < 3; i++) { + await d.select('Which code host', 3); + await d.answer('Git clone URL', `https://${canary}.example.invalid/repo-${i}`); + await d.answer('Add another code host?', i === 2 ? 'n' : 'y'); + } + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?', `https://${canary}.example.invalid/path?secret=yes`); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.deepEqual(result.events.filter(e => e.event === 'setup_sourcebot_configured_code_source').map(e => e.properties.configurationIndex), [1, 2, 3]); + assert.equal(prop(result, 'configured_code_sources').repositoryCount, 3); + assert.equal(prop(result, 'configured_code_sources').uniqueCodeHostCount, 1); + assert.equal(prop(result, 'configured_hosted_url').hostCategory, 'address'); +}); + +for (const status of [401, 403, 500, 'reset', 'malformed']) { + test(`GitHub autocomplete fallback: ${status}`, async () => { + const result = await scenario(packed, { searchStatus: status === 'malformed' ? 200 : status, search: status === 'malformed' ? { items: [null] } : undefined }, async d => { + await d.answer('What directory would you like'); + await d.select('Which code host'); + await d.answer('GitHub URL'); + await d.answer('GitHub Personal Access Token', `${canary}-token`); + await d.check('What do you want to index?'); + await d.multi('Repositories to index'); + await d.answer('Add another code host?', 'n'); + await d.answer('Would you like to configure AI features?', 'n'); + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.equal(prop(result, 'configured_code_source').repositoryCount, 1); + assert.equal(prop(result, 'failed').failureCategory, 'network'); + assert.equal(prop(result, 'failed').recoverable, true); + assert.equal(result.events.at(-1).event, 'setup_sourcebot_completed'); + }); +} + +for (const mode of ['catalog', 'custom', 'malformed', 'http_failure']) { + test(`model catalog ${mode}, repeated-provider credential reuse`, async () => { + const catalog = { anthropic: { models: { fixture: { id: `${canary}-model`, name: `${canary}-name` } } } }; + if (mode === 'malformed') { + catalog.anthropic.models.fixture = null; + } + const result = await scenario(packed, { catalog, catalogStatus: mode === 'http_failure' ? 503 : 200 }, async d => { + await minimal(d, { ai: true }); + for (let i = 0; i < 2; i++) { + await d.select('Which AI provider?'); + if (mode === 'catalog' || mode === 'custom') { + await d.wait('Model name'); + await sleep(350); + if (mode === 'custom') { + d.write(`${canary}-custom`); + await sleep(350); + } + d.write('\r'); + } else { + await d.answer('Model name', `${canary}-fallback`); + } + if (!i) { + await d.answer('API key (', `${canary}-key`); + } + await d.answer('Display name'); + await d.answer('Add another model?', i ? 'n' : 'y'); + } + await d.answer('What URL will Sourcebot be hosted at?'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + assert.equal(prop(result, 'ai_setup_completed').aiConfigurationCount, 2); + assert.equal(prop(result, 'ai_setup_completed').uniqueProviderCount, 1); + assert.equal(prop(result, 'generated_configs').credentialVariableCount, 1); + assert.equal(prop(result, 'configured_ai_provider').modelSelectionMethod, mode === 'catalog' ? 'catalog' : mode === 'custom' ? 'custom_entry' : 'manual_fallback'); + }); +} + +for (const stage of ['code_sources', 'ai_setup', 'hosted_url', 'compose_file']) { + test(`Ctrl+C at stage ${stage}`, async () => { + const result = await scenario(packed, {}, async d => { + if (stage === 'code_sources') { + await d.answer('What directory would you like'); + await d.wait('Which code host'); + } else if (stage === 'ai_setup') { + await minimal(d, { ai: true }); + await d.wait('Which AI provider?'); + } else if (stage === 'hosted_url') { + await minimal(d, { pauseBeforeHosted: true }); + await d.wait('What URL will Sourcebot be hosted at?'); + } else { + await minimal(d); + await d.wait('Download docker-compose.yml?'); + } + d.write('\x03'); + await d.finish(130); + }); + assert.equal(prop(result, 'cancelled').stage, stage); + }); +} diff --git a/packages/setupWizard/tests/e2e/wizard.test.mjs b/packages/setupWizard/tests/e2e/wizard.test.mjs new file mode 100644 index 000000000..d53aa09c6 --- /dev/null +++ b/packages/setupWizard/tests/e2e/wizard.test.mjs @@ -0,0 +1,143 @@ +import assert from 'node:assert/strict'; +import { before, after, test } from 'node:test'; +import { randomUUID } from 'node:crypto'; +import { artifact, scenario, minimal, defaultCompose } from './harness.mjs'; + +let packed; +before(() => { packed = artifact(); console.log(`Packed artifact SHA-256: ${packed.digest}`); }); +after(() => packed?.cleanup()); +const names = result => result.events.map(e => e.event.replace('setup_sourcebot_', '')); +const props = (result, name) => result.events.find(e => e.event === `setup_sourcebot_${name}`)?.properties; +const funnel = ['started', 'chose_setup_directory', 'configured_code_source', 'configured_code_sources', 'ai_setup_completed', 'configured_hosted_url', 'generated_configs', 'resolved_compose_file', 'validated_docker_state', 'completed']; + +test('packed CLI: full manual funnel, exact identity and only Sourcebot files persisted', async () => { + const result = await scenario(packed, {}, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + assert.deepEqual(names(result), funnel); + assert.deepEqual(Object.keys(result.files).sort(), ['.env', 'config.json']); + assert.equal(JSON.parse(result.files['config.json']).connections.git.type, 'git'); + assert.equal(result.files['.env'].match(/^SOURCEBOT_INSTALL_ID=(.+)$/m)[1], result.events[0].distinct_id); + assert.equal(props(result, 'validated_docker_state').runningComposeContainerCount, null); + assert.equal(props(result, 'completed').sourcebotStartOutcome, 'not_offered'); + assert.equal(result.dockerCalls.length, 0); +}); + +for (const compose of ['download', 'existing', 'declined', 404, 500]) { + test(`packed compose resolution: ${compose}`, async () => { + const existing = compose === 'existing'; + const result = await scenario(packed, { files: existing ? { 'docker-compose.yml': defaultCompose } : undefined, composeStatus: typeof compose === 'number' ? compose : undefined }, async d => { + await minimal(d, { existing }); + if (!existing) { + await d.answer('Download docker-compose.yml?', compose === 'declined' ? 'n' : 'y'); + } + if (existing || compose === 'download') { + await d.answer('Start Sourcebot now?', 'n'); + } + }); + const expected = existing ? 'already_present' : compose === 'download' ? 'downloaded' : compose === 'declined' ? 'declined' : 'download_failed'; + assert.equal(props(result, 'resolved_compose_file').outcome, expected); + assert.equal(props(result, 'completed').sourcebotStartOutcome, existing || compose === 'download' ? 'declined' : 'not_offered'); + assert.equal(names(result).filter(n => n === 'failed').length, typeof compose === 'number' ? 1 : 0); + }); +} + +for (const identity of ['valid', 'invalid', 'absent']) { + test(`existing env identity: ${identity}`, async () => { + const old = randomUUID(); + const result = await scenario(packed, { files: { '.env': identity === 'absent' ? '' : `SOURCEBOT_INSTALL_ID=${identity === 'valid' ? old : 'invalid'}\n`, 'config.json': '{}' } }, async d => { + await minimal(d, { existing: true }); + await d.answer('config.json already exists. Overwrite?', 'y'); + await d.answer('.env already exists. Overwrite?', 'y'); + await d.answer('Download docker-compose.yml?', 'n'); + }); + const id = result.files['.env'].match(/^SOURCEBOT_INSTALL_ID=(.+)$/m)[1]; + assert.equal(id, identity === 'valid' ? old : result.events[0].distinct_id); + assert.equal(JSON.stringify(result.events).includes(old), false); + assert.equal(props(result, 'generated_configs').deploymentIdentityAction, identity === 'valid' ? 'preserved_existing' : 'created_from_setup_session'); + }); +} + +for (const parentSignal of [false, true]) { + test(`prompt cancellation ${parentSignal ? 'parent SIGINT' : 'PTY Ctrl+C'}`, async () => { + const result = await scenario(packed, {}, async d => { + await d.wait('What directory would you like'); + const began = Date.now(); + if (parentSignal) { + d.interrupt(); + } else { + d.write('\x03'); + } + await d.finish(130); + assert.ok(Date.now() - began < 3500); + }); + assert.deepEqual(names(result), ['started', 'cancelled']); + assert.deepEqual(result.files, {}); + }); +} + +test('completion is committed at spawn; Ctrl+C still exits without a cancellation event', async () => { + const result = await scenario(packed, { docker: { stall: ['compose up'], stubborn: true } }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + const began = Date.now(); + while (!d.events.some(e => e.event === 'setup_sourcebot_completed')) { + assert.ok(Date.now() - began < 3000); + await new Promise(resolve => setTimeout(resolve, 20)); + } + d.interrupt(); + await d.finish(130); + }); + assert.deepEqual(names(result), funnel); + assert.equal(props(result, 'completed').sourcebotStartOutcome, 'spawned'); +}); + +for (const telemetry of ['reject', 'stall', 'reset']) { + test(`telemetry ${telemetry} does not change files or successful completion`, async () => { + const began = Date.now(); + const result = await scenario(packed, { telemetry }, async d => { await minimal(d); await d.answer('Download docker-compose.yml?', 'n'); }); + assert.equal(result.exitCode, 0); + assert.ok(Date.now() - began < 10000); + assert.deepEqual(Object.keys(result.files).sort(), ['.env', 'config.json']); + }); +} + +test('Ctrl+C at start prompt cancels before Docker can spawn', async () => { + const result = await scenario(packed, {}, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.wait('Start Sourcebot now?'); + d.interrupt(); + await d.finish(130); + }); + assert.equal(props(result, 'cancelled').stage, 'start'); + assert.equal(props(result, 'completed'), undefined); + assert.equal(result.dockerCalls.some(args => args.join(' ') === 'compose up'), false); +}); + +test('repeated Ctrl+C forces prompt exit despite a stalled SDK shutdown', async () => { + const result = await scenario(packed, { telemetry: 'stall' }, async d => { + await d.wait('What directory would you like'); + const began = Date.now(); + d.interrupt(); + await new Promise(resolve => setTimeout(resolve, 50)); + d.interrupt(); + await d.finish(130); + assert.ok(Date.now() - began < 1500, 'Second interrupt must not restart the shutdown deadline'); + }); + const attempted = result.requests.flatMap(request => request.batch); + assert.ok(attempted.filter(event => event.event === 'setup_sourcebot_cancelled').length <= 1); + assert.deepEqual(result.files, {}); +}); + +test('Ctrl+C after completion still cleans stubborn Docker while telemetry stalls', async () => { + await scenario(packed, { telemetry: 'stall', docker: { stall: ['compose up'], stubborn: true } }, async d => { + await minimal(d); + await d.answer('Download docker-compose.yml?', 'y'); + await d.answer('Start Sourcebot now?', 'y'); + await new Promise(resolve => setTimeout(resolve, 100)); + const began = Date.now(); + d.interrupt(); + await d.finish(130); + assert.ok(Date.now() - began < 3500); + }); +}); diff --git a/packages/setupWizard/tests/integration/sdk.test.mjs b/packages/setupWizard/tests/integration/sdk.test.mjs new file mode 100644 index 000000000..b60fd2a3b --- /dev/null +++ b/packages/setupWizard/tests/integration/sdk.test.mjs @@ -0,0 +1,47 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { createServer } from 'node:http'; +import { once } from 'node:events'; +import { gunzipSync } from 'node:zlib'; +import { PostHog } from 'posthog-node'; +import { Telemetry, POSTHOG_OPTIONS } from '../../dist/telemetry.js'; + +test('real SDK transport envelope keeps its permitted metadata and identity', async () => { + const requests = []; + const server = createServer(async (req, res) => { + const chunks = []; + for await (const chunk of req) { + chunks.push(chunk); + } + const bytes = Buffer.concat(chunks); + requests.push(JSON.parse((req.headers['content-encoding'] === 'gzip' ? gunzipSync(bytes) : bytes).toString())); + res.writeHead(200, { 'content-type': 'application/json' }).end('{"status":1}'); + }).listen(0, '127.0.0.1'); + await once(server, 'listening'); + try { + const telemetry = new Telemetry(() => new PostHog('phc_test', { ...POSTHOG_OPTIONS, host: `http://127.0.0.1:${server.address().port}` })); + telemetry.capture('started', { invocationMethod: 'unknown', isInteractive: true }); + telemetry.capture('cancelled', { stage: 'setup_directory', reason: 'keyboard_interrupt' }); + await telemetry.shutdown(); + const events = requests.flatMap(r => r.batch); + assert.equal(events.length, 2); + assert.ok(Date.parse(events[1].timestamp) > Date.parse(events[0].timestamp)); + for (const e of events) { + assert.equal(e.distinct_id, telemetry.setupSessionId); + assert.equal(e.properties.install_id, telemetry.setupSessionId); + assert.deepEqual(e.properties.$groups, { company: telemetry.setupSessionId }); + assert.equal(e.properties.$lib, 'posthog-node'); + assert.equal(e.properties.$lib_version, '5.52.1'); + assert.equal(e.properties.$geoip_disable, true); + assert.equal(e.properties.$ignore_sent_at, true); + assert.equal(e.properties.$is_server, undefined); + assert.equal(e.properties.$process_person_profile, undefined); + assert.equal(e.properties.$set, undefined); + assert.ok(!Number.isNaN(Date.parse(e.timestamp))); + assert.match(e.uuid, /^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/); + } + } finally { + server.closeAllConnections(); + await new Promise(resolve => server.close(resolve)); + } +}); diff --git a/packages/setupWizard/tests/schemaSnapshot.mjs b/packages/setupWizard/tests/schemaSnapshot.mjs new file mode 100644 index 000000000..cf8b1c145 --- /dev/null +++ b/packages/setupWizard/tests/schemaSnapshot.mjs @@ -0,0 +1,56 @@ +import ts from 'typescript'; +import { readFileSync } from 'node:fs'; + +// Extract the declared schema for comparison with an independently checked-in +// snapshot. Tests must not silently accept new properties just because the +// production allowlist also changed. +export function declaredSchema() { + const source = ts.createSourceFile('telemetryEvents.ts', readFileSync(new URL('../src/telemetryEvents.ts', import.meta.url), 'utf8'), ts.ScriptTarget.Latest, true); + const bindings = new Map(); + for (const statement of source.statements) { + if (ts.isVariableStatement(statement)) { + for (const declaration of statement.declarationList.declarations) { + bindings.set(declaration.name.getText(source), declaration.initializer); + } + } + } + function resolve(node) { + if (!node) { + throw Error('Unrecognized telemetry declaration'); + } + if (ts.isIdentifier(node)) { + return resolve(bindings.get(node.text)); + } + if (ts.isAsExpression(node) || ts.isParenthesizedExpression(node)) { + return resolve(node.expression); + } + if (ts.isObjectLiteralExpression(node)) { + return Object.assign({}, ...node.properties.map(property => { + if (ts.isSpreadAssignment(property)) { + return resolve(property.expression); + } + const name = property.name.getText(source); + return { [name]: resolve(ts.isShorthandPropertyAssignment(property) ? property.name : property.initializer) }; + })); + } + if (ts.isCallExpression(node)) { + const name = node.expression.getText(source); + if (name === 'choice') { + return { enum: node.arguments.map(value => { + if (!ts.isStringLiteral(value)) { + throw Error('Review nonliteral telemetry enum'); + } + return value.text; + }) }; + } + if (name === 'nullable' || name === 'array') { + return { [name]: resolve(node.arguments[0]) }; + } + if (name === 'rule') { + return { type: node.typeArguments?.[0]?.getText(source) ?? 'unknown' }; + } + } + throw Error(`Unrecognized telemetry schema expression: ${node.getText(source)}`); + } + return resolve(bindings.get('eventSchemas')); +} diff --git a/packages/setupWizard/tests/unit/bootstrap.test.mjs b/packages/setupWizard/tests/unit/bootstrap.test.mjs new file mode 100644 index 000000000..9e4f846e7 --- /dev/null +++ b/packages/setupWizard/tests/unit/bootstrap.test.mjs @@ -0,0 +1,29 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { spawnSync } from 'node:child_process'; +import { copyFileSync, mkdtempSync, rmSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; + +test('bootstrap admits dependency-compatible versions and rejects unsupported gaps before import', () => { + const root = mkdtempSync(join(tmpdir(), 'setup-bootstrap-')); + const bin = join(root, 'bin.cjs'); + // Deliberately omit dist: entering the supported branch must stop at its + // handled missing-module error, never run the wizard or send telemetry. + copyFileSync(new URL('../../bin.cjs', import.meta.url), bin); + try { + for (const [version, supported] of [ + ['18.20.8', false], ['20.19.9', false], ['20.20.0', true], ['20.21.0', true], + ['21.7.3', false], ['22.21.9', false], ['22.22.0', true], + ['23.4.9', false], ['23.5.0', true], ['24.0.0', true], ['26.0.0', true], + ]) { + const result = spawnSync(process.execPath, ['-e', `Object.defineProperty(process.versions, 'node', { value: ${JSON.stringify(version)} }); require(${JSON.stringify(bin)});`], { cwd: root, encoding: 'utf8', env: { HOME: root, PATH: '' }, timeout: 10000 }); + assert.equal(result.status, 1, version); + assert.equal(result.stdout, '', version); + assert.match(result.stderr, supported ? /Unable to start setup-sourcebot/ : /requires Node.js 20\.20\+/, version); + } + assert.deepEqual(readdirSync(root), ['bin.cjs']); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/packages/setupWizard/tests/unit/dockerStartFailure.test.mjs b/packages/setupWizard/tests/unit/dockerStartFailure.test.mjs new file mode 100644 index 000000000..d81264fa5 --- /dev/null +++ b/packages/setupWizard/tests/unit/dockerStartFailure.test.mjs @@ -0,0 +1,42 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { DockerStartFailure } from '../../dist/dockerStartFailure.js'; +import { Lifecycle } from '../../dist/lifecycle.js'; +import { eventSchemas, validateFields } from '../../dist/telemetryEvents.js'; + +test('Docker reasons are bounded, chunk-safe, ANSI-safe and never contain runtime text', () => { + const classifier = new DockerStartFailure(); + const message = '\u001b[31mError response from daemon: Conflict. The container name "/secret" is already in use by container "secret-id".\u001b[0m'; + for (const char of message) classifier.write(Buffer.from(char)); + assert.equal(classifier.reason(), 'container_name_conflict'); + const huge = new DockerStartFailure(); + huge.write(Buffer.from('secret'.repeat(100000))); + assert.ok(huge.pending.length <= 8192); + assert.equal(huge.reason(), 'unknown'); + const props = { failurePhase: 'compose_exit', failureCategory: 'docker_command', failureReason: 'unknown', stderr: message }; + assert.deepEqual(Object.keys(validateFields(eventSchemas.start_failed, props)).sort(), ['failureCategory', 'failurePhase', 'failureReason']); + assert.throws(() => validateFields(eventSchemas.start_failed, { ...props, failureReason: message })); +}); + +test('only one start diagnostic may follow completion; SDK stays open until process cleanup', async () => { + const events = []; + let shutdowns = 0; + const life = new Lifecycle({ capture: (name, props) => events.push({ name, props }), shutdown: async () => { shutdowns++; } }); + await life.complete({}, true); + assert.equal(shutdowns, 0); + assert.equal(life.terminal, 'completed'); + const failure = { failurePhase: 'compose_exit', failureCategory: 'docker_command', failureReason: 'unknown' }; + life.startFailed(failure); + life.startFailed(failure); + life.fail('docker_command', false); + life.capture('started', {}); + assert.deepEqual(events.map(e => e.name), ['completed', 'start_failed']); + await life.telemetry.shutdown(); + assert.equal(shutdowns, 1); + for (const terminal of [undefined, 'completed', 'cancelled', 'failed']) { + const suppressed = new Lifecycle({ capture() { assert.fail('Unexpected event'); } }); + suppressed.terminal = terminal; + suppressed.interrupted = true; + suppressed.startFailed(failure); + } +}); diff --git a/packages/setupWizard/tests/unit/schema.test.mjs b/packages/setupWizard/tests/unit/schema.test.mjs new file mode 100644 index 000000000..8009758a2 --- /dev/null +++ b/packages/setupWizard/tests/unit/schema.test.mjs @@ -0,0 +1,9 @@ +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { test } from 'node:test'; +import { declaredSchema } from '../schemaSnapshot.mjs'; + +test('telemetry fields and enum values match the reviewed independent schema snapshot', () => { + const approved = JSON.parse(readFileSync(new URL('../approvedSchema.json', import.meta.url), 'utf8')); + assert.deepEqual(declaredSchema(), approved, 'Review telemetry plan and update the approved snapshot when changing the wire contract'); +}); diff --git a/packages/setupWizard/tests/unit/telemetry.test.mjs b/packages/setupWizard/tests/unit/telemetry.test.mjs new file mode 100644 index 000000000..611ab71c7 --- /dev/null +++ b/packages/setupWizard/tests/unit/telemetry.test.mjs @@ -0,0 +1,165 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { randomUUID } from 'node:crypto'; +import { Telemetry, INSTALL_ID_PATTERN, POSTHOG_OPTIONS, systemProperties } from '../../dist/telemetry.js'; +import { eventSchemas, validateFields } from '../../dist/telemetryEvents.js'; +import { sourceSummary, aggregateSources, aggregateAi, deployment, hostCategory, selectInstallId, emptyDockerSummary, dockerOutcome, discoveredBucket } from '../../dist/telemetrySummary.js'; +import { Lifecycle } from '../../dist/lifecycle.js'; + +test('UUID contract and persisted identity precedence', () => { + for (let i = 0; i < 100; i++) { + const id = new Telemetry(() => { throw Error(); }).setupSessionId; + assert.match(id, INSTALL_ID_PATTERN); + assert.equal(id.length, 36); + } + const old = randomUUID(); + const next = randomUUID(); + for (const value of [old, `"${old}"`, `'${old}'`, `${old} # comment`]) { + assert.deepEqual(selectInstallId(`SOURCEBOT_INSTALL_ID=${value}`, next), { id: old, action: 'preserved_existing' }); + } + for (const value of ['', old.toUpperCase(), 'not-an-id', '$(danger)', `${old}\nSOURCEBOT_INSTALL_ID=invalid`]) { + assert.deepEqual(selectInstallId(`SOURCEBOT_INSTALL_ID=${value}`, next), { id: next, action: 'created_from_setup_session' }); + } +}); + +test('code host normalization matches boundaries and formatting, not arbitrary suffixes', () => { + for (const input of ['github.com', ' HTTPS://WWW.GITHUB.COM./path ', 'http://github.com:8080']) { + assert.equal(deployment('github', input), 'cloud'); + } + assert.equal(deployment('github', 'tenant.ghe.com'), 'cloud'); + assert.equal(deployment('github', 'github.com.attacker.invalid'), 'self_hosted'); + assert.equal(deployment('gitlab', 'gitlab.com'), 'cloud'); + assert.equal(deployment('gitlab', 'tenant.gitlab-dedicated.com'), 'cloud'); + assert.equal(deployment('gitlab', 'internal.example'), 'unknown'); + assert.equal(deployment('gitea', 'www.gitea.com'), 'cloud'); + assert.equal(deployment('gitea', 'internal.example'), 'self_hosted'); + for (const input of ['', '://', 'ssh://github.com']) { + assert.equal(deployment('github', input), 'unknown'); + } +}); + +test('host classification never performs DNS or exposes an address', () => { + for (const input of ['http://localhost', 'https://LOCALHOST.:3000', 'http://x.localhost', 'http://127.0.0.0', 'http://127.9.8.7', 'http://[::1]']) { + assert.equal(hostCategory(input), 'localhost'); + } + for (const input of ['http://192.168.0.1', 'https://secret.example', 'http://localhost.attacker.invalid', 'http://[::2]']) { + assert.equal(hostCategory(input), 'address'); + } + assert.equal(hostCategory('not a URL'), 'unknown'); +}); + +test('aggregates reflect selections, counts, unique providers, and local wildcard collapse', () => { + const aggregate = aggregateSources([ + sourceSummary('github', { deploymentType: 'cloud', repositoryCount: 2, credentialMode: 'personal_access_token' }), + sourceSummary('github', { deploymentType: 'self_hosted', organizationCount: 1, indexAll: true }), + sourceSummary('local_git', { deploymentType: 'local', repositoryCount: 9, generatedConnectionCount: 3 }), + ]); + assert.equal(aggregate.repositoryCount, 11); + assert.equal(aggregate.generatedConnectionCount, 5); + assert.equal(aggregate.codeSourceConfigurationCount, 3); + assert.deepEqual(aggregate.codeHostTypes, ['github', 'local_git']); + assert.equal(aggregate.credentialedCodeSourceCount, 1); + assert.equal(aggregate.indexAllCodeSourceCount, 1); + assert.deepEqual([1, 2, 5, 6, 20, 21, 100, 101].map(discoveredBucket), ['1', '2-5', '2-5', '6-20', '6-20', '21-100', '21-100', '101+']); + assert.deepEqual(aggregateAi([]), { aiConfigured: false, aiConfigurationCount: 0, uniqueProviderCount: 0, providerTypes: [], usesCustomEndpoint: false, credentialModes: [], modelSelectionMethods: [] }); + const model = { provider: 'openai', credentialMode: 'api_key', modelSelectionMethod: 'catalog', usesCustomEndpoint: false, hasDisplayName: true }; + assert.equal(aggregateAi([model, model]).uniqueProviderCount, 1); + assert.equal(aggregateAi([model, model]).aiConfigurationCount, 2); +}); + +test('Docker skipped/failed measurements are null and outcome precedence is explicit', () => { + const state = emptyDockerSummary(); + for (const key of Object.keys(state).filter(k => k.endsWith('Count'))) { + assert.equal(state[key], null); + } + assert.equal(dockerOutcome(state, false, false), 'skipped_no_compose'); + assert.equal(dockerOutcome(state, true, true), 'validation_failed'); + state.leftExistingDeploymentRunning = true; + assert.equal(dockerOutcome(state, true, false), 'skipped_existing_deployment_running'); + assert.equal(dockerOutcome(state, true, true), 'validation_failed'); + state.leftExistingDeploymentRunning = false; + state.remainingPortConflictCount = 2; + assert.equal(dockerOutcome(state, true, false), 'unresolved_conflicts'); + state.remainingPortConflictCount = 0; + state.volumeAction = 'removed'; + assert.equal(dockerOutcome(state, true, false), 'passed_after_cleanup'); +}); + +test('allowlists reject wrong types and strip arbitrary user values', () => { + const fields = { configurationIndex: 1, ...sourceSummary('github'), token: 'CANARY_SECRET', repository: 'CANARY_REPO' }; + const output = validateFields(eventSchemas.configured_code_source, fields); + assert.equal(JSON.stringify(output).includes('CANARY'), false); + for (const change of [{ configurationIndex: NaN }, { repositoryCount: -1 }, { codeHost: 'CANARY_HOST' }, { scopeTypes: ['CANARY_SCOPE'] }, { indexAll: 'true' }]) { + assert.throws(() => validateFields(eventSchemas.configured_code_source, { ...fields, ...change })); + } +}); + +test('telemetry is fail-open, uses exact identity, default SDK transport and no opt-out coupling', async () => { + const sent = []; + const client = { capture: e => sent.push(e), on() {}, async shutdown() {} }; + const previous = process.env.SOURCEBOT_TELEMETRY_DISABLED; + process.env.SOURCEBOT_TELEMETRY_DISABLED = 'true'; + try { + const telemetry = new Telemetry(() => client); + telemetry.capture('started', { invocationMethod: 'unknown', isInteractive: true }); + assert.equal(sent.length, 1); + const event = sent[0]; + assert.equal(event.distinctId, telemetry.setupSessionId); + assert.equal(event.properties.install_id, event.distinctId); + assert.deepEqual(event.groups, { company: event.distinctId }); + assert.equal(event.properties.$geoip_disable, true); + assert.equal(event.properties.$process_person_profile, undefined); + assert.ok(event.properties.elapsedMs >= 0); + await telemetry.shutdown(); + telemetry.capture('started', { invocationMethod: 'unknown', isInteractive: true }); + assert.equal(sent.length, 1); + for (const broken of [() => { throw Error(); }, () => ({ ...client, capture() { throw Error(); }, shutdown() { throw Error(); } })]) { + const t = new Telemetry(broken); + assert.doesNotThrow(() => t.capture('started', { invocationMethod: 'unknown', isInteractive: true })); + await t.shutdown(); + } + assert.deepEqual(POSTHOG_OPTIONS, { host: 'https://us.i.posthog.com', flushAt: 1, flushInterval: 0, disableGeoip: true, isServer: false }); + assert.ok(['npm', 'yarn', 'pnpm', 'bun', 'unknown'].includes(systemProperties().packageManager)); + } finally { + if (previous === undefined) { + delete process.env.SOURCEBOT_TELEMETRY_DISABLED; + } else { + process.env.SOURCEBOT_TELEMETRY_DISABLED = previous; + } + } +}); + +test('terminal latch is independent of multiple recoverable failures', async () => { + const events = []; + const t = { capture: (name, props) => events.push({ name, props }), shutdown: async () => {} }; + const life = new Lifecycle(t); + life.fail('docker_command', true); + life.fail('docker_unavailable', true); + await life.complete({}); + await life.decline('keyboard_interrupt'); + life.fail('unknown', false); + assert.deepEqual(events.map(e => e.name), ['failed', 'failed', 'completed']); + assert.equal(life.terminal, 'completed'); + const fatal = new Lifecycle(t); + fatal.fail('filesystem', false); + await fatal.complete({}); + assert.equal(events.at(-1).name, 'failed'); + assert.equal(events.at(-1).props.recoverable, false); +}); + +test('each system property falls back independently and elapsed time is monotonic', () => { + const runtime = { platform: 'linux', arch: 'arm64', versions: { node: '24.1.0' }, env: { npm_config_user_agent: 'pnpm/10.0', CI: 'true' } }; + assert.deepEqual(systemProperties(runtime), { platform: 'linux', arch: 'arm64', nodeMajorVersion: 24, packageManager: 'pnpm', isCI: true }); + for (const [key, field, fallback] of [['platform', 'platform', 'other'], ['arch', 'arch', 'other'], ['versions', 'nodeMajorVersion', null], ['env', 'packageManager', 'unknown']]) { + const broken = { ...runtime }; + Object.defineProperty(broken, key, { get() { throw Error('fixture'); } }); + assert.equal(systemProperties(broken)[field], fallback); + } + const brokenEnv = { ...runtime, get env() { throw Error(); } }; + assert.equal(systemProperties(brokenEnv).isCI, null); + assert.equal(systemProperties({ ...runtime, versions: { node: 'invalid' } }).nodeMajorVersion, null); + let now = 100; + const t = new Telemetry(() => { throw Error(); }, randomUUID, () => now); + now = 123.6; + assert.equal(t.elapsed(), 24); +}); diff --git a/yarn.lock b/yarn.lock index c07ff3465..16aa3ddd3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4849,6 +4849,15 @@ __metadata: languageName: node linkType: hard +"@posthog/core@npm:^1.53.1": + version: 1.53.2 + resolution: "@posthog/core@npm:1.53.2" + dependencies: + "@posthog/types": "npm:^1.411.1" + checksum: 10c0/63248d839b140887dd7d4af81096909f9920a566f09bec945806c3ece7a8d047c3644569cee8962f100decf6b60a42188016319dd11ec87458eddbf7bcb5fd3a + languageName: node + linkType: hard + "@posthog/types@npm:1.369.0": version: 1.369.0 resolution: "@posthog/types@npm:1.369.0" @@ -4870,6 +4879,13 @@ __metadata: languageName: node linkType: hard +"@posthog/types@npm:^1.411.1": + version: 1.411.1 + resolution: "@posthog/types@npm:1.411.1" + checksum: 10c0/fd4cf6bd38abea89ff37fc2af8f329373ba6da8bfc99a1743f69f665428d43bcf6be5c3ef01196292e7b14f928df0b0de64ec3e401576ca15632c21b3b1b1703 + languageName: node + linkType: hard + "@preact/signals-core@npm:^1.7.0": version: 1.14.0 resolution: "@preact/signals-core@npm:1.14.0" @@ -10020,6 +10036,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:^20.19.43": + version: 20.19.43 + resolution: "@types/node@npm:20.19.43" + dependencies: + undici-types: "npm:~6.21.0" + checksum: 10c0/9bcec3b5295bdd77ff0b44a528a69f7e22028c347507ba2c69be47ec84e30299f45043b222e9c86c510e138c9c53b2419dd5cd34920602a4a5a381c288075318 + languageName: node + linkType: hard + "@types/nodemailer@npm:^6.4.17": version: 6.4.17 resolution: "@types/nodemailer@npm:6.4.17" @@ -18473,6 +18498,15 @@ __metadata: languageName: node linkType: hard +"node-addon-api@npm:^7.1.0": + version: 7.1.1 + resolution: "node-addon-api@npm:7.1.1" + dependencies: + node-gyp: "npm:latest" + checksum: 10c0/fb32a206276d608037fa1bcd7e9921e177fe992fc610d098aa3128baca3c0050fc1e014fa007e9b3874cf865ddb4f5bd9f43ccb7cbbbe4efaff6a83e920b17e9 + languageName: node + linkType: hard + "node-cleanup@npm:^2.1.2": version: 2.1.2 resolution: "node-cleanup@npm:2.1.2" @@ -18545,6 +18579,16 @@ __metadata: languageName: node linkType: hard +"node-pty@npm:^1.1.0": + version: 1.1.0 + resolution: "node-pty@npm:1.1.0" + dependencies: + node-addon-api: "npm:^7.1.0" + node-gyp: "npm:latest" + checksum: 10c0/e527305263da36d554b7d2116aee6430b898675fc50e7bd82fbd762f14cb6b948fb8d997736af1f70d76bf9ffd4f5179bc1ed1d8002567dfcf3b29f039291798 + languageName: node + linkType: hard + "node-releases@npm:^2.0.53": version: 2.0.54 resolution: "node-releases@npm:2.0.54" @@ -19516,6 +19560,20 @@ __metadata: languageName: node linkType: hard +"posthog-node@npm:5.52.1": + version: 5.52.1 + resolution: "posthog-node@npm:5.52.1" + dependencies: + "@posthog/core": "npm:^1.53.1" + peerDependencies: + rxjs: ^7.0.0 + peerDependenciesMeta: + rxjs: + optional: true + checksum: 10c0/dfb7f71792156a817aaddf478e844f6d91f3f53c1f91f8aa496a1a4f30b414dfee40baec3cad35570ca383ef6e0a5ec316e59f687fa303960dedd2aeb2d09218 + languageName: node + linkType: hard + "posthog-node@npm:^5.51.2": version: 5.51.2 resolution: "posthog-node@npm:5.51.2" @@ -21240,15 +21298,18 @@ __metadata: dependencies: "@inquirer/prompts": "npm:^8.4.3" "@sourcebot/schemas": "workspace:^" - "@types/node": "npm:^22.7.5" + "@types/node": "npm:^20.19.43" chalk: "npm:^5.6.2" inquirer-select-pro: "npm:^1.0.0-alpha.9" + node-pty: "npm:^1.1.0" ora: "npm:^9.4.0" + posthog-node: "npm:5.52.1" reo-census: "npm:^1.2.10" tsx: "npm:^4.21.0" typescript: "npm:^5.6.2" + undici: "npm:^7" bin: - setup-sourcebot: ./dist/index.js + setup-sourcebot: ./bin.cjs languageName: unknown linkType: soft @@ -22971,6 +23032,13 @@ __metadata: languageName: node linkType: hard +"undici@npm:^7": + version: 7.29.1 + resolution: "undici@npm:7.29.1" + checksum: 10c0/5a419b1364531071ebdfd93748f7f2392d4eb6d68dd99814e1020f8fc6a517e3298129004c871516ee204bac68aba93541f587200b04a7449b8d45683a7d86f6 + languageName: node + linkType: hard + "unified@npm:^11.0.0": version: 11.0.5 resolution: "unified@npm:11.0.5"