From 75feb75f30901d44f8c27a8b5e5900d761eb3697 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Thu, 6 Aug 2026 15:55:40 +0200 Subject: [PATCH 1/3] feat: add pypi ecosystem to blast Signed-off-by: Umberto Sgueglia --- .../src/api/public/v1/packages/blastRadius.ts | 1 + .../__tests__/ecosystemSupport.test.ts | 12 +- .../src/blast-radius/agent/pypiPrompts.ts | 125 +++++++++++ .../clients/__tests__/pypiSource.test.ts | 178 +++++++++++++++ .../src/blast-radius/clients/pypiSource.ts | 208 ++++++++++++++++++ .../src/blast-radius/ecosystemSupport.ts | 10 +- .../src/blast-radius/packageIdentifier.ts | 27 +++ .../stages/__tests__/dispatch.test.ts | 38 ++++ .../src/blast-radius/stages/ecosystems.ts | 8 + .../pypi/__tests__/pypiConstraint.test.ts | 70 ++++++ .../stages/pypi/dependentsPyPi.ts | 115 ++++++++++ .../stages/pypi/dependentsScanPyPi.ts | 69 ++++++ .../src/blast-radius/stages/pypi/intelPyPi.ts | 187 ++++++++++++++++ .../stages/pypi/pypiConstraint.ts | 137 ++++++++++++ .../stages/pypi/reachabilityConfig.ts | 26 +++ .../src/osv/__tests__/versionCompare.test.ts | 28 ++- .../packages_worker/src/osv/versionCompare.ts | 154 +++++++++++++ .../apps/packages_worker/src/pypi/types.ts | 10 + .../data-access-layer/src/packages/osv.ts | 6 + 19 files changed, 1404 insertions(+), 5 deletions(-) create mode 100644 services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts create mode 100644 services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts create mode 100644 services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts create mode 100644 services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts create mode 100644 services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsPyPi.ts create mode 100644 services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsScanPyPi.ts create mode 100644 services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts create mode 100644 services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts create mode 100644 services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts diff --git a/backend/src/api/public/v1/packages/blastRadius.ts b/backend/src/api/public/v1/packages/blastRadius.ts index 6cc5528f45..b4b78f0c94 100644 --- a/backend/src/api/public/v1/packages/blastRadius.ts +++ b/backend/src/api/public/v1/packages/blastRadius.ts @@ -7,6 +7,7 @@ export const SUPPORTED_BLAST_RADIUS_ECOSYSTEMS = [ 'cargo', 'nuget', 'rubygems', + 'pypi', ] as const // Always exactly one job per request — advisory-wide (package omitted) or narrowed diff --git a/services/apps/packages_worker/src/blast-radius/__tests__/ecosystemSupport.test.ts b/services/apps/packages_worker/src/blast-radius/__tests__/ecosystemSupport.test.ts index 491727896d..886403e2bd 100644 --- a/services/apps/packages_worker/src/blast-radius/__tests__/ecosystemSupport.test.ts +++ b/services/apps/packages_worker/src/blast-radius/__tests__/ecosystemSupport.test.ts @@ -4,8 +4,16 @@ import { describe, expect, it } from 'vitest' import { SUPPORTED_ECOSYSTEMS, buildEcosystemNotSupportedFailure } from '../ecosystemSupport' describe('SUPPORTED_ECOSYSTEMS', () => { - it('includes cargo, nuget, and rubygems alongside npm, go, and maven', () => { - expect(SUPPORTED_ECOSYSTEMS).toEqual(['npm', 'go', 'maven', 'cargo', 'nuget', 'rubygems']) + it('includes cargo, nuget, rubygems, and pypi alongside npm, go, and maven', () => { + expect(SUPPORTED_ECOSYSTEMS).toEqual([ + 'npm', + 'go', + 'maven', + 'cargo', + 'nuget', + 'rubygems', + 'pypi', + ]) }) }) diff --git a/services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts b/services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts new file mode 100644 index 0000000000..bcc20a465d --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts @@ -0,0 +1,125 @@ +// Parallels cargoPrompts.ts — schema shape and the intel prompt builder are shared via +// promptKit.ts; only the Python-specific keys/enum and system-prompt prose live here. +import { + buildIntelPrompt, + buildIntelSchema, + buildReachabilitySymbolsBlock, + buildVerdictSchema, +} from './promptKit' +import { SymbolSpec } from './prompts' + +// ---------- STAGE 1: INTEL ---------- + +const IMPORT_SIGNATURE_KEYS = ['import_module', 'from_import', 'attribute_access', 'dynamic_import'] + +export const PYPI_INTEL_SCHEMA = buildIntelSchema(IMPORT_SIGNATURE_KEYS) + +export const PYPI_INTEL_SYSTEM_PROMPT = `You are a vulnerability analyst. Your working directory contains the FULL SOURCE of the +vulnerable version of a Python package. You are given the security advisory and the patch +(diff) that fixed the vulnerability. + +Your job is to determine, precisely, WHAT is vulnerable — so that downstream analysts can +check whether other packages actually reach the vulnerable code. + +Rules: +- Identify the exact vulnerable function(s)/method(s)/class(es) from the patch and the + source. Be minimal and precise: do NOT include similar-but-unaffected symbols. If the + patch only touches a private (\`_\`-prefixed) helper, trace which public symbols route + through it and list those as the reachable surface (note the helper in \`notes\`). +- Python has no compile-time visibility — determine what's actually importable by a + dependent: check \`__all__\` in the defining module's \`__init__.py\` (if present, only + names listed there are the package's public API), the \`_\`-prefix convention for private + names, and whether the symbol is re-exported through a package's \`__init__.py\` (which + counts as \`reexport\`). Note the exact module path each symbol lives in (e.g. + \`package.submodule.Symbol\`). +- Build \`import_signatures\`: concrete code patterns a dependent package would contain if + it uses the vulnerable symbol. Cover: \`import package.module\` followed by fully-qualified + attribute access, \`from package.module import Symbol\` followed by bare \`Symbol\` usage, + attribute access on an imported module/class instance, and dynamic import via + \`importlib.import_module(...)\` or \`__import__(...)\`. These are the patterns analysts will + grep for — make them literal and greppable, not prose. +- \`reachability_notes\` must state what does NOT count (e.g. sibling functions that look + similar but are not affected, usage confined to \`tests/\`, \`test/\`, \`docs/\`, or + \`examples/\`) and any conditions required for exploitability (e.g. an optional extra must + be installed, or a specific argument/config must be set). +- Set \`confidence\` for your identification: 0.9+ only if the patch unambiguously + identifies the symbol(s); lower if you had to infer from indirect evidence.` + +export const buildPyPiIntelPrompt = buildIntelPrompt + +// ---------- STAGE 3: REACHABILITY ---------- + +const IMPORT_STYLE_ENUM = [ + 'import-module', + 'from-import', + 'attribute-access', + 'dynamic-import', + 'reexport', + 'none', +] + +export const PYPI_VERDICT_SCHEMA = buildVerdictSchema(IMPORT_STYLE_ENUM) + +export function buildPyPiReachabilitySystemPrompt(spec: SymbolSpec): string { + const { symbolsText, signatures } = buildReachabilitySymbolsBlock(spec) + + return `You are a security reachability analyst. Your working directory contains the published +source of ONE Python package (the "dependent") that declares a dependency on +\`${spec.package}\`, which has a known vulnerability (${spec.vuln_id}). + +## The vulnerability +${spec.summary} + +Vulnerable symbol(s) in \`${spec.package}\`: +${symbolsText} + +Exploit preconditions: ${spec.exploit_preconditions} + +Analyst notes: ${spec.reachability_notes} + +## Import signatures to look for +${signatures} + +## Your task +Decide whether THIS dependent's own code actually reaches the vulnerable symbol(s). + +Scope rules — follow strictly: +1. Only the dependent's OWN shipped code counts (its package source, not vendored/third- + party code bundled inside it). Usage of the vulnerable symbol inside the dependent's + OTHER dependencies (its own \`pyproject.toml\`/\`setup.py\`/\`requirements*.txt\` deps) is + OUT OF SCOPE (that is second-level analysis, done separately). +2. Merely declaring a dependency on \`${spec.package}\` (present in \`pyproject.toml\`, + \`setup.py\`, or a \`requirements*.txt\`) is NOT enough — the vulnerable symbol itself must + be reached. Uses of other items from the package are irrelevant. +3. Usage only in \`tests/\`, \`test/\`, \`docs/\`, or \`examples/\` that is not part of the + shipped runtime code → \`not_affected\` (explain in reasoning). +4. If the dependent RE-EXPORTS the vulnerable symbol to its own consumers (via its + \`__init__.py\`, or a thin wrapper function/class that passes arguments through), that + DOES count as \`affected\` with \`import_style: "reexport"\` — it propagates the + vulnerable surface. +5. Watch for indirect reachability inside the dependent's own code: fully-qualified + attribute access (\`package.module.Symbol\`), subclassing the vulnerable class, and + dynamic import via \`importlib.import_module\` or \`__import__\`. +6. \`import_style\` describes how the VULNERABLE SYMBOL is reached, not how the package is + declared: report \`none\` whenever the vulnerable symbol itself is not reached, even if + the package is a dependency for other functionality. + +Method: grep for the import signatures (and the bare symbol names) across the source, open +every hit, and trace whether the symbol is actually invoked. Check \`pyproject.toml\`, +\`setup.py\`, or \`requirements*.txt\` to confirm the declared dependency, its version +requirement, and whether an optional extra gates the vulnerable code path. Exclude +\`tests/\`, \`test/\`, \`docs/\`, and \`examples/\` from consideration. + +## Confidence calibration +- 0.8–1.0: direct evidence — you found (or ruled out) the import AND the call site + explicitly; source was readable. +- 0.4–0.8: symbol is imported but the call path is ambiguous (dynamic dispatch, conditional + imports, generated code). +- <0.4 and/or \`unclear\`: source is generated/absent, or indirection you could not resolve. + +Report evidence as exact file paths, line numbers, and short verbatim snippets.` +} + +export const PYPI_REACHABILITY_PROMPT = + 'Analyze this package per your instructions and produce the structured verdict. ' + + 'Start by listing the package structure and grepping for the import signatures.' diff --git a/services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts b/services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts new file mode 100644 index 0000000000..5157402d05 --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts @@ -0,0 +1,178 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' +import { Readable } from 'stream' +import * as tar from 'tar' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { PypiSourceNotFoundError, downloadAndExtractPypiSource } from '../pypiSource' + +import { buildStoredZip } from './zipFixture' + +vi.mock('../downloadLimits', async (importOriginal) => { + const actual = await importOriginal() + return { ...actual, MAX_EXTRACTED_FILES: 5 } +}) + +async function buildFixtureSdist( + name: string, + version: string, + files: Record, +): Promise { + const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pypisdist-')) + try { + const wrapperDir = `${name}-${version}` + for (const [relPath, content] of Object.entries(files)) { + const full = path.join(workDir, wrapperDir, relPath) + fs.mkdirSync(path.dirname(full), { recursive: true }) + fs.writeFileSync(full, content) + } + + const tarPath = path.join(workDir, 'fixture.tar.gz') + await tar.create({ gzip: true, cwd: workDir, file: tarPath }, [wrapperDir]) + return fs.readFileSync(tarPath) + } finally { + fs.rmSync(workDir, { recursive: true, force: true }) + } +} + +interface FixtureResponse { + status?: number + ok?: boolean + body?: Buffer | null + json?: unknown +} + +function mockFetchSequence(responses: FixtureResponse[]): void { + let call = 0 + vi.stubGlobal( + 'fetch', + vi.fn().mockImplementation(() => { + const r = responses[Math.min(call, responses.length - 1)] + call++ + return Promise.resolve({ + status: r.status ?? 200, + ok: r.ok ?? true, + statusText: 'OK', + json: () => Promise.resolve(r.json), + body: r.body ? Readable.toWeb(Readable.from(r.body)) : null, + }) + }), + ) +} + +describe('downloadAndExtractPypiSource', () => { + let destDir: string + + beforeEach(() => { + destDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pypidest-')) + }) + + afterEach(() => { + fs.rmSync(destDir, { recursive: true, force: true }) + vi.unstubAllGlobals() + }) + + it('prefers sdist over wheel when both are present', async () => { + const sdistBuf = await buildFixtureSdist('flask', '3.0.0', { 'src/flask/__init__.py': 'x=1' }) + mockFetchSequence([ + { + json: { + info: { name: 'flask' }, + urls: [ + { + packagetype: 'bdist_wheel', + url: 'https://files.pythonhosted.org/wheel.whl', + filename: 'flask-3.0.0-py3-none-any.whl', + }, + { + packagetype: 'sdist', + url: 'https://files.pythonhosted.org/sdist.tar.gz', + filename: 'flask-3.0.0.tar.gz', + }, + ], + }, + }, + { body: sdistBuf }, + ]) + + await downloadAndExtractPypiSource('flask', '3.0.0', destDir) + + expect(fs.readFileSync(path.join(destDir, 'src/flask/__init__.py'), 'utf8')).toBe('x=1') + expect(fetch).toHaveBeenNthCalledWith( + 2, + 'https://files.pythonhosted.org/sdist.tar.gz', + expect.anything(), + ) + }) + + it('falls back to a wheel when no sdist is present', async () => { + const wheelBuf = buildStoredZip([ + { path: 'flask/__init__.py', content: 'x=1' }, + { path: 'flask-3.0.0.dist-info/METADATA', content: 'Metadata-Version: 2.1' }, + ]) + mockFetchSequence([ + { + json: { + info: { name: 'flask' }, + urls: [ + { + packagetype: 'bdist_wheel', + url: 'https://files.pythonhosted.org/wheel.whl', + filename: 'flask-3.0.0-py3-none-any.whl', + }, + ], + }, + }, + { body: wheelBuf }, + ]) + + await downloadAndExtractPypiSource('flask', '3.0.0', destDir) + + expect(fs.readFileSync(path.join(destDir, 'flask/__init__.py'), 'utf8')).toBe('x=1') + expect(fs.readFileSync(path.join(destDir, 'flask-3.0.0.dist-info/METADATA'), 'utf8')).toBe( + 'Metadata-Version: 2.1', + ) + }) + + it('throws PypiSourceNotFoundError when neither sdist nor wheel is present', async () => { + mockFetchSequence([{ json: { info: { name: 'flask' }, urls: [] } }]) + + await expect(downloadAndExtractPypiSource('flask', '3.0.0', destDir)).rejects.toThrow( + PypiSourceNotFoundError, + ) + }) + + it('throws PypiSourceNotFoundError on a 404 for the version metadata', async () => { + mockFetchSequence([{ status: 404, ok: false }]) + + await expect(downloadAndExtractPypiSource('flask', '999.999.999', destDir)).rejects.toThrow( + PypiSourceNotFoundError, + ) + }) + + it('aborts sdist extraction when the file count exceeds the limit', async () => { + const files: Record = {} + for (let i = 0; i < 7; i++) files[`src/file${i}.py`] = `# ${i}` + const sdistBuf = await buildFixtureSdist('bloated', '1.0.0', files) + mockFetchSequence([ + { + json: { + info: { name: 'bloated' }, + urls: [ + { + packagetype: 'sdist', + url: 'https://files.pythonhosted.org/bloated.tar.gz', + filename: 'bloated-1.0.0.tar.gz', + }, + ], + }, + }, + { body: sdistBuf }, + ]) + + await expect(downloadAndExtractPypiSource('bloated', '1.0.0', destDir)).rejects.toThrow( + 'sdist exceeded size/file limits', + ) + }) +}) diff --git a/services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts b/services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts new file mode 100644 index 0000000000..58c3ed9d66 --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts @@ -0,0 +1,208 @@ +import { createWriteStream, mkdirSync, rmSync } from 'fs' +import * as path from 'path' +import { Readable } from 'stream' +import { pipeline } from 'stream/promises' +import type { ReadableStream as NodeWebReadableStream } from 'stream/web' +import * as tar from 'tar' +import unzipper from 'unzipper' + +import type { PyPiProject, PyPiUrlInfo } from '../../pypi/types' + +import { + FETCH_TIMEOUT_MS, + MAX_EXTRACTED_BYTES, + MAX_EXTRACTED_FILES, + createDownloadLimiter, +} from './downloadLimits' + +// Thrown when no downloadable sdist/wheel exists for this name/version — the reachability +// stage turns this into a clean "no source" verdict rather than a retry. +export class PypiSourceNotFoundError extends Error { + constructor(packageName: string, version: string) { + super(`No downloadable sdist or wheel for ${packageName}@${version}`) + this.name = 'PypiSourceNotFoundError' + } +} + +async function fetchVersionProject(name: string, version: string): Promise { + const url = `https://pypi.org/pypi/${encodeURIComponent(name)}/${encodeURIComponent(version)}/json` + const controller = new AbortController() + const timeoutHandle = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS) + + try { + let res: Response + try { + res = await fetch(url, { signal: controller.signal }) + } catch { + throw new PypiSourceNotFoundError(name, version) + } + if (res.status === 404) { + throw new PypiSourceNotFoundError(name, version) + } + if (!res.ok) { + throw new Error( + `Failed to fetch PyPI project ${name}@${version}: ${res.status} ${res.statusText}`, + ) + } + return (await res.json()) as PyPiProject + } finally { + clearTimeout(timeoutHandle) + } +} + +function selectDistribution(urls: PyPiUrlInfo[] | undefined): PyPiUrlInfo | null { + if (!urls || urls.length === 0) return null + const sdist = urls.find((u) => u.packagetype === 'sdist') + if (sdist) return sdist + return urls.find((u) => u.packagetype === 'bdist_wheel') ?? null +} + +// A sdist tarball wraps a single "name-version/" directory — strip:1 drops it so +// destDir ends up holding the package contents directly. +async function downloadSdist( + url: string, + destDir: string, + packageName: string, + version: string, +): Promise { + const controller = new AbortController() + const timeoutHandle = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS) + + try { + let res: Response + try { + res = await fetch(url, { signal: controller.signal }) + } catch (e) { + throw new Error( + `Failed to fetch sdist for ${packageName}@${version}: ${(e as Error).message}`, + ) + } + if (!res.ok || !res.body) { + throw new Error( + `Failed to fetch sdist for ${packageName}@${version}: ${res.status} ${res.statusText}`, + ) + } + + let extractedFiles = 0 + let extractedBytes = 0 + const extract = tar.extract({ + cwd: destDir, + strip: 1, + strict: true, + onentry: (entry) => { + extractedFiles++ + extractedBytes += entry.size ?? 0 + if (extractedFiles > MAX_EXTRACTED_FILES || extractedBytes > MAX_EXTRACTED_BYTES) { + extract.abort(new Error('sdist exceeded size/file limits')) + } + }, + }) + + await new Promise((resolve, reject) => { + Readable.fromWeb(res.body as unknown as NodeWebReadableStream) + .on('error', reject) + .pipe(createDownloadLimiter('sdist download exceeded size limit')) + .on('error', reject) + .pipe(extract as unknown as NodeJS.WritableStream) + .on('finish', resolve) + .on('error', reject) + }) + } finally { + clearTimeout(timeoutHandle) + } +} + +// A .whl is a zip with no wrapper directory. Zip's central directory sits at the end of +// the file, so unlike tar we can't stream-extract incrementally — download to a scratch +// file first, then extract, same as goModuleZip.ts. +async function downloadWheel( + url: string, + destDir: string, + packageName: string, + version: string, +): Promise { + const controller = new AbortController() + const timeoutHandle = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS) + const zipPath = `${destDir}.whl` + + try { + let res: Response + try { + res = await fetch(url, { signal: controller.signal }) + } catch (e) { + throw new Error( + `Failed to fetch wheel for ${packageName}@${version}: ${(e as Error).message}`, + ) + } + if (!res.ok || !res.body) { + throw new Error( + `Failed to fetch wheel for ${packageName}@${version}: ${res.status} ${res.statusText}`, + ) + } + + await pipeline( + Readable.fromWeb(res.body as unknown as NodeWebReadableStream), + createDownloadLimiter('Wheel download exceeded size limit'), + createWriteStream(zipPath), + ) + + let directory: unzipper.CentralDirectory + try { + directory = await unzipper.Open.file(zipPath) + } catch (err) { + throw new Error(`Malformed wheel from ${url}: ${(err as Error).message}`) + } + + const extractedByteCounter = { bytes: 0 } + let extractedFiles = 0 + + for (const entry of directory.files) { + if (entry.type !== 'File') continue + + // Wheel contents originate from a third-party package — guard path traversal + // defensively rather than trust the archive (no tar-style preservePaths here). + const resolvedPath = path.resolve(destDir, entry.path) + if (resolvedPath !== destDir && !resolvedPath.startsWith(destDir + path.sep)) { + throw new Error(`Wheel entry escapes destination dir: ${entry.path}`) + } + + extractedFiles++ + if (extractedFiles > MAX_EXTRACTED_FILES) { + throw new Error('Wheel extraction exceeded size/file limits') + } + + mkdirSync(path.dirname(resolvedPath), { recursive: true }) + + const extractionLimiter = createDownloadLimiter( + 'Wheel extraction exceeded size/file limits', + MAX_EXTRACTED_BYTES, + extractedByteCounter, + ) + + await pipeline(entry.stream(), extractionLimiter, createWriteStream(resolvedPath)) + } + } finally { + clearTimeout(timeoutHandle) + rmSync(zipPath, { force: true }) + } +} + +export async function downloadAndExtractPypiSource( + packageName: string, + version: string, + destDir: string, +): Promise { + const project = await fetchVersionProject(packageName, version) + const dist = selectDistribution(project.urls) + if (!dist) { + throw new PypiSourceNotFoundError(packageName, version) + } + + mkdirSync(destDir, { recursive: true }) + + if (dist.packagetype === 'sdist') { + await downloadSdist(dist.url, destDir, packageName, version) + } else { + await downloadWheel(dist.url, destDir, packageName, version) + } +} diff --git a/services/apps/packages_worker/src/blast-radius/ecosystemSupport.ts b/services/apps/packages_worker/src/blast-radius/ecosystemSupport.ts index e6d1178838..f4ced0c381 100644 --- a/services/apps/packages_worker/src/blast-radius/ecosystemSupport.ts +++ b/services/apps/packages_worker/src/blast-radius/ecosystemSupport.ts @@ -2,7 +2,15 @@ import { ApplicationFailure } from '@temporalio/workflow' // Single source of truth for supported ecosystems — kept in this leaf, I/O-free file // (no activities/DAL imports) so the workflow bundle stays deterministic-safe. -export const SUPPORTED_ECOSYSTEMS = ['npm', 'go', 'maven', 'cargo', 'nuget', 'rubygems'] as const +export const SUPPORTED_ECOSYSTEMS = [ + 'npm', + 'go', + 'maven', + 'cargo', + 'nuget', + 'rubygems', + 'pypi', +] as const export type Ecosystem = (typeof SUPPORTED_ECOSYSTEMS)[number] // Pure so it's testable outside the workflow sandbox (Workflow.log/context calls diff --git a/services/apps/packages_worker/src/blast-radius/packageIdentifier.ts b/services/apps/packages_worker/src/blast-radius/packageIdentifier.ts index 76be236d85..057197e2b3 100644 --- a/services/apps/packages_worker/src/blast-radius/packageIdentifier.ts +++ b/services/apps/packages_worker/src/blast-radius/packageIdentifier.ts @@ -118,6 +118,33 @@ export function toDbCargoName(name: string): string { return name.toLowerCase().replace(/-/g, '_') } +export function toBarePypiName(input: string): string { + let name = input.trim() + + name = stripQueryAndFragment(name) + + try { + name = decodeURIComponent(name) + } catch { + // Continue normalizing even if decoding fails — purl stripping and version removal + // are independent of decoding success. + } + + if (name.startsWith('pkg:pypi/')) { + name = name.slice('pkg:pypi/'.length) + } + + name = name.replace(/@[^/@]+$/, '') + + return name +} + +// PEP 503 normalization: packages.purl is always normalized, packages.name is not — +// normalize before purl lookup. +export function toPypiNormalizedName(name: string): string { + return name.toLowerCase().replace(/[-_.]+/g, '-') +} + // Maven has no single "bare name" — accepts either the "groupId:artifactId" coordinate // (OSV's package.name spelling) or a purl (pkg:maven/groupId/artifactId@version). export function toBareMavenCoordinate(input: string): { groupId: string; artifactId: string } { diff --git a/services/apps/packages_worker/src/blast-radius/stages/__tests__/dispatch.test.ts b/services/apps/packages_worker/src/blast-radius/stages/__tests__/dispatch.test.ts index f8adc3b15b..7e8a1e72c9 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/__tests__/dispatch.test.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/__tests__/dispatch.test.ts @@ -19,6 +19,9 @@ import { npmReachabilityConfig } from '../npm/reachabilityConfig' import { runDependentsStageNuGet } from '../nuget/dependentsNuGet' import { runIntelStageNuGet } from '../nuget/intelNuGet' import { nugetReachabilityConfig } from '../nuget/reachabilityConfig' +import { runDependentsStagePyPi } from '../pypi/dependentsPyPi' +import { runIntelStagePyPi } from '../pypi/intelPyPi' +import { pypiReachabilityConfig } from '../pypi/reachabilityConfig' import { runReachabilityStage } from '../reachability' import { runReachabilityStage as runReachabilityStageWithConfig } from '../reachabilityStage' import { runDependentsStageRubyGems } from '../rubygems/dependentsRubyGems' @@ -42,6 +45,9 @@ vi.mock('../nuget/intelNuGet', () => ({ vi.mock('../rubygems/intelRubyGems', () => ({ runIntelStageRubyGems: vi.fn().mockResolvedValue(undefined), })) +vi.mock('../pypi/intelPyPi', () => ({ + runIntelStagePyPi: vi.fn().mockResolvedValue(undefined), +})) vi.mock('../go/dependentsGo', () => ({ runDependentsStageGo: vi.fn().mockResolvedValue(undefined), })) @@ -60,6 +66,9 @@ vi.mock('../nuget/dependentsNuGet', () => ({ vi.mock('../rubygems/dependentsRubyGems', () => ({ runDependentsStageRubyGems: vi.fn().mockResolvedValue(undefined), })) +vi.mock('../pypi/dependentsPyPi', () => ({ + runDependentsStagePyPi: vi.fn().mockResolvedValue(undefined), +})) vi.mock('../reachabilityStage', () => ({ runReachabilityStage: vi.fn().mockResolvedValue(undefined), })) @@ -125,6 +134,15 @@ describe('stage dispatchers (EcosystemConfig registry)', () => { expect(runIntelStageMaven).not.toHaveBeenCalled() }) + it('routes intel to the PyPI body when ecosystem is pypi', async () => { + mockGetAnalysisDetail.mockResolvedValue({ ecosystem: 'pypi' } as never) + await runIntelStage(qx, 'analysis-1', 'GHSA-xxxx', undefined) + expect(runIntelStagePyPi).toHaveBeenCalledWith(qx, 'analysis-1', 'GHSA-xxxx', undefined) + expect(runIntelStageGo).not.toHaveBeenCalled() + expect(runIntelStageNpm).not.toHaveBeenCalled() + expect(runIntelStageMaven).not.toHaveBeenCalled() + }) + it('routes intel to the npm body when ecosystem is missing/unknown', async () => { mockGetAnalysisDetail.mockResolvedValue(null) await runIntelStage(qx, 'analysis-1', 'GHSA-xxxx', undefined) @@ -177,6 +195,15 @@ describe('stage dispatchers (EcosystemConfig registry)', () => { expect(runDependentsStageMaven).not.toHaveBeenCalled() }) + it('routes dependents to the PyPI body when ecosystem is pypi', async () => { + mockGetAnalysisDetail.mockResolvedValue({ ecosystem: 'pypi' } as never) + await runDependentsStage(qx, 'analysis-1', undefined, undefined) + expect(runDependentsStagePyPi).toHaveBeenCalled() + expect(runDependentsStageGo).not.toHaveBeenCalled() + expect(runDependentsStageNpm).not.toHaveBeenCalled() + expect(runDependentsStageMaven).not.toHaveBeenCalled() + }) + it('routes dependents to the npm body for npm/unknown ecosystems', async () => { mockGetAnalysisDetail.mockResolvedValue({ ecosystem: 'npm' } as never) await runDependentsStage(qx, 'analysis-1', undefined, undefined) @@ -241,6 +268,17 @@ describe('stage dispatchers (EcosystemConfig registry)', () => { ) }) + it('routes reachability to the PyPI config when ecosystem is pypi', async () => { + mockGetAnalysisDetail.mockResolvedValue({ ecosystem: 'pypi' } as never) + await runReachabilityStage(qx, 'analysis-1', undefined) + expect(mockRunReachabilityStageWithConfig).toHaveBeenCalledWith( + qx, + 'analysis-1', + pypiReachabilityConfig, + undefined, + ) + }) + it('routes reachability to the npm config for npm/unknown ecosystems', async () => { mockGetAnalysisDetail.mockResolvedValue({ ecosystem: 'npm' } as never) await runReachabilityStage(qx, 'analysis-1', undefined) diff --git a/services/apps/packages_worker/src/blast-radius/stages/ecosystems.ts b/services/apps/packages_worker/src/blast-radius/stages/ecosystems.ts index a64c02ef03..87f808412d 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/ecosystems.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/ecosystems.ts @@ -17,6 +17,9 @@ import { npmReachabilityConfig } from './npm/reachabilityConfig' import { runDependentsStageNuGet } from './nuget/dependentsNuGet' import { runIntelStageNuGet } from './nuget/intelNuGet' import { nugetReachabilityConfig } from './nuget/reachabilityConfig' +import { runDependentsStagePyPi } from './pypi/dependentsPyPi' +import { runIntelStagePyPi } from './pypi/intelPyPi' +import { pypiReachabilityConfig } from './pypi/reachabilityConfig' import { ReachabilitySourceConfig } from './reachabilityStage' import { runDependentsStageRubyGems } from './rubygems/dependentsRubyGems' import { runIntelStageRubyGems } from './rubygems/intelRubyGems' @@ -71,6 +74,11 @@ const ECOSYSTEMS: Record = { runDependents: runDependentsStageRubyGems, reachability: rubygemsReachabilityConfig, }, + pypi: { + runIntel: runIntelStagePyPi, + runDependents: runDependentsStagePyPi, + reachability: pypiReachabilityConfig, + }, } export function getEcosystemConfig(ecosystem: string | null | undefined): EcosystemConfig { diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts new file mode 100644 index 0000000000..f459e188e4 --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' + +import { pypiConstraintMayInclude, pypiDependencyMayIncludeVuln } from '../pypiConstraint' + +describe('pypiConstraintMayInclude', () => { + it('expands "~=" into a bounded range', () => { + expect(pypiConstraintMayInclude('~=2.2', ['2.2.0', '2.3.0'])).toBe('matched') + expect(pypiConstraintMayInclude('~=2.2', ['2.1.0'])).toBe('excluded') + expect(pypiConstraintMayInclude('~=2.2', ['3.0.0'])).toBe('excluded') + expect(pypiConstraintMayInclude('~=1.4.5', ['1.4.5', '1.4.9'])).toBe('matched') + expect(pypiConstraintMayInclude('~=1.4.5', ['1.4.4'])).toBe('excluded') + expect(pypiConstraintMayInclude('~=1.4.5', ['1.5.0'])).toBe('excluded') + }) + + it('treats "~=" with a single release segment as unparseable', () => { + expect(pypiConstraintMayInclude('~=2', ['2.0.0'])).toBe('unparseable-included') + }) + + it('matches "==" with a ".*" wildcard as a prefix', () => { + expect(pypiConstraintMayInclude('==2.2.*', ['2.2.0', '2.2.9'])).toBe('matched') + expect(pypiConstraintMayInclude('==2.2.*', ['2.3.0'])).toBe('excluded') + }) + + it('matches "!=" with a ".*" wildcard as exclusion of the prefix', () => { + expect(pypiConstraintMayInclude('!=2.2.*', ['2.3.0'])).toBe('matched') + expect(pypiConstraintMayInclude('!=2.2.*', ['2.2.0'])).toBe('excluded') + }) + + it('matches exact "==" and "!="', () => { + expect(pypiConstraintMayInclude('==1.0.0', ['1.0.0'])).toBe('matched') + expect(pypiConstraintMayInclude('==1.0.0', ['1.0.1'])).toBe('excluded') + expect(pypiConstraintMayInclude('!=1.0.0', ['1.0.1'])).toBe('matched') + expect(pypiConstraintMayInclude('!=1.0.0', ['1.0.0'])).toBe('excluded') + }) + + it('matches "===" via arbitrary case-insensitive string equality, unnormalized', () => { + expect(pypiConstraintMayInclude('===1.0.0', ['1.0.0'])).toBe('matched') + expect(pypiConstraintMayInclude('===1.0.0', ['1.0.0.0'])).toBe('excluded') + }) + + it('ANDs comma-separated clauses', () => { + expect(pypiConstraintMayInclude('>=1.0,<2.0', ['1.5.0'])).toBe('matched') + expect(pypiConstraintMayInclude('>=1.0,<2.0', ['2.0.0'])).toBe('excluded') + expect(pypiConstraintMayInclude('>=1.0,<2.0', ['0.9.0'])).toBe('excluded') + }) + + it('handles prerelease bounds', () => { + expect(pypiConstraintMayInclude('<1.0', ['1.0rc1'])).toBe('matched') + expect(pypiConstraintMayInclude('>=1.0', ['1.0rc1'])).toBe('excluded') + }) + + it('is over-inclusive on a null or malformed constraint', () => { + expect(pypiConstraintMayInclude(null, ['1.0.0'])).toBe('unparseable-included') + expect(pypiConstraintMayInclude('', ['1.0.0'])).toBe('unparseable-included') + expect(pypiConstraintMayInclude('not-a-specifier', ['1.0.0'])).toBe('unparseable-included') + expect(pypiConstraintMayInclude('>=1.0,', ['1.0.0'])).toBe('unparseable-included') + }) +}) + +describe('pypiDependencyMayIncludeVuln', () => { + it('prefers the resolved version as ground truth', () => { + expect(pypiDependencyMayIncludeVuln('1.0.0', '<1.0.0', ['1.0.0'])).toBe('matched') + expect(pypiDependencyMayIncludeVuln('2.0.0', '<1.0.0', ['1.0.0'])).toBe('excluded') + }) + + it('falls back to the constraint when no resolved version is present', () => { + expect(pypiDependencyMayIncludeVuln(null, '==1.0.0', ['1.0.0'])).toBe('matched') + expect(pypiDependencyMayIncludeVuln(null, null, ['1.0.0'])).toBe('unparseable-included') + }) +}) diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsPyPi.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsPyPi.ts new file mode 100644 index 0000000000..f342be4663 --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsPyPi.ts @@ -0,0 +1,115 @@ +import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' +import { findPackageIdsByName } from '@crowd/data-access-layer/src/packages/osv' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { scanPyPiDependents } from './dependentsScanPyPi' + +export async function runDependentsStagePyPi( + qx: QueryExecutor, + analysisId: string, + onProgress?: () => void, + signal?: AbortSignal, +): Promise { + const startTime = Date.now() + + try { + const existingStatus = await blastRadiusDal.getStageRunStatus(qx, analysisId, 'dependents') + if (existingStatus === 'succeeded') { + return + } + + await blastRadiusDal.startStageRun(qx, { + analysisId, + stage: 'dependents', + status: 'running', + model: null, + }) + + const spec = await blastRadiusDal.getSymbolSpec(qx, analysisId) + if (!spec) { + throw new Error('Symbol spec not found; stage 1 (intel) must run first') + } + + // See dependentsNpm.ts for why this unconditional clear is safe: reachability + // hasn't produced any verdicts yet at this point in the pipeline. + await blastRadiusDal.deleteDependents(qx, analysisId) + + const analysis = await blastRadiusDal.getAnalysis(qx, analysisId) + if (!analysis?.package_id) { + throw new Error('Vulnerable project package_id not resolved; stage 1 (intel) must run first') + } + + const vulnerableVersions = (spec.vulnerable_versions || []) as string[] + const relatedAffectedPackages = (spec.related_affected_packages || []) as string[] + + onProgress?.() + + const scanResult = await scanPyPiDependents( + qx, + String(analysis.package_id), + vulnerableVersions, + 25, + relatedAffectedPackages, + ) + + if (signal?.aborted) { + throw new Error('Dependents scan cancelled') + } + onProgress?.() + + // scanResult.analyzed's names come straight from packages.name via getReverseDependents, + // so they round-trip consistently — no purl-normalization dance needed here. + const packageIdsByName = await findPackageIdsByName( + qx, + 'pypi', + scanResult.analyzed.map((d) => d.name), + ) + + const dependentInputs = [ + ...scanResult.analyzed.map((d) => ({ + analysisId, + packageId: packageIdsByName.get(d.name) ?? null, + name: d.name, + version: d.version, + downloads: d.downloads, + declaredRange: d.declaredRange, + dependencyKind: d.dependencyKind, + rangeIncludesVuln: d.rangeIncludesVuln, + rangeCheck: d.rangeCheck, + tarballUrl: d.tarballUrl, + excludedByRange: false, + exclusionReason: null, + })), + ...scanResult.excludedByRange.map((d) => ({ + analysisId, + packageId: null, + name: d.name, + version: d.version, + downloads: d.downloads, + declaredRange: d.declaredRange, + dependencyKind: d.dependencyKind, + rangeIncludesVuln: d.rangeIncludesVuln, + rangeCheck: d.rangeCheck, + tarballUrl: d.tarballUrl, + excludedByRange: true, + exclusionReason: `Requirement does not include vulnerable versions (${d.rangeCheck})`, + })), + ] + + await blastRadiusDal.insertDependents(qx, dependentInputs) + await blastRadiusDal.setDependentsMeta( + qx, + analysisId, + scanResult.source, + scanResult.candidatesConsidered, + ) + + const duration = Date.now() - startTime + await blastRadiusDal.completeStageRun(qx, analysisId, 'dependents', duration, 0) + } catch (err) { + const duration = Date.now() - startTime + const errorMsg = err instanceof Error ? err.message : String(err) + await blastRadiusDal.failStageRun(qx, analysisId, 'dependents', duration, errorMsg) + throw err + } +} diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsScanPyPi.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsScanPyPi.ts new file mode 100644 index 0000000000..ff80cd779f --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/dependentsScanPyPi.ts @@ -0,0 +1,69 @@ +import { getReverseDependents } from '@crowd/data-access-layer/src/packages/blastRadiusDependents' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { DependentCandidate, ScanDependentsResult } from '../../dependentsScan' +import { toPypiNormalizedName } from '../../packageIdentifier' + +import { pypiDependencyMayIncludeVuln } from './pypiConstraint' + +// PyPI is a deps.dev EDGE ecosystem, same as npm/Maven/Cargo — dependents come from our +// own DB (package_dependencies), with a resolved version preferred as ground truth. +export async function scanPyPiDependents( + qx: QueryExecutor, + vulnerablePackageId: string, + vulnerableVersions: string[], + topN: number, + relatedAffectedPackages?: string[], +): Promise { + if (vulnerableVersions.length === 0) { + return { + source: 'package_dependencies', + candidatesConsidered: 0, + analyzed: [], + excludedByRange: [], + excludedByRangeCount: 0, + } + } + + // Cap distinct from topN: gather a wider pool so excludedByRange candidates are + // still visible for diagnostics, same pattern as npm's/cargo's scanLimit. + const scanLimit = Math.max(topN * 8, 200) + const rows = await getReverseDependents(qx, vulnerablePackageId, 'pypi', scanLimit) + const relatedPackageNames = new Set((relatedAffectedPackages || []).map(toPypiNormalizedName)) + + const included: DependentCandidate[] = [] + const excluded: DependentCandidate[] = [] + + for (const row of rows) { + if (relatedPackageNames.has(toPypiNormalizedName(row.name))) continue + + const rangeCheck = pypiDependencyMayIncludeVuln( + row.resolvedVersionNumber, + row.versionConstraint, + vulnerableVersions, + ) + const candidate: DependentCandidate = { + name: row.name, + version: row.versionNumber, + downloads: row.dependentReposCount ?? row.dependentCount ?? null, + declaredRange: row.versionConstraint, + dependencyKind: row.dependencyKind, + rangeIncludesVuln: rangeCheck !== 'excluded', + rangeCheck, + tarballUrl: null, + } + if (candidate.rangeIncludesVuln) { + included.push(candidate) + } else { + excluded.push(candidate) + } + } + + return { + source: 'package_dependencies', + candidatesConsidered: included.length + excluded.length, + analyzed: included.slice(0, topN), + excludedByRange: excluded.slice(0, 200), + excludedByRangeCount: excluded.length, + } +} diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts new file mode 100644 index 0000000000..d7b3635a22 --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts @@ -0,0 +1,187 @@ +import * as fs from 'fs' +import * as os from 'os' +import * as path from 'path' + +import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' +import { getVersionNumbers } from '@crowd/data-access-layer/src/packages/blastRadiusDependents' +import { findPackageIdByPurl } from '@crowd/data-access-layer/src/packages/osv' +import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' + +import { fetchProject } from '../../../pypi/fetchProject' +import { isFetchError } from '../../../pypi/types' +import { + PYPI_INTEL_SCHEMA, + PYPI_INTEL_SYSTEM_PROMPT, + buildPyPiIntelPrompt, +} from '../../agent/pypiPrompts' +import { runAnalysisAgent } from '../../agent/runner' +import { fetchPatch } from '../../clients/githubPatch' +import { + affectedEntriesForEcosystem, + fetchOsvVuln, + fixReferenceUrls, +} from '../../clients/osvClient' +import { downloadAndExtractPypiSource } from '../../clients/pypiSource' +import { toBarePypiName, toPypiNormalizedName } from '../../packageIdentifier' +import { ecosystemRangeEvents, highestVersion, versionsInRanges } from '../ecosystemVersions' +import { selectAdvisoryEntry } from '../selectAdvisoryEntry' + +// OSV spells the PyPI ecosystem 'PyPI' (mixed case), unlike our DB's lowercase 'pypi' — +// see ADR-0001 §OSV "Ecosystem normalization" for the DB-side convention. +const OSV_PYPI_ECOSYSTEM = 'PyPI' + +export async function runIntelStagePyPi( + qx: QueryExecutor, + analysisId: string, + advisoryOsvId: string, + onProgress?: () => void, +): Promise { + const startTime = Date.now() + + try { + // Guard on stage_run status to avoid stuck failed/running state if intel crashes + // between upsertSymbolSpec and completeStageRun. + const existingStatus = await blastRadiusDal.getStageRunStatus(qx, analysisId, 'intel') + if (existingStatus === 'succeeded') { + return + } + + await blastRadiusDal.startStageRun(qx, { + analysisId, + stage: 'intel', + status: 'running', + model: 'claude-opus-4-8', + }) + + const osv = await fetchOsvVuln(advisoryOsvId) + + const pypiEntries = affectedEntriesForEcosystem(osv, OSV_PYPI_ECOSYSTEM) + if (pypiEntries.length === 0) { + throw new Error(`No PyPI entries found in advisory ${advisoryOsvId}`) + } + + // Pick the project entry the analysis was requested for; see selectAdvisoryEntry for + // rejection rules on non-matching or omitted requests. + const analysisDetail = await blastRadiusDal.getAnalysisDetail(qx, analysisId) + const requestedName = analysisDetail?.package_name + ? toBarePypiName(analysisDetail.package_name) + : null + const requestedNormalized = requestedName !== null ? toPypiNormalizedName(requestedName) : null + const { entry, relatedAffectedPackages } = selectAdvisoryEntry( + pypiEntries, + requestedName, + (e) => toPypiNormalizedName(e.package.name) === requestedNormalized, + advisoryOsvId, + ) + const project = entry.package.name + const normalizedName = toPypiNormalizedName(project) + const ecosystem = 'pypi' + + // Resolve vulnerable versions from OSV ranges first (PyPI OSV ranges are + // ECOSYSTEM-typed, not SEMVER — PEP 440 versions don't follow semver ordering). + const ranges = ecosystemRangeEvents(entry) + + // packages.name can drift from the canonical PEP 503 spelling for pypi rows (a + // pre-existing data-quality issue — see the blast-radius PyPI plan's name-casing + // section); packages.purl is always PEP 503-normalized, so look up by purl instead. + const packageId = await findPackageIdByPurl(qx, `pkg:pypi/${normalizedName}`) + + // pypi.org's JSON API is the authoritative version list; fall back to our own + // ingested `versions` rows (deps.dev) if the registry is unreachable and the + // project is already known to us. + const projectResult = await fetchProject(normalizedName) + let allVersions: string[] + if (!isFetchError(projectResult)) { + allVersions = Object.keys(projectResult.releases ?? {}) + } else if (packageId !== null) { + allVersions = await getVersionNumbers(qx, String(packageId)) + } else { + throw new Error( + `Failed to fetch PyPI versions for ${project} (${projectResult.message}) and project is not in our DB`, + ) + } + + const vulnerableVersions = versionsInRanges(ecosystem, allVersions, ranges) + + const analyzed = highestVersion(ecosystem, vulnerableVersions) + if (!analyzed) { + throw new Error(`Could not determine analyzed version for ${project}`) + } + + const pkgsrcDir = fs.mkdtempSync(path.join(os.tmpdir(), 'pypisrc-')) + const patches: Record = {} + + try { + await downloadAndExtractPypiSource(normalizedName, analyzed, pkgsrcDir) + + const patchUrls = fixReferenceUrls(osv) + const patchResults = await Promise.allSettled( + patchUrls.slice(0, 3).map(async (url) => { + const patchText = await fetchPatch(url) + const slug = new URL(url).pathname.split('/').filter(Boolean).join('-') + return { slug, patchText } + }), + ) + patchResults.forEach((result) => { + if (result.status === 'fulfilled') { + patches[result.value.slug] = result.value.patchText + } + }) + + const agentPrompt = buildPyPiIntelPrompt( + osv.id || advisoryOsvId, + osv.aliases || [], + osv.details || osv.summary || '', + analyzed, + patches, + ) + + const agentResult = await runAnalysisAgent({ + prompt: agentPrompt, + systemPrompt: PYPI_INTEL_SYSTEM_PROMPT, + cwd: pkgsrcDir, + model: 'claude-opus-4-8', + schema: PYPI_INTEL_SCHEMA, + maxTurns: 15, + timeoutMs: 600_000, + onProgress, + }) + + if (agentResult.isError || !agentResult.structuredOutput) { + throw new Error(`Agent failed: ${agentResult.errorMessage}`) + } + + const output = agentResult.structuredOutput + await blastRadiusDal.upsertSymbolSpec(qx, { + analysisId, + vulnId: osv.id || advisoryOsvId, + aliases: osv.aliases || [], + package: project, + ecosystem, + affectedRanges: ranges as unknown as Record[], + vulnerableVersions, + analyzedVersion: analyzed, + relatedAffectedPackages, + vulnerableSymbols: (output.vulnerable_symbols || []) as Record[], + importSignatures: (output.import_signatures || {}) as Record, + exploitPreconditions: String(output.exploit_preconditions || ''), + reachabilityNotes: String(output.reachability_notes || ''), + confidence: Number(output.confidence ?? 0.5), + sources: [advisoryOsvId], + summary: String(output.summary || ''), + }) + + await blastRadiusDal.resolveAdvisoryAndPackageIds(qx, analysisId, advisoryOsvId, packageId) + + const duration = Date.now() - startTime + await blastRadiusDal.completeStageRun(qx, analysisId, 'intel', duration, agentResult.costUsd) + } finally { + fs.rmSync(pkgsrcDir, { recursive: true, force: true }) + } + } catch (err) { + const duration = Date.now() - startTime + const errorMsg = err instanceof Error ? err.message : String(err) + await blastRadiusDal.failStageRun(qx, analysisId, 'intel', duration, errorMsg) + throw err + } +} diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts new file mode 100644 index 0000000000..af78595cc0 --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts @@ -0,0 +1,137 @@ +import { compareVersion } from '../../../osv/versionCompare' + +export type PypiConstraintMatch = 'matched' | 'excluded' | 'unparseable-included' + +type Op = '~=' | '===' | '==' | '!=' | '<=' | '>=' | '<' | '>' + +interface PypiClause { + op: Op + version: string + wildcard: boolean +} + +// Longest operators first — a naive scan would let "==" swallow the first two chars of "===". +const OPERATORS: Op[] = ['~=', '===', '==', '!=', '<=', '>=', '<', '>'] + +// "~= V.N" expands to ">= V.N, == V.*" (drop the last release segment for the wildcard +// bound) per PEP 440's compatible-release operator; invalid with fewer than two segments. +function expandCompatible(version: string): PypiClause[] | null { + const releaseMatch = version.match(/^[0-9]+(?:\.[0-9]+)*/) + if (!releaseMatch) return null + const segments = releaseMatch[0].split('.') + if (segments.length < 2) return null + + const prefix = segments.slice(0, -1).join('.') + return [ + { op: '>=', version, wildcard: false }, + { op: '==', version: prefix, wildcard: true }, + ] +} + +// PEP 440 specifiers always carry an operator — unlike RubyGems there is no bare-version +// shorthand. Returns multiple clauses only for "~=", which expands to an AND pair. +function parseClause(raw: string): PypiClause[] | null { + const trimmed = raw.trim() + if (!trimmed) return null + + for (const op of OPERATORS) { + if (!trimmed.startsWith(op)) continue + + let version = trimmed.slice(op.length).trim() + if (!version) return null + + if (op === '~=') return expandCompatible(version) + + let wildcard = false + if (version.endsWith('.*')) { + if (op !== '==' && op !== '!=') return null + wildcard = true + version = version.slice(0, -2) + } + return [{ op, version, wildcard }] + } + + return null +} + +function parseSpecifierSet(constraint: string | null): PypiClause[] | null { + // package_dependencies.version_constraint is nullable (deps.dev fill path) — treat a + // missing constraint the same as an unparseable one, not a crash on .split. + if (constraint == null) return null + const trimmed = constraint.trim() + if (!trimmed) return null + + const clauses: PypiClause[] = [] + for (const part of trimmed.split(',')) { + const parsed = parseClause(part) + if (!parsed) return null + clauses.push(...parsed) + } + return clauses +} + +// Wildcard matching reduces to a public-version string prefix check — deliberately +// over-inclusive rather than spec-exact, consistent with every other clause here. +function normalizedForWildcard(version: string): string { + return version.trim().toLowerCase().replace(/^v/, '').split('+')[0] +} + +function clauseMatches(clause: PypiClause, version: string): boolean { + if (clause.op === '===') { + return version.trim().toLowerCase() === clause.version.trim().toLowerCase() + } + + if (clause.wildcard) { + const matches = normalizedForWildcard(version).startsWith(clause.version.toLowerCase()) + return clause.op === '==' ? matches : !matches + } + + const c = compareVersion('pypi', version, clause.version) + if (c === null) return true // unparseable bound — over-inclusive + + switch (clause.op) { + case '==': + return c === 0 + case '!=': + return c !== 0 + case '<': + return c < 0 + case '>': + return c > 0 + case '<=': + return c <= 0 + case '>=': + return c >= 0 + case '~=': + return true // unreachable — expandCompatible never leaves '~=' on a leaf clause + } +} + +// Over-inclusive by design — reachability is the real precision filter. Checks every +// vulnerable version, not just the highest: a bounded "~=" can include an older one. +export function pypiConstraintMayInclude( + constraint: string | null, + vulnerableVersions: string[], +): PypiConstraintMatch { + const clauses = parseSpecifierSet(constraint) + if (!clauses) return 'unparseable-included' + + const matched = vulnerableVersions.some((version) => + clauses.every((clause) => clauseMatches(clause, version)), + ) + return matched ? 'matched' : 'excluded' +} + +// Prefers resolved version over the declared specifier (ground truth vs. declared) — PyPI +// is a deps.dev EDGE ecosystem, so a resolved version is usually available; mirrors +// cargoDependencyMayIncludeVuln. +export function pypiDependencyMayIncludeVuln( + resolvedVersion: string | null, + constraint: string | null, + vulnerableVersions: string[], +): PypiConstraintMatch { + if (resolvedVersion) { + return vulnerableVersions.includes(resolvedVersion) ? 'matched' : 'excluded' + } + return pypiConstraintMayInclude(constraint, vulnerableVersions) +} diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts new file mode 100644 index 0000000000..37982e97cb --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts @@ -0,0 +1,26 @@ +import { + PYPI_REACHABILITY_PROMPT, + PYPI_VERDICT_SCHEMA, + buildPyPiReachabilitySystemPrompt, +} from '../../agent/pypiPrompts' +import { downloadAndExtractPypiSource } from '../../clients/pypiSource' +import { toPypiNormalizedName } from '../../packageIdentifier' +import { ReachabilitySourceConfig } from '../reachabilityStage' + +// PyPI is a deps.dev EDGE ecosystem, so dep.version is the dependent's own resolved +// version and is never null — no canonical-name round trip is needed the way Cargo's +// resolveCargoCanonical requires, since the PyPI JSON API accepts normalized names. +export const pypiReachabilityConfig: ReachabilitySourceConfig = { + prompt: PYPI_REACHABILITY_PROMPT, + schema: PYPI_VERDICT_SCHEMA, + buildSystemPrompt: buildPyPiReachabilitySystemPrompt, + prepareSource: async (dep) => { + if (!dep.version) return null + const normalizedName = toPypiNormalizedName(dep.name) + return { + download: (destDir) => downloadAndExtractPypiSource(normalizedName, dep.version, destDir), + } + }, + noSourceMessage: 'Could not resolve a concrete PyPI project version', + downloadErrorPrefix: 'PyPI source download failed', +} diff --git a/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts b/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts index 0fa0fc515f..a5361ce21d 100644 --- a/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts +++ b/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts @@ -161,11 +161,35 @@ describe('compareVersion — rubygems (Gem::Version-style)', () => { }) }) -describe('compareVersion — unsupported ecosystems', () => { - it('returns null for ecosystems we have no comparator for', () => { +describe('compareVersion — pypi (PEP 440)', () => { + it.each([ + ['1!1.0', '2.0', 1], // epoch always outranks the release tuple + ['1.0.dev1', '1.0a1', -1], + ['1.0a1', '1.0b1', -1], + ['1.0b1', '1.0rc1', -1], + ['1.0rc1', '1.0', -1], + ['1.0', '1.0.post1', -1], + ['1.0a1.dev1', '1.0a1', -1], // a pre-release's own dev build sorts below it + ['1.0alpha1', '1.0a1', 0], // normalization alias + ['1.0.beta.2', '1.0b2', 0], // separators before a phase marker are optional + ['1.0', '1.0.0', 0], // release tuple zero-padded to equal length + ['1.0+local', '1.0', 1], // local version sorts above the same plain version + ['1.0+a.2', '1.0+a.10', -1], // numeric local segments compare numerically + ])('compareVersion("pypi", %s, %s) sign = %s', (a, b, expected) => { + expect(sign(compareVersion('pypi', a, b))).toBe(expected) + }) + + it('returns null for unparseable pypi versions', () => { + expect(compareVersion('pypi', '', '1.0.0')).toBeNull() + expect(compareVersion('pypi', 'not-a-version', '1.0.0')).toBeNull() + }) + + it('rejects titlecase "PyPI" — production storage is always lowercase', () => { expect(compareVersion('PyPI', '1.0.0', '2.0.0')).toBeNull() }) +}) +describe('compareVersion — unsupported ecosystems', () => { it('rejects titlecase "Maven" — production storage is always lowercase', () => { // Regression guard for the casing bug Fix 1 missed: deriveCriticalFlag // reads `ecosystem` from packages-db where it's lowercase. The comparator diff --git a/services/apps/packages_worker/src/osv/versionCompare.ts b/services/apps/packages_worker/src/osv/versionCompare.ts index d9971574b1..ed797d6499 100644 --- a/services/apps/packages_worker/src/osv/versionCompare.ts +++ b/services/apps/packages_worker/src/osv/versionCompare.ts @@ -195,6 +195,159 @@ function compareRubyGems(a: string, b: string): number | null { return 0 } +interface Pep440Pre { + letter: 'a' | 'b' | 'rc' + num: number +} + +interface Pep440Version { + epoch: number + release: number[] + pre: Pep440Pre | null + post: number | null + dev: number | null + local: (number | string)[] +} + +// Longest-alias-first: JS regex alternation picks the first matching alternative, not the +// longest, so "a" would swallow only the first letter of "alpha" if it came first. +const PEP440_PRE_ALIASES: Record = { + alpha: 'a', + a: 'a', + beta: 'b', + b: 'b', + preview: 'rc', + pre: 'rc', + rc: 'rc', + c: 'rc', +} +const PEP440_PRE_RE = /^[-_.]?(alpha|beta|preview|pre|rc|a|b|c)[-_.]?([0-9]*)/ +const PEP440_POST_RE = /^[-_.]?(?:post|rev|r)[-_.]?([0-9]*)/ +const PEP440_DEV_RE = /^[-_.]?dev[-_.]?([0-9]*)/ +const PEP440_LOCAL_RE = /^\+([a-z0-9]+(?:[-_.][a-z0-9]+)*)/ + +// https://peps.python.org/pep-0440/#version-scheme — consumes the grammar left to right; +// anything left unconsumed means the input isn't a valid PEP 440 version. +function parsePep440(raw: string): Pep440Version | null { + let s = raw.trim().toLowerCase().replace(/^v/, '') + + let epoch = 0 + const epochMatch = s.match(/^([0-9]+)!/) + if (epochMatch) { + epoch = parseInt(epochMatch[1], 10) + s = s.slice(epochMatch[0].length) + } + + const releaseMatch = s.match(/^[0-9]+(?:\.[0-9]+)*/) + if (!releaseMatch) return null + const release = releaseMatch[0].split('.').map((n) => parseInt(n, 10)) + s = s.slice(releaseMatch[0].length) + + let pre: Pep440Pre | null = null + const preMatch = s.match(PEP440_PRE_RE) + if (preMatch) { + pre = { + letter: PEP440_PRE_ALIASES[preMatch[1]], + num: preMatch[2] ? parseInt(preMatch[2], 10) : 0, + } + s = s.slice(preMatch[0].length) + } + + let post: number | null = null + const implicitPostMatch = s.match(/^-([0-9]+)/) + if (implicitPostMatch) { + post = parseInt(implicitPostMatch[1], 10) + s = s.slice(implicitPostMatch[0].length) + } else { + const postMatch = s.match(PEP440_POST_RE) + if (postMatch) { + post = postMatch[1] ? parseInt(postMatch[1], 10) : 0 + s = s.slice(postMatch[0].length) + } + } + + let dev: number | null = null + const devMatch = s.match(PEP440_DEV_RE) + if (devMatch) { + dev = devMatch[1] ? parseInt(devMatch[1], 10) : 0 + s = s.slice(devMatch[0].length) + } + + let local: (number | string)[] = [] + const localMatch = s.match(PEP440_LOCAL_RE) + if (localMatch) { + local = localMatch[1] + .split(/[-_.]/) + .map((seg) => (/^[0-9]+$/.test(seg) ? parseInt(seg, 10) : seg)) + s = s.slice(localMatch[0].length) + } + + if (s.length > 0) return null + + return { epoch, release, pre, post, dev, local } +} + +const PEP440_PRE_RANK: Record<'a' | 'b' | 'rc', number> = { a: 0, b: 1, rc: 2 } + +// Bare dev release sorts below pre-releases; release with no pre-release sorts above them — +// see PEP 440's documented order .devN < aN < bN < rcN < final < .postN. +function pep440PreOrder(v: Pep440Version): [number, number, number] { + if (v.pre) return [0, PEP440_PRE_RANK[v.pre.letter], v.pre.num] + if (v.post === null && v.dev !== null) return [-1, 0, 0] + return [1, 0, 0] +} + +function comparePep440Local(a: (number | string)[], b: (number | string)[]): number { + const len = Math.max(a.length, b.length) + for (let i = 0; i < len; i++) { + if (i >= a.length) return -1 + if (i >= b.length) return 1 + const [x, y] = [a[i], b[i]] + if (typeof x === 'number' && typeof y === 'number') { + if (x !== y) return x < y ? -1 : 1 + continue + } + if (typeof x === 'number') return 1 + if (typeof y === 'number') return -1 + if (x !== y) return x < y ? -1 : 1 + } + return 0 +} + +function comparePep440(a: string, b: string): number | null { + const pa = parsePep440(a) + const pb = parsePep440(b) + if (!pa || !pb) return null + + if (pa.epoch !== pb.epoch) return pa.epoch < pb.epoch ? -1 : 1 + + const relLen = Math.max(pa.release.length, pb.release.length) + for (let i = 0; i < relLen; i++) { + const ra = pa.release[i] ?? 0 + const rb = pb.release[i] ?? 0 + if (ra !== rb) return ra < rb ? -1 : 1 + } + + const preA = pep440PreOrder(pa) + const preB = pep440PreOrder(pb) + for (let i = 0; i < 3; i++) { + if (preA[i] !== preB[i]) return preA[i] < preB[i] ? -1 : 1 + } + + const postA = pa.post ?? -Infinity + const postB = pb.post ?? -Infinity + if (postA !== postB) return postA < postB ? -1 : 1 + + const devA = pa.dev ?? Infinity + const devB = pb.dev ?? Infinity + if (devA !== devB) return devA < devB ? -1 : 1 + + if (pa.local.length === 0 && pb.local.length === 0) return 0 + if (pa.local.length === 0) return -1 + if (pb.local.length === 0) return 1 + return comparePep440Local(pa.local, pb.local) +} + const SEMVER_ECOSYSTEMS = new Set(['npm', 'cargo', 'nuget', 'go']) // Ecosystem names are stored lowercase in packages-db per ADR-0001 §OSV @@ -204,5 +357,6 @@ export function compareVersion(ecosystem: string, a: string, b: string): number if (SEMVER_ECOSYSTEMS.has(ecosystem)) return compareSemver(a, b) if (ecosystem === 'maven') return compareMaven(a, b) if (ecosystem === 'rubygems') return compareRubyGems(a, b) + if (ecosystem === 'pypi') return comparePep440(a, b) return null } diff --git a/services/apps/packages_worker/src/pypi/types.ts b/services/apps/packages_worker/src/pypi/types.ts index 9269d4746e..561a8a54b0 100644 --- a/services/apps/packages_worker/src/pypi/types.ts +++ b/services/apps/packages_worker/src/pypi/types.ts @@ -38,8 +38,18 @@ export interface PyPiInfo { yanked?: boolean } +// Per-version endpoint only; URL must be read here, never constructed (contains content hash). +export interface PyPiUrlInfo { + packagetype: string + url: string + filename: string + size?: number + yanked?: boolean +} + export interface PyPiProject { info: PyPiInfo // version string -> array of distribution file objects (may be empty). releases?: Record + urls?: PyPiUrlInfo[] } diff --git a/services/libs/data-access-layer/src/packages/osv.ts b/services/libs/data-access-layer/src/packages/osv.ts index f02284e92d..ba76ae1c54 100644 --- a/services/libs/data-access-layer/src/packages/osv.ts +++ b/services/libs/data-access-layer/src/packages/osv.ts @@ -119,6 +119,12 @@ export async function findPackageId( return (row?.id as number | undefined) ?? null } +// Use purl as key for ecosystems where name drifts; purl is UNIQUE NOT NULL and always normalized. +export async function findPackageIdByPurl(qx: QueryExecutor, purl: string): Promise { + const row = await qx.selectOneOrNone(`SELECT id FROM packages WHERE purl = $(purl)`, { purl }) + return (row?.id as number | undefined) ?? null +} + // Batched form of findPackageId for a flat list of full package names (e.g. // "lodash", "@babel/core") — one round-trip instead of one query per name. // Mirrors getNpmPurlsForChangedNames's namespace/name reconstruction join. From 2cbd3cac3249a78c665a91494b78e55e63d092a8 Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 7 Aug 2026 11:16:32 +0200 Subject: [PATCH 2/3] feat: add pypi ecosystem to blast Signed-off-by: Umberto Sgueglia --- .../public/v1/packages/blastRadius.test.ts | 10 +- .../v1/packages/submitBlastRadiusJob.test.ts | 18 ++- .../submitBlastRadiusJobBatch.test.ts | 12 +- .../__tests__/packageIdentifier.test.ts | 53 ++++++- .../agent/__tests__/agentAuth.test.ts | 147 ++++++++++++++++++ .../src/blast-radius/agent/pypiPrompts.ts | 4 - .../clients/__tests__/pypiSource.test.ts | 23 +++ .../src/blast-radius/clients/pypiSource.ts | 34 ++-- .../pypi/__tests__/pypiConstraint.test.ts | 10 ++ .../src/blast-radius/stages/pypi/intelPyPi.ts | 16 +- .../stages/pypi/pypiConstraint.ts | 21 ++- .../stages/pypi/reachabilityConfig.ts | 4 +- .../maven/scripts/importMaintainersFromCsv.ts | 2 +- services/libs/common/src/agentAuth.ts | 97 ++++++++++++ .../src/osspckgs/packages.ts | 6 +- .../src/packages/blastRadius.ts | 2 +- .../data-access-layer/src/packages/osv.ts | 6 - 17 files changed, 417 insertions(+), 48 deletions(-) create mode 100644 services/apps/packages_worker/src/blast-radius/agent/__tests__/agentAuth.test.ts create mode 100644 services/libs/common/src/agentAuth.ts diff --git a/backend/src/api/public/v1/packages/blastRadius.test.ts b/backend/src/api/public/v1/packages/blastRadius.test.ts index dce7782fb6..7e073a65c9 100644 --- a/backend/src/api/public/v1/packages/blastRadius.test.ts +++ b/backend/src/api/public/v1/packages/blastRadius.test.ts @@ -85,10 +85,18 @@ describe('blastRadiusJobRequestSchema', () => { it('rejects an unsupported ecosystem', () => { const result = blastRadiusJobRequestSchema.safeParse({ advisoryId: 'GHSA-jf85-cpcp-j695', - ecosystem: 'pypi', + ecosystem: 'homebrew', }) expect(result.success).toBe(false) }) + + it('accepts pypi as a supported ecosystem', () => { + const result = blastRadiusJobRequestSchema.safeParse({ + advisoryId: 'GHSA-jf85-cpcp-j695', + ecosystem: 'pypi', + }) + expect(result.success).toBe(true) + }) }) describe('toBlastRadiusJobEntry', () => { diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts index eb70ad1bd9..a0c765ff00 100644 --- a/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJob.test.ts @@ -109,7 +109,7 @@ describe('submitBlastRadiusJob', () => { it('rejects an unsupported ecosystem without starting a workflow', async () => { const { req, res, start } = mockReqRes({ advisoryId: 'GHSA-jf85-cpcp-j695', - ecosystem: 'pypi', + ecosystem: 'homebrew', }) await expect(submitBlastRadiusJob(req, res)).rejects.toThrow(/not supported/) @@ -117,6 +117,22 @@ describe('submitBlastRadiusJob', () => { expect(createAnalysis).not.toHaveBeenCalled() }) + it('starts a workflow for a pypi ecosystem request', async () => { + const { req, res, start } = mockReqRes({ + advisoryId: 'GHSA-jf85-cpcp-j695', + ecosystem: 'pypi', + }) + + await submitBlastRadiusJob(req, res) + + expect(start).toHaveBeenCalledTimes(1) + const [, options] = start.mock.calls[0] + expect(options.args[0]).toMatchObject({ + advisoryId: 'GHSA-jf85-cpcp-j695', + ecosystem: 'pypi', + }) + }) + it('rejects a missing ecosystem without starting a workflow', async () => { const { req, res, start } = mockReqRes({ advisoryId: 'GHSA-jf85-cpcp-j695' }) diff --git a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts index 91296ae4ad..8794ff7a7b 100644 --- a/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts +++ b/backend/src/api/public/v1/packages/submitBlastRadiusJobBatch.test.ts @@ -94,7 +94,7 @@ describe('submitBlastRadiusJobBatch', () => { const { req, res, start } = mockReqRes({ jobs: [ { advisoryId: 'GHSA-jf85-cpcp-j695', ecosystem: 'npm' }, - { advisoryId: 'GHSA-652q-gvq3-74qv', ecosystem: 'pypi' }, + { advisoryId: 'GHSA-652q-gvq3-74qv', ecosystem: 'homebrew' }, ], }) @@ -103,6 +103,16 @@ describe('submitBlastRadiusJobBatch', () => { expect(createAnalysis).not.toHaveBeenCalled() }) + it('accepts a batch containing a pypi job', async () => { + const { req, res, start } = mockReqRes({ + jobs: [{ advisoryId: 'GHSA-652q-gvq3-74qv', ecosystem: 'pypi' }], + }) + + await submitBlastRadiusJobBatch(req, res) + + expect(start).toHaveBeenCalledTimes(1) + }) + it('rejects a batch with more than 20 jobs without submitting any job', async () => { const jobs = Array.from({ length: 21 }, () => ({ advisoryId: 'GHSA-jf85-cpcp-j695', diff --git a/services/apps/packages_worker/src/blast-radius/__tests__/packageIdentifier.test.ts b/services/apps/packages_worker/src/blast-radius/__tests__/packageIdentifier.test.ts index 4f98341cf3..50a1f740f0 100644 --- a/services/apps/packages_worker/src/blast-radius/__tests__/packageIdentifier.test.ts +++ b/services/apps/packages_worker/src/blast-radius/__tests__/packageIdentifier.test.ts @@ -1,6 +1,13 @@ import { describe, expect, it } from 'vitest' -import { toBareGemName, toBareNpmName, toBareNuGetId, toDbCargoName } from '../packageIdentifier' +import { + toBareGemName, + toBareNpmName, + toBareNuGetId, + toBarePypiName, + toDbCargoName, + toPypiNormalizedName, +} from '../packageIdentifier' describe('toBareNpmName', () => { it('returns a bare name unchanged', () => { @@ -91,3 +98,47 @@ describe('toBareGemName', () => { expect(toBareGemName('pkg:gem/RedCloth@4.3.2')).toBe('RedCloth') }) }) + +describe('toBarePypiName', () => { + it('returns a bare name unchanged', () => { + expect(toBarePypiName('flask')).toBe('flask') + }) + + it('strips the pkg:pypi/ prefix', () => { + expect(toBarePypiName('pkg:pypi/flask')).toBe('flask') + }) + + it('strips a trailing version', () => { + expect(toBarePypiName('pkg:pypi/flask@3.0.0')).toBe('flask') + }) + + it('strips qualifiers and subpath', () => { + expect(toBarePypiName('pkg:pypi/flask@3.0.0?foo=bar#sub')).toBe('flask') + }) + + it('does not lowercase the name — preserves the publisher-cased spelling', () => { + expect(toBarePypiName('pkg:pypi/Jinja2@3.1.2')).toBe('Jinja2') + }) + + it('decodes a percent-encoded name', () => { + expect(toBarePypiName('pkg:pypi/py%2Dyaml')).toBe('py-yaml') + }) + + it('keeps normalizing after a decode failure', () => { + expect(toBarePypiName('pkg:pypi/flask@3.0.0%')).toBe('flask') + }) +}) + +describe('toPypiNormalizedName', () => { + it('applies PEP 503 normalization', () => { + expect(toPypiNormalizedName('Foo_Bar.Baz')).toBe('foo-bar-baz') + }) + + it('collapses repeated separators into a single dash', () => { + expect(toPypiNormalizedName('foo--bar__baz')).toBe('foo-bar-baz') + }) + + it('is idempotent on an already-normalized name', () => { + expect(toPypiNormalizedName('jinja2')).toBe('jinja2') + }) +}) diff --git a/services/apps/packages_worker/src/blast-radius/agent/__tests__/agentAuth.test.ts b/services/apps/packages_worker/src/blast-radius/agent/__tests__/agentAuth.test.ts new file mode 100644 index 0000000000..94dca4b004 --- /dev/null +++ b/services/apps/packages_worker/src/blast-radius/agent/__tests__/agentAuth.test.ts @@ -0,0 +1,147 @@ +import { afterEach, describe, expect, it } from 'vitest' + +import { resolveAgentAuth } from '@crowd/common' + +const AKRITES_BEDROCK_ENV_VAR_NAMES = { + accessKeyId: 'AKRITES_AWS_BEDROCK_ACCESS_KEY_ID', + secretAccessKey: 'AKRITES_AWS_BEDROCK_SECRET_ACCESS_KEY', + region: 'AKRITES_AWS_BEDROCK_REGION', +} + +const ALL_ENV_VARS = [ + 'CROWD_AWS_BEDROCK_ACCESS_KEY_ID', + 'CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY', + 'CROWD_AWS_BEDROCK_REGION', + 'AKRITES_AWS_BEDROCK_ACCESS_KEY_ID', + 'AKRITES_AWS_BEDROCK_SECRET_ACCESS_KEY', + 'AKRITES_AWS_BEDROCK_REGION', + 'BLAST_RADIUS_ANTHROPIC_API_KEY', + 'BLAST_RADIUS_ANTHROPIC_BASE_URL', + 'ANTHROPIC_API_KEY', +] + +function clearAuthEnv() { + for (const key of ALL_ENV_VARS) { + delete process.env[key] + } +} + +describe('resolveAgentAuth', () => { + afterEach(() => { + clearAuthEnv() + }) + + it('resolves bedrock mode using the default (CROWD_AWS_BEDROCK_*) env vars', () => { + clearAuthEnv() + process.env.CROWD_AWS_BEDROCK_ACCESS_KEY_ID = 'AKIA_TEST' + process.env.CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY = 'secret' + process.env.ANTHROPIC_API_KEY = 'sk-ant-should-not-survive' + + const auth = resolveAgentAuth() + + expect(auth.mode).toBe('bedrock') + expect(auth.env.CLAUDE_CODE_USE_BEDROCK).toBe('1') + expect(auth.env.AWS_ACCESS_KEY_ID).toBe('AKIA_TEST') + expect(auth.env.AWS_SECRET_ACCESS_KEY).toBe('secret') + expect(auth.env.AWS_REGION).toBe('us-east-1') + expect(auth.env.ANTHROPIC_API_KEY).toBeUndefined() + }) + + it('resolves bedrock mode using a caller-supplied bedrockEnvVarNames (e.g. Akrites)', () => { + clearAuthEnv() + process.env.AKRITES_AWS_BEDROCK_ACCESS_KEY_ID = 'AKIA_AKRITES' + process.env.AKRITES_AWS_BEDROCK_SECRET_ACCESS_KEY = 'akrites-secret' + // A CROWD_AWS_BEDROCK_* credential being set for an unrelated consumer must not + // leak into an Akrites-scoped caller. + process.env.CROWD_AWS_BEDROCK_ACCESS_KEY_ID = 'AKIA_UNRELATED' + process.env.CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY = 'unrelated-secret' + + const auth = resolveAgentAuth({ bedrockEnvVarNames: AKRITES_BEDROCK_ENV_VAR_NAMES }) + + expect(auth.mode).toBe('bedrock') + expect(auth.env.AWS_ACCESS_KEY_ID).toBe('AKIA_AKRITES') + expect(auth.env.AWS_SECRET_ACCESS_KEY).toBe('akrites-secret') + }) + + it('uses the region env var named by bedrockEnvVarNames when set', () => { + clearAuthEnv() + process.env.AKRITES_AWS_BEDROCK_ACCESS_KEY_ID = 'AKIA_AKRITES' + process.env.AKRITES_AWS_BEDROCK_SECRET_ACCESS_KEY = 'akrites-secret' + process.env.AKRITES_AWS_BEDROCK_REGION = 'us-west-2' + + const auth = resolveAgentAuth({ bedrockEnvVarNames: AKRITES_BEDROCK_ENV_VAR_NAMES }) + + expect(auth.env.AWS_REGION).toBe('us-west-2') + }) + + it('does not fall into bedrock mode with only one of the two credentials', () => { + clearAuthEnv() + process.env.AKRITES_AWS_BEDROCK_ACCESS_KEY_ID = 'AKIA_AKRITES' + process.env.BLAST_RADIUS_ANTHROPIC_API_KEY = 'sk-ant-fallback' + + const auth = resolveAgentAuth({ bedrockEnvVarNames: AKRITES_BEDROCK_ENV_VAR_NAMES }) + + expect(auth.mode).toBe('anthropic-api-key') + }) + + it('resolves anthropic-api-key mode when no bedrock credentials are set', () => { + clearAuthEnv() + process.env.BLAST_RADIUS_ANTHROPIC_API_KEY = 'sk-ant-test' + process.env.BLAST_RADIUS_ANTHROPIC_BASE_URL = 'https://litellm.internal' + + const auth = resolveAgentAuth({ bedrockEnvVarNames: AKRITES_BEDROCK_ENV_VAR_NAMES }) + + expect(auth.mode).toBe('anthropic-api-key') + expect(auth.env.ANTHROPIC_API_KEY).toBe('sk-ant-test') + expect(auth.env.ANTHROPIC_BASE_URL).toBe('https://litellm.internal') + expect(auth.resolveModel('claude-opus-4-8')).toBe('claude-opus-4-8') + }) + + it('supports a custom api key env var name', () => { + clearAuthEnv() + process.env.CUSTOM_ANTHROPIC_API_KEY = 'sk-ant-custom' + + const auth = resolveAgentAuth({ apiKeyEnvVar: 'CUSTOM_ANTHROPIC_API_KEY' }) + + expect(auth.mode).toBe('anthropic-api-key') + expect(auth.env.ANTHROPIC_API_KEY).toBe('sk-ant-custom') + + delete process.env.CUSTOM_ANTHROPIC_API_KEY + }) + + it('falls back to cli auth when nothing is configured', () => { + clearAuthEnv() + + const auth = resolveAgentAuth({ bedrockEnvVarNames: AKRITES_BEDROCK_ENV_VAR_NAMES }) + + expect(auth.mode).toBe('cli-fallback') + expect(auth.env).toBeUndefined() + expect(auth.resolveModel('claude-sonnet-5')).toBe('claude-sonnet-5') + }) + + it('translates model IDs via modelBedrockMap in bedrock mode', () => { + clearAuthEnv() + process.env.AKRITES_AWS_BEDROCK_ACCESS_KEY_ID = 'AKIA_AKRITES' + process.env.AKRITES_AWS_BEDROCK_SECRET_ACCESS_KEY = 'akrites-secret' + + const auth = resolveAgentAuth({ + bedrockEnvVarNames: AKRITES_BEDROCK_ENV_VAR_NAMES, + modelBedrockMap: { 'claude-opus-4-8': 'us.anthropic.claude-opus-4-8-v1:0' }, + }) + + expect(auth.resolveModel('claude-opus-4-8')).toBe('us.anthropic.claude-opus-4-8-v1:0') + }) + + it('throws on an unmapped model when a modelBedrockMap is provided', () => { + clearAuthEnv() + process.env.AKRITES_AWS_BEDROCK_ACCESS_KEY_ID = 'AKIA_AKRITES' + process.env.AKRITES_AWS_BEDROCK_SECRET_ACCESS_KEY = 'akrites-secret' + + const auth = resolveAgentAuth({ + bedrockEnvVarNames: AKRITES_BEDROCK_ENV_VAR_NAMES, + modelBedrockMap: { 'claude-opus-4-8': 'us.anthropic.claude-opus-4-8-v1:0' }, + }) + + expect(() => auth.resolveModel('claude-haiku-9000')).toThrow() + }) +}) diff --git a/services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts b/services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts index bcc20a465d..13e805b8df 100644 --- a/services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts +++ b/services/apps/packages_worker/src/blast-radius/agent/pypiPrompts.ts @@ -8,8 +8,6 @@ import { } from './promptKit' import { SymbolSpec } from './prompts' -// ---------- STAGE 1: INTEL ---------- - const IMPORT_SIGNATURE_KEYS = ['import_module', 'from_import', 'attribute_access', 'dynamic_import'] export const PYPI_INTEL_SCHEMA = buildIntelSchema(IMPORT_SIGNATURE_KEYS) @@ -47,8 +45,6 @@ Rules: export const buildPyPiIntelPrompt = buildIntelPrompt -// ---------- STAGE 3: REACHABILITY ---------- - const IMPORT_STYLE_ENUM = [ 'import-module', 'from-import', diff --git a/services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts b/services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts index 5157402d05..0493c479ae 100644 --- a/services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts +++ b/services/apps/packages_worker/src/blast-radius/clients/__tests__/pypiSource.test.ts @@ -135,6 +135,29 @@ describe('downloadAndExtractPypiSource', () => { ) }) + it('extracts a zip-format sdist, stripping the wrapper directory', async () => { + const sdistBuf = buildStoredZip([{ path: 'flask-3.0.0/src/flask/__init__.py', content: 'x=1' }]) + mockFetchSequence([ + { + json: { + info: { name: 'flask' }, + urls: [ + { + packagetype: 'sdist', + url: 'https://files.pythonhosted.org/sdist.zip', + filename: 'flask-3.0.0.zip', + }, + ], + }, + }, + { body: sdistBuf }, + ]) + + await downloadAndExtractPypiSource('flask', '3.0.0', destDir) + + expect(fs.readFileSync(path.join(destDir, 'src/flask/__init__.py'), 'utf8')).toBe('x=1') + }) + it('throws PypiSourceNotFoundError when neither sdist nor wheel is present', async () => { mockFetchSequence([{ json: { info: { name: 'flask' }, urls: [] } }]) diff --git a/services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts b/services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts index 58c3ed9d66..2555432d32 100644 --- a/services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts +++ b/services/apps/packages_worker/src/blast-radius/clients/pypiSource.ts @@ -112,14 +112,19 @@ async function downloadSdist( } } -// A .whl is a zip with no wrapper directory. Zip's central directory sits at the end of -// the file, so unlike tar we can't stream-extract incrementally — download to a scratch -// file first, then extract, same as goModuleZip.ts. -async function downloadWheel( +// A sdist can ship as either .tar.gz/.tgz or .zip — packagetype only tells us +// "source distribution", not the archive format, so dispatch on the actual filename. +function isZipArchive(filename: string): boolean { + return filename.toLowerCase().endsWith('.zip') +} + +// Zip's central directory is at the end; download first, then extract (see goModuleZip.ts). +async function downloadZip( url: string, destDir: string, packageName: string, version: string, + stripTopLevelDir: boolean, ): Promise { const controller = new AbortController() const timeoutHandle = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS) @@ -159,22 +164,25 @@ async function downloadWheel( for (const entry of directory.files) { if (entry.type !== 'File') continue - // Wheel contents originate from a third-party package — guard path traversal + const relativePath = stripTopLevelDir ? entry.path.split('/').slice(1).join('/') : entry.path + if (!relativePath) continue + + // Archive contents originate from a third-party package — guard path traversal // defensively rather than trust the archive (no tar-style preservePaths here). - const resolvedPath = path.resolve(destDir, entry.path) + const resolvedPath = path.resolve(destDir, relativePath) if (resolvedPath !== destDir && !resolvedPath.startsWith(destDir + path.sep)) { - throw new Error(`Wheel entry escapes destination dir: ${entry.path}`) + throw new Error(`Zip entry escapes destination dir: ${entry.path}`) } extractedFiles++ if (extractedFiles > MAX_EXTRACTED_FILES) { - throw new Error('Wheel extraction exceeded size/file limits') + throw new Error('Zip extraction exceeded size/file limits') } mkdirSync(path.dirname(resolvedPath), { recursive: true }) const extractionLimiter = createDownloadLimiter( - 'Wheel extraction exceeded size/file limits', + 'Zip extraction exceeded size/file limits', MAX_EXTRACTED_BYTES, extractedByteCounter, ) @@ -201,8 +209,12 @@ export async function downloadAndExtractPypiSource( mkdirSync(destDir, { recursive: true }) if (dist.packagetype === 'sdist') { - await downloadSdist(dist.url, destDir, packageName, version) + if (isZipArchive(dist.filename)) { + await downloadZip(dist.url, destDir, packageName, version, true) + } else { + await downloadSdist(dist.url, destDir, packageName, version) + } } else { - await downloadWheel(dist.url, destDir, packageName, version) + await downloadZip(dist.url, destDir, packageName, version, false) } } diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts index f459e188e4..e7fcfcb292 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts @@ -21,6 +21,11 @@ describe('pypiConstraintMayInclude', () => { expect(pypiConstraintMayInclude('==2.2.*', ['2.3.0'])).toBe('excluded') }) + it('does not let wildcard prefix matching cross a release-segment boundary', () => { + expect(pypiConstraintMayInclude('==2.2.*', ['2.20.0'])).toBe('excluded') + expect(pypiConstraintMayInclude('~=1.4.5', ['1.40.0'])).toBe('excluded') + }) + it('matches "!=" with a ".*" wildcard as exclusion of the prefix', () => { expect(pypiConstraintMayInclude('!=2.2.*', ['2.3.0'])).toBe('matched') expect(pypiConstraintMayInclude('!=2.2.*', ['2.2.0'])).toBe('excluded') @@ -67,4 +72,9 @@ describe('pypiDependencyMayIncludeVuln', () => { expect(pypiDependencyMayIncludeVuln(null, '==1.0.0', ['1.0.0'])).toBe('matched') expect(pypiDependencyMayIncludeVuln(null, null, ['1.0.0'])).toBe('unparseable-included') }) + + it('recognizes PEP 440-equivalent resolved versions, not just exact strings', () => { + expect(pypiDependencyMayIncludeVuln('1.0', '<2.0', ['1.0.0'])).toBe('matched') + expect(pypiDependencyMayIncludeVuln('1.0alpha1', '<2.0', ['1.0a1'])).toBe('matched') + }) }) diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts index d7b3635a22..abcd20d810 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts @@ -2,9 +2,9 @@ import * as fs from 'fs' import * as os from 'os' import * as path from 'path' +import { findPackageIdsByPurl } from '@crowd/data-access-layer/src/osspckgs/packages' import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' import { getVersionNumbers } from '@crowd/data-access-layer/src/packages/blastRadiusDependents' -import { findPackageIdByPurl } from '@crowd/data-access-layer/src/packages/osv' import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { fetchProject } from '../../../pypi/fetchProject' @@ -81,20 +81,18 @@ export async function runIntelStagePyPi( // ECOSYSTEM-typed, not SEMVER — PEP 440 versions don't follow semver ordering). const ranges = ecosystemRangeEvents(entry) - // packages.name can drift from the canonical PEP 503 spelling for pypi rows (a - // pre-existing data-quality issue — see the blast-radius PyPI plan's name-casing - // section); packages.purl is always PEP 503-normalized, so look up by purl instead. - const packageId = await findPackageIdByPurl(qx, `pkg:pypi/${normalizedName}`) + // packages.purl is always normalized; packages.name can drift from canonical PEP 503. + // See: blast-radius PyPI plan's name-casing section. + const purl = `pkg:pypi/${normalizedName}` + const packageId = (await findPackageIdsByPurl(qx, [purl])).get(purl) ?? null - // pypi.org's JSON API is the authoritative version list; fall back to our own - // ingested `versions` rows (deps.dev) if the registry is unreachable and the - // project is already known to us. + // pypi.org's JSON API is authoritative; fall back to ingested versions if unreachable. const projectResult = await fetchProject(normalizedName) let allVersions: string[] if (!isFetchError(projectResult)) { allVersions = Object.keys(projectResult.releases ?? {}) } else if (packageId !== null) { - allVersions = await getVersionNumbers(qx, String(packageId)) + allVersions = await getVersionNumbers(qx, packageId) } else { throw new Error( `Failed to fetch PyPI versions for ${project} (${projectResult.message}) and project is not in our DB`, diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts index af78595cc0..0f6f57d97a 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts @@ -70,19 +70,27 @@ function parseSpecifierSet(constraint: string | null): PypiClause[] | null { return clauses } -// Wildcard matching reduces to a public-version string prefix check — deliberately -// over-inclusive rather than spec-exact, consistent with every other clause here. function normalizedForWildcard(version: string): string { return version.trim().toLowerCase().replace(/^v/, '').split('+')[0] } +// PEP 440 wildcard matching is a release-segment prefix match, not a raw string prefix — +// "==2.2.*" must not match "2.20.0", so the char right after the prefix can't be a digit. +function matchesWildcardPrefix(version: string, prefix: string): boolean { + const normalized = normalizedForWildcard(version) + const normalizedPrefix = prefix.toLowerCase() + if (!normalized.startsWith(normalizedPrefix)) return false + const boundaryChar = normalized[normalizedPrefix.length] + return boundaryChar === undefined || !/[0-9]/.test(boundaryChar) +} + function clauseMatches(clause: PypiClause, version: string): boolean { if (clause.op === '===') { return version.trim().toLowerCase() === clause.version.trim().toLowerCase() } if (clause.wildcard) { - const matches = normalizedForWildcard(version).startsWith(clause.version.toLowerCase()) + const matches = matchesWildcardPrefix(version, clause.version) return clause.op === '==' ? matches : !matches } @@ -122,16 +130,15 @@ export function pypiConstraintMayInclude( return matched ? 'matched' : 'excluded' } -// Prefers resolved version over the declared specifier (ground truth vs. declared) — PyPI -// is a deps.dev EDGE ecosystem, so a resolved version is usually available; mirrors -// cargoDependencyMayIncludeVuln. +// Prefers resolved version (ground truth); PyPI is a deps.dev EDGE ecosystem. export function pypiDependencyMayIncludeVuln( resolvedVersion: string | null, constraint: string | null, vulnerableVersions: string[], ): PypiConstraintMatch { if (resolvedVersion) { - return vulnerableVersions.includes(resolvedVersion) ? 'matched' : 'excluded' + const matched = vulnerableVersions.some((v) => compareVersion('pypi', resolvedVersion, v) === 0) + return matched ? 'matched' : 'excluded' } return pypiConstraintMayInclude(constraint, vulnerableVersions) } diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts index 37982e97cb..c4e3dd563f 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/reachabilityConfig.ts @@ -7,9 +7,7 @@ import { downloadAndExtractPypiSource } from '../../clients/pypiSource' import { toPypiNormalizedName } from '../../packageIdentifier' import { ReachabilitySourceConfig } from '../reachabilityStage' -// PyPI is a deps.dev EDGE ecosystem, so dep.version is the dependent's own resolved -// version and is never null — no canonical-name round trip is needed the way Cargo's -// resolveCargoCanonical requires, since the PyPI JSON API accepts normalized names. +// PyPI is a deps.dev EDGE ecosystem; dep.version is the dependent's resolved version. export const pypiReachabilityConfig: ReachabilitySourceConfig = { prompt: PYPI_REACHABILITY_PROMPT, schema: PYPI_VERDICT_SCHEMA, diff --git a/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts b/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts index cae07d38bf..deda3accda 100644 --- a/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts +++ b/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts @@ -139,7 +139,7 @@ async function main() { const role = normaliseRole(r.role) const insertedId = await insertPackageMaintainerLink( tqx, - packageId, + Number(packageId), maintainerId, role, 'manual_csv', diff --git a/services/libs/common/src/agentAuth.ts b/services/libs/common/src/agentAuth.ts new file mode 100644 index 0000000000..425ba1cc53 --- /dev/null +++ b/services/libs/common/src/agentAuth.ts @@ -0,0 +1,97 @@ +export type AgentAuthMode = 'bedrock' | 'anthropic-api-key' | 'cli-fallback' + +export interface AgentAuth { + mode: AgentAuthMode + env: Record | undefined + resolveModel(model: string): string +} + +export interface BedrockEnvVarNames { + accessKeyId: string + secretAccessKey: string + region: string +} + +export interface ResolveAgentAuthOptions { + apiKeyEnvVar?: string + modelBedrockMap?: Record + // Which env vars carry the Bedrock credential for this caller. There is no single + // org-wide Bedrock credential: CROWD_AWS_BEDROCK_* (the default) is the one shared by + // the enrichment workers, unrelated to Akrites. Callers under Akrites must pass their + // own AKRITES_* var names explicitly rather than relying on this default. + bedrockEnvVarNames?: BedrockEnvVarNames +} + +const DEFAULT_BEDROCK_ENV_VAR_NAMES: BedrockEnvVarNames = { + accessKeyId: 'CROWD_AWS_BEDROCK_ACCESS_KEY_ID', + secretAccessKey: 'CROWD_AWS_BEDROCK_SECRET_ACCESS_KEY', + region: 'CROWD_AWS_BEDROCK_REGION', +} + +const DEFAULT_BEDROCK_REGION = 'us-east-1' + +// The CLI prefers ANTHROPIC_API_KEY over Bedrock env vars when both are present in the +// subprocess env, so it must be stripped explicitly — otherwise Bedrock mode silently +// never activates even with valid AWS credentials. +function bedrockEnv(accessKeyId: string, secretAccessKey: string, region: string) { + const env = { ...process.env } as Record + delete env.ANTHROPIC_API_KEY + env.CLAUDE_CODE_USE_BEDROCK = '1' + env.AWS_ACCESS_KEY_ID = accessKeyId + env.AWS_SECRET_ACCESS_KEY = secretAccessKey + env.AWS_REGION = region + return env +} + +function resolveModelWith(modelBedrockMap: Record | undefined) { + return (model: string): string => { + if (!modelBedrockMap) { + return model + } + const resolved = modelBedrockMap[model] + if (!resolved) { + throw new Error(`No Bedrock model ID mapped for agent model "${model}"`) + } + return resolved + } +} + +export function resolveAgentAuth(opts: ResolveAgentAuthOptions = {}): AgentAuth { + const { + apiKeyEnvVar = 'BLAST_RADIUS_ANTHROPIC_API_KEY', + modelBedrockMap, + bedrockEnvVarNames = DEFAULT_BEDROCK_ENV_VAR_NAMES, + } = opts + + const bedrockAccessKeyId = process.env[bedrockEnvVarNames.accessKeyId] + const bedrockSecretAccessKey = process.env[bedrockEnvVarNames.secretAccessKey] + + if (bedrockAccessKeyId && bedrockSecretAccessKey) { + const region = process.env[bedrockEnvVarNames.region] || DEFAULT_BEDROCK_REGION + return { + mode: 'bedrock', + env: bedrockEnv(bedrockAccessKeyId, bedrockSecretAccessKey, region), + resolveModel: resolveModelWith(modelBedrockMap), + } + } + + const apiKey = process.env[apiKeyEnvVar] + if (apiKey) { + const baseUrl = process.env.BLAST_RADIUS_ANTHROPIC_BASE_URL + return { + mode: 'anthropic-api-key', + env: { + ...process.env, + ANTHROPIC_API_KEY: apiKey, + ...(baseUrl ? { ANTHROPIC_BASE_URL: baseUrl } : {}), + } as Record, + resolveModel: (model) => model, + } + } + + return { + mode: 'cli-fallback', + env: undefined, + resolveModel: (model) => model, + } +} diff --git a/services/libs/data-access-layer/src/osspckgs/packages.ts b/services/libs/data-access-layer/src/osspckgs/packages.ts index 03f986ef34..1331700ecf 100644 --- a/services/libs/data-access-layer/src/osspckgs/packages.ts +++ b/services/libs/data-access-layer/src/osspckgs/packages.ts @@ -2,15 +2,17 @@ import { QueryExecutor } from '../queryExecutor' import { IDbPackageUniverse, IDbPackageUpsert, IDbSonatypePopularityUpsert } from './types' +// Returns IDs as strings: packages.id is BIGSERIAL (int8) and pg-promise leaves int8 +// values as strings to avoid silently losing precision via Number() coercion. export async function findPackageIdsByPurl( qx: QueryExecutor, purls: string[], -): Promise> { +): Promise> { if (purls.length === 0) return new Map() const rows = await qx.select(`SELECT id, purl FROM packages WHERE purl = ANY($(purls))`, { purls, }) - return new Map(rows.map((r: { purl: string; id: number }) => [r.purl, r.id])) + return new Map(rows.map((r: { purl: string; id: string }) => [r.purl, r.id])) } // ─── packages_universe ──────────────────────────────────────────────────────── diff --git a/services/libs/data-access-layer/src/packages/blastRadius.ts b/services/libs/data-access-layer/src/packages/blastRadius.ts index 0cac869b2a..f86a697e4e 100644 --- a/services/libs/data-access-layer/src/packages/blastRadius.ts +++ b/services/libs/data-access-layer/src/packages/blastRadius.ts @@ -220,7 +220,7 @@ export async function resolveAdvisoryAndPackageIds( qx: QueryExecutor, analysisId: string, advisoryOsvId: string, - packageId: number | null, + packageId: number | string | null, ): Promise { await qx.result( ` diff --git a/services/libs/data-access-layer/src/packages/osv.ts b/services/libs/data-access-layer/src/packages/osv.ts index ba76ae1c54..f02284e92d 100644 --- a/services/libs/data-access-layer/src/packages/osv.ts +++ b/services/libs/data-access-layer/src/packages/osv.ts @@ -119,12 +119,6 @@ export async function findPackageId( return (row?.id as number | undefined) ?? null } -// Use purl as key for ecosystems where name drifts; purl is UNIQUE NOT NULL and always normalized. -export async function findPackageIdByPurl(qx: QueryExecutor, purl: string): Promise { - const row = await qx.selectOneOrNone(`SELECT id FROM packages WHERE purl = $(purl)`, { purl }) - return (row?.id as number | undefined) ?? null -} - // Batched form of findPackageId for a flat list of full package names (e.g. // "lodash", "@babel/core") — one round-trip instead of one query per name. // Mirrors getNpmPurlsForChangedNames's namespace/name reconstruction join. From bb3e7c412dbae3c544c2340a91b07b9d314bfa7f Mon Sep 17 00:00:00 2001 From: Umberto Sgueglia Date: Fri, 7 Aug 2026 17:41:45 +0200 Subject: [PATCH 3/3] fix: comments Signed-off-by: Umberto Sgueglia --- .../pypi/__tests__/pypiConstraint.test.ts | 6 ++ .../src/blast-radius/stages/pypi/intelPyPi.ts | 4 +- .../stages/pypi/pypiConstraint.ts | 6 +- .../maven/scripts/importMaintainersFromCsv.ts | 2 +- .../src/osv/__tests__/versionCompare.test.ts | 6 ++ .../packages_worker/src/osv/versionCompare.ts | 76 ++++++++++--------- services/libs/common/src/index.ts | 1 + .../src/osspckgs/packages.ts | 6 +- .../data-access-layer/src/packages/osv.ts | 17 +++++ 9 files changed, 80 insertions(+), 44 deletions(-) diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts index e7fcfcb292..afdd41dc7c 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/__tests__/pypiConstraint.test.ts @@ -77,4 +77,10 @@ describe('pypiDependencyMayIncludeVuln', () => { expect(pypiDependencyMayIncludeVuln('1.0', '<2.0', ['1.0.0'])).toBe('matched') expect(pypiDependencyMayIncludeVuln('1.0alpha1', '<2.0', ['1.0a1'])).toBe('matched') }) + + it('is over-inclusive when the resolved version cannot be parsed', () => { + expect(pypiDependencyMayIncludeVuln('not-a-version', '<2.0', ['1.0.0'])).toBe( + 'unparseable-included', + ) + }) }) diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts index abcd20d810..318b081427 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/intelPyPi.ts @@ -2,9 +2,9 @@ import * as fs from 'fs' import * as os from 'os' import * as path from 'path' -import { findPackageIdsByPurl } from '@crowd/data-access-layer/src/osspckgs/packages' import * as blastRadiusDal from '@crowd/data-access-layer/src/packages/blastRadius' import { getVersionNumbers } from '@crowd/data-access-layer/src/packages/blastRadiusDependents' +import { findPackageIdByPurl } from '@crowd/data-access-layer/src/packages/osv' import { QueryExecutor } from '@crowd/data-access-layer/src/queryExecutor' import { fetchProject } from '../../../pypi/fetchProject' @@ -84,7 +84,7 @@ export async function runIntelStagePyPi( // packages.purl is always normalized; packages.name can drift from canonical PEP 503. // See: blast-radius PyPI plan's name-casing section. const purl = `pkg:pypi/${normalizedName}` - const packageId = (await findPackageIdsByPurl(qx, [purl])).get(purl) ?? null + const packageId = await findPackageIdByPurl(qx, purl) // pypi.org's JSON API is authoritative; fall back to ingested versions if unreachable. const projectResult = await fetchProject(normalizedName) diff --git a/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts b/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts index 0f6f57d97a..50b6182915 100644 --- a/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts +++ b/services/apps/packages_worker/src/blast-radius/stages/pypi/pypiConstraint.ts @@ -137,8 +137,10 @@ export function pypiDependencyMayIncludeVuln( vulnerableVersions: string[], ): PypiConstraintMatch { if (resolvedVersion) { - const matched = vulnerableVersions.some((v) => compareVersion('pypi', resolvedVersion, v) === 0) - return matched ? 'matched' : 'excluded' + const comparisons = vulnerableVersions.map((v) => compareVersion('pypi', resolvedVersion, v)) + if (comparisons.includes(0)) return 'matched' + if (comparisons.includes(null)) return 'unparseable-included' + return 'excluded' } return pypiConstraintMayInclude(constraint, vulnerableVersions) } diff --git a/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts b/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts index deda3accda..cae07d38bf 100644 --- a/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts +++ b/services/apps/packages_worker/src/maven/scripts/importMaintainersFromCsv.ts @@ -139,7 +139,7 @@ async function main() { const role = normaliseRole(r.role) const insertedId = await insertPackageMaintainerLink( tqx, - Number(packageId), + packageId, maintainerId, role, 'manual_csv', diff --git a/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts b/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts index a5361ce21d..428584ecb5 100644 --- a/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts +++ b/services/apps/packages_worker/src/osv/__tests__/versionCompare.test.ts @@ -184,6 +184,12 @@ describe('compareVersion — pypi (PEP 440)', () => { expect(compareVersion('pypi', 'not-a-version', '1.0.0')).toBeNull() }) + it('distinguishes release segments beyond Number.MAX_SAFE_INTEGER', () => { + expect(sign(compareVersion('pypi', '1.9007199254740992', '1.9007199254740993'))).toBe(-1) + expect(sign(compareVersion('pypi', '1.9007199254740993', '1.9007199254740992'))).toBe(1) + expect(compareVersion('pypi', '1.9007199254740992', '1.9007199254740992')).toBe(0) + }) + it('rejects titlecase "PyPI" — production storage is always lowercase', () => { expect(compareVersion('PyPI', '1.0.0', '2.0.0')).toBeNull() }) diff --git a/services/apps/packages_worker/src/osv/versionCompare.ts b/services/apps/packages_worker/src/osv/versionCompare.ts index ed797d6499..c6ee6a0b47 100644 --- a/services/apps/packages_worker/src/osv/versionCompare.ts +++ b/services/apps/packages_worker/src/osv/versionCompare.ts @@ -197,16 +197,16 @@ function compareRubyGems(a: string, b: string): number | null { interface Pep440Pre { letter: 'a' | 'b' | 'rc' - num: number + num: bigint } interface Pep440Version { - epoch: number - release: number[] + epoch: bigint + release: bigint[] pre: Pep440Pre | null - post: number | null - dev: number | null - local: (number | string)[] + post: bigint | null + dev: bigint | null + local: (bigint | string)[] } // Longest-alias-first: JS regex alternation picks the first matching alternative, not the @@ -231,16 +231,16 @@ const PEP440_LOCAL_RE = /^\+([a-z0-9]+(?:[-_.][a-z0-9]+)*)/ function parsePep440(raw: string): Pep440Version | null { let s = raw.trim().toLowerCase().replace(/^v/, '') - let epoch = 0 + let epoch = BigInt(0) const epochMatch = s.match(/^([0-9]+)!/) if (epochMatch) { - epoch = parseInt(epochMatch[1], 10) + epoch = BigInt(epochMatch[1]) s = s.slice(epochMatch[0].length) } const releaseMatch = s.match(/^[0-9]+(?:\.[0-9]+)*/) if (!releaseMatch) return null - const release = releaseMatch[0].split('.').map((n) => parseInt(n, 10)) + const release = releaseMatch[0].split('.').map((n) => BigInt(n)) s = s.slice(releaseMatch[0].length) let pre: Pep440Pre | null = null @@ -248,37 +248,35 @@ function parsePep440(raw: string): Pep440Version | null { if (preMatch) { pre = { letter: PEP440_PRE_ALIASES[preMatch[1]], - num: preMatch[2] ? parseInt(preMatch[2], 10) : 0, + num: preMatch[2] ? BigInt(preMatch[2]) : BigInt(0), } s = s.slice(preMatch[0].length) } - let post: number | null = null + let post: bigint | null = null const implicitPostMatch = s.match(/^-([0-9]+)/) if (implicitPostMatch) { - post = parseInt(implicitPostMatch[1], 10) + post = BigInt(implicitPostMatch[1]) s = s.slice(implicitPostMatch[0].length) } else { const postMatch = s.match(PEP440_POST_RE) if (postMatch) { - post = postMatch[1] ? parseInt(postMatch[1], 10) : 0 + post = postMatch[1] ? BigInt(postMatch[1]) : BigInt(0) s = s.slice(postMatch[0].length) } } - let dev: number | null = null + let dev: bigint | null = null const devMatch = s.match(PEP440_DEV_RE) if (devMatch) { - dev = devMatch[1] ? parseInt(devMatch[1], 10) : 0 + dev = devMatch[1] ? BigInt(devMatch[1]) : BigInt(0) s = s.slice(devMatch[0].length) } - let local: (number | string)[] = [] + let local: (bigint | string)[] = [] const localMatch = s.match(PEP440_LOCAL_RE) if (localMatch) { - local = localMatch[1] - .split(/[-_.]/) - .map((seg) => (/^[0-9]+$/.test(seg) ? parseInt(seg, 10) : seg)) + local = localMatch[1].split(/[-_.]/).map((seg) => (/^[0-9]+$/.test(seg) ? BigInt(seg) : seg)) s = s.slice(localMatch[0].length) } @@ -287,28 +285,32 @@ function parsePep440(raw: string): Pep440Version | null { return { epoch, release, pre, post, dev, local } } -const PEP440_PRE_RANK: Record<'a' | 'b' | 'rc', number> = { a: 0, b: 1, rc: 2 } +const PEP440_PRE_RANK: Record<'a' | 'b' | 'rc', bigint> = { + a: BigInt(0), + b: BigInt(1), + rc: BigInt(2), +} // Bare dev release sorts below pre-releases; release with no pre-release sorts above them — // see PEP 440's documented order .devN < aN < bN < rcN < final < .postN. -function pep440PreOrder(v: Pep440Version): [number, number, number] { - if (v.pre) return [0, PEP440_PRE_RANK[v.pre.letter], v.pre.num] - if (v.post === null && v.dev !== null) return [-1, 0, 0] - return [1, 0, 0] +function pep440PreOrder(v: Pep440Version): [bigint, bigint, bigint] { + if (v.pre) return [BigInt(0), PEP440_PRE_RANK[v.pre.letter], v.pre.num] + if (v.post === null && v.dev !== null) return [BigInt(-1), BigInt(0), BigInt(0)] + return [BigInt(1), BigInt(0), BigInt(0)] } -function comparePep440Local(a: (number | string)[], b: (number | string)[]): number { +function comparePep440Local(a: (bigint | string)[], b: (bigint | string)[]): number { const len = Math.max(a.length, b.length) for (let i = 0; i < len; i++) { if (i >= a.length) return -1 if (i >= b.length) return 1 const [x, y] = [a[i], b[i]] - if (typeof x === 'number' && typeof y === 'number') { + if (typeof x === 'bigint' && typeof y === 'bigint') { if (x !== y) return x < y ? -1 : 1 continue } - if (typeof x === 'number') return 1 - if (typeof y === 'number') return -1 + if (typeof x === 'bigint') return 1 + if (typeof y === 'bigint') return -1 if (x !== y) return x < y ? -1 : 1 } return 0 @@ -323,8 +325,8 @@ function comparePep440(a: string, b: string): number | null { const relLen = Math.max(pa.release.length, pb.release.length) for (let i = 0; i < relLen; i++) { - const ra = pa.release[i] ?? 0 - const rb = pb.release[i] ?? 0 + const ra = pa.release[i] ?? BigInt(0) + const rb = pb.release[i] ?? BigInt(0) if (ra !== rb) return ra < rb ? -1 : 1 } @@ -334,13 +336,17 @@ function comparePep440(a: string, b: string): number | null { if (preA[i] !== preB[i]) return preA[i] < preB[i] ? -1 : 1 } - const postA = pa.post ?? -Infinity - const postB = pb.post ?? -Infinity + const postA = pa.post ?? BigInt(-1) + const postB = pb.post ?? BigInt(-1) if (postA !== postB) return postA < postB ? -1 : 1 - const devA = pa.dev ?? Infinity - const devB = pb.dev ?? Infinity - if (devA !== devB) return devA < devB ? -1 : 1 + const devA = pa.dev + const devB = pb.dev + if (devA !== devB) { + if (devA === null) return 1 + if (devB === null) return -1 + return devA < devB ? -1 : 1 + } if (pa.local.length === 0 && pb.local.length === 0) return 0 if (pa.local.length === 0) return -1 diff --git a/services/libs/common/src/index.ts b/services/libs/common/src/index.ts index 6a4eb89884..968975a31f 100644 --- a/services/libs/common/src/index.ts +++ b/services/libs/common/src/index.ts @@ -21,6 +21,7 @@ import { export { getDbConstraint } from './errors/db' +export * from './agentAuth' export * from './env' export * from './timing' export * from './utils' diff --git a/services/libs/data-access-layer/src/osspckgs/packages.ts b/services/libs/data-access-layer/src/osspckgs/packages.ts index 1331700ecf..03f986ef34 100644 --- a/services/libs/data-access-layer/src/osspckgs/packages.ts +++ b/services/libs/data-access-layer/src/osspckgs/packages.ts @@ -2,17 +2,15 @@ import { QueryExecutor } from '../queryExecutor' import { IDbPackageUniverse, IDbPackageUpsert, IDbSonatypePopularityUpsert } from './types' -// Returns IDs as strings: packages.id is BIGSERIAL (int8) and pg-promise leaves int8 -// values as strings to avoid silently losing precision via Number() coercion. export async function findPackageIdsByPurl( qx: QueryExecutor, purls: string[], -): Promise> { +): Promise> { if (purls.length === 0) return new Map() const rows = await qx.select(`SELECT id, purl FROM packages WHERE purl = ANY($(purls))`, { purls, }) - return new Map(rows.map((r: { purl: string; id: string }) => [r.purl, r.id])) + return new Map(rows.map((r: { purl: string; id: number }) => [r.purl, r.id])) } // ─── packages_universe ──────────────────────────────────────────────────────── diff --git a/services/libs/data-access-layer/src/packages/osv.ts b/services/libs/data-access-layer/src/packages/osv.ts index f02284e92d..e80874164d 100644 --- a/services/libs/data-access-layer/src/packages/osv.ts +++ b/services/libs/data-access-layer/src/packages/osv.ts @@ -119,6 +119,23 @@ export async function findPackageId( return (row?.id as number | undefined) ?? null } +// purl-based counterpart of findPackageId, for ecosystems (e.g. PyPI) where +// packages.name can drift from the canonical normalized form but packages.purl +// is always normalized. Returns the ID as a string: packages.id is BIGSERIAL +// (int8) and pg-promise leaves int8 values as strings to avoid silently losing +// precision via Number() coercion. +export async function findPackageIdByPurl(qx: QueryExecutor, purl: string): Promise { + const row = await qx.selectOneOrNone( + ` + SELECT id + FROM packages + WHERE purl = $(purl) + `, + { purl }, + ) + return (row?.id as string | undefined) ?? null +} + // Batched form of findPackageId for a flat list of full package names (e.g. // "lodash", "@babel/core") — one round-trip instead of one query per name. // Mirrors getNpmPurlsForChangedNames's namespace/name reconstruction join.