diff --git a/PR_BODY_COMPUTE_RERUN_FEASIBILITY_16.md b/PR_BODY_COMPUTE_RERUN_FEASIBILITY_16.md new file mode 100644 index 00000000..fed579db --- /dev/null +++ b/PR_BODY_COMPUTE_RERUN_FEASIBILITY_16.md @@ -0,0 +1,34 @@ +/claim #16 + +## Summary + +Adds `research-compute-rerun-feasibility-guard`, a standalone local package for the AI-Powered Research Assistant Suite. + +The guard prevents AI assistant reproducibility language from being released until the project has enough compute-resource evidence for a practical reviewer rerun. It checks hardware specs, RAM/VRAM feasibility, runtime budget, checkpoint/resume evidence, container/lockfile/runtime digests, accelerator determinism, large data-transfer fixture availability, paid cloud/HPC disclosure, and long-run resource notes. + +## Demo Video + +`research-compute-rerun-feasibility-guard/reports/demo.mp4` + +The MP4 is a synthetic FFmpeg-rendered slate. It does not capture a desktop, private data, credentials, real manuscripts, or paid services. + +## Validation + +From `research-compute-rerun-feasibility-guard`: + +- `npm run check` +- `npm test` +- `npm run demo` +- `npm run video` + +Repository-level: + +- `git diff --check` + +## Notes + +- No external APIs. +- No paid cloud resources. +- No credentials or private data. +- Synthetic fixtures only. +- Scope is distinct from existing #16 slices around general dependency locks, evidence grounding, external validity, image integrity, statistical/model assumptions, data fabrication, and generic reproducibility attempts. This package specifically gates AI assistant output on practical compute rerun feasibility and resource disclosure. diff --git a/research-compute-rerun-feasibility-guard/README.md b/research-compute-rerun-feasibility-guard/README.md new file mode 100644 index 00000000..4e4255a6 --- /dev/null +++ b/research-compute-rerun-feasibility-guard/README.md @@ -0,0 +1,47 @@ +# Research Compute Rerun Feasibility Guard + +This package adds a local release gate for the SCIBASE AI-Powered Research Assistant Suite. It checks whether an AI assistant can safely describe a manuscript or project as practically reproducible before the assistant output reaches authors, reviewers, journals, or funders. + +The guard focuses on compute feasibility rather than general dependency locking or claim grounding. It asks whether a reviewer can actually rerun the work with the stated resources, time budget, determinism controls, and cost disclosure. + +## What It Checks + +- Hardware profile completeness: CPU, RAM, accelerator, and VRAM. +- Dataset working set vs RAM and GPU memory. +- Runtime estimate vs reviewer budget. +- Long-run checkpoint/resume evidence. +- Container, lockfile, runtime, or notebook environment digests. +- Accelerator determinism controls and seed policy. +- Large data transfer and fixture availability. +- Paid cloud or queued HPC requirements and disclosure. +- Resource-impact note for long reruns. + +## Release Decisions + +- `RELEASE_ASSISTANT_OUTPUT`: the assistant can present the rerun as reviewer-feasible. +- `REVISE_ASSISTANT_OUTPUT`: wording must be softened until missing resource evidence is repaired. +- `HOLD_ASSISTANT_OUTPUT`: the assistant must not endorse reproducibility because the compute plan is materially unsafe. + +## Local Demo + +```bash +npm run check +npm test +npm run demo +npm run video +``` + +Generated artifacts: + +- `reports/summary.json` +- `reports/reviewer-packet.md` +- `reports/summary.svg` +- `reports/demo.mp4` + +All fixtures are synthetic. The package uses no external APIs, paid cloud, credentials, private data, real manuscripts, or desktop capture. + +## Why This Fits Issue #16 + +Issue #16 includes a reproducibility checker as a core AI research assistant capability. A research assistant can mislead reviewers if it says a project is reproducible while the rerun secretly requires unavailable GPUs, huge memory, long queues, paid cloud resources, or nondeterministic accelerator settings. + +This guard adds a practical compute/resource release gate so the assistant can distinguish a reviewer-feasible rerun from an overclaimed reproducibility endorsement. diff --git a/research-compute-rerun-feasibility-guard/data/sample-rerun-packets.json b/research-compute-rerun-feasibility-guard/data/sample-rerun-packets.json new file mode 100644 index 00000000..9927a9b4 --- /dev/null +++ b/research-compute-rerun-feasibility-guard/data/sample-rerun-packets.json @@ -0,0 +1,143 @@ +[ + { + "id": "paper-cpu-fixture-pass", + "title": "Open fixture rerun for a tabular treatment-effect study", + "assistantOutput": "pre-submission reproducibility checker", + "dataset": { + "sizeGb": 1.2, + "downloadGb": 1.2, + "workingSetMultiplier": 1.8, + "syntheticFixtureAvailable": true, + "requiresGpuResidency": false + }, + "compute": { + "cpuCores": 8, + "ramGb": 32, + "accelerator": "cpu", + "gpuCount": 0, + "gpuVramGb": 0, + "estimatedRuntimeHours": 1.5, + "reviewerBudgetHours": 4, + "paidCloudRequired": false, + "hpcQueueRequired": false + }, + "environment": { + "containerDigest": "sha256:fixture-pass-container", + "lockfileDigest": "sha256:fixture-pass-lock" + }, + "reproducibility": { + "seedPolicy": "fixed seed family with three rerun seeds", + "determinismControls": ["seed manifest", "variance tolerance report"], + "checkpointResumeEvidence": "not required for short CPU rerun" + }, + "disclosure": { + "estimatedCostUsd": 0, + "carbonOrResourceNote": "CPU fixture rerun completes locally under two hours." + } + }, + { + "id": "paper-gpu-overclaim-hold", + "title": "Large multimodal training rerun marked reproducible without resources", + "assistantOutput": "AI reviewer reproducibility endorsement", + "dataset": { + "sizeGb": 480, + "downloadGb": 620, + "workingSetMultiplier": 2.2, + "syntheticFixtureAvailable": false, + "requiresGpuResidency": true + }, + "compute": { + "cpuCores": 16, + "ramGb": 64, + "accelerator": "cuda", + "gpuCount": 1, + "gpuVramGb": 24, + "estimatedRuntimeHours": 38, + "reviewerBudgetHours": 8, + "paidCloudRequired": true, + "hpcQueueRequired": false + }, + "environment": {}, + "reproducibility": { + "seedPolicy": "", + "determinismControls": [], + "checkpointResumeEvidence": "" + }, + "disclosure": { + "estimatedCostUsd": 0, + "carbonOrResourceNote": "" + } + }, + { + "id": "paper-hpc-revise", + "title": "Queued HPC simulation with enough hardware but weak disclosure", + "assistantOutput": "journal internal reviewer copilot", + "dataset": { + "sizeGb": 32, + "downloadGb": 40, + "workingSetMultiplier": 1.5, + "syntheticFixtureAvailable": true, + "requiresGpuResidency": false + }, + "compute": { + "cpuCores": 48, + "ramGb": 256, + "accelerator": "cpu", + "gpuCount": 0, + "gpuVramGb": 0, + "estimatedRuntimeHours": 12, + "reviewerBudgetHours": 8, + "paidCloudRequired": false, + "hpcQueueRequired": true + }, + "environment": { + "runtimeDigest": "sha256:hpc-module-stack" + }, + "reproducibility": { + "seedPolicy": "fixed Monte Carlo seed manifest", + "determinismControls": ["seed manifest", "variance tolerance"], + "checkpointResumeEvidence": "checkpoint manifest for every 90 simulated minutes" + }, + "disclosure": { + "estimatedCostUsd": 0, + "queuePolicy": "institutional queue, expected wait not stated", + "carbonOrResourceNote": "" + } + }, + { + "id": "paper-vram-fixture-needed", + "title": "Single-cell embedding rerun with GPU memory shortfall", + "assistantOutput": "research-gap assistant reproducibility sidebar", + "dataset": { + "sizeGb": 78, + "downloadGb": 92, + "workingSetMultiplier": 1.4, + "syntheticFixtureAvailable": false, + "requiresGpuResidency": true + }, + "compute": { + "cpuCores": 24, + "ramGb": 192, + "accelerator": "cuda", + "gpuCount": 1, + "gpuVramGb": 40, + "estimatedRuntimeHours": 7, + "reviewerBudgetHours": 6, + "paidCloudRequired": true, + "hpcQueueRequired": true + }, + "environment": { + "containerDigest": "sha256:single-cell-container" + }, + "reproducibility": { + "seedPolicy": "seeded embedding initialization", + "determinismControls": ["seed manifest", "cuda deterministic kernels"], + "checkpointResumeEvidence": "" + }, + "disclosure": { + "estimatedCostUsd": 48, + "queuePolicy": "shared GPU queue", + "carbonOrResourceNote": "" + } + } +] diff --git a/research-compute-rerun-feasibility-guard/package.json b/research-compute-rerun-feasibility-guard/package.json new file mode 100644 index 00000000..c8515ccc --- /dev/null +++ b/research-compute-rerun-feasibility-guard/package.json @@ -0,0 +1,21 @@ +{ + "name": "research-compute-rerun-feasibility-guard", + "version": "1.0.0", + "description": "Local guard that checks whether AI research assistant reproducibility claims are compute-feasible before reviewer release.", + "type": "module", + "main": "src/index.js", + "scripts": { + "check": "node --check src/index.js && node --check scripts/demo.js && node --check scripts/render-demo-video.js && node --check test/compute-rerun-feasibility.test.js", + "test": "node --test", + "demo": "node scripts/demo.js", + "video": "node scripts/render-demo-video.js" + }, + "keywords": [ + "research-assistant", + "reproducibility", + "compute-feasibility", + "peer-review", + "scibase" + ], + "license": "MIT" +} diff --git a/research-compute-rerun-feasibility-guard/reports/demo.mp4 b/research-compute-rerun-feasibility-guard/reports/demo.mp4 new file mode 100644 index 00000000..daea6bb2 Binary files /dev/null and b/research-compute-rerun-feasibility-guard/reports/demo.mp4 differ diff --git a/research-compute-rerun-feasibility-guard/reports/reviewer-packet.md b/research-compute-rerun-feasibility-guard/reports/reviewer-packet.md new file mode 100644 index 00000000..6178792d --- /dev/null +++ b/research-compute-rerun-feasibility-guard/reports/reviewer-packet.md @@ -0,0 +1,20 @@ +# Research Compute Rerun Feasibility Guard + +Generated: 2026-08-16T19:40:17.361Z + +This packet is a local synthetic demonstration for SCIBASE issue #16. It checks whether an AI-powered research assistant should release reproducibility language before the compute plan is practically rerunnable by a reviewer. + +| Packet | Release gate | Score | Working set | Top findings | +| --- | --- | ---: | ---: | --- | +| paper-cpu-fixture-pass | RELEASE_ASSISTANT_OUTPUT | 100 | 2.16 GB | None | +| paper-gpu-overclaim-hold | HOLD_ASSISTANT_OUTPUT | 0 | 1056 GB | CONTAINER_OR_LOCKFILE_MISSING (high); DATASET_MEMORY_EXCEEDS_NODE (critical); GPU_VRAM_FEASIBILITY_GAP (high) | +| paper-hpc-revise | REVISE_ASSISTANT_OUTPUT | 40 | 48 GB | RUNTIME_BUDGET_UNREALISTIC (medium); CLOUD_OR_HPC_COST_UNDISCLOSED (high); RESOURCE_IMPACT_NOTE_MISSING (low) | +| paper-vram-fixture-needed | HOLD_ASSISTANT_OUTPUT | 0 | 109.2 GB | GPU_VRAM_FEASIBILITY_GAP (high); RUNTIME_BUDGET_UNREALISTIC (medium); CHECKPOINT_RESUME_MISSING (high) | + +## Release Policy + +- `RELEASE_ASSISTANT_OUTPUT`: the assistant can present the rerun as reviewer-feasible. +- `REVISE_ASSISTANT_OUTPUT`: assistant wording must be softened until missing resource evidence is repaired. +- `HOLD_ASSISTANT_OUTPUT`: the assistant must not endorse reproducibility because compute feasibility is materially unsafe. + +No external API, private data, paid cloud, real manuscript, or user desktop capture is used. diff --git a/research-compute-rerun-feasibility-guard/reports/summary.json b/research-compute-rerun-feasibility-guard/reports/summary.json new file mode 100644 index 00000000..225aa5f5 --- /dev/null +++ b/research-compute-rerun-feasibility-guard/reports/summary.json @@ -0,0 +1,224 @@ +{ + "generatedAt": "2026-08-16T19:40:17.361Z", + "packetCount": 4, + "decisionCounts": { + "RELEASE_ASSISTANT_OUTPUT": 1, + "HOLD_ASSISTANT_OUTPUT": 2, + "REVISE_ASSISTANT_OUTPUT": 1 + }, + "results": [ + { + "id": "paper-cpu-fixture-pass", + "title": "Open fixture rerun for a tabular treatment-effect study", + "decision": "RELEASE_ASSISTANT_OUTPUT", + "score": 100, + "summary": { + "packetId": "paper-cpu-fixture-pass", + "title": "Open fixture rerun for a tabular treatment-effect study", + "assistantOutput": "pre-submission reproducibility checker", + "estimatedWorkingSetGb": 2.16, + "ramGb": 32, + "totalGpuVramGb": 0, + "findingCount": 0, + "worstSeverity": "none" + }, + "findings": [], + "releaseGate": { + "canReleaseAssistantOutput": true, + "reviewerMessage": "Compute evidence is sufficient for the assistant to present the rerun as reviewer-feasible." + } + }, + { + "id": "paper-gpu-overclaim-hold", + "title": "Large multimodal training rerun marked reproducible without resources", + "decision": "HOLD_ASSISTANT_OUTPUT", + "score": 0, + "summary": { + "packetId": "paper-gpu-overclaim-hold", + "title": "Large multimodal training rerun marked reproducible without resources", + "assistantOutput": "AI reviewer reproducibility endorsement", + "estimatedWorkingSetGb": 1056, + "ramGb": 64, + "totalGpuVramGb": 24, + "findingCount": 10, + "worstSeverity": "critical" + }, + "findings": [ + { + "code": "CONTAINER_OR_LOCKFILE_MISSING", + "severity": "high", + "title": "Executable environment evidence is missing", + "evidence": "No container digest, lockfile digest, runtime digest, or notebook environment digest was provided.", + "remediation": "Attach a digest-pinned container, lockfile, runtime manifest, or archived notebook environment." + }, + { + "code": "DATASET_MEMORY_EXCEEDS_NODE", + "severity": "critical", + "title": "Dataset working set exceeds stated system memory", + "evidence": "1056 GB estimated working set vs 64 GB RAM with 4 GB reserved headroom.", + "remediation": "Provide a smaller reviewer fixture, streaming pipeline proof, shard plan, or larger reproducible hardware profile." + }, + { + "code": "GPU_VRAM_FEASIBILITY_GAP", + "severity": "high", + "title": "GPU-resident workload exceeds available VRAM", + "evidence": "1056 GB working set marked GPU-resident vs 24 GB total VRAM.", + "remediation": "Document gradient accumulation, checkpointing, offload, or a verified smaller fixture before release." + }, + { + "code": "RUNTIME_BUDGET_UNREALISTIC", + "severity": "high", + "title": "Reviewer rerun time exceeds the stated review budget", + "evidence": "38 hour rerun estimate vs 8 hour reviewer budget.", + "remediation": "Split the rerun into smoke, fixture, and full modes or disclose that full reproduction requires extended resources." + }, + { + "code": "CHECKPOINT_RESUME_MISSING", + "severity": "high", + "title": "Long rerun lacks checkpoint or resume evidence", + "evidence": "38 hour run has no checkpoint/resume evidence in the packet.", + "remediation": "Attach a checkpoint manifest, restart transcript, or resumable workflow proof." + }, + { + "code": "NONDETERMINISTIC_ACCELERATOR_PATH", + "severity": "high", + "title": "Accelerator rerun lacks determinism controls", + "evidence": "cuda path is present without seed policy plus deterministic kernel/runtime controls.", + "remediation": "Document seed handling, deterministic kernel settings, tolerance windows, and accepted variance." + }, + { + "code": "SEED_POLICY_MISSING", + "severity": "medium", + "title": "Seed or variance policy is missing", + "evidence": "The assistant packet has no seed policy and no deterministic control list.", + "remediation": "Add seed policy, variance acceptance bands, and a rerun transcript showing stable outputs." + }, + { + "code": "DATA_ACCESS_BANDWIDTH_GAP", + "severity": "medium", + "title": "Large data transfer lacks a reviewer fixture", + "evidence": "620 GB download is required and no synthetic or reduced reviewer fixture is listed.", + "remediation": "Provide a small fixture, cached digest, or staged data-access plan with expected transfer time." + }, + { + "code": "CLOUD_OR_HPC_COST_UNDISCLOSED", + "severity": "high", + "title": "Paid cloud or queued HPC requirement is not disclosed", + "evidence": "The packet requires paid cloud/HPC resources but does not disclose expected reviewer cost.", + "remediation": "State cost, queue assumptions, no-cost fixture alternative, and who bears the rerun cost." + }, + { + "code": "RESOURCE_IMPACT_NOTE_MISSING", + "severity": "low", + "title": "Long compute rerun lacks resource-impact disclosure", + "evidence": "No carbon/resource note is attached for a long-running rerun.", + "remediation": "Add a short resource-impact note and reviewer-facing rerun alternatives." + } + ], + "releaseGate": { + "canReleaseAssistantOutput": false, + "reviewerMessage": "Hold or revise the assistant's reproducibility wording until compute feasibility evidence is repaired." + } + }, + { + "id": "paper-hpc-revise", + "title": "Queued HPC simulation with enough hardware but weak disclosure", + "decision": "REVISE_ASSISTANT_OUTPUT", + "score": 40, + "summary": { + "packetId": "paper-hpc-revise", + "title": "Queued HPC simulation with enough hardware but weak disclosure", + "assistantOutput": "journal internal reviewer copilot", + "estimatedWorkingSetGb": 48, + "ramGb": 256, + "totalGpuVramGb": 0, + "findingCount": 3, + "worstSeverity": "high" + }, + "findings": [ + { + "code": "RUNTIME_BUDGET_UNREALISTIC", + "severity": "medium", + "title": "Reviewer rerun time exceeds the stated review budget", + "evidence": "12 hour rerun estimate vs 8 hour reviewer budget.", + "remediation": "Split the rerun into smoke, fixture, and full modes or disclose that full reproduction requires extended resources." + }, + { + "code": "CLOUD_OR_HPC_COST_UNDISCLOSED", + "severity": "high", + "title": "Paid cloud or queued HPC requirement is not disclosed", + "evidence": "The packet requires paid cloud/HPC resources but does not disclose expected reviewer cost.", + "remediation": "State cost, queue assumptions, no-cost fixture alternative, and who bears the rerun cost." + }, + { + "code": "RESOURCE_IMPACT_NOTE_MISSING", + "severity": "low", + "title": "Long compute rerun lacks resource-impact disclosure", + "evidence": "No carbon/resource note is attached for a long-running rerun.", + "remediation": "Add a short resource-impact note and reviewer-facing rerun alternatives." + } + ], + "releaseGate": { + "canReleaseAssistantOutput": false, + "reviewerMessage": "Hold or revise the assistant's reproducibility wording until compute feasibility evidence is repaired." + } + }, + { + "id": "paper-vram-fixture-needed", + "title": "Single-cell embedding rerun with GPU memory shortfall", + "decision": "HOLD_ASSISTANT_OUTPUT", + "score": 0, + "summary": { + "packetId": "paper-vram-fixture-needed", + "title": "Single-cell embedding rerun with GPU memory shortfall", + "assistantOutput": "research-gap assistant reproducibility sidebar", + "estimatedWorkingSetGb": 109.2, + "ramGb": 192, + "totalGpuVramGb": 40, + "findingCount": 5, + "worstSeverity": "high" + }, + "findings": [ + { + "code": "GPU_VRAM_FEASIBILITY_GAP", + "severity": "high", + "title": "GPU-resident workload exceeds available VRAM", + "evidence": "109.2 GB working set marked GPU-resident vs 40 GB total VRAM.", + "remediation": "Document gradient accumulation, checkpointing, offload, or a verified smaller fixture before release." + }, + { + "code": "RUNTIME_BUDGET_UNREALISTIC", + "severity": "medium", + "title": "Reviewer rerun time exceeds the stated review budget", + "evidence": "7 hour rerun estimate vs 6 hour reviewer budget.", + "remediation": "Split the rerun into smoke, fixture, and full modes or disclose that full reproduction requires extended resources." + }, + { + "code": "CHECKPOINT_RESUME_MISSING", + "severity": "high", + "title": "Long rerun lacks checkpoint or resume evidence", + "evidence": "7 hour run has no checkpoint/resume evidence in the packet.", + "remediation": "Attach a checkpoint manifest, restart transcript, or resumable workflow proof." + }, + { + "code": "NO_LOW_COST_REVIEWER_MODE", + "severity": "high", + "title": "Expensive rerun lacks a low-cost review mode", + "evidence": "$48.00 estimated cost without a synthetic fixture or low-cost mode.", + "remediation": "Add an inexpensive smoke test and fixture-mode result before the AI assistant marks it reproducible." + }, + { + "code": "RESOURCE_IMPACT_NOTE_MISSING", + "severity": "low", + "title": "Long compute rerun lacks resource-impact disclosure", + "evidence": "No carbon/resource note is attached for a long-running rerun.", + "remediation": "Add a short resource-impact note and reviewer-facing rerun alternatives." + } + ], + "releaseGate": { + "canReleaseAssistantOutput": false, + "reviewerMessage": "Hold or revise the assistant's reproducibility wording until compute feasibility evidence is repaired." + } + } + ] +} diff --git a/research-compute-rerun-feasibility-guard/reports/summary.svg b/research-compute-rerun-feasibility-guard/reports/summary.svg new file mode 100644 index 00000000..16c5b4f2 --- /dev/null +++ b/research-compute-rerun-feasibility-guard/reports/summary.svg @@ -0,0 +1,44 @@ + + + Compute Rerun Feasibility Guard + AI assistant release gate for practical reproducibility claims + + + paper-cpu-fixture-pass + + + 100/100 + RELEASE_ASSISTANT_OUTPUT + + + + paper-gpu-overclaim-hold + + + 0/100 + HOLD_ASSISTANT_OUTPUT + + + + paper-hpc-revise + + + 40/100 + REVISE_ASSISTANT_OUTPUT + + + + paper-vram-fixture-needed + + + 0/100 + HOLD_ASSISTANT_OUTPUT + + + diff --git a/research-compute-rerun-feasibility-guard/scripts/demo.js b/research-compute-rerun-feasibility-guard/scripts/demo.js new file mode 100644 index 00000000..692510af --- /dev/null +++ b/research-compute-rerun-feasibility-guard/scripts/demo.js @@ -0,0 +1,95 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { analyzeRerunPackets } from "../src/index.js"; + +const root = process.cwd(); +const reportsDir = join(root, "reports"); +const packets = JSON.parse(await readFile(join(root, "data", "sample-rerun-packets.json"), "utf8")); +const report = analyzeRerunPackets(packets); + +await mkdir(reportsDir, { recursive: true }); +await writeFile(join(reportsDir, "summary.json"), `${JSON.stringify(report, null, 2)}\n`); +await writeFile(join(reportsDir, "reviewer-packet.md"), renderMarkdown(report)); +await writeFile(join(reportsDir, "summary.svg"), renderSvg(report)); + +console.log(`Wrote ${report.packetCount} packet report to ${reportsDir}`); +console.log(JSON.stringify(report.decisionCounts, null, 2)); + +function renderMarkdown(reportData) { + const rows = reportData.results + .map((result) => { + const topFindings = + result.findings.length === 0 + ? "None" + : result.findings + .slice(0, 3) + .map((finding) => `${finding.code} (${finding.severity})`) + .join("; "); + return `| ${result.id} | ${result.decision} | ${result.score} | ${result.summary.estimatedWorkingSetGb} GB | ${topFindings} |`; + }) + .join("\n"); + + return `# Research Compute Rerun Feasibility Guard + +Generated: ${reportData.generatedAt} + +This packet is a local synthetic demonstration for SCIBASE issue #16. It checks whether an AI-powered research assistant should release reproducibility language before the compute plan is practically rerunnable by a reviewer. + +| Packet | Release gate | Score | Working set | Top findings | +| --- | --- | ---: | ---: | --- | +${rows} + +## Release Policy + +- \`RELEASE_ASSISTANT_OUTPUT\`: the assistant can present the rerun as reviewer-feasible. +- \`REVISE_ASSISTANT_OUTPUT\`: assistant wording must be softened until missing resource evidence is repaired. +- \`HOLD_ASSISTANT_OUTPUT\`: the assistant must not endorse reproducibility because compute feasibility is materially unsafe. + +No external API, private data, paid cloud, real manuscript, or user desktop capture is used. +`; +} + +function renderSvg(reportData) { + const color = { + RELEASE_ASSISTANT_OUTPUT: "#1f9d55", + REVISE_ASSISTANT_OUTPUT: "#c27803", + HOLD_ASSISTANT_OUTPUT: "#c2410c" + }; + const rows = reportData.results + .map((result, index) => { + const y = 112 + index * 86; + const barWidth = Math.max(16, result.score * 5.1); + return ` + + ${escapeXml(result.id)} + + + ${result.score}/100 + ${result.decision} + `; + }) + .join("\n"); + + return ` + + Compute Rerun Feasibility Guard + AI assistant release gate for practical reproducibility claims +${rows} + + +`; +} + +function escapeXml(value) { + return String(value) + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">") + .replaceAll('"', """); +} diff --git a/research-compute-rerun-feasibility-guard/scripts/render-demo-video.js b/research-compute-rerun-feasibility-guard/scripts/render-demo-video.js new file mode 100644 index 00000000..05c04a14 --- /dev/null +++ b/research-compute-rerun-feasibility-guard/scripts/render-demo-video.js @@ -0,0 +1,68 @@ +import { mkdir, readFile } from "node:fs/promises"; +import { spawn } from "node:child_process"; +import { join, resolve } from "node:path"; + +const root = process.cwd(); +const reportsDir = join(root, "reports"); +const output = join(reportsDir, "demo.mp4"); +const ffmpeg = resolve(root, "..", "..", "tool_downloads", "video_tools", "node_modules", "ffmpeg-static", "ffmpeg.exe"); +const font = "C\\:/Windows/Fonts/arial.ttf"; + +await mkdir(reportsDir, { recursive: true }); +const report = JSON.parse(await readFile(join(reportsDir, "summary.json"), "utf8").catch(async () => { + throw new Error("Run npm run demo before npm run video so reports/summary.json exists."); +})); + +const counts = report.decisionCounts || {}; +const release = counts.RELEASE_ASSISTANT_OUTPUT || 0; +const revise = counts.REVISE_ASSISTANT_OUTPUT || 0; +const hold = counts.HOLD_ASSISTANT_OUTPUT || 0; +const headline = "SCIBASE #16: Compute Rerun Feasibility Guard"; +const subhead = `Release ${release} | Revise ${revise} | Hold ${hold}`; +const caption = "Synthetic demo: no desktop capture, private data, API keys, or paid cloud."; + +const drawText = [ + `drawtext=fontfile=${font}:text='${escapeDrawText(headline)}':fontsize=42:fontcolor=white:x=60:y=80`, + `drawtext=fontfile=${font}:text='${escapeDrawText(subhead)}':fontsize=34:fontcolor=0x8ee6a7:x=60:y=160`, + `drawtext=fontfile=${font}:text='${escapeDrawText("Gate AI reproducibility claims on hardware, memory, runtime, determinism, and disclosure evidence.")}':fontsize=24:fontcolor=white:x=60:y=250`, + `drawtext=fontfile=${font}:text='${escapeDrawText(caption)}':fontsize=22:fontcolor=0xcbd5e1:x=60:y=610` +].join(","); + +await run(ffmpeg, [ + "-y", + "-f", + "lavfi", + "-i", + "color=c=0x101827:s=1280x720:d=8:r=30", + "-vf", + drawText, + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + output +]); + +console.log(`Wrote ${output}`); + +function run(command, args) { + return new Promise((resolvePromise, reject) => { + const child = spawn(command, args, { stdio: "inherit" }); + child.on("error", reject); + child.on("exit", (code) => { + if (code === 0) { + resolvePromise(); + } else { + reject(new Error(`${command} exited with ${code}`)); + } + }); + }); +} + +function escapeDrawText(value) { + return String(value) + .replaceAll("\\", "\\\\") + .replaceAll(":", "\\:") + .replaceAll("'", "\\'") + .replaceAll(",", "\\,"); +} diff --git a/research-compute-rerun-feasibility-guard/src/index.js b/research-compute-rerun-feasibility-guard/src/index.js new file mode 100644 index 00000000..71c55c7b --- /dev/null +++ b/research-compute-rerun-feasibility-guard/src/index.js @@ -0,0 +1,366 @@ +const DECISIONS = Object.freeze({ + RELEASE: "RELEASE_ASSISTANT_OUTPUT", + REVISE: "REVISE_ASSISTANT_OUTPUT", + HOLD: "HOLD_ASSISTANT_OUTPUT" +}); + +const SEVERITY_RANK = Object.freeze({ + low: 1, + medium: 2, + high: 3, + critical: 4 +}); + +const DEFAULTS = Object.freeze({ + datasetMultiplier: 2.5, + minimumRamHeadroomGb: 4, + runtimeBudgetHours: 8, + expensiveCloudThresholdUsd: 25, + longRunHours: 6, + largeDatasetGb: 100 +}); + +function toNumber(value, fallback = 0) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : fallback; +} + +function asArray(value) { + return Array.isArray(value) ? value : []; +} + +function createFinding(code, severity, title, evidence, remediation) { + return { code, severity, title, evidence, remediation }; +} + +function datasetWorkingSetGb(packet) { + const dataset = packet.dataset || {}; + const sizeGb = toNumber(dataset.sizeGb); + const multiplier = toNumber(dataset.workingSetMultiplier, DEFAULTS.datasetMultiplier); + return Number((sizeGb * multiplier).toFixed(2)); +} + +function computeRamGb(packet) { + const compute = packet.compute || {}; + const ramGb = toNumber(compute.ramGb); + const gpuVramGb = toNumber(compute.gpuVramGb); + const gpuCount = toNumber(compute.gpuCount); + return { + ramGb, + totalGpuVramGb: Number((gpuVramGb * Math.max(1, gpuCount || 0)).toFixed(2)) + }; +} + +function hasHardwareSpec(packet) { + const compute = packet.compute || {}; + return Boolean( + compute.cpuCores && + compute.ramGb && + (compute.accelerator === "none" || compute.gpuVramGb || compute.accelerator === "cpu") + ); +} + +function hasContainerEvidence(packet) { + const environment = packet.environment || {}; + return Boolean( + environment.containerDigest || + environment.lockfileDigest || + environment.runtimeDigest || + environment.notebookEnvironmentDigest + ); +} + +function hasDeterminismEvidence(packet) { + const reproducibility = packet.reproducibility || {}; + const controls = asArray(reproducibility.determinismControls); + return Boolean(reproducibility.seedPolicy && controls.length > 0); +} + +function resourceDisclosure(packet) { + const disclosure = packet.disclosure || {}; + return { + hasCost: disclosure.estimatedCostUsd !== undefined, + costUsd: toNumber(disclosure.estimatedCostUsd), + hasQueue: Boolean(disclosure.queuePolicy || disclosure.hpcQueuePolicy), + hasCarbon: Boolean(disclosure.carbonOrResourceNote) + }; +} + +function evaluateMemory(packet) { + const dataset = packet.dataset || {}; + const compute = computeRamGb(packet); + const findings = []; + const workingSetGb = datasetWorkingSetGb(packet); + const ramCapacity = compute.ramGb - DEFAULTS.minimumRamHeadroomGb; + + if (toNumber(dataset.sizeGb) === 0) { + return findings; + } + + if (compute.ramGb && workingSetGb > ramCapacity) { + findings.push( + createFinding( + "DATASET_MEMORY_EXCEEDS_NODE", + "critical", + "Dataset working set exceeds stated system memory", + `${workingSetGb} GB estimated working set vs ${compute.ramGb} GB RAM with ${DEFAULTS.minimumRamHeadroomGb} GB reserved headroom.`, + "Provide a smaller reviewer fixture, streaming pipeline proof, shard plan, or larger reproducible hardware profile." + ) + ); + } + + if (dataset.requiresGpuResidency && compute.totalGpuVramGb && workingSetGb > compute.totalGpuVramGb) { + findings.push( + createFinding( + "GPU_VRAM_FEASIBILITY_GAP", + "high", + "GPU-resident workload exceeds available VRAM", + `${workingSetGb} GB working set marked GPU-resident vs ${compute.totalGpuVramGb} GB total VRAM.`, + "Document gradient accumulation, checkpointing, offload, or a verified smaller fixture before release." + ) + ); + } + + return findings; +} + +function evaluateRuntime(packet) { + const compute = packet.compute || {}; + const findings = []; + const runtimeHours = toNumber(compute.estimatedRuntimeHours); + const budgetHours = toNumber(compute.reviewerBudgetHours, DEFAULTS.runtimeBudgetHours); + + if (runtimeHours > budgetHours) { + findings.push( + createFinding( + "RUNTIME_BUDGET_UNREALISTIC", + runtimeHours > budgetHours * 2 ? "high" : "medium", + "Reviewer rerun time exceeds the stated review budget", + `${runtimeHours} hour rerun estimate vs ${budgetHours} hour reviewer budget.`, + "Split the rerun into smoke, fixture, and full modes or disclose that full reproduction requires extended resources." + ) + ); + } + + if (runtimeHours >= DEFAULTS.longRunHours && !packet.reproducibility?.checkpointResumeEvidence) { + findings.push( + createFinding( + "CHECKPOINT_RESUME_MISSING", + "high", + "Long rerun lacks checkpoint or resume evidence", + `${runtimeHours} hour run has no checkpoint/resume evidence in the packet.`, + "Attach a checkpoint manifest, restart transcript, or resumable workflow proof." + ) + ); + } + + return findings; +} + +function evaluateEnvironment(packet) { + const findings = []; + + if (!hasHardwareSpec(packet)) { + findings.push( + createFinding( + "HARDWARE_SPEC_MISSING", + "high", + "Hardware profile is incomplete", + "The packet does not state enough CPU, RAM, accelerator, and VRAM information for reviewer rerun planning.", + "Publish a reviewer hardware profile with minimum and validated configurations." + ) + ); + } + + if (!hasContainerEvidence(packet)) { + findings.push( + createFinding( + "CONTAINER_OR_LOCKFILE_MISSING", + "high", + "Executable environment evidence is missing", + "No container digest, lockfile digest, runtime digest, or notebook environment digest was provided.", + "Attach a digest-pinned container, lockfile, runtime manifest, or archived notebook environment." + ) + ); + } + + return findings; +} + +function evaluateDeterminism(packet) { + const findings = []; + const compute = packet.compute || {}; + const reproducibility = packet.reproducibility || {}; + + if (compute.accelerator && compute.accelerator !== "none" && compute.accelerator !== "cpu") { + const controls = asArray(reproducibility.determinismControls).join(" ").toLowerCase(); + const mentionsNondeterminism = controls.includes("deterministic") || controls.includes("cudnn") || controls.includes("seed"); + if (!mentionsNondeterminism || !reproducibility.seedPolicy) { + findings.push( + createFinding( + "NONDETERMINISTIC_ACCELERATOR_PATH", + "high", + "Accelerator rerun lacks determinism controls", + `${compute.accelerator} path is present without seed policy plus deterministic kernel/runtime controls.`, + "Document seed handling, deterministic kernel settings, tolerance windows, and accepted variance." + ) + ); + } + } + + if (!hasDeterminismEvidence(packet)) { + findings.push( + createFinding( + "SEED_POLICY_MISSING", + "medium", + "Seed or variance policy is missing", + "The assistant packet has no seed policy and no deterministic control list.", + "Add seed policy, variance acceptance bands, and a rerun transcript showing stable outputs." + ) + ); + } + + return findings; +} + +function evaluateAccessAndDisclosure(packet) { + const findings = []; + const dataset = packet.dataset || {}; + const compute = packet.compute || {}; + const disclosure = resourceDisclosure(packet); + const downloadGb = toNumber(dataset.downloadGb, dataset.sizeGb); + + if (downloadGb >= DEFAULTS.largeDatasetGb && !dataset.syntheticFixtureAvailable) { + findings.push( + createFinding( + "DATA_ACCESS_BANDWIDTH_GAP", + "medium", + "Large data transfer lacks a reviewer fixture", + `${downloadGb} GB download is required and no synthetic or reduced reviewer fixture is listed.`, + "Provide a small fixture, cached digest, or staged data-access plan with expected transfer time." + ) + ); + } + + if ((compute.paidCloudRequired || compute.hpcQueueRequired) && (!disclosure.hasCost || disclosure.costUsd === 0)) { + findings.push( + createFinding( + "CLOUD_OR_HPC_COST_UNDISCLOSED", + "high", + "Paid cloud or queued HPC requirement is not disclosed", + "The packet requires paid cloud/HPC resources but does not disclose expected reviewer cost.", + "State cost, queue assumptions, no-cost fixture alternative, and who bears the rerun cost." + ) + ); + } + + if (disclosure.costUsd > DEFAULTS.expensiveCloudThresholdUsd && !packet.dataset?.syntheticFixtureAvailable) { + findings.push( + createFinding( + "NO_LOW_COST_REVIEWER_MODE", + "high", + "Expensive rerun lacks a low-cost review mode", + `$${disclosure.costUsd.toFixed(2)} estimated cost without a synthetic fixture or low-cost mode.`, + "Add an inexpensive smoke test and fixture-mode result before the AI assistant marks it reproducible." + ) + ); + } + + if (toNumber(packet.compute?.estimatedRuntimeHours) >= DEFAULTS.longRunHours && !disclosure.hasCarbon) { + findings.push( + createFinding( + "RESOURCE_IMPACT_NOTE_MISSING", + "low", + "Long compute rerun lacks resource-impact disclosure", + "No carbon/resource note is attached for a long-running rerun.", + "Add a short resource-impact note and reviewer-facing rerun alternatives." + ) + ); + } + + return findings; +} + +function decisionForFindings(findings) { + const worst = findings.reduce((max, finding) => Math.max(max, SEVERITY_RANK[finding.severity] || 0), 0); + const highOrWorse = findings.filter((finding) => SEVERITY_RANK[finding.severity] >= SEVERITY_RANK.high).length; + + if (worst >= SEVERITY_RANK.critical || highOrWorse >= 3) { + return DECISIONS.HOLD; + } + + if (worst >= SEVERITY_RANK.medium || findings.length > 0) { + return DECISIONS.REVISE; + } + + return DECISIONS.RELEASE; +} + +function scoreForFindings(findings) { + const penalty = findings.reduce((total, finding) => total + (SEVERITY_RANK[finding.severity] || 0) * 10, 0); + return Math.max(0, 100 - penalty); +} + +function summarizeReadiness(packet, findings) { + const workingSetGb = datasetWorkingSetGb(packet); + const compute = computeRamGb(packet); + return { + packetId: packet.id, + title: packet.title, + assistantOutput: packet.assistantOutput || "reproducibility-checker", + estimatedWorkingSetGb: workingSetGb, + ramGb: compute.ramGb, + totalGpuVramGb: compute.totalGpuVramGb, + findingCount: findings.length, + worstSeverity: + findings + .map((finding) => finding.severity) + .sort((a, b) => SEVERITY_RANK[b] - SEVERITY_RANK[a])[0] || "none" + }; +} + +export function analyzeRerunPacket(packet) { + const findings = [ + ...evaluateEnvironment(packet), + ...evaluateMemory(packet), + ...evaluateRuntime(packet), + ...evaluateDeterminism(packet), + ...evaluateAccessAndDisclosure(packet) + ]; + + const decision = decisionForFindings(findings); + const score = scoreForFindings(findings); + + return { + id: packet.id, + title: packet.title, + decision, + score, + summary: summarizeReadiness(packet, findings), + findings, + releaseGate: { + canReleaseAssistantOutput: decision === DECISIONS.RELEASE, + reviewerMessage: + decision === DECISIONS.RELEASE + ? "Compute evidence is sufficient for the assistant to present the rerun as reviewer-feasible." + : "Hold or revise the assistant's reproducibility wording until compute feasibility evidence is repaired." + } + }; +} + +export function analyzeRerunPackets(packets) { + const results = asArray(packets).map(analyzeRerunPacket); + const decisionCounts = results.reduce((counts, result) => { + counts[result.decision] = (counts[result.decision] || 0) + 1; + return counts; + }, {}); + + return { + generatedAt: new Date().toISOString(), + packetCount: results.length, + decisionCounts, + results + }; +} + +export { DECISIONS }; diff --git a/research-compute-rerun-feasibility-guard/test/compute-rerun-feasibility.test.js b/research-compute-rerun-feasibility-guard/test/compute-rerun-feasibility.test.js new file mode 100644 index 00000000..87e162f2 --- /dev/null +++ b/research-compute-rerun-feasibility-guard/test/compute-rerun-feasibility.test.js @@ -0,0 +1,49 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import assert from "node:assert/strict"; +import test from "node:test"; +import { DECISIONS, analyzeRerunPacket, analyzeRerunPackets } from "../src/index.js"; + +const fixtures = JSON.parse( + await readFile(join(process.cwd(), "data", "sample-rerun-packets.json"), "utf8") +); + +test("classifies synthetic rerun packets by release gate", () => { + const report = analyzeRerunPackets(fixtures); + assert.equal(report.packetCount, 4); + assert.equal(report.decisionCounts[DECISIONS.RELEASE], 1); + assert.equal(report.decisionCounts[DECISIONS.REVISE], 1); + assert.equal(report.decisionCounts[DECISIONS.HOLD], 2); +}); + +test("holds assistant output when memory and cloud disclosures are unsafe", () => { + const result = analyzeRerunPacket(fixtures.find((packet) => packet.id === "paper-gpu-overclaim-hold")); + assert.equal(result.decision, DECISIONS.HOLD); + assert.equal(result.releaseGate.canReleaseAssistantOutput, false); + assert(result.findings.some((finding) => finding.code === "DATASET_MEMORY_EXCEEDS_NODE")); + assert(result.findings.some((finding) => finding.code === "GPU_VRAM_FEASIBILITY_GAP")); + assert(result.findings.some((finding) => finding.code === "CLOUD_OR_HPC_COST_UNDISCLOSED")); +}); + +test("revises long queued runs when timing and resource notes are incomplete", () => { + const result = analyzeRerunPacket(fixtures.find((packet) => packet.id === "paper-hpc-revise")); + assert.equal(result.decision, DECISIONS.REVISE); + assert(result.findings.some((finding) => finding.code === "RUNTIME_BUDGET_UNREALISTIC")); + assert(result.findings.some((finding) => finding.code === "RESOURCE_IMPACT_NOTE_MISSING")); +}); + +test("releases assistant output for complete low-cost reviewer fixtures", () => { + const result = analyzeRerunPacket(fixtures.find((packet) => packet.id === "paper-cpu-fixture-pass")); + assert.equal(result.decision, DECISIONS.RELEASE); + assert.equal(result.findings.length, 0); + assert.equal(result.releaseGate.canReleaseAssistantOutput, true); + assert(result.score >= 90); +}); + +test("summaries include reviewer-relevant compute facts", () => { + const result = analyzeRerunPacket(fixtures.find((packet) => packet.id === "paper-vram-fixture-needed")); + assert.equal(result.decision, DECISIONS.HOLD); + assert.equal(result.summary.estimatedWorkingSetGb, 109.2); + assert.equal(result.summary.totalGpuVramGb, 40); + assert(result.summary.findingCount >= 3); +});