diff --git a/bounty-ai-assistance-disclosure-guard/README.md b/bounty-ai-assistance-disclosure-guard/README.md new file mode 100644 index 00000000..afe40c54 --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/README.md @@ -0,0 +1,56 @@ +# Bounty AI Assistance Disclosure Guard + +This module adds a focused solver AI-assistance disclosure and provenance guard for the Scientific Bounty System in issue `#18`. + +Scientific bounty platforms increasingly receive submissions that mix human research, generated code, generated reports, model-assisted analysis, and tool-produced artifacts. Sponsors and reviewers need a deterministic pre-scoring gate that answers a narrow question: is this AI-assisted submission safe and transparent enough to score? + +## Scope + +- Checks whether AI assistance is allowed by the challenge policy. +- Requires explicit AI-use disclosure when the policy requires it. +- Checks approved AI/tool lists and tool license compatibility. +- Flags prohibited data classes sent through AI workflows. +- Holds packets with private sponsor data, prompt leakage, or secret-like text. +- Checks human review attestation for AI-assisted deliverables. +- Checks generated components for disclosure, human review, provenance hashes, and submission-license fit. +- Requires reproducibility commands or environment notes for AI-assisted work. +- Produces JSON, Markdown, SVG, and MP4 reviewer artifacts. +- Uses synthetic data only. +- Uses no external AI APIs, credentials, private submissions, payment systems, or network calls. + +## Requirement Map + +| Issue #18 capability | Implementation | +| --- | --- | +| Submission engine | Validates solver submission packets before sponsor/reviewer scoring. | +| Secure private workspaces | Detects private data and prompt/secret leakage before reviewer handoff. | +| Arbitration and reviewer checklists | Emits severity-coded findings, decisions, confidence, evidence, and remediation. | +| IP and licensing options | Checks generated component and tool licenses against challenge policy. | +| Audit logs and reproducibility | Requires provenance hashes, human review, and reproducibility commands. | +| Multi-phase challenge support | Decisions can be run before proposal, prototype, final scoring, award release, or sponsor handoff. | + +## Run + +```bash +npm test +npm run demo +npm run video +npm run check +``` + +Demo artifacts are written to `reports/`: + +- `summary.json` +- `reviewer-packet.md` +- `summary.svg` +- `demo.mp4` + +## Decisions + +- `RELEASE`: disclosure, provenance, human review, and reproducibility evidence are scoring-ready. +- `REVIEW`: missing or incomplete provenance can be repaired before scoring. +- `HOLD`: critical policy violations such as undisclosed AI, prohibited private data, prompt leakage, or AI use in a no-AI challenge block scoring and award release. + +## AI Assistance Disclosure + +Implemented with Codex assistance and locally verified with deterministic tests and generated artifacts. diff --git a/bounty-ai-assistance-disclosure-guard/data/sample_submission_packets.json b/bounty-ai-assistance-disclosure-guard/data/sample_submission_packets.json new file mode 100644 index 00000000..b8b166f9 --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/data/sample_submission_packets.json @@ -0,0 +1,224 @@ +[ + { + "id": "release-transparent-ai", + "title": "Transparent AI-assisted climate forecast submission", + "challenge": { + "title": "Regional climate forecasting bounty" + }, + "challengePolicy": { + "allowsAiAssistance": true, + "requiresDisclosure": true, + "requiresAiToolList": true, + "approvedTools": ["Codex", "OpenModelLab"], + "allowedToolLicenses": ["MIT", "Apache-2.0"], + "allowedSubmissionLicenses": ["MIT", "CC-BY-4.0"], + "prohibitedDataClasses": ["private sponsor data", "participant health data", "export controlled data"] + }, + "aiAssistance": { + "used": true, + "disclosed": true, + "tools": [ + { + "name": "Codex", + "type": "coding agent", + "version": "local", + "license": "MIT", + "purpose": "test generation", + "disclosed": true + } + ], + "dataClassesUsed": ["public benchmark data"], + "privateDataUsed": false, + "promptLeakage": false + }, + "humanReview": { + "completed": true, + "reviewer": "team lead", + "reviewedComponents": ["model.py", "report.md"] + }, + "generatedComponents": [ + { + "path": "model.py", + "type": "code", + "aiGenerated": true, + "disclosed": true, + "humanReviewed": true, + "provenanceHash": "sha256:111111", + "license": "MIT" + } + ], + "reproducibility": { + "commands": "python -m pytest && python run_forecast.py --seed 17", + "environment": "python 3.12", + "artifactHash": "sha256:222222" + }, + "submission": { + "license": "MIT", + "includesPrivateData": false, + "promptLeakage": false, + "notes": "AI assistance disclosed in solver packet." + } + }, + { + "id": "review-missing-component-provenance", + "title": "AI-assisted materials model with incomplete provenance", + "challenge": { + "title": "Materials discovery bounty" + }, + "challengePolicy": { + "allowsAiAssistance": true, + "requiresDisclosure": true, + "requiresAiToolList": true, + "approvedTools": ["Codex"], + "allowedToolLicenses": ["MIT"], + "allowedSubmissionLicenses": ["MIT"], + "prohibitedDataClasses": ["private sponsor data"] + }, + "aiAssistance": { + "used": true, + "disclosed": true, + "tools": [ + { + "name": "Codex", + "type": "coding agent", + "license": "MIT", + "purpose": "feature extraction", + "disclosed": true + } + ], + "dataClassesUsed": ["public crystal dataset"], + "privateDataUsed": false, + "promptLeakage": false + }, + "humanReview": { + "completed": true, + "reviewer": "principal investigator" + }, + "generatedComponents": [ + { + "path": "features.py", + "type": "code", + "aiGenerated": true, + "disclosed": true, + "humanReviewed": false, + "license": "MIT" + } + ], + "reproducibility": { + "commands": "python -m pytest", + "environment": "python 3.12" + }, + "submission": { + "license": "MIT", + "includesPrivateData": false, + "promptLeakage": false, + "notes": "Needs final review signoff." + } + }, + { + "id": "hold-undisclosed-ai", + "title": "Undisclosed AI-generated solver report", + "challenge": { + "title": "Single-cell biomarker bounty" + }, + "challengePolicy": { + "allowsAiAssistance": true, + "requiresDisclosure": true, + "requiresAiToolList": true, + "approvedTools": ["Codex"], + "allowedSubmissionLicenses": ["MIT"], + "prohibitedDataClasses": ["private sponsor data", "participant health data"] + }, + "aiAssistance": { + "used": true, + "disclosed": false, + "tools": [ + { + "name": "Unknown LLM", + "type": "LLM", + "purpose": "drafted report", + "disclosed": false + } + ], + "dataClassesUsed": ["public benchmark data"], + "privateDataUsed": false, + "promptLeakage": false + }, + "humanReview": { + "completed": false + }, + "generatedComponents": [ + { + "path": "report.md", + "type": "whitepaper", + "aiGenerated": true, + "disclosed": false, + "humanReviewed": false, + "license": "MIT" + } + ], + "reproducibility": {}, + "submission": { + "license": "MIT", + "includesPrivateData": false, + "promptLeakage": false, + "notes": "No disclosure included." + } + }, + { + "id": "hold-private-data-leak", + "title": "Private sponsor data sent through AI workflow", + "challenge": { + "title": "Protein design bounty" + }, + "challengePolicy": { + "allowsAiAssistance": true, + "requiresDisclosure": true, + "approvedTools": ["Codex"], + "allowedSubmissionLicenses": ["Apache-2.0"], + "prohibitedDataClasses": ["private sponsor data", "export controlled data"] + }, + "aiAssistance": { + "used": true, + "disclosed": true, + "tools": [ + { + "name": "Codex", + "type": "coding agent", + "license": "MIT", + "purpose": "analysis rewrite", + "disclosed": true + } + ], + "dataClassesUsed": ["private sponsor data"], + "privateDataUsed": true, + "promptLeakage": true, + "promptExcerpt": "analysis with bearer abcdefghijklmnopqrstuvwxyz" + }, + "humanReview": { + "completed": true, + "reviewer": "compliance reviewer" + }, + "generatedComponents": [ + { + "path": "analysis.ipynb", + "type": "notebook", + "aiGenerated": true, + "disclosed": true, + "humanReviewed": true, + "provenanceHash": "sha256:333333", + "license": "MIT" + } + ], + "reproducibility": { + "commands": "python rerun.py", + "environment": "python 3.12" + }, + "submission": { + "license": "MIT", + "includesPrivateData": true, + "promptLeakage": true, + "notes": "Reviewer packet includes prompt material." + } + } +] diff --git a/bounty-ai-assistance-disclosure-guard/package.json b/bounty-ai-assistance-disclosure-guard/package.json new file mode 100644 index 00000000..0f93819b --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/package.json @@ -0,0 +1,13 @@ +{ + "name": "bounty-ai-assistance-disclosure-guard", + "version": "1.0.0", + "description": "Dependency-free AI assistance disclosure and provenance guard for scientific bounty submissions.", + "main": "src/index.js", + "scripts": { + "test": "node --test", + "demo": "node scripts/demo.js", + "video": "node scripts/render-demo-video.js", + "check": "node --check src/index.js && node --check scripts/demo.js && node --check scripts/render-demo-video.js" + }, + "license": "MIT" +} diff --git a/bounty-ai-assistance-disclosure-guard/reports/demo.mp4 b/bounty-ai-assistance-disclosure-guard/reports/demo.mp4 new file mode 100644 index 00000000..3e62b34e Binary files /dev/null and b/bounty-ai-assistance-disclosure-guard/reports/demo.mp4 differ diff --git a/bounty-ai-assistance-disclosure-guard/reports/reviewer-packet.md b/bounty-ai-assistance-disclosure-guard/reports/reviewer-packet.md new file mode 100644 index 00000000..52ad2f43 --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/reports/reviewer-packet.md @@ -0,0 +1,72 @@ +# Bounty AI Assistance Disclosure Review + +Generated: 2026-08-15T20:59:43.334Z + +## Decision Counts + +- RELEASE: 1 +- REVIEW: 1 +- HOLD: 2 + +## Packet Reviews + +### Transparent AI-assisted climate forecast submission +Challenge: Regional climate forecasting bounty +Decision: RELEASE +Confidence: 0.96 +AI used: true +Generated components: 1 +Reviewer note: Regional climate forecasting bounty: AI-assistance disclosure, provenance, human review, and reproducibility evidence are scoring-ready. + +### AI-assisted materials model with incomplete provenance +Challenge: Materials discovery bounty +Decision: REVIEW +Confidence: 0.7 +AI used: true +Generated components: 1 +Reviewer note: Materials discovery bounty: REVIEW before scoring because an ai-generated component lacks human review attestation. +- MAJOR GENERATED_COMPONENT_NOT_HUMAN_REVIEWED: An AI-generated component lacks human review attestation. + Remediation: Add human review evidence before the component can be scored or released to a sponsor. +- MINOR GENERATED_COMPONENT_MISSING_PROVENANCE_HASH: An AI-generated component does not include a provenance or artifact hash. + Remediation: Add a stable artifact hash, prompt digest, or build provenance digest. + +### Undisclosed AI-generated solver report +Challenge: Single-cell biomarker bounty +Decision: HOLD +Confidence: 0.1 +AI used: true +Generated components: 1 +Reviewer note: Single-cell biomarker bounty: HOLD before scoring because the submission used ai assistance without the disclosure required by the challenge policy. +- CRITICAL AI_ASSISTANCE_UNDISCLOSED: The submission used AI assistance without the disclosure required by the challenge policy. + Remediation: Require an explicit AI-use disclosure before scoring, award release, or sponsor handoff. +- MAJOR MISSING_HUMAN_REVIEW_ATTESTATION: AI-assisted work lacks a completed human review attestation. + Remediation: Add named reviewer, review date, reviewed components, and residual-risk notes. +- MAJOR AI_ASSISTED_WORK_NOT_REPRODUCIBLE: The submission lacks reproducibility commands or environment notes for AI-assisted deliverables. + Remediation: Add commands, environment, seeds, and artifact hashes sufficient for sponsor-side reproduction. +- MAJOR UNAPPROVED_AI_TOOL_USED: The submission used an AI/tooling system that is not listed in the challenge-approved tool policy. + Remediation: Add sponsor approval evidence for the tool or rerun the affected work with an approved toolchain. +- MAJOR AI_TOOL_NOT_LISTED_IN_DISCLOSURE: An apparent AI tool is present in provenance records but not listed as disclosed. + Remediation: Update the solver disclosure packet so reviewers can see which AI tools touched the deliverable. +- MAJOR GENERATED_COMPONENT_NOT_DISCLOSED: An AI-generated component is not clearly disclosed in the submission packet. + Remediation: Mark the generated component and explain how it was produced and reviewed. +- MAJOR GENERATED_COMPONENT_NOT_HUMAN_REVIEWED: An AI-generated component lacks human review attestation. + Remediation: Add human review evidence before the component can be scored or released to a sponsor. +- MINOR GENERATED_COMPONENT_MISSING_PROVENANCE_HASH: An AI-generated component does not include a provenance or artifact hash. + Remediation: Add a stable artifact hash, prompt digest, or build provenance digest. + +### Private sponsor data sent through AI workflow +Challenge: Protein design bounty +Decision: HOLD +Confidence: 0.1 +AI used: true +Generated components: 1 +Reviewer note: Protein design bounty: HOLD before scoring because ai assistance appears to have used data classes prohibited by the challenge policy. +- CRITICAL PROHIBITED_DATA_SENT_TO_AI: AI assistance appears to have used data classes prohibited by the challenge policy. + Remediation: Hold the packet, notify the sponsor, and require a clean-room rerun or documented exception. +- CRITICAL PRIVATE_DATA_IN_AI_WORKFLOW: The AI workflow or final packet includes private or sponsor-restricted data. + Remediation: Remove private data, regenerate affected artifacts, and provide reviewer-safe provenance evidence. +- CRITICAL PROMPT_OR_SECRET_LEAKAGE: The packet appears to expose prompts, secrets, bearer tokens, or private workflow instructions. + Remediation: Redact the packet, rotate any exposed credentials if real, and resubmit reviewer-safe evidence only. +- MAJOR GENERATED_COMPONENT_LICENSE_MISMATCH: A generated component uses a license outside the challenge submission policy. + Remediation: Relicense, remove, or obtain sponsor approval before scoring the submission. + diff --git a/bounty-ai-assistance-disclosure-guard/reports/summary.json b/bounty-ai-assistance-disclosure-guard/reports/summary.json new file mode 100644 index 00000000..463625d8 --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/reports/summary.json @@ -0,0 +1,198 @@ +{ + "generatedAt": "2026-08-15T20:59:43.334Z", + "packetCount": 4, + "decisionCounts": { + "RELEASE": 1, + "REVIEW": 1, + "HOLD": 2 + }, + "reviews": [ + { + "id": "release-transparent-ai", + "title": "Transparent AI-assisted climate forecast submission", + "challenge": "Regional climate forecasting bounty", + "decision": "RELEASE", + "confidence": 0.96, + "aiUsed": true, + "componentCount": 1, + "findingCount": 0, + "findings": [], + "reviewerNote": "Regional climate forecasting bounty: AI-assistance disclosure, provenance, human review, and reproducibility evidence are scoring-ready." + }, + { + "id": "review-missing-component-provenance", + "title": "AI-assisted materials model with incomplete provenance", + "challenge": "Materials discovery bounty", + "decision": "REVIEW", + "confidence": 0.7, + "aiUsed": true, + "componentCount": 1, + "findingCount": 2, + "findings": [ + { + "code": "GENERATED_COMPONENT_NOT_HUMAN_REVIEWED", + "severity": "major", + "message": "An AI-generated component lacks human review attestation.", + "evidence": { + "path": "features.py" + }, + "remediation": "Add human review evidence before the component can be scored or released to a sponsor." + }, + { + "code": "GENERATED_COMPONENT_MISSING_PROVENANCE_HASH", + "severity": "minor", + "message": "An AI-generated component does not include a provenance or artifact hash.", + "evidence": { + "path": "features.py" + }, + "remediation": "Add a stable artifact hash, prompt digest, or build provenance digest." + } + ], + "reviewerNote": "Materials discovery bounty: REVIEW before scoring because an ai-generated component lacks human review attestation." + }, + { + "id": "hold-undisclosed-ai", + "title": "Undisclosed AI-generated solver report", + "challenge": "Single-cell biomarker bounty", + "decision": "HOLD", + "confidence": 0.1, + "aiUsed": true, + "componentCount": 1, + "findingCount": 8, + "findings": [ + { + "code": "AI_ASSISTANCE_UNDISCLOSED", + "severity": "critical", + "message": "The submission used AI assistance without the disclosure required by the challenge policy.", + "evidence": { + "aiUsed": true, + "disclosed": false + }, + "remediation": "Require an explicit AI-use disclosure before scoring, award release, or sponsor handoff." + }, + { + "code": "MISSING_HUMAN_REVIEW_ATTESTATION", + "severity": "major", + "message": "AI-assisted work lacks a completed human review attestation.", + "evidence": { + "humanReviewCompleted": false + }, + "remediation": "Add named reviewer, review date, reviewed components, and residual-risk notes." + }, + { + "code": "AI_ASSISTED_WORK_NOT_REPRODUCIBLE", + "severity": "major", + "message": "The submission lacks reproducibility commands or environment notes for AI-assisted deliverables.", + "evidence": { + "commands": null + }, + "remediation": "Add commands, environment, seeds, and artifact hashes sufficient for sponsor-side reproduction." + }, + { + "code": "UNAPPROVED_AI_TOOL_USED", + "severity": "major", + "message": "The submission used an AI/tooling system that is not listed in the challenge-approved tool policy.", + "evidence": { + "tool": "Unknown LLM", + "approvedTools": [ + "Codex" + ] + }, + "remediation": "Add sponsor approval evidence for the tool or rerun the affected work with an approved toolchain." + }, + { + "code": "AI_TOOL_NOT_LISTED_IN_DISCLOSURE", + "severity": "major", + "message": "An apparent AI tool is present in provenance records but not listed as disclosed.", + "evidence": { + "tool": "Unknown LLM", + "type": "LLM" + }, + "remediation": "Update the solver disclosure packet so reviewers can see which AI tools touched the deliverable." + }, + { + "code": "GENERATED_COMPONENT_NOT_DISCLOSED", + "severity": "major", + "message": "An AI-generated component is not clearly disclosed in the submission packet.", + "evidence": { + "path": "report.md", + "type": "whitepaper" + }, + "remediation": "Mark the generated component and explain how it was produced and reviewed." + }, + { + "code": "GENERATED_COMPONENT_NOT_HUMAN_REVIEWED", + "severity": "major", + "message": "An AI-generated component lacks human review attestation.", + "evidence": { + "path": "report.md" + }, + "remediation": "Add human review evidence before the component can be scored or released to a sponsor." + }, + { + "code": "GENERATED_COMPONENT_MISSING_PROVENANCE_HASH", + "severity": "minor", + "message": "An AI-generated component does not include a provenance or artifact hash.", + "evidence": { + "path": "report.md" + }, + "remediation": "Add a stable artifact hash, prompt digest, or build provenance digest." + } + ], + "reviewerNote": "Single-cell biomarker bounty: HOLD before scoring because the submission used ai assistance without the disclosure required by the challenge policy." + }, + { + "id": "hold-private-data-leak", + "title": "Private sponsor data sent through AI workflow", + "challenge": "Protein design bounty", + "decision": "HOLD", + "confidence": 0.1, + "aiUsed": true, + "componentCount": 1, + "findingCount": 4, + "findings": [ + { + "code": "PROHIBITED_DATA_SENT_TO_AI", + "severity": "critical", + "message": "AI assistance appears to have used data classes prohibited by the challenge policy.", + "evidence": { + "prohibitedData": [ + "private sponsor data" + ] + }, + "remediation": "Hold the packet, notify the sponsor, and require a clean-room rerun or documented exception." + }, + { + "code": "PRIVATE_DATA_IN_AI_WORKFLOW", + "severity": "critical", + "message": "The AI workflow or final packet includes private or sponsor-restricted data.", + "evidence": { + "privateDataUsed": true, + "includesPrivateData": true + }, + "remediation": "Remove private data, regenerate affected artifacts, and provide reviewer-safe provenance evidence." + }, + { + "code": "PROMPT_OR_SECRET_LEAKAGE", + "severity": "critical", + "message": "The packet appears to expose prompts, secrets, bearer tokens, or private workflow instructions.", + "evidence": { + "promptLeakage": true + }, + "remediation": "Redact the packet, rotate any exposed credentials if real, and resubmit reviewer-safe evidence only." + }, + { + "code": "GENERATED_COMPONENT_LICENSE_MISMATCH", + "severity": "major", + "message": "A generated component uses a license outside the challenge submission policy.", + "evidence": { + "path": "analysis.ipynb", + "license": "MIT" + }, + "remediation": "Relicense, remove, or obtain sponsor approval before scoring the submission." + } + ], + "reviewerNote": "Protein design bounty: HOLD before scoring because ai assistance appears to have used data classes prohibited by the challenge policy." + } + ] +} diff --git a/bounty-ai-assistance-disclosure-guard/reports/summary.svg b/bounty-ai-assistance-disclosure-guard/reports/summary.svg new file mode 100644 index 00000000..4abbc3d5 --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/reports/summary.svg @@ -0,0 +1,14 @@ + + + Bounty AI Assistance Disclosure + Disclosure, approved tools, human review, provenance hashes, private-data leakage, and license fit. + RELEASE: 1 + + + REVIEW: 1 + + + HOLD: 2 + + + \ No newline at end of file diff --git a/bounty-ai-assistance-disclosure-guard/scripts/demo.js b/bounty-ai-assistance-disclosure-guard/scripts/demo.js new file mode 100644 index 00000000..844ff37a --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/scripts/demo.js @@ -0,0 +1,77 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { analyzePackets } = require("../src"); + +const root = path.resolve(__dirname, ".."); +const packets = JSON.parse(fs.readFileSync(path.join(root, "data", "sample_submission_packets.json"), "utf8")); +const report = analyzePackets(packets); +const outDir = path.join(root, "reports"); + +fs.mkdirSync(outDir, { recursive: true }); +fs.writeFileSync(path.join(outDir, "summary.json"), `${JSON.stringify(report, null, 2)}\n`); +fs.writeFileSync(path.join(outDir, "reviewer-packet.md"), renderMarkdown(report)); +fs.writeFileSync(path.join(outDir, "summary.svg"), renderSvg(report)); + +console.log(JSON.stringify({ + packetCount: report.packetCount, + decisionCounts: report.decisionCounts, + reports: [ + "reports/summary.json", + "reports/reviewer-packet.md", + "reports/summary.svg" + ] +}, null, 2)); + +function renderMarkdown(report) { + const lines = [ + "# Bounty AI Assistance Disclosure Review", + "", + `Generated: ${report.generatedAt}`, + "", + "## Decision Counts", + "", + `- RELEASE: ${report.decisionCounts.RELEASE}`, + `- REVIEW: ${report.decisionCounts.REVIEW}`, + `- HOLD: ${report.decisionCounts.HOLD}`, + "", + "## Packet Reviews", + "" + ]; + + for (const review of report.reviews) { + lines.push(`### ${review.title}`); + lines.push(`Challenge: ${review.challenge}`); + lines.push(`Decision: ${review.decision}`); + lines.push(`Confidence: ${review.confidence}`); + lines.push(`AI used: ${review.aiUsed}`); + lines.push(`Generated components: ${review.componentCount}`); + lines.push(`Reviewer note: ${review.reviewerNote}`); + for (const item of review.findings) { + lines.push(`- ${item.severity.toUpperCase()} ${item.code}: ${item.message}`); + lines.push(` Remediation: ${item.remediation}`); + } + lines.push(""); + } + + return `${lines.join("\n")}\n`; +} + +function renderSvg(report) { + return ` + + Bounty AI Assistance Disclosure + Disclosure, approved tools, human review, provenance hashes, private-data leakage, and license fit. + ${bar("RELEASE", report.decisionCounts.RELEASE, 48, 148, "#36af8f")} + ${bar("REVIEW", report.decisionCounts.REVIEW, 48, 218, "#d4a235")} + ${bar("HOLD", report.decisionCounts.HOLD, 48, 288, "#d65d73")} +`; +} + +function bar(label, count, x, y, color) { + const width = 96 + count * 116; + return `${label}: ${count} + + `; +} diff --git a/bounty-ai-assistance-disclosure-guard/scripts/render-demo-video.js b/bounty-ai-assistance-disclosure-guard/scripts/render-demo-video.js new file mode 100644 index 00000000..b1bfc69b --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/scripts/render-demo-video.js @@ -0,0 +1,60 @@ +"use strict"; + +const fs = require("node:fs"); +const path = require("node:path"); +const { spawnSync } = require("node:child_process"); + +const root = path.resolve(__dirname, ".."); +const ffmpeg = path.resolve( + root, + "..", + "..", + "tool_downloads", + "video_tools", + "node_modules", + "ffmpeg-static", + "ffmpeg.exe" +); +const outDir = path.join(root, "reports"); +const out = path.join(outDir, "demo.mp4"); +const font = "C\\:/Windows/Fonts/arial.ttf"; + +fs.mkdirSync(outDir, { recursive: true }); + +const draw = [ + `drawtext=fontfile=${font}:text='Bounty AI Assistance Disclosure':x=64:y=56:fontsize=42:fontcolor=black`, + `drawtext=fontfile=${font}:text='Scientific bounty guard for solver AI-use provenance':x=64:y=120:fontsize=25:fontcolor=0x34413b`, + `drawtext=fontfile=${font}:text='Synthetic submissions analyzed 4':x=64:y=210:fontsize=34:fontcolor=black`, + `drawtext=fontfile=${font}:text='RELEASE 1 REVIEW 1 HOLD 2':x=64:y=268:fontsize=38:fontcolor=0x245fd6`, + `drawtext=fontfile=${font}:text='Checks AI disclosure approved tools human review provenance hashes':x=64:y=358:fontsize=26:fontcolor=0x34413b`, + `drawtext=fontfile=${font}:text='Flags private data prompt leakage license mismatch and missing review':x=64:y=422:fontsize=26:fontcolor=0x34413b`, + `drawtext=fontfile=${font}:text='Synthetic generated slate only no desktop capture or private data':x=64:y=486:fontsize=26:fontcolor=0x34413b` +].join(","); + +if (!fs.existsSync(ffmpeg)) { + throw new Error(`ffmpeg binary not found at ${ffmpeg}`); +} + +const result = spawnSync(ffmpeg, [ + "-y", + "-f", + "lavfi", + "-i", + "color=c=0xf8faf9:s=1280x720:d=8:r=30", + "-vf", + draw, + "-c:v", + "libx264", + "-pix_fmt", + "yuv420p", + "-movflags", + "+faststart", + out +], { encoding: "utf8" }); + +if (result.status !== 0) { + process.stderr.write(result.stderr || result.stdout); + process.exit(result.status || 1); +} + +console.log(JSON.stringify({ out, bytes: fs.statSync(out).size }, null, 2)); diff --git a/bounty-ai-assistance-disclosure-guard/src/index.js b/bounty-ai-assistance-disclosure-guard/src/index.js new file mode 100644 index 00000000..fdaf6438 --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/src/index.js @@ -0,0 +1,310 @@ +"use strict"; + +const SECRET_PATTERNS = [ + /api[_-]?key\s*=/i, + /secret\s*=/i, + /bearer\s+[A-Za-z0-9._-]{16,}/i, + /-----BEGIN [A-Z ]+PRIVATE KEY-----/i, + /sk-[A-Za-z0-9]{16,}/i +]; + +const AI_TOOL_TERMS = [ + "llm", + "gpt", + "claude", + "gemini", + "copilot", + "codex", + "diffusion", + "agent" +]; + +function finding(code, severity, message, evidence, remediation) { + return { code, severity, message, evidence, remediation }; +} + +function normalizeText(value) { + return String(value || "").trim().toLowerCase(); +} + +function normalizePacket(packet) { + return { + id: packet.id, + title: packet.title || "Untitled scientific bounty submission", + challenge: packet.challenge || {}, + challengePolicy: packet.challengePolicy || {}, + aiAssistance: packet.aiAssistance || {}, + humanReview: packet.humanReview || {}, + generatedComponents: Array.isArray(packet.generatedComponents) ? packet.generatedComponents : [], + reproducibility: packet.reproducibility || {}, + submission: packet.submission || {} + }; +} + +function list(value) { + return Array.isArray(value) ? value : []; +} + +function containsSecretLikeText(value) { + const text = String(value || ""); + return SECRET_PATTERNS.some((pattern) => pattern.test(text)); +} + +function toolLooksAi(tool) { + const haystack = `${tool.name || ""} ${tool.type || ""} ${tool.purpose || ""}`; + return AI_TOOL_TERMS.some((term) => normalizeText(haystack).includes(term)); +} + +function licenseAllowed(license, allowedLicenses) { + if (!license || allowedLicenses.length === 0) return true; + return allowedLicenses.map(normalizeText).includes(normalizeText(license)); +} + +function policyAllowsTool(tool, policy) { + const approvedTools = list(policy.approvedTools).map(normalizeText); + if (approvedTools.length === 0) return true; + return approvedTools.includes(normalizeText(tool.name)); +} + +function usedProhibitedData(aiAssistance, policy) { + const used = list(aiAssistance.dataClassesUsed).map(normalizeText); + const prohibited = list(policy.prohibitedDataClasses).map(normalizeText); + return used.filter((item) => prohibited.includes(item)); +} + +function analyzeTools(packet) { + const findings = []; + const policy = packet.challengePolicy; + const tools = list(packet.aiAssistance.tools); + + for (const tool of tools) { + if (!policyAllowsTool(tool, policy)) { + findings.push(finding( + "UNAPPROVED_AI_TOOL_USED", + "major", + "The submission used an AI/tooling system that is not listed in the challenge-approved tool policy.", + { tool: tool.name || null, approvedTools: policy.approvedTools || [] }, + "Add sponsor approval evidence for the tool or rerun the affected work with an approved toolchain." + )); + } + + if (toolLooksAi(tool) && tool.disclosed !== true && packet.challengePolicy.requiresAiToolList === true) { + findings.push(finding( + "AI_TOOL_NOT_LISTED_IN_DISCLOSURE", + "major", + "An apparent AI tool is present in provenance records but not listed as disclosed.", + { tool: tool.name || null, type: tool.type || null }, + "Update the solver disclosure packet so reviewers can see which AI tools touched the deliverable." + )); + } + + if (tool.license && !licenseAllowed(tool.license, list(policy.allowedToolLicenses))) { + findings.push(finding( + "TOOL_LICENSE_NOT_ALLOWED", + "major", + "A tool license is outside the licenses allowed by the challenge policy.", + { tool: tool.name || null, license: tool.license, allowedToolLicenses: policy.allowedToolLicenses || [] }, + "Replace the tool, document a sponsor exception, or update the challenge policy before scoring." + )); + } + } + + return findings; +} + +function analyzeComponents(packet) { + const findings = []; + const generated = packet.generatedComponents; + + for (const component of generated) { + if (component.aiGenerated === true && component.disclosed !== true) { + findings.push(finding( + "GENERATED_COMPONENT_NOT_DISCLOSED", + "major", + "An AI-generated component is not clearly disclosed in the submission packet.", + { path: component.path || null, type: component.type || null }, + "Mark the generated component and explain how it was produced and reviewed." + )); + } + + if (component.aiGenerated === true && component.humanReviewed !== true) { + findings.push(finding( + "GENERATED_COMPONENT_NOT_HUMAN_REVIEWED", + "major", + "An AI-generated component lacks human review attestation.", + { path: component.path || null }, + "Add human review evidence before the component can be scored or released to a sponsor." + )); + } + + if (component.aiGenerated === true && !component.provenanceHash) { + findings.push(finding( + "GENERATED_COMPONENT_MISSING_PROVENANCE_HASH", + "minor", + "An AI-generated component does not include a provenance or artifact hash.", + { path: component.path || null }, + "Add a stable artifact hash, prompt digest, or build provenance digest." + )); + } + + if (component.license && !licenseAllowed(component.license, list(packet.challengePolicy.allowedSubmissionLicenses))) { + findings.push(finding( + "GENERATED_COMPONENT_LICENSE_MISMATCH", + "major", + "A generated component uses a license outside the challenge submission policy.", + { path: component.path || null, license: component.license }, + "Relicense, remove, or obtain sponsor approval before scoring the submission." + )); + } + } + + return findings; +} + +function analyzePacket(input) { + const packet = normalizePacket(input); + const findings = []; + const policy = packet.challengePolicy; + const ai = packet.aiAssistance; + const submission = packet.submission; + + if (policy.allowsAiAssistance === false && ai.used === true) { + findings.push(finding( + "AI_ASSISTANCE_PROHIBITED", + "critical", + "The challenge policy prohibits AI assistance, but the submission declares or shows AI use.", + { allowsAiAssistance: policy.allowsAiAssistance, aiUsed: ai.used }, + "Hold scoring until the sponsor allows AI assistance or the solver resubmits without AI-generated work." + )); + } + + if (ai.used === true && policy.requiresDisclosure !== false && ai.disclosed !== true) { + findings.push(finding( + "AI_ASSISTANCE_UNDISCLOSED", + "critical", + "The submission used AI assistance without the disclosure required by the challenge policy.", + { aiUsed: ai.used, disclosed: ai.disclosed }, + "Require an explicit AI-use disclosure before scoring, award release, or sponsor handoff." + )); + } + + const prohibitedData = usedProhibitedData(ai, policy); + if (prohibitedData.length > 0) { + findings.push(finding( + "PROHIBITED_DATA_SENT_TO_AI", + "critical", + "AI assistance appears to have used data classes prohibited by the challenge policy.", + { prohibitedData }, + "Hold the packet, notify the sponsor, and require a clean-room rerun or documented exception." + )); + } + + if (ai.privateDataUsed === true || submission.includesPrivateData === true) { + findings.push(finding( + "PRIVATE_DATA_IN_AI_WORKFLOW", + "critical", + "The AI workflow or final packet includes private or sponsor-restricted data.", + { privateDataUsed: ai.privateDataUsed, includesPrivateData: submission.includesPrivateData }, + "Remove private data, regenerate affected artifacts, and provide reviewer-safe provenance evidence." + )); + } + + if (ai.promptLeakage === true || submission.promptLeakage === true || containsSecretLikeText(ai.promptExcerpt) || containsSecretLikeText(submission.notes)) { + findings.push(finding( + "PROMPT_OR_SECRET_LEAKAGE", + "critical", + "The packet appears to expose prompts, secrets, bearer tokens, or private workflow instructions.", + { promptLeakage: ai.promptLeakage || submission.promptLeakage || false }, + "Redact the packet, rotate any exposed credentials if real, and resubmit reviewer-safe evidence only." + )); + } + + if (ai.used === true && packet.humanReview.completed !== true) { + findings.push(finding( + "MISSING_HUMAN_REVIEW_ATTESTATION", + "major", + "AI-assisted work lacks a completed human review attestation.", + { humanReviewCompleted: packet.humanReview.completed || false }, + "Add named reviewer, review date, reviewed components, and residual-risk notes." + )); + } + + if (ai.used === true && !packet.reproducibility.commands) { + findings.push(finding( + "AI_ASSISTED_WORK_NOT_REPRODUCIBLE", + "major", + "The submission lacks reproducibility commands or environment notes for AI-assisted deliverables.", + { commands: packet.reproducibility.commands || null }, + "Add commands, environment, seeds, and artifact hashes sufficient for sponsor-side reproduction." + )); + } + + findings.push(...analyzeTools(packet)); + findings.push(...analyzeComponents(packet)); + + const decision = decisionFromFindings(findings); + return { + id: packet.id, + title: packet.title, + challenge: packet.challenge.title || "scientific bounty", + decision, + confidence: confidenceFromFindings(findings), + aiUsed: ai.used === true, + componentCount: packet.generatedComponents.length, + findingCount: findings.length, + findings, + reviewerNote: reviewerNote(packet, decision, findings) + }; +} + +function severityWeight(severity) { + if (severity === "critical") return 5; + if (severity === "major") return 3; + return 1; +} + +function decisionFromFindings(findings) { + if (findings.some((item) => item.severity === "critical")) return "HOLD"; + if (findings.some((item) => item.severity === "major" || item.severity === "minor")) return "REVIEW"; + return "RELEASE"; +} + +function confidenceFromFindings(findings) { + const penalty = findings.reduce((total, item) => total + severityWeight(item.severity), 0); + return Math.max(0.1, Number((0.96 - penalty * 0.065).toFixed(2))); +} + +function reviewerNote(packet, decision, findings) { + if (decision === "RELEASE") { + return `${packet.challenge.title || "scientific bounty"}: AI-assistance disclosure, provenance, human review, and reproducibility evidence are scoring-ready.`; + } + const first = findings[0]; + return `${packet.challenge.title || "scientific bounty"}: ${decision} before scoring because ${first.message.toLowerCase()}`; +} + +function analyzePackets(packets) { + const reviews = packets.map(analyzePacket); + const counts = reviews.reduce((acc, review) => { + acc[review.decision] = (acc[review.decision] || 0) + 1; + return acc; + }, {}); + + return { + generatedAt: new Date().toISOString(), + packetCount: reviews.length, + decisionCounts: { + RELEASE: counts.RELEASE || 0, + REVIEW: counts.REVIEW || 0, + HOLD: counts.HOLD || 0 + }, + reviews + }; +} + +module.exports = { + SECRET_PATTERNS, + analyzePacket, + analyzePackets, + containsSecretLikeText, + decisionFromFindings +}; diff --git a/bounty-ai-assistance-disclosure-guard/test/bounty-ai-assistance-disclosure-guard.test.js b/bounty-ai-assistance-disclosure-guard/test/bounty-ai-assistance-disclosure-guard.test.js new file mode 100644 index 00000000..0ab3d077 --- /dev/null +++ b/bounty-ai-assistance-disclosure-guard/test/bounty-ai-assistance-disclosure-guard.test.js @@ -0,0 +1,173 @@ +"use strict"; + +const assert = require("node:assert/strict"); +const test = require("node:test"); +const { analyzePacket, analyzePackets, containsSecretLikeText, decisionFromFindings } = require("../src"); + +test("detects secret-like prompt leakage", () => { + assert.equal(containsSecretLikeText("bearer abcdefghijklmnopqrstuvwxyz"), true); + assert.equal(containsSecretLikeText("public benchmark summary"), false); +}); + +test("releases transparent AI-assisted submissions with human review and provenance", () => { + const review = analyzePacket({ + id: "clean", + title: "Clean AI-assisted packet", + challenge: { title: "Climate bounty" }, + challengePolicy: { + allowsAiAssistance: true, + requiresDisclosure: true, + requiresAiToolList: true, + approvedTools: ["Codex"], + allowedToolLicenses: ["MIT"], + allowedSubmissionLicenses: ["MIT"], + prohibitedDataClasses: ["private sponsor data"] + }, + aiAssistance: { + used: true, + disclosed: true, + tools: [{ name: "Codex", type: "coding agent", license: "MIT", disclosed: true }], + dataClassesUsed: ["public data"], + privateDataUsed: false, + promptLeakage: false + }, + humanReview: { completed: true, reviewer: "lead" }, + generatedComponents: [ + { + path: "model.py", + aiGenerated: true, + disclosed: true, + humanReviewed: true, + provenanceHash: "sha256:abc", + license: "MIT" + } + ], + reproducibility: { commands: "python -m pytest" }, + submission: { includesPrivateData: false, promptLeakage: false } + }); + + assert.equal(review.decision, "RELEASE"); + assert.equal(review.findingCount, 0); +}); + +test("holds undisclosed AI assistance", () => { + const review = analyzePacket({ + id: "undisclosed", + title: "Undisclosed AI", + challengePolicy: { + allowsAiAssistance: true, + requiresDisclosure: true, + approvedTools: ["Codex"], + prohibitedDataClasses: [] + }, + aiAssistance: { + used: true, + disclosed: false, + tools: [{ name: "Codex", type: "coding agent", disclosed: false }] + }, + humanReview: { completed: true }, + generatedComponents: [], + reproducibility: { commands: "npm test" }, + submission: {} + }); + + assert.equal(review.decision, "HOLD"); + assert.ok(review.findings.some((item) => item.code === "AI_ASSISTANCE_UNDISCLOSED")); +}); + +test("holds prohibited private data and prompt leakage", () => { + const review = analyzePacket({ + id: "private", + title: "Private data AI workflow", + challengePolicy: { + allowsAiAssistance: true, + requiresDisclosure: true, + prohibitedDataClasses: ["private sponsor data"] + }, + aiAssistance: { + used: true, + disclosed: true, + dataClassesUsed: ["private sponsor data"], + privateDataUsed: true, + promptExcerpt: "api_key=abc1234567890" + }, + humanReview: { completed: true }, + generatedComponents: [], + reproducibility: { commands: "python rerun.py" }, + submission: { includesPrivateData: true } + }); + + assert.equal(review.decision, "HOLD"); + assert.ok(review.findings.some((item) => item.code === "PROHIBITED_DATA_SENT_TO_AI")); + assert.ok(review.findings.some((item) => item.code === "PRIVATE_DATA_IN_AI_WORKFLOW")); + assert.ok(review.findings.some((item) => item.code === "PROMPT_OR_SECRET_LEAKAGE")); +}); + +test("routes missing human review and missing provenance to review", () => { + const review = analyzePacket({ + id: "review", + title: "Needs review", + challengePolicy: { + allowsAiAssistance: true, + requiresDisclosure: true, + approvedTools: ["Codex"], + allowedSubmissionLicenses: ["MIT"] + }, + aiAssistance: { + used: true, + disclosed: true, + tools: [{ name: "Codex", type: "coding agent", disclosed: true }] + }, + humanReview: { completed: false }, + generatedComponents: [ + { + path: "features.py", + aiGenerated: true, + disclosed: true, + humanReviewed: false, + license: "MIT" + } + ], + reproducibility: { commands: "npm test" }, + submission: {} + }); + + assert.equal(review.decision, "REVIEW"); + assert.ok(review.findings.some((item) => item.code === "MISSING_HUMAN_REVIEW_ATTESTATION")); + assert.ok(review.findings.some((item) => item.code === "GENERATED_COMPONENT_NOT_HUMAN_REVIEWED")); + assert.ok(review.findings.some((item) => item.code === "GENERATED_COMPONENT_MISSING_PROVENANCE_HASH")); +}); + +test("summarizes packet decisions deterministically", () => { + const report = analyzePackets([ + { + id: "release", + title: "Release", + challengePolicy: { allowsAiAssistance: true, requiresDisclosure: true }, + aiAssistance: { used: false }, + humanReview: {}, + generatedComponents: [], + reproducibility: {}, + submission: {} + }, + { + id: "hold", + title: "Hold", + challengePolicy: { allowsAiAssistance: false, requiresDisclosure: true }, + aiAssistance: { used: true, disclosed: true }, + generatedComponents: [], + reproducibility: { commands: "npm test" }, + submission: {} + } + ]); + + assert.equal(report.packetCount, 2); + assert.equal(report.decisionCounts.RELEASE, 1); + assert.equal(report.decisionCounts.HOLD, 1); +}); + +test("decision helper maps critical findings to HOLD", () => { + assert.equal(decisionFromFindings([{ severity: "critical" }]), "HOLD"); + assert.equal(decisionFromFindings([{ severity: "major" }]), "REVIEW"); + assert.equal(decisionFromFindings([]), "RELEASE"); +});