diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d2c28ca..4bc9a3e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,9 @@ # Default code owners. Enforced via branch protection (required review). * @abrichr + +# Production backup and restore trust boundary. +/.github/workflows/db-backup.yml @abrichr +/ops/backup/ @abrichr +/scripts/database_backup_contract.py @abrichr +/scripts/run_database_restore_drill.sh @abrichr +/tests/test_database_backup_contract.py @abrichr diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml index 73434bd..3beafd1 100644 --- a/.github/workflows/db-backup.yml +++ b/.github/workflows/db-backup.yml @@ -1,124 +1,140 @@ name: Production DB logical backup -# $0 recovery point for the hosted control plane (AUDIT.md finding b: -# pitr_enabled=false, zero provider physical backups as of 2026-07-25). -# Daily `supabase db dump` (roles + schema + data, the exact triple the -# cloud data-safety runbook prescribes), encrypted to an age public key -# committed in ops/backup/age-recipients.txt, uploaded as a workflow -# artifact with 90-day retention. -# -# The private key exists ONLY with the founder (see -# ops/backup/RESTORE_DRILL.md). This repo is PUBLIC: artifacts are -# downloadable by any logged-in GitHub user, so the age encryption is -# load-bearing, not defense-in-depth. Never upload plaintext. -# -# Cost: the repo is public, so Actions minutes and artifact storage are -# free. Daily cadence => RPO up to 24h (logical only). Anything better -# (RPO minutes) requires the paid Supabase PITR add-on — see the -# tradeoff table in ops/backup/RESTORE_DRILL.md. -# -# FAIL-CLOSED: this workflow fails loudly when the DB secret or the age -# recipient is missing, and refuses to upload an empty or schema-only -# dump. A backup job that silently succeeds without a usable backup is -# worse than a red run. +# Daily, off-provider logical recovery point. The database URL is a protected +# production-backup environment secret. Only age ciphertext and a redacted +# integrity manifest enter the private, public-access-blocked S3 bucket. +# Maximum RPO: 24 hours. Retention: 90 days. This does not cover Storage +# objects and does not replace provider PITR. on: workflow_dispatch: schedule: - - cron: '23 7 * * *' # daily 07:23 UTC (off-peak; odd minute to avoid the top-of-hour scheduler crush) + - cron: '23 7 * * *' permissions: contents: read + id-token: write concurrency: - group: db-backup + group: production-db-backup cancel-in-progress: false jobs: dump: runs-on: ubuntu-latest timeout-minutes: 30 + environment: production-backup + env: + AWS_REGION: us-east-1 + BACKUP_BUCKET: ${{ vars.AWS_BACKUP_BUCKET }} + SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + SUPABASE_PROJECT_REF: ${{ secrets.SUPABASE_PROJECT_REF }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Require configuration (fail closed) - env: - SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} - run: | - fail=0 - if [ -z "$SUPABASE_DB_URL" ]; then - echo '::error::Secret SUPABASE_DB_URL is not set. Founder action: gh secret set SUPABASE_DB_URL --repo OpenAdaptAI/openadapt-ops (value: the production Postgres connection string from Supabase -> Project Settings -> Database).' - fail=1 - fi - if ! grep -Eq '^age1[0-9a-z]+$' ops/backup/age-recipients.txt; then - echo '::error::ops/backup/age-recipients.txt contains no age recipient. Founder action: generate the keypair per ops/backup/RESTORE_DRILL.md section 1 and commit the PUBLIC key line.' - fail=1 - fi - exit "$fail" + - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + aws-region: ${{ env.AWS_REGION }} + role-to-assume: ${{ vars.AWS_BACKUP_ROLE_ARN }} + allowed-account-ids: '992382684924' + role-session-name: openadapt-db-backup-${{ github.run_id }} - uses: supabase/setup-cli@46f7f98c7f948ad727d22c1e67fab04c223a0520 # v3.0.0 with: - version: latest # tracks server-version support in `supabase db dump`; the dump SQL itself is what we archive + version: 2.75.0 - name: Install age run: | sudo apt-get update -qq sudo apt-get install -y -qq age - age --version - - name: Dump roles, schema, and data (cloud runbook triple) - env: - SUPABASE_DB_URL: ${{ secrets.SUPABASE_DB_URL }} + - name: Validate the exact source, private target, and recipient run: | set -euo pipefail - mkdir -p dump - # Same commands as openadapt-cloud docs/RUNBOOK_DATA_SAFETY.md - # section 2 step 5: a single unflagged dump is schema-only and - # is NOT a recoverable backup. - supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/roles.sql --role-only - supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/schema.sql - supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/data.sql --use-copy --data-only \ - -x 'storage.buckets_vectors' -x 'storage.vector_indexes' - - - name: Refuse an empty or schema-only dump (fail closed) - run: | - set -euo pipefail - test -s dump/roles.sql || { echo '::error::roles.sql is empty'; exit 1; } - grep -q 'CREATE' dump/schema.sql || { echo '::error::schema.sql has no CREATE statements'; exit 1; } - copies=$(grep -c '^COPY ' dump/data.sql || true) - bytes=$(wc -c < dump/data.sql) - echo "data.sql: ${copies} COPY blocks, ${bytes} bytes" - if [ "$copies" -lt 1 ]; then - echo '::error::data.sql contains no COPY blocks — this is schema-only or empty; refusing to upload a worthless artifact.' + if [ -z "$SUPABASE_DB_URL" ] || [ -z "$SUPABASE_PROJECT_REF" ]; then + echo '::error::The production-backup environment needs SUPABASE_DB_URL and SUPABASE_PROJECT_REF.' + exit 1 + fi + if [ -z "$BACKUP_BUCKET" ]; then + echo '::error::The production-backup environment needs AWS_BACKUP_BUCKET.' exit 1 fi + test "$(aws sts get-caller-identity --query Account --output text)" = '992382684924' + aws s3api get-public-access-block --bucket "$BACKUP_BUCKET" \ + --query 'PublicAccessBlockConfiguration.[BlockPublicAcls,IgnorePublicAcls,BlockPublicPolicy,RestrictPublicBuckets]' \ + --output text | grep -q $'True\tTrue\tTrue\tTrue' + python scripts/database_backup_contract.py validate-source \ + --db-url "$SUPABASE_DB_URL" \ + --project-ref "$SUPABASE_PROJECT_REF" \ + --recipients ops/backup/age-recipients.txt - - name: Encrypt to the committed age public key + - name: Dump, validate, encrypt, and upload run: | set -euo pipefail + umask 077 + mkdir -p dump + plain='' + cipher='' + cleanup() { + find dump -type f -delete 2>/dev/null || true + if [ -n "$plain" ]; then rm -f "$plain"; fi + if [ -n "$cipher" ]; then rm -f "$cipher"; fi + rm -f artifact-manifest.json + } + trap cleanup EXIT + stamp=$(date -u +%Y%m%dT%H%M%SZ) - tar -czf "db-backup-${stamp}.tar.gz" -C dump roles.sql schema.sql data.sql - plain_sha=$(sha256sum "db-backup-${stamp}.tar.gz" | cut -d' ' -f1) - age -R ops/backup/age-recipients.txt -o "db-backup-${stamp}.tar.gz.age" "db-backup-${stamp}.tar.gz" - rm -f "db-backup-${stamp}.tar.gz" dump/roles.sql dump/schema.sql dump/data.sql - cipher_sha=$(sha256sum "db-backup-${stamp}.tar.gz.age" | cut -d' ' -f1) - { - echo "created_at_utc=${stamp}" - echo "workflow_run=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - echo "repo_commit=${GITHUB_SHA}" - echo "plaintext_tar_sha256=${plain_sha}" - echo "ciphertext_sha256=${cipher_sha}" - echo "recipients_file_sha256=$(sha256sum ops/backup/age-recipients.txt | cut -d' ' -f1)" - } > manifest.txt - cat manifest.txt - echo "STAMP=${stamp}" >> "$GITHUB_ENV" - - - name: Upload encrypted artifact (90-day retention) - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 - with: - name: db-backup-${{ env.STAMP }} - path: | - db-backup-*.tar.gz.age - manifest.txt - retention-days: 90 - if-no-files-found: error + created_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + plain="db-backup-${stamp}.tar.gz" + cipher="${plain}.age" + prefix="daily/${stamp}" + + # This is the maintained Supabase roles/schema/data procedure. One + # unflagged dump would be schema-only and is not a recovery point. + supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/roles.sql --role-only + supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/schema.sql + supabase db dump --db-url "$SUPABASE_DB_URL" -f dump/data.sql \ + --use-copy --data-only \ + -x 'storage.buckets_vectors' -x 'storage.vector_indexes' + + python scripts/database_backup_contract.py create-contract \ + --project-ref "$SUPABASE_PROJECT_REF" \ + --recipients ops/backup/age-recipients.txt \ + --dump-dir dump \ + --created-at "$created_at" \ + --supabase-cli-version "$(supabase --version)" \ + --maximum-rpo-seconds 86400 \ + --retention-days 90 \ + --output dump/backup-contract.json + + tar -czf "$plain" -C dump \ + roles.sql schema.sql data.sql backup-contract.json + age -R ops/backup/age-recipients.txt -o "$cipher" "$plain" + + python scripts/database_backup_contract.py create-manifest \ + --contract dump/backup-contract.json \ + --plaintext-archive "$plain" \ + --ciphertext-archive "$cipher" \ + --repository-commit "$GITHUB_SHA" \ + --workflow-run-id "$GITHUB_RUN_ID" \ + --output artifact-manifest.json + python scripts/database_backup_contract.py verify-artifact \ + --manifest artifact-manifest.json \ + --ciphertext-archive "$cipher" + + local_sha=$(sha256sum "$cipher" | cut -d' ' -f1) + local_checksum=$(openssl dgst -sha256 -binary "$cipher" | base64) + aws s3 cp "$cipher" "s3://${BACKUP_BUCKET}/${prefix}/${cipher}" \ + --only-show-errors --sse AES256 --metadata "sha256=${local_sha}" \ + --checksum-algorithm SHA256 + aws s3 cp artifact-manifest.json \ + "s3://${BACKUP_BUCKET}/${prefix}/artifact-manifest.json" \ + --only-show-errors --sse AES256 \ + --content-type application/json --checksum-algorithm SHA256 + + remote_checksum=$(aws s3api get-object-attributes \ + --bucket "$BACKUP_BUCKET" --key "${prefix}/${cipher}" \ + --object-attributes Checksum \ + --query 'Checksum.ChecksumSHA256' --output text) + test "$remote_checksum" = "$local_checksum" + echo "Encrypted database backup stored at s3://${BACKUP_BUCKET}/${prefix}/" diff --git a/.github/workflows/prod-health-alert.yml b/.github/workflows/prod-health-alert.yml index 9028eff..54b86ec 100644 --- a/.github/workflows/prod-health-alert.yml +++ b/.github/workflows/prod-health-alert.yml @@ -17,7 +17,7 @@ name: Production health alert # - GitHub auto-disables scheduled workflows after 60 days WITHOUT # repo activity. sync.yml commits docs daily, which counts as # activity; if that sync ever stops, these schedules die silently -# ~60 days later. See ops/backup/RESTORE_DRILL.md section 6. +# ~60 days later. See the alerts section in ops/backup/RESTORE_DRILL.md. # - Failure emails go to the last committer of this file; keep that a # monitored account. diff --git a/ops/backup/RESTORE_DRILL.md b/ops/backup/RESTORE_DRILL.md index 9caf73d..b09e7e7 100644 --- a/ops/backup/RESTORE_DRILL.md +++ b/ops/backup/RESTORE_DRILL.md @@ -1,141 +1,268 @@ -# Production DB backup: keypair, restore drill, and paid-vs-free tradeoff +# Production database backup and restore drill -Companion to `.github/workflows/db-backup.yml` (daily encrypted logical backup) -and `.github/workflows/prod-health-alert.yml` (30-minute health pager). -Authoritative context: `openadapt-cloud` `AUDIT.md` finding (b) and -`docs/RUNBOOK_DATA_SAFETY.md` (the provider reported `pitr_enabled=false` and -zero physical backups on 2026-07-25). +This runbook owns the daily off-provider logical database backup in +`.github/workflows/db-backup.yml`. It complements the complete database and +private-Storage restore drill in `openadapt-cloud/docs/RUNBOOK_DATA_SAFETY.md`. -**Threat model note — this repo is PUBLIC.** Workflow artifacts on a public -repo are downloadable by any logged-in GitHub user. The backup artifact is -therefore ciphertext only (`age`), encrypted to the public key committed in -`ops/backup/age-recipients.txt`. The private key exists only offline with the -founder. If the private key is ever exposed, rotate (section 5) and treat all -retained artifacts as exposed. +## Current state -## 1. One-time founder setup: generate the age keypair +The design target is: -On a trusted machine (not CI): +- one logical database backup each day; +- client-side `age` encryption before network transfer; +- a private S3 bucket in OpenAdapt AWS account `992382684924`; +- a maximum recovery-point objective (RPO) of 24 hours when consecutive jobs + succeed; +- 90-day S3 retention; +- a local restore to a separate Supabase scratch project; +- measured database-only RPO and recovery-time objective (RTO) evidence; and +- the separate Cloud drill before any complete recovery claim. + +This is not yet a proven recovery path. As of 2026-08-08: + +- no daily backup has completed; +- no scratch restore has completed; +- no measured RTO exists; +- provider PITR is not enabled; +- the AWS stack is not deployed; +- the production database URL is not configured in the GitHub environment; +- no scratch Supabase project is configured; and +- one local private `age` key exists, but its required second vault or offline + copy is not confirmed. + +Do not describe the database as recoverable until one scheduled backup and one +isolated restore drill pass. + +## Security boundary + +This repository is public. Production database bytes must never enter a +GitHub artifact, workflow log, pull-request attachment, or public release. + +The workflow writes only these objects to a private S3 bucket: + +- `daily//db-backup-.tar.gz.age`, which is `age` + ciphertext; and +- `daily//artifact-manifest.json`, which contains sizes, digests, + the source commit, and the workflow run ID. It contains no project name, + database URL, table data, or credential. + +S3 also encrypts each object with SSE-S3. This server-side layer does not +replace `age`. The private `age` key stays on a trusted operator device and in +a separate vault or offline copy. It does not enter GitHub. + +The logical database backup does not contain private Supabase Storage objects. +It also uses the maintained Supabase roles, schema, and data commands, which +take separate logical snapshots. Do not run it during a schema migration. The +complete Cloud drill pauses writes, exports and rechecks Storage, restores both +boundaries, and produces the canonical retention receipt. + +## One-time AWS setup + +The CloudFormation template creates: + +- a private, versioned S3 bucket; +- public-access blocking and a TLS-only bucket policy; +- 90-day retention for daily backups; +- 365-day retention for database-only drill evidence; +- a GitHub OIDC writer role bound to the exact `production-backup` + environment; and +- a local restore role bound to one exact AWS operator principal. + +The expected S3 Standard storage price is approximately USD 0.023 per GB each +month, plus small request charges. For example, retaining 90 daily 100 MB +backups is approximately 9 GB, or USD 0.21 each month before requests. The +template does not create a paid KMS key and does not enable Supabase PITR. + +An AWS principal with CloudFormation, IAM, and S3 administration rights must +run: ```sh -brew install age # or: apt-get install age -umask 077 -age-keygen -o ~/openadapt-db-backup.agekey -``` +export AWS_PROFILE=openadapt +aws sts get-caller-identity --query Account --output text +# Must print 992382684924. -`age-keygen` prints one line `# public key: age1...`. +aws cloudformation deploy \ + --stack-name openadapt-production-db-backup \ + --template-file ops/backup/aws-backup-target.yml \ + --capabilities CAPABILITY_NAMED_IAM \ + --tags owner=OpenAdapt environment=production purpose=database-backup +``` -1. Store `~/openadapt-db-backup.agekey` (the PRIVATE key) in the team vault - (e.g. 1Password) AND on an offline copy. It never goes into this repo, - GitHub secrets, or any CI log. Without it, every backup artifact is - permanently unreadable. -2. Commit ONLY the public key: append the `age1...` line (just the key, no - `# public key:` prefix) to `ops/backup/age-recipients.txt` on a branch and - open a PR. -3. Set the database secret (value: the production Postgres connection string, - Supabase dashboard -> Project Settings -> Database -> Connection string, - direct or session-pooler URI): +The current `claude-ops` principal cannot create this stack because it does +not have `cloudformation:CreateChangeSet`. Do not bypass the template with +manual S3 or IAM changes. Use an authorized administration principal, then +record these outputs: ```sh -gh secret set SUPABASE_DB_URL --repo OpenAdaptAI/openadapt-ops +aws cloudformation describe-stacks \ + --stack-name openadapt-production-db-backup \ + --query 'Stacks[0].Outputs' ``` -The backup workflow fails loudly until both steps are done. That is -intentional: a red daily run is the reminder that production still has no -recovery point. +## One-time GitHub setup -## 2. What each daily artifact contains +First protect the repository and the environment: -Artifact `db-backup-` (retention 90 days): +1. Protect `main` with a repository ruleset. Require a pull request and code + owner review for the backup trust-boundary files in `.github/CODEOWNERS`. +2. Create the `production-backup` GitHub environment. +3. Restrict that environment to the protected `main` branch. Do not require a + manual deployment approval because it would prevent the daily schedule. +4. Set environment variables from the CloudFormation outputs: + - `AWS_BACKUP_BUCKET` + - `AWS_BACKUP_ROLE_ARN` +5. Set environment secrets: + - `SUPABASE_DB_URL`: the production direct or session-pooler PostgreSQL URL + - `SUPABASE_PROJECT_REF`: the exact production project reference -- `db-backup-.tar.gz.age` — age-encrypted tarball of the runbook - triple: `roles.sql`, `schema.sql`, `data.sql` (`--use-copy --data-only`, - excluding `storage.buckets_vectors` / `storage.vector_indexes`). -- `manifest.txt` — timestamps, run URL, plaintext and ciphertext SHA-256. +The workflow validates that the URL belongs to the declared Supabase project. +It also checks AWS account `992382684924`, complete S3 public-access blocking, +and the committed `age` recipient before it reads the database. -Storage buckets (bundles/reports/recordings) are NOT in this backup; database -PITR would not cover them either. Bucket export/restore stays with the cloud -runbook's `retention` drill tooling. +The public recipient is in `ops/backup/age-recipients.txt`. Store its private +key with mode `0600` on an encrypted trusted device. Make a second copy in a +team vault or an offline medium before the first backup. Without a second copy, +one device loss makes all retained backups unreadable. -## 3. Restore drill (run on a scratch database, never prod) +## Run and verify the first backup -Quarterly, or before any risky migration. Mirrors `RUNBOOK_DATA_SAFETY.md` -section 2 steps 6-8; the scratch project must be new, disposable, and in the -same regional/data-handling boundary as production. +After the AWS and GitHub setup, dispatch the workflow once. Do not wait for the +next schedule. ```sh -umask 077 -mkdir -p /secure/offline/openadapt-restore && cd /secure/offline/openadapt-restore - -# 1. Download the newest artifact (list runs, then download by run id): -gh run list --repo OpenAdaptAI/openadapt-ops --workflow db-backup.yml --limit 5 -gh run download --repo OpenAdaptAI/openadapt-ops - -# 2. Verify + decrypt with the founder-held private key: -cd db-backup- -sha256sum db-backup-.tar.gz.age # must equal ciphertext_sha256 in manifest.txt -age -d -i ~/openadapt-db-backup.agekey \ - -o db-backup-.tar.gz db-backup-.tar.gz.age -sha256sum db-backup-.tar.gz # must equal plaintext_tar_sha256 in manifest.txt -tar -xzf db-backup-.tar.gz # -> roles.sql schema.sql data.sql - -# 3. Create a brand-new scratch Supabase project (dashboard). Then restore -# in one transaction, same order and flags as the cloud runbook: -psql --single-transaction --variable ON_ERROR_STOP=1 \ - --file roles.sql \ - --file schema.sql \ - --command 'SET session_replication_role = replica' \ - --file data.sql \ - --dbname "$SCRATCH_DB_URL" - -# 4. Validate: row counts on runs/orgs/usage vs production expectations; -# spot-check one recent run row. Record the wall-clock restore time (RTO). - -# 5. Decommission the scratch project and delete the local plaintext: -rm -f roles.sql schema.sql data.sql db-backup-.tar.gz +gh workflow run db-backup.yml --repo OpenAdaptAI/openadapt-ops --ref main +gh run list --repo OpenAdaptAI/openadapt-ops \ + --workflow db-backup.yml --limit 5 ``` -Restore caveats (same as PITR): after any real restore, reconcile Stripe by -replaying webhooks from the Stripe dashboard (`stripe_events` is idempotent), -and expect up to 24h of lost writes (see RPO below). - -## 4. Can we get to a recovery point for $0 — and what does paying buy? - -**Yes.** Once section 1 is done, this workflow gives a daily, encrypted, -off-provider logical recovery point at $0 (public repo: Actions minutes and -artifact storage are free; the Supabase Free plan itself includes no backups). - -| Option | Monthly cost (Supabase public pricing, checked 2026-08-02) | RPO | Restore | Covers | -|---|---|---|---|---| -| This workflow (`pg_dump` daily, encrypted artifact, 90d retention) | $0 | up to 24h | manual, logical (`psql`), drill above; hours | DB only (roles+schema+data) | -| Supabase Pro (daily physical backups, 7-day retention) | $25 | up to 24h | dashboard-driven physical restore; also unlocks restore-to-new-project | DB only | -| Supabase Pro + PITR add-on | $25 + $100 per 7 days of PITR retention (+ compute add-on if the project is below the required tier, ~$10-15 net of Pro's $10 compute credit) | minutes | dashboard PITR to timestamp | DB only (WAL) | - -**Recommendation:** buy Supabase Pro ($25/mo) now — it is the cheapest change -that adds provider-side physical backups and makes `provider-status ---require-recovery` pass, and hosted retention/deletion gating in -openadapt-cloud wants provider recovery evidence. Defer the PITR add-on -(~$110-140/mo all-in) until there is at least one paying pilot; at N=0 -customers, a 24h RPO with a drilled restore path is a defensible posture, and -this workflow keeps an independent, off-provider copy either way. Revisit the -moment real customer billing/usage state is at stake. - -## 5. Key rotation / compromise - -1. Generate a new keypair (section 1); commit the new public key line. -2. Keep the old private key in the vault until every artifact encrypted to it - has aged out (90 days), then destroy it. -3. If the private key was exposed: rotate immediately AND rotate the database - credentials in `SUPABASE_DB_URL` (the artifacts contain full table data). - -## 6. Alerting caveats (prod-health-alert.yml) - -- GitHub emails the actor who last modified the workflow file when a scheduled - run fails. Keep that account monitored. -- GitHub auto-disables scheduled workflows after 60 days with no repo - activity. `sync.yml` commits docs daily, which keeps schedules alive; if the - doc sync ever stops, both schedules here die silently ~60 days later — - check Actions -> enabled state during any quarterly drill. -- Optional Telegram page: set the crier bot's `TELEGRAM_BOT_TOKEN` and - `TELEGRAM_OWNER_ID` as repo secrets (commands in the workflow log) and the - failure step will also message the founder directly. +Require all of these results: + +- the workflow succeeds on the exact `main` commit; +- S3 contains one ciphertext object and one redacted manifest below the same + UTC stamp; +- the stored SHA-256 checksum equals the local upload checksum; and +- no GitHub Actions artifact exists for the run. + +A successful upload proves backup creation and storage. It does not prove that +the backup can restore. + +## Run the database-only scratch restore + +Create a new disposable Supabase project in the required region. Never use the +production project, an existing customer project, or a shared development +project. The script never creates or deletes a project. + +Install `aws`, `age`, `psql`, and Supabase CLI 2.75.0 on the trusted operator +device. Then set: + +```sh +export AWS_PROFILE=openadapt +export AWS_BACKUP_BUCKET='' +export AWS_RESTORE_ROLE_ARN='' +export BACKUP_STAMP='' +export PRODUCTION_PROJECT_REF='' +export SCRATCH_PROJECT_REF='' +export CONFIRM_SCRATCH_PROJECT_REF="$SCRATCH_PROJECT_REF" +export SCRATCH_DB_URL='' +export AGE_SECRET_KEY_FILE='' + +scripts/run_database_restore_drill.sh +``` + +The script: + +1. validates that the scratch URL belongs to the repeated scratch project; +2. assumes the least-privilege restore role in AWS account `992382684924`; +3. checks S3 public-access blocking; +4. downloads the exact timestamped ciphertext and manifest; +5. verifies the S3 metadata digest and the artifact contract; +6. decrypts into a private temporary directory; +7. extracts only four exact regular files and rejects unsafe archive members; +8. restores with `ON_ERROR_STOP` in one transaction; +9. dumps the scratch schema and data again and compares their digests; +10. writes a new database-only evidence file without overwriting old evidence; +11. uploads that metadata-only evidence below `drills/database-only/`; and +12. removes all temporary plaintext. + +RTO starts before AWS role assumption and download. It ends after the scratch +database redump and validation. RPO is measured from the backup recovery point +to the same start time. + +The script does not delete the scratch project. Review the evidence first. +Then decommission the project through its authorized owner process. + +## Complete recovery evidence + +Database-only evidence has: + +```json +{ + "database_restored": true, + "storage_restored": false +} +``` + +It cannot unlock destructive hosted retention and is not the canonical Cloud +restore receipt. After the database-only drill, run the database and private +Storage procedure in `openadapt-cloud/docs/RUNBOOK_DATA_SAFETY.md`: + +1. `drill-storage-export` +2. `drill-prepare` +3. the exact roles, schema, and data dumps +4. `drill-storage-check-source` +5. scratch database restore +6. `drill-storage-restore` +7. `drill-verify` without `--record` +8. review the result +9. repeat `drill-verify` with `--record` + +Only that result proves the complete declared database and Storage boundary. + +## Recovery cost options + +Prices can change. Verify the provider price before purchase. + +| Option | Approximate monthly cost | Recovery point | Boundary | +|---|---:|---|---| +| This daily logical backup | S3 bytes and requests only; approximately USD 0.023/GB-month | Up to 24 hours when consecutive jobs pass | Database only | +| Supabase Pro | USD 25 base plan | Daily provider backup with seven-day history | Database only | +| Supabase PITR | About USD 100 for seven days, plus the required paid plan and minimum compute | Minutes, based on retained WAL | Database only | + +References: + +- +- +- +- + +Do not enable PITR without a recorded cost decision. A logical backup and a +successful complete scratch drill are still required because provider database +recovery does not cover private Storage objects. + +## Key rotation + +For routine rotation: + +1. Generate a new private key on a trusted device. +2. store it on the device and in the second vault or offline location; +3. replace the committed public recipient; +4. merge the change through code owner review; +5. confirm the next backup uses only the new recipient; and +6. keep the old private key until every old S3 object expires after 90 days. + +If a private key is compromised, remove its public recipient immediately. +Rotate the production database credential because retained backups contain the +complete logical database. Treat all backups encrypted to the compromised key +as exposed. Do not keep encrypting new backups to both old and new recipients. + +## Alerts and scheduled-workflow limits + +A failed backup job stays red and GitHub sends workflow failure notifications +according to repository notification settings. The founder must monitor this +signal. A later change should add a direct freshness alert from an independent +system after the first successful object exists. + +GitHub can disable schedules after 60 days without repository activity. The +daily documentation sync currently keeps this repository active. Verify the +backup workflow enabled state during each quarterly drill. A schedule status +is not backup evidence; check the newest S3 recovery point and drill receipt. diff --git a/ops/backup/age-recipients.txt b/ops/backup/age-recipients.txt index f78e4b0..971472c 100644 --- a/ops/backup/age-recipients.txt +++ b/ops/backup/age-recipients.txt @@ -3,7 +3,8 @@ # This file must contain exactly the PUBLIC key line(s) of the founder's # age keypair — one `age1...` recipient per line. The PRIVATE key never # enters this repo, GitHub secrets, or CI. Generation procedure: -# ops/backup/RESTORE_DRILL.md section 1. +# ops/backup/RESTORE_DRILL.md, "One-time GitHub setup". # # The backup workflow FAILS (on purpose) until a real recipient line # exists below. Do not put anything secret in this file. +age1cw6u268e7vrjsl224w69may8ujxvqhqfymz79xm99dup5mf5y9jqv0vcqu diff --git a/ops/backup/aws-backup-target.yml b/ops/backup/aws-backup-target.yml new file mode 100644 index 0000000..7fde61a --- /dev/null +++ b/ops/backup/aws-backup-target.yml @@ -0,0 +1,165 @@ +AWSTemplateFormatVersion: '2010-09-09' +Description: Private encrypted database-backup target with exact writer and restore roles. + +Parameters: + GitHubOrganization: + Type: String + Default: OpenAdaptAI + GitHubRepository: + Type: String + Default: openadapt-ops + RestorePrincipalArn: + Type: String + Default: arn:aws:iam::992382684924:user/claude-ops + AllowedPattern: ^arn:aws:iam::992382684924:(user|role)/[A-Za-z0-9+=,.@_/-]+$ + Description: Exact local operator principal that can assume the restore role. + +Resources: + GitHubActionsOidcProvider: + Type: AWS::IAM::OIDCProvider + Properties: + Url: https://token.actions.githubusercontent.com + ClientIdList: + - sts.amazonaws.com + ThumbprintList: + - 6938fd4d98bab03faadb97b34396831e3780aea1 + + BackupBucket: + Type: AWS::S3::Bucket + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + BucketName: !Sub openadapt-production-db-backups-${AWS::AccountId} + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + VersioningConfiguration: + Status: Enabled + LifecycleConfiguration: + Rules: + - Id: DeleteExpiredBackups + Status: Enabled + Prefix: daily/ + ExpirationInDays: 90 + NoncurrentVersionExpiration: + NoncurrentDays: 7 + AbortIncompleteMultipartUpload: + DaysAfterInitiation: 1 + - Id: DeleteExpiredDrillEvidence + Status: Enabled + Prefix: drills/ + ExpirationInDays: 365 + NoncurrentVersionExpiration: + NoncurrentDays: 7 + Tags: + - Key: data-classification + Value: production-confidential + - Key: managed-by + Value: openadapt-ops + + BackupBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref BackupBucket + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: DenyInsecureTransport + Effect: Deny + Principal: '*' + Action: s3:* + Resource: + - !GetAtt BackupBucket.Arn + - !Sub ${BackupBucket.Arn}/* + Condition: + Bool: + aws:SecureTransport: false + - Sid: RequireSseS3 + Effect: Deny + Principal: '*' + Action: s3:PutObject + Resource: !Sub ${BackupBucket.Arn}/daily/* + Condition: + StringNotEquals: + s3:x-amz-server-side-encryption: AES256 + - Sid: RequireSseS3ForDrillEvidence + Effect: Deny + Principal: '*' + Action: s3:PutObject + Resource: !Sub ${BackupBucket.Arn}/drills/* + Condition: + StringNotEquals: + s3:x-amz-server-side-encryption: AES256 + + BackupWriterRole: + Type: AWS::IAM::Role + Properties: + RoleName: openadapt-ops-production-db-backup-writer + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Federated: !Ref GitHubActionsOidcProvider + Action: sts:AssumeRoleWithWebIdentity + Condition: + StringEquals: + token.actions.githubusercontent.com:aud: sts.amazonaws.com + token.actions.githubusercontent.com:sub: !Sub repo:${GitHubOrganization}/${GitHubRepository}:environment:production-backup + Policies: + - PolicyName: WriteEncryptedDailyBackups + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: s3:GetBucketPublicAccessBlock + Resource: !GetAtt BackupBucket.Arn + - Effect: Allow + Action: + - s3:AbortMultipartUpload + - s3:GetObjectAttributes + - s3:PutObject + Resource: !Sub ${BackupBucket.Arn}/daily/* + + BackupRestoreRole: + Type: AWS::IAM::Role + Properties: + RoleName: openadapt-ops-production-db-backup-reader + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: !Ref RestorePrincipalArn + Action: sts:AssumeRole + Policies: + - PolicyName: ReadEncryptedDailyBackups + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: s3:GetBucketPublicAccessBlock + Resource: !GetAtt BackupBucket.Arn + - Effect: Allow + Action: s3:GetObject + Resource: !Sub ${BackupBucket.Arn}/daily/* + - Effect: Allow + Action: s3:PutObject + Resource: !Sub ${BackupBucket.Arn}/drills/* + +Outputs: + BackupBucketName: + Value: !Ref BackupBucket + BackupWriterRoleArn: + Value: !GetAtt BackupWriterRole.Arn + BackupRestoreRoleArn: + Value: !GetAtt BackupRestoreRole.Arn diff --git a/scripts/database_backup_contract.py b/scripts/database_backup_contract.py new file mode 100644 index 0000000..78d8a0e --- /dev/null +++ b/scripts/database_backup_contract.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +"""Fail-closed contracts for encrypted production database backups. + +This module never connects to a database. The workflows keep database access in +the Supabase CLI and psql. This module validates target identity, verifies dump +components, creates a redacted integrity manifest, and records restore evidence. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +import sys +import tarfile +from datetime import datetime, timezone +from pathlib import Path +from urllib.parse import unquote, urlsplit + + +CONTRACT_SCHEMA = "openadapt.database-backup-contract/v2" +ARTIFACT_SCHEMA = "openadapt.database-backup-artifact/v2" +RESTORE_EVIDENCE_SCHEMA = "openadapt.database-restore-evidence/v1" +PROJECT_REF = re.compile(r"^[a-z0-9]{8,64}$") +AGE_RECIPIENT = re.compile(r"^age1[0-9a-z]+$") +REQUIRED_DUMPS = ("roles.sql", "schema.sql", "data.sql") +ARCHIVE_MEMBERS = (*REQUIRED_DUMPS, "backup-contract.json") +MAX_ARCHIVE_MEMBER_BYTES = 2 * 1024 * 1024 * 1024 +UNSAFE_PSQL_COMMAND = re.compile( + r"^\\(?:!|cd|copy|e(?:dit)?|i|ir|o|out|qecho|setenv|w(?:rite)?)(?:\s|$)", + re.MULTILINE | re.IGNORECASE, +) +COPY_PROGRAM = re.compile(r"\bCOPY\b[^;]*\bPROGRAM\b", re.IGNORECASE | re.DOTALL) + + +class ContractError(ValueError): + """A backup or restore contract is unsafe or incomplete.""" + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def sha256_text(value: str) -> str: + return hashlib.sha256(value.encode()).hexdigest() + + +def stable_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def project_ref(value: str, name: str) -> str: + value = value.strip() + if not PROJECT_REF.fullmatch(value): + raise ContractError(f"{name} is not a valid Supabase project reference") + return value + + +def database_identity(url: str, expected_ref: str, name: str) -> dict[str, str]: + expected_ref = project_ref(expected_ref, f"{name} project reference") + parsed = urlsplit(url.strip()) + if parsed.scheme not in {"postgres", "postgresql"}: + raise ContractError(f"{name} must use a postgres connection URL") + if not parsed.hostname or not parsed.username or not parsed.path.strip("/"): + raise ContractError(f"{name} is incomplete") + if parsed.hostname in {"localhost", "127.0.0.1", "::1"}: + raise ContractError(f"{name} must name the declared Supabase project") + + username = unquote(parsed.username) + direct_host = parsed.hostname == f"db.{expected_ref}.supabase.co" + pooler_user = username == f"postgres.{expected_ref}" + pooler_host = parsed.hostname.endswith(".pooler.supabase.com") + if not direct_host and not (pooler_user and pooler_host): + raise ContractError(f"{name} does not match the declared Supabase project") + + # This digest lets an operator compare identities without publishing the + # project reference, hostname, username, or connection string. + canonical = f"{expected_ref}|{parsed.hostname}|{username}|{parsed.path.strip('/')}" + return { + "identity_sha256": sha256_text(canonical), + "project_ref_sha256": sha256_text(expected_ref), + } + + +def recipients(path: Path) -> list[str]: + values = [ + line.strip() + for line in path.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + if not values or any(not AGE_RECIPIENT.fullmatch(value) for value in values): + raise ContractError("the recipient file must contain only age public recipients") + if len(values) != len(set(values)): + raise ContractError("the recipient file contains a duplicate recipient") + return sorted(values) + + +def dump_inventory(root: Path) -> list[dict[str, object]]: + result: list[dict[str, object]] = [] + for name in REQUIRED_DUMPS: + path = root / name + if not path.is_file() or path.stat().st_size <= 0: + raise ContractError(f"database dump component is missing or empty: {name}") + result.append( + {"name": name, "bytes": path.stat().st_size, "sha256": sha256_file(path)} + ) + + schema = (root / "schema.sql").read_text(encoding="utf-8", errors="replace") + data = (root / "data.sql").read_text(encoding="utf-8", errors="replace") + for name in REQUIRED_DUMPS: + sql = (root / name).read_text(encoding="utf-8", errors="replace") + if UNSAFE_PSQL_COMMAND.search(sql) or COPY_PROGRAM.search(sql): + raise ContractError(f"database dump contains an unsafe local command: {name}") + if not re.search(r"^CREATE\s", schema, re.MULTILINE | re.IGNORECASE): + raise ContractError("schema.sql contains no CREATE statement") + if not re.search(r"^COPY\s", data, re.MULTILINE | re.IGNORECASE): + raise ContractError("data.sql contains no COPY statement") + return result + + +def parse_time(value: str, name: str) -> datetime: + try: + result = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError as error: + raise ContractError(f"{name} is not an ISO-8601 timestamp") from error + if result.tzinfo is None: + raise ContractError(f"{name} must include a timezone") + return result.astimezone(timezone.utc) + + +def read_contract(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if value.get("schema") != CONTRACT_SCHEMA: + raise ContractError("the backup contract schema is not supported") + contract = value.get("contract") + if not isinstance(contract, dict): + raise ContractError("the backup contract is missing") + expected = sha256_text(stable_json(contract)) + if value.get("contract_sha256") != expected: + raise ContractError("the backup contract digest is invalid") + return value + + +def read_manifest(path: Path) -> dict[str, object]: + value = json.loads(path.read_text(encoding="utf-8")) + if value.get("schema") != ARTIFACT_SCHEMA: + raise ContractError("the artifact manifest schema is not supported") + manifest = value.get("artifact") + if not isinstance(manifest, dict): + raise ContractError("the artifact manifest is missing") + expected = sha256_text(stable_json(manifest)) + if value.get("artifact_sha256") != expected: + raise ContractError("the artifact manifest digest is invalid") + return value + + +def validate_source(args: argparse.Namespace) -> None: + identity = database_identity(args.db_url, args.project_ref, "production database") + keys = recipients(Path(args.recipients)) + print( + json.dumps( + { + "valid": True, + **identity, + "recipient_count": len(keys), + "recipients_sha256": sha256_text("\n".join(keys) + "\n"), + }, + sort_keys=True, + ) + ) + + +def create_contract(args: argparse.Namespace) -> None: + source_ref = project_ref(args.project_ref, "production project reference") + keys = recipients(Path(args.recipients)) + created = parse_time(args.created_at, "created-at") + dump_files = dump_inventory(Path(args.dump_dir)) + contract: dict[str, object] = { + "source_project_ref_sha256": sha256_text(source_ref), + "created_at": created.isoformat().replace("+00:00", "Z"), + "maximum_rpo_seconds": args.maximum_rpo_seconds, + "retention_days": args.retention_days, + "dump_files": dump_files, + "recipients_sha256": sha256_text("\n".join(keys) + "\n"), + "supabase_cli_version": args.supabase_cli_version, + } + envelope = { + "schema": CONTRACT_SCHEMA, + "contract": contract, + "contract_sha256": sha256_text(stable_json(contract)), + } + output = Path(args.output) + output.write_text(json.dumps(envelope, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"contract": str(output), "contract_sha256": envelope["contract_sha256"]})) + + +def create_manifest(args: argparse.Namespace) -> None: + contract = read_contract(Path(args.contract)) + plaintext = Path(args.plaintext_archive) + ciphertext = Path(args.ciphertext_archive) + if not plaintext.is_file() or plaintext.stat().st_size <= 0: + raise ContractError("the plaintext archive is missing or empty") + if not ciphertext.is_file() or ciphertext.stat().st_size <= 0: + raise ContractError("the encrypted archive is missing or empty") + artifact: dict[str, object] = { + "backup_contract_sha256": contract["contract_sha256"], + "plaintext_archive": { + "bytes": plaintext.stat().st_size, + "sha256": sha256_file(plaintext), + }, + "ciphertext_archive": { + "bytes": ciphertext.stat().st_size, + "sha256": sha256_file(ciphertext), + }, + "repository_commit": args.repository_commit, + "workflow_run_id": args.workflow_run_id, + } + manifest = { + "schema": ARTIFACT_SCHEMA, + "artifact": artifact, + "artifact_sha256": sha256_text(stable_json(artifact)), + } + output = Path(args.output) + output.write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"manifest": str(output), "artifact_sha256": manifest["artifact_sha256"]})) + + +def verify_artifact(args: argparse.Namespace) -> None: + manifest = read_manifest(Path(args.manifest)) + artifact = manifest["artifact"] + assert isinstance(artifact, dict) + ciphertext = Path(args.ciphertext_archive) + expected = artifact.get("ciphertext_archive") + if not isinstance(expected, dict): + raise ContractError("the encrypted archive contract is missing") + if expected.get("bytes") != ciphertext.stat().st_size or expected.get("sha256") != sha256_file(ciphertext): + raise ContractError("the encrypted archive does not match the manifest") + print(json.dumps({"valid": True, "artifact_sha256": manifest["artifact_sha256"]})) + + +def validate_restore_target(args: argparse.Namespace) -> None: + source_ref = project_ref(args.source_project_ref, "production project reference") + scratch_ref = project_ref(args.scratch_project_ref, "scratch project reference") + if source_ref == scratch_ref: + raise ContractError("the restore target is the production project") + identity = database_identity(args.scratch_db_url, scratch_ref, "scratch database") + print(json.dumps({"valid": True, **identity}, sort_keys=True)) + + +def extract_artifact(args: argparse.Namespace) -> None: + archive = Path(args.plaintext_archive) + manifest = read_manifest(Path(args.manifest)) + artifact = manifest["artifact"] + assert isinstance(artifact, dict) + expected_archive = artifact.get("plaintext_archive") + if not isinstance(expected_archive, dict): + raise ContractError("the plaintext archive contract is missing") + if ( + expected_archive.get("bytes") != archive.stat().st_size + or expected_archive.get("sha256") != sha256_file(archive) + ): + raise ContractError("the decrypted archive does not match the manifest") + output = Path(args.output_dir) + if output.exists() and any(output.iterdir()): + raise ContractError("the extraction directory is not empty") + output.mkdir(mode=0o700, parents=True, exist_ok=True) + + with tarfile.open(archive, mode="r:gz") as bundle: + members = bundle.getmembers() + names = [member.name for member in members] + if sorted(names) != sorted(ARCHIVE_MEMBERS): + raise ContractError("the archive member allowlist does not match") + for member in members: + if not member.isfile(): + raise ContractError("the archive contains a non-regular member") + if member.name.startswith("/") or ".." in Path(member.name).parts: + raise ContractError("the archive contains an unsafe member path") + if member.size <= 0 or member.size > MAX_ARCHIVE_MEMBER_BYTES: + raise ContractError("the archive contains an invalid member size") + source = bundle.extractfile(member) + if source is None: + raise ContractError("the archive member is not readable") + destination = output / member.name + with destination.open("xb") as target: + while chunk := source.read(1024 * 1024): + target.write(chunk) + destination.chmod(0o600) + + contract = read_contract(output / "backup-contract.json") + if artifact.get("backup_contract_sha256") != contract.get("contract_sha256"): + raise ContractError("the decrypted backup contract does not match the manifest") + inventory = dump_inventory(output) + expected = contract["contract"].get("dump_files") + if stable_json(inventory) != stable_json(expected): + raise ContractError("the extracted dump files do not match the backup contract") + print(json.dumps({"valid": True, "contract_sha256": contract["contract_sha256"]})) + + +def verify_restored_dumps(args: argparse.Namespace) -> None: + source = {entry["name"]: entry for entry in dump_inventory(Path(args.source_dir))} + restored: dict[str, dict[str, object]] = {} + for name in ("schema.sql", "data.sql"): + path = Path(args.restored_dir) / name + if not path.is_file() or path.stat().st_size <= 0: + raise ContractError(f"the restored dump is missing or empty: {name}") + restored[name] = { + "name": name, + "bytes": path.stat().st_size, + "sha256": sha256_file(path), + } + # Roles are target-specific. Schema and data must reproduce exactly. + for name in ("schema.sql", "data.sql"): + if source[name]["sha256"] != restored[name]["sha256"]: + raise ContractError(f"the restored {name} does not match the backup") + print( + json.dumps( + { + "valid": True, + "schema_sha256": source["schema.sql"]["sha256"], + "data_sha256": source["data.sql"]["sha256"], + }, + sort_keys=True, + ) + ) + + +def record_restore(args: argparse.Namespace) -> None: + manifest = read_manifest(Path(args.manifest)) + contract = read_contract(Path(args.contract)) + source_ref = project_ref(args.source_project_ref, "production project reference") + scratch_ref = project_ref(args.scratch_project_ref, "scratch project reference") + if source_ref == scratch_ref: + raise ContractError("the restore target is the production project") + artifact = manifest["artifact"] + assert isinstance(artifact, dict) + payload = contract["contract"] + assert isinstance(payload, dict) + if artifact.get("backup_contract_sha256") != contract.get("contract_sha256"): + raise ContractError("the artifact and backup contract do not match") + if payload.get("source_project_ref_sha256") != sha256_text(source_ref): + raise ContractError("the backup does not belong to the declared production project") + + verification = json.loads(Path(args.verification).read_text(encoding="utf-8")) + if verification.get("valid") is not True: + raise ContractError("the scratch restore verification did not pass") + dump_files = payload.get("dump_files") + if not isinstance(dump_files, list): + raise ContractError("the backup contract has no dump inventory") + expected_digests = { + entry.get("name"): entry.get("sha256") + for entry in dump_files + if isinstance(entry, dict) + } + for name in ("schema", "data"): + if verification.get(f"{name}_sha256") != expected_digests.get(f"{name}.sql"): + raise ContractError("the restore verification does not match the backup contract") + started = parse_time(args.started_at, "started-at") + completed = parse_time(args.completed_at, "completed-at") + if completed < started: + raise ContractError("the restore completion time precedes the start time") + recovery_point = parse_time(str(payload.get("created_at")), "backup recovery point") + receipt = { + "schema": RESTORE_EVIDENCE_SCHEMA, + "backup_contract_sha256": contract["contract_sha256"], + "artifact_sha256": manifest["artifact_sha256"], + "source_project_ref_sha256": sha256_text(source_ref), + "scratch_project_ref_sha256": sha256_text(scratch_ref), + "recovery_point_at": recovery_point.isoformat().replace("+00:00", "Z"), + "started_at": started.isoformat().replace("+00:00", "Z"), + "completed_at": completed.isoformat().replace("+00:00", "Z"), + "rpo_seconds_at_start": max(0, int((started - recovery_point).total_seconds())), + "rto_seconds": int((completed - started).total_seconds()), + "schema_sha256": verification["schema_sha256"], + "data_sha256": verification["data_sha256"], + "database_restored": True, + "storage_restored": False, + } + output = Path(args.output) + try: + with output.open("x", encoding="utf-8") as stream: + json.dump(receipt, stream, indent=2, sort_keys=True) + stream.write("\n") + except FileExistsError as error: + raise ContractError("the restore evidence output already exists") from error + print(json.dumps({"receipt": str(output), "rto_seconds": receipt["rto_seconds"]})) + + +def parser() -> argparse.ArgumentParser: + root = argparse.ArgumentParser() + commands = root.add_subparsers(dest="command", required=True) + + source = commands.add_parser("validate-source") + source.add_argument("--db-url", required=True) + source.add_argument("--project-ref", required=True) + source.add_argument("--recipients", required=True) + source.set_defaults(run=validate_source) + + contract = commands.add_parser("create-contract") + contract.add_argument("--project-ref", required=True) + contract.add_argument("--recipients", required=True) + contract.add_argument("--dump-dir", required=True) + contract.add_argument("--created-at", required=True) + contract.add_argument("--supabase-cli-version", required=True) + contract.add_argument("--maximum-rpo-seconds", type=int, default=86400) + contract.add_argument("--retention-days", type=int, default=90) + contract.add_argument("--output", required=True) + contract.set_defaults(run=create_contract) + + manifest = commands.add_parser("create-manifest") + manifest.add_argument("--contract", required=True) + manifest.add_argument("--plaintext-archive", required=True) + manifest.add_argument("--ciphertext-archive", required=True) + manifest.add_argument("--repository-commit", required=True) + manifest.add_argument("--workflow-run-id", required=True) + manifest.add_argument("--output", required=True) + manifest.set_defaults(run=create_manifest) + + artifact = commands.add_parser("verify-artifact") + artifact.add_argument("--manifest", required=True) + artifact.add_argument("--ciphertext-archive", required=True) + artifact.set_defaults(run=verify_artifact) + + target = commands.add_parser("validate-restore-target") + target.add_argument("--source-project-ref", required=True) + target.add_argument("--scratch-project-ref", required=True) + target.add_argument("--scratch-db-url", required=True) + target.set_defaults(run=validate_restore_target) + + extract = commands.add_parser("extract-artifact") + extract.add_argument("--plaintext-archive", required=True) + extract.add_argument("--manifest", required=True) + extract.add_argument("--output-dir", required=True) + extract.set_defaults(run=extract_artifact) + + restored = commands.add_parser("verify-restored-dumps") + restored.add_argument("--source-dir", required=True) + restored.add_argument("--restored-dir", required=True) + restored.set_defaults(run=verify_restored_dumps) + + receipt = commands.add_parser("record-restore") + receipt.add_argument("--manifest", required=True) + receipt.add_argument("--contract", required=True) + receipt.add_argument("--verification", required=True) + receipt.add_argument("--source-project-ref", required=True) + receipt.add_argument("--scratch-project-ref", required=True) + receipt.add_argument("--started-at", required=True) + receipt.add_argument("--completed-at", required=True) + receipt.add_argument("--output", required=True) + receipt.set_defaults(run=record_restore) + return root + + +def main() -> int: + args = parser().parse_args() + try: + args.run(args) + except (ContractError, OSError, json.JSONDecodeError) as error: + print(f"error: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_database_restore_drill.sh b/scripts/run_database_restore_drill.sh new file mode 100755 index 0000000..b28d800 --- /dev/null +++ b/scripts/run_database_restore_drill.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Restore one encrypted production backup to a separate scratch Supabase +# project. This stays on an operator-controlled machine so the age private key +# never enters GitHub. It never creates or deletes a project. + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +cd "$repo_root" + +required=( + AWS_BACKUP_BUCKET + AWS_RESTORE_ROLE_ARN + BACKUP_STAMP + PRODUCTION_PROJECT_REF + SCRATCH_DB_URL + SCRATCH_PROJECT_REF + AGE_SECRET_KEY_FILE +) +for name in "${required[@]}"; do + if [[ -z "${!name:-}" ]]; then + echo "error: ${name} is required" >&2 + exit 2 + fi +done +if [[ ! "$BACKUP_STAMP" =~ ^[0-9]{8}T[0-9]{6}Z$ ]]; then + echo 'error: BACKUP_STAMP is invalid' >&2 + exit 2 +fi +if [[ "${CONFIRM_SCRATCH_PROJECT_REF:-}" != "$SCRATCH_PROJECT_REF" ]]; then + echo 'error: repeat SCRATCH_PROJECT_REF in CONFIRM_SCRATCH_PROJECT_REF' >&2 + exit 2 +fi +if [[ ! -f "$AGE_SECRET_KEY_FILE" ]]; then + echo 'error: AGE_SECRET_KEY_FILE does not exist' >&2 + exit 2 +fi +if [[ ! "$AWS_RESTORE_ROLE_ARN" =~ ^arn:aws:iam::992382684924:role/[A-Za-z0-9+=,.@_/-]+$ ]]; then + echo 'error: AWS_RESTORE_ROLE_ARN is not an OpenAdapt account role' >&2 + exit 2 +fi +if [[ "$(uname -s)" == 'Darwin' ]]; then + key_mode=$(stat -f '%Lp' "$AGE_SECRET_KEY_FILE") +else + key_mode=$(stat -c '%a' "$AGE_SECRET_KEY_FILE") +fi +if (( (8#$key_mode & 077) != 0 )); then + echo 'error: AGE_SECRET_KEY_FILE must not be readable by group or other users' >&2 + exit 2 +fi +private_recipient=$(age-keygen -y "$AGE_SECRET_KEY_FILE") +if ! grep -Fqx "$private_recipient" ops/backup/age-recipients.txt; then + echo 'error: AGE_SECRET_KEY_FILE does not match a committed backup recipient' >&2 + exit 2 +fi + +root=$(mktemp -d "${TMPDIR:-/tmp}/openadapt-db-restore.XXXXXX") +chmod 700 "$root" +cleanup() { + find "$root" -type f -exec chmod 600 {} + 2>/dev/null || true + rm -rf "$root" +} +trap cleanup EXIT INT TERM + +started_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +python scripts/database_backup_contract.py validate-restore-target \ + --source-project-ref "$PRODUCTION_PROJECT_REF" \ + --scratch-project-ref "$SCRATCH_PROJECT_REF" \ + --scratch-db-url "$SCRATCH_DB_URL" + +read -r AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN < <( + AWS_PROFILE="${AWS_PROFILE:-openadapt}" aws sts assume-role \ + --role-arn "$AWS_RESTORE_ROLE_ARN" \ + --role-session-name "openadapt-db-restore-${BACKUP_STAMP}" \ + --duration-seconds 3600 \ + --query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \ + --output text +) +export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN +unset AWS_PROFILE + +account=$(aws sts get-caller-identity --query Account --output text) +if [[ "$account" != '992382684924' ]]; then + echo 'error: AWS_PROFILE does not resolve to OpenAdapt account 992382684924' >&2 + exit 2 +fi +block=$(aws s3api get-public-access-block \ + --bucket "$AWS_BACKUP_BUCKET" \ + --query 'PublicAccessBlockConfiguration.[BlockPublicAcls,IgnorePublicAcls,BlockPublicPolicy,RestrictPublicBuckets]' \ + --output text) +if [[ "$block" != $'True\tTrue\tTrue\tTrue' ]]; then + echo 'error: backup bucket public-access blocking is incomplete' >&2 + exit 2 +fi + +cipher="db-backup-${BACKUP_STAMP}.tar.gz.age" +plain="db-backup-${BACKUP_STAMP}.tar.gz" +prefix="daily/${BACKUP_STAMP}" +aws s3 cp \ + "s3://${AWS_BACKUP_BUCKET}/${prefix}/${cipher}" "$root/$cipher" --only-show-errors +aws s3 cp \ + "s3://${AWS_BACKUP_BUCKET}/${prefix}/artifact-manifest.json" \ + "$root/artifact-manifest.json" --only-show-errors + +remote_sha=$(aws s3api head-object \ + --bucket "$AWS_BACKUP_BUCKET" --key "${prefix}/${cipher}" \ + --query 'Metadata.sha256' --output text) +local_sha=$(shasum -a 256 "$root/$cipher" | awk '{print $1}') +if [[ "$remote_sha" != "$local_sha" ]]; then + echo 'error: downloaded ciphertext digest does not match S3 metadata' >&2 + exit 2 +fi +python scripts/database_backup_contract.py verify-artifact \ + --manifest "$root/artifact-manifest.json" \ + --ciphertext-archive "$root/$cipher" + +age -d -i "$AGE_SECRET_KEY_FILE" -o "$root/$plain" "$root/$cipher" +mkdir -m 700 "$root/recovered" "$root/redump" +python scripts/database_backup_contract.py extract-artifact \ + --plaintext-archive "$root/$plain" \ + --manifest "$root/artifact-manifest.json" \ + --output-dir "$root/recovered" + +PGDATABASE="$SCRATCH_DB_URL" psql \ + --single-transaction --variable ON_ERROR_STOP=1 \ + --file "$root/recovered/roles.sql" \ + --file "$root/recovered/schema.sql" \ + --command 'SET session_replication_role = replica' \ + --file "$root/recovered/data.sql" +supabase db dump --db-url "$SCRATCH_DB_URL" -f "$root/redump/schema.sql" +supabase db dump --db-url "$SCRATCH_DB_URL" -f "$root/redump/data.sql" \ + --use-copy --data-only \ + -x 'storage.buckets_vectors' -x 'storage.vector_indexes' +python scripts/database_backup_contract.py verify-restored-dumps \ + --source-dir "$root/recovered" --restored-dir "$root/redump" \ + > "$root/verification.json" +completed_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) + +output=${RESTORE_EVIDENCE_OUTPUT:-restore-evidence-${BACKUP_STAMP}.json} +python scripts/database_backup_contract.py record-restore \ + --manifest "$root/artifact-manifest.json" \ + --contract "$root/recovered/backup-contract.json" \ + --verification "$root/verification.json" \ + --source-project-ref "$PRODUCTION_PROJECT_REF" \ + --scratch-project-ref "$SCRATCH_PROJECT_REF" \ + --started-at "$started_at" --completed-at "$completed_at" \ + --output "$output" +chmod 600 "$output" +aws s3 cp "$output" \ + "s3://${AWS_BACKUP_BUCKET}/drills/database-only/${BACKUP_STAMP}/$(basename "$output")" \ + --only-show-errors --sse AES256 --content-type application/json \ + --checksum-algorithm SHA256 +echo "Database-only restore evidence: $output" +echo 'Run the openadapt-cloud database-plus-Storage drill before recording the canonical retention receipt.' diff --git a/tests/test_database_backup_contract.py b/tests/test_database_backup_contract.py new file mode 100644 index 0000000..3d9bc3b --- /dev/null +++ b/tests/test_database_backup_contract.py @@ -0,0 +1,341 @@ +import importlib.util +import io +import json +import tarfile +from argparse import Namespace +from pathlib import Path + +import pytest + + +MODULE_PATH = Path(__file__).parents[1] / "scripts" / "database_backup_contract.py" +SPEC = importlib.util.spec_from_file_location("database_backup_contract", MODULE_PATH) +assert SPEC and SPEC.loader +backup = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(backup) + +SOURCE_REF = "abcdefghijklmnopqrst" +SCRATCH_REF = "zyxwvutsrqponmlkjihg" +RECIPIENT = "age1cw6u268e7vrjsl224w69may8ujxvqhqfymz79xm99dup5mf5y9jqv0vcqu" + + +def write_dumps(root: Path, *, data: str = "COPY public.runs (id) FROM stdin;\n1\n\\.\n") -> None: + root.mkdir(parents=True, exist_ok=True) + (root / "roles.sql").write_text("CREATE ROLE example;\n") + (root / "schema.sql").write_text("CREATE TABLE public.runs (id integer);\n") + (root / "data.sql").write_text(data) + + +def write_recipient(path: Path, value: str = RECIPIENT) -> None: + path.write_text(f"# public only\n{value}\n") + + +def make_contract(tmp_path: Path) -> tuple[Path, Path, Path]: + dumps = tmp_path / "dump" + write_dumps(dumps) + recipients = tmp_path / "recipients.txt" + write_recipient(recipients) + contract = dumps / "backup-contract.json" + backup.create_contract( + Namespace( + project_ref=SOURCE_REF, + recipients=str(recipients), + dump_dir=str(dumps), + created_at="2026-08-08T07:23:00Z", + supabase_cli_version="2.75.0", + maximum_rpo_seconds=86400, + retention_days=90, + output=str(contract), + ) + ) + return dumps, recipients, contract + + +def test_source_must_match_declared_supabase_project(tmp_path: Path) -> None: + recipients = tmp_path / "recipients.txt" + write_recipient(recipients) + direct = f"postgresql://postgres:secret@db.{SOURCE_REF}.supabase.co:5432/postgres" + backup.database_identity(direct, SOURCE_REF, "production") + + pooler = ( + f"postgresql://postgres.{SOURCE_REF}:secret@aws-0-us-east-1.pooler.supabase.com:5432/postgres" + ) + backup.database_identity(pooler, SOURCE_REF, "production") + + with pytest.raises(backup.ContractError, match="does not match"): + backup.database_identity(direct, SCRATCH_REF, "production") + with pytest.raises(backup.ContractError, match="declared Supabase project"): + backup.database_identity("postgresql://postgres:x@127.0.0.1/postgres", SOURCE_REF, "production") + + +def test_recipient_and_dump_contract_fail_closed(tmp_path: Path) -> None: + recipient_file = tmp_path / "recipients.txt" + recipient_file.write_text("# none\n") + with pytest.raises(backup.ContractError, match="age public recipients"): + backup.recipients(recipient_file) + + dumps = tmp_path / "dump" + write_dumps(dumps, data="-- schema-only accident\n") + with pytest.raises(backup.ContractError, match="no COPY"): + backup.dump_inventory(dumps) + + write_dumps(dumps, data="COPY public.runs (id) FROM PROGRAM 'id';\n") + with pytest.raises(backup.ContractError, match="unsafe local command"): + backup.dump_inventory(dumps) + + write_dumps(dumps) + (dumps / "schema.sql").write_text( + "CREATE TABLE public.runs (id integer);\n\\! touch /tmp/not-allowed\n" + ) + with pytest.raises(backup.ContractError, match="unsafe local command"): + backup.dump_inventory(dumps) + + +def test_manifest_detects_ciphertext_tampering(tmp_path: Path) -> None: + _, _, contract = make_contract(tmp_path) + plaintext = tmp_path / "backup.tar.gz" + ciphertext = tmp_path / "backup.tar.gz.age" + plaintext.write_bytes(b"plain") + ciphertext.write_bytes(b"ciphertext") + manifest = tmp_path / "artifact-manifest.json" + backup.create_manifest( + Namespace( + contract=str(contract), + plaintext_archive=str(plaintext), + ciphertext_archive=str(ciphertext), + repository_commit="a" * 40, + workflow_run_id="123", + output=str(manifest), + ) + ) + backup.verify_artifact( + Namespace(manifest=str(manifest), ciphertext_archive=str(ciphertext)) + ) + ciphertext.write_bytes(b"changed") + with pytest.raises(backup.ContractError, match="does not match"): + backup.verify_artifact( + Namespace(manifest=str(manifest), ciphertext_archive=str(ciphertext)) + ) + + +def test_safe_extraction_accepts_only_the_exact_regular_file_set(tmp_path: Path) -> None: + dumps, _, contract = make_contract(tmp_path) + ciphertext = tmp_path / "backup.tar.gz.age" + ciphertext.write_bytes(b"synthetic-ciphertext") + + def manifest_for(archive: Path, name: str) -> Path: + manifest = tmp_path / name + backup.create_manifest( + Namespace( + contract=str(contract), + plaintext_archive=str(archive), + ciphertext_archive=str(ciphertext), + repository_commit="a" * 40, + workflow_run_id="123", + output=str(manifest), + ) + ) + return manifest + + archive = tmp_path / "backup.tar.gz" + with tarfile.open(archive, "w:gz") as bundle: + for name in backup.ARCHIVE_MEMBERS: + bundle.add(dumps / name, arcname=name) + manifest = manifest_for(archive, "manifest.json") + output = tmp_path / "out" + backup.extract_artifact( + Namespace( + plaintext_archive=str(archive), manifest=str(manifest), output_dir=str(output) + ) + ) + assert sorted(path.name for path in output.iterdir()) == sorted(backup.ARCHIVE_MEMBERS) + assert all(path.stat().st_mode & 0o077 == 0 for path in output.iterdir()) + + with archive.open("ab") as stream: + stream.write(b"tamper") + with pytest.raises(backup.ContractError, match="decrypted archive does not match"): + backup.extract_artifact( + Namespace( + plaintext_archive=str(archive), + manifest=str(manifest), + output_dir=str(tmp_path / "tampered-out"), + ) + ) + + other = tmp_path / "other" + write_dumps(other) + other_contract = other / "backup-contract.json" + recipient_file = tmp_path / "other-recipients.txt" + write_recipient(recipient_file) + backup.create_contract( + Namespace( + project_ref=SCRATCH_REF, + recipients=str(recipient_file), + dump_dir=str(other), + created_at="2026-08-08T07:23:00Z", + supabase_cli_version="2.75.0", + maximum_rpo_seconds=86400, + retention_days=90, + output=str(other_contract), + ) + ) + substituted = tmp_path / "substituted.tar.gz" + with tarfile.open(substituted, "w:gz") as bundle: + for name in backup.ARCHIVE_MEMBERS: + bundle.add(other / name, arcname=name) + # The outer manifest is valid for these bytes but names the first backup + # contract. Extraction must bind the decrypted inner contract before SQL. + substituted_manifest = manifest_for(substituted, "substituted-manifest.json") + with pytest.raises(backup.ContractError, match="contract does not match"): + backup.extract_artifact( + Namespace( + plaintext_archive=str(substituted), + manifest=str(substituted_manifest), + output_dir=str(tmp_path / "substituted-out"), + ) + ) + + unsafe = tmp_path / "unsafe.tar.gz" + with tarfile.open(unsafe, "w:gz") as bundle: + info = tarfile.TarInfo("../roles.sql") + payload = b"CREATE ROLE example;\n" + info.size = len(payload) + bundle.addfile(info, io.BytesIO(payload)) + unsafe_manifest = manifest_for(unsafe, "unsafe-manifest.json") + with pytest.raises(backup.ContractError, match="allowlist"): + backup.extract_artifact( + Namespace( + plaintext_archive=str(unsafe), + manifest=str(unsafe_manifest), + output_dir=str(tmp_path / "unsafe-out"), + ) + ) + + symlink = tmp_path / "symlink.tar.gz" + with tarfile.open(symlink, "w:gz") as bundle: + for name in backup.ARCHIVE_MEMBERS: + if name == "roles.sql": + info = tarfile.TarInfo(name) + info.type = tarfile.SYMTYPE + info.linkname = "/etc/passwd" + bundle.addfile(info) + else: + bundle.add(dumps / name, arcname=name) + symlink_manifest = manifest_for(symlink, "symlink-manifest.json") + with pytest.raises(backup.ContractError, match="non-regular"): + backup.extract_artifact( + Namespace( + plaintext_archive=str(symlink), + manifest=str(symlink_manifest), + output_dir=str(tmp_path / "symlink-out"), + ) + ) + + +def test_restore_target_can_never_be_production() -> None: + scratch_url = f"postgresql://postgres:secret@db.{SOURCE_REF}.supabase.co/postgres" + with pytest.raises(backup.ContractError, match="production project"): + backup.validate_restore_target( + Namespace( + source_project_ref=SOURCE_REF, + scratch_project_ref=SOURCE_REF, + scratch_db_url=scratch_url, + ) + ) + + +def test_restore_evidence_is_bound_to_exact_backup_and_scratch(tmp_path: Path) -> None: + dumps, _, contract = make_contract(tmp_path) + restored = tmp_path / "restored" + write_dumps(restored) + verification = tmp_path / "verification.json" + source = {item["name"]: item for item in backup.dump_inventory(dumps)} + verification.write_text( + json.dumps( + { + "valid": True, + "schema_sha256": source["schema.sql"]["sha256"], + "data_sha256": source["data.sql"]["sha256"], + } + ) + ) + plaintext = tmp_path / "backup.tar.gz" + ciphertext = tmp_path / "backup.tar.gz.age" + plaintext.write_bytes(b"plain") + ciphertext.write_bytes(b"ciphertext") + manifest = tmp_path / "manifest.json" + backup.create_manifest( + Namespace( + contract=str(contract), + plaintext_archive=str(plaintext), + ciphertext_archive=str(ciphertext), + repository_commit="a" * 40, + workflow_run_id="123", + output=str(manifest), + ) + ) + evidence = tmp_path / "restore-evidence.json" + backup.record_restore( + Namespace( + manifest=str(manifest), + contract=str(contract), + verification=str(verification), + source_project_ref=SOURCE_REF, + scratch_project_ref=SCRATCH_REF, + started_at="2026-08-08T08:00:00Z", + completed_at="2026-08-08T08:05:00Z", + output=str(evidence), + ) + ) + value = json.loads(evidence.read_text()) + assert value["database_restored"] is True + assert value["storage_restored"] is False + assert value["rto_seconds"] == 300 + assert value["rpo_seconds_at_start"] == 2220 + + verification.write_text( + json.dumps( + { + "valid": True, + "schema_sha256": "0" * 64, + "data_sha256": source["data.sql"]["sha256"], + } + ) + ) + with pytest.raises(backup.ContractError, match="does not match the backup contract"): + backup.record_restore( + Namespace( + manifest=str(manifest), + contract=str(contract), + verification=str(verification), + source_project_ref=SOURCE_REF, + scratch_project_ref=SCRATCH_REF, + started_at="2026-08-08T08:00:00Z", + completed_at="2026-08-08T08:05:00Z", + output=str(tmp_path / "forged-evidence.json"), + ) + ) + + verification.write_text( + json.dumps( + { + "valid": True, + "schema_sha256": source["schema.sql"]["sha256"], + "data_sha256": source["data.sql"]["sha256"], + } + ) + ) + + with pytest.raises(backup.ContractError, match="already exists"): + backup.record_restore( + Namespace( + manifest=str(manifest), + contract=str(contract), + verification=str(verification), + source_project_ref=SOURCE_REF, + scratch_project_ref=SCRATCH_REF, + started_at="2026-08-08T08:00:00Z", + completed_at="2026-08-08T08:05:00Z", + output=str(evidence), + ) + )