Migrate API v1 persistence to SQLAlchemy and Alembic

.env.example

@@ -7,6 +7,9 @@
 # Password for connecting to the PolicyEngine database
 POLICYENGINE_DB_PASSWORD=policyengine_db_password
 
+# Cloud SQL instance targeted by remote database connections
+POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=policyengine-api:us-central1:policyengine-api-data
+
 # Github Microdata Token
 POLICYENGINE_GITHUB_MICRODATA_AUTH_TOKEN=policyengine_github_token
 

.github/copilot-instructions.md

@@ -10,5 +10,8 @@ or migration guard changes, read
 For tests, read `docs/engineering/skills/testing.md` before adding, moving, or
 reviewing test files.
 
+For SQLAlchemy model or Alembic migration work, read
+`docs/engineering/skills/alembic-migrations.md`.
+
 For pull requests, read `docs/engineering/skills/github-prs.md` before opening,
 replacing, or sharing a PR.

.github/scripts/cloud_run_env.sh

@@ -9,7 +9,6 @@ cloud_run_set_defaults() {
   # image built by the staging track, so it must not embed the service name.
   CLOUD_RUN_IMAGE_NAME="${CLOUD_RUN_IMAGE_NAME:-policyengine-api}"
   CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT="${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT:-policyengine-api-cr-runtime@policyengine-api.iam.gserviceaccount.com}"
-  CLOUD_RUN_CLOUD_SQL_INSTANCE="${CLOUD_RUN_CLOUD_SQL_INSTANCE:-policyengine-api:us-central1:policyengine-api-data}"
   CLOUD_RUN_CPU="${CLOUD_RUN_CPU:-4}"
   CLOUD_RUN_MEMORY="${CLOUD_RUN_MEMORY:-16Gi}"
   CLOUD_RUN_TIMEOUT="${CLOUD_RUN_TIMEOUT:-300}"
@@ -57,7 +56,6 @@ cloud_run_set_defaults() {
   export CLOUD_RUN_ARTIFACT_REPOSITORY
   export CLOUD_RUN_IMAGE_NAME
   export CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT
-  export CLOUD_RUN_CLOUD_SQL_INSTANCE
   export CLOUD_RUN_CPU
   export CLOUD_RUN_MEMORY
   export CLOUD_RUN_TIMEOUT

.github/scripts/create_cloud_sql_backup.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME:?POLICYENGINE_DB_INSTANCE_CONNECTION_NAME is required}"
+
+instance_id="${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME##*:}"
+description="policyengine-api-v1-alembic-${GITHUB_SHA:-manual}-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}"
+gcloud sql backups create \
+  --project policyengine-api \
+  --instance "${instance_id}" \
+  --description "${description}" \
+  --quiet >&2
+
+# `gcloud sql backups create` waits for completion but does not consistently
+# emit the created resource with value-format output. Recover the ID from the
+# unique workflow description and require the service-reported successful state.
+backup_id="$(
+  gcloud sql backups list \
+    --project policyengine-api \
+    --instance "${instance_id}" \
+    --filter="description=${description} AND status=SUCCESSFUL" \
+    --sort-by='~startTime' \
+    --limit=1 \
+    --format='value(id)'
+)"
+
+if [[ -z "${backup_id}" ]]; then
+  echo "Cloud SQL did not return a completed backup ID." >&2
+  exit 1
+fi
+
+printf '%s\n' "${backup_id}"

.github/scripts/deploy_cloud_run_candidate.sh

@@ -8,7 +8,7 @@ cloud_run_set_defaults
 bash .github/scripts/validate_cloud_run_deploy_env.sh
 
 env_vars=(
-  "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=${CLOUD_RUN_CLOUD_SQL_INSTANCE}"
+  "POLICYENGINE_DB_INSTANCE_CONNECTION_NAME=${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}"
   "POLICYENGINE_DB_USER=${POLICYENGINE_DB_USER:-policyengine}"
   "POLICYENGINE_DB_NAME=${POLICYENGINE_DB_NAME:-policyengine}"
   "GATEWAY_AUTH_REQUIRED=1"
@@ -54,7 +54,7 @@ cloud_run_run gcloud run deploy "${CLOUD_RUN_SERVICE}" \
   --allow-unauthenticated \
   --execution-environment gen2 \
   --service-account "${CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT}" \
-  --add-cloudsql-instances "${CLOUD_RUN_CLOUD_SQL_INSTANCE}" \
+  --add-cloudsql-instances "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \
   --port "${CLOUD_RUN_PORT}" \
   --cpu "${CLOUD_RUN_CPU}" \
   --cpu-boost \

.github/scripts/migrate_v1_cloud_sql.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME:?POLICYENGINE_DB_INSTANCE_CONNECTION_NAME is required}"
+
+readonly_password="$(
+  gcloud secrets versions access latest \
+    --secret policyengine-api-prod-db-readonly-password \
+    --project policyengine-api
+)"
+migration_password="$(
+  gcloud secrets versions access latest \
+    --secret policyengine-api-prod-db-migration-password \
+    --project policyengine-api
+)"
+
+if [[ -z "${readonly_password}" || -z "${migration_password}" ]]; then
+  echo "Cloud SQL database credentials must not be empty." >&2
+  exit 1
+fi
+
+printf '::add-mask::%s\n' "${readonly_password}"
+printf '::add-mask::%s\n' "${migration_password}"
+export POLICYENGINE_DB_READONLY_PASSWORD="${readonly_password}"
+export POLICYENGINE_DB_MIGRATION_PASSWORD="${migration_password}"
+
+# Never allow a job-level URL to bypass the credentials fetched for this release.
+unset STAGE7_EXISTING_DATABASE_URL ALEMBIC_DATABASE_URL
+
+database_state="$(python scripts/v1_database_migration.py --mode state)"
+echo "Detected v1 database state: ${database_state}"
+
+case "${database_state}" in
+  head)
+    ;;
+  pending)
+    backup_id="$(bash .github/scripts/create_cloud_sql_backup.sh)"
+    python scripts/v1_database_migration.py \
+      --mode upgrade \
+      --backup-id "${backup_id}"
+    ;;
+  unversioned | invalid)
+    echo "database is unversioned or has invalid Alembic state; automatic baseline stamping is disabled and manual recovery is required" >&2
+    exit 1
+    ;;
+  *)
+    echo "Unrecognized v1 database state: ${database_state}" >&2
+    exit 1
+    ;;
+esac
+
+python scripts/v1_database_migration.py --mode verify-head

.github/scripts/start_cloud_sql_proxy.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+: "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME:?POLICYENGINE_DB_INSTANCE_CONNECTION_NAME is required}"
+
+proxy_version="2.25.0"
+proxy_sha256="091a9a12eddab6c028b6c563a4f2dacd067e8f7689c25a3fb4afce397e1f0c60"
+proxy_path="${RUNNER_TEMP:-/tmp}/cloud-sql-proxy"
+pid_path="${RUNNER_TEMP:-/tmp}/cloud-sql-proxy.pid"
+log_path="${RUNNER_TEMP:-/tmp}/cloud-sql-proxy.log"
+
+curl -fsSL \
+  "https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v${proxy_version}/cloud-sql-proxy.linux.amd64" \
+  --output "${proxy_path}"
+printf '%s  %s\n' "${proxy_sha256}" "${proxy_path}" | sha256sum --check --status
+chmod +x "${proxy_path}"
+
+"${proxy_path}" \
+  --quota-project policyengine-api \
+  --address 127.0.0.1 \
+  --port 3307 \
+  "${POLICYENGINE_DB_INSTANCE_CONNECTION_NAME}" \
+  >"${log_path}" 2>&1 &
+proxy_pid="$!"
+printf '%s\n' "${proxy_pid}" >"${pid_path}"
+
+for _ in $(seq 1 30); do
+  if ! kill -0 "${proxy_pid}" 2>/dev/null; then
+    echo "Cloud SQL Auth Proxy exited before becoming ready." >&2
+    sed -n '1,120p' "${log_path}" >&2
+    exit 1
+  fi
+  if python -c 'import socket; socket.create_connection(("127.0.0.1", 3307), 1).close()' 2>/dev/null; then
+    exit 0
+  fi
+  sleep 1
+done
+
+echo "Cloud SQL Auth Proxy did not become ready." >&2
+exit 1

.github/scripts/stop_cloud_sql_proxy.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+pid_path="${RUNNER_TEMP:-/tmp}/cloud-sql-proxy.pid"
+if [[ ! -f "${pid_path}" ]]; then
+  exit 0
+fi
+
+proxy_pid="$(cat "${pid_path}")"
+if kill -0 "${proxy_pid}" 2>/dev/null; then
+  kill "${proxy_pid}"
+  wait "${proxy_pid}" 2>/dev/null || true
+fi

.github/scripts/validate_app_engine_deploy_env.sh

@@ -9,6 +9,7 @@ selected_url_env="$(
 )"
 
 required=(
+  POLICYENGINE_DB_INSTANCE_CONNECTION_NAME
   SIM_ENTRYPOINT
   "${selected_url_env}"
   GATEWAY_AUTH_ISSUER

.github/scripts/validate_cloud_run_deploy_env.sh

@@ -23,7 +23,7 @@ cloud_run_require_env \
   CLOUD_RUN_IMAGE_URI \
   CLOUD_RUN_TAG \
   CLOUD_RUN_RUNTIME_SERVICE_ACCOUNT \
-  CLOUD_RUN_CLOUD_SQL_INSTANCE \
+  POLICYENGINE_DB_INSTANCE_CONNECTION_NAME \
   CLOUD_RUN_POLICYENGINE_DB_PASSWORD_SECRET \
   CLOUD_RUN_GITHUB_MICRODATA_TOKEN_SECRET \
   CLOUD_RUN_ANTHROPIC_API_KEY_SECRET \

.github/workflows/alembic-v1-check.yml

@@ -0,0 +1,49 @@
+name: Alembic v1 checks
+
+on:
+  workflow_call:
+  workflow_dispatch:
+
+jobs:
+  mysql-lifecycle:
+    name: Alembic MySQL lifecycle
+    runs-on: ubuntu-latest
+    services:
+      mysql:
+        image: mysql:8.4
+        # This database exists only for this job, so these test credentials are
+        # intentionally non-secret and safe to expose in the workflow.
+        env:
+          MYSQL_ROOT_PASSWORD: policyengine_test
+          MYSQL_DATABASE: policyengine_alembic_test
+        ports:
+          - 3306:3306
+        options: >-
+          --health-cmd="mysqladmin ping --silent"
+          --health-interval=5s
+          --health-timeout=5s
+          --health-retries=20
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dependencies
+        run: make install
+      - name: Test Alembic configuration and MySQL lifecycle
+        run: >-
+          python -m pytest tests/unit/data/test_alembic_baseline.py
+          tests/unit/data/test_v1_database_migration.py
+          tests/integration/test_alembic_mysql_lifecycle.py -q
+        env:
+          ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test
+      - name: Require database at all v1 heads
+        run: python -m alembic -c alembic-v1.ini current --check-heads
+        env:
+          ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test
+      - name: Require no ungenerated v1 operations
+        run: python -m alembic -c alembic-v1.ini check
+        env:
+          ALEMBIC_DATABASE_URL: mysql+pymysql://root:policyengine_test@127.0.0.1:3306/policyengine_alembic_test

.github/workflows/pr.yml

@@ -50,6 +50,10 @@ jobs:
           python-version: "3.12"
       - name: Run quality guards
         run: python scripts/run_quality_guards.py
+
+  alembic-v1-check:
+    name: Alembic v1 qualification
+    uses: ./.github/workflows/alembic-v1-check.yml
   check-changelog:
     name: Check changelog fragment
     runs-on: ubuntu-latest