diff --git a/.github/actions/docker-build/action.yml b/.github/actions/docker-build/action.yml index 72d90e8e8bb..8e720e18ea9 100644 --- a/.github/actions/docker-build/action.yml +++ b/.github/actions/docker-build/action.yml @@ -19,6 +19,9 @@ inputs: tags: description: Comma-separated list of tags to push. required: true + build-args: + description: Newline-separated Docker build arguments. + required: false max-cache-size-mb: description: >- Layer cache to retain after this action prunes, in MB. Must stay above one @@ -30,8 +33,8 @@ inputs: bypass an input `default:` entirely. required: false -# Registry logins must precede this action. provenance/sbom stay off: attestation -# manifests break `imagetools create` retagging in promote-images. +# Registry logins must precede this action. Keep the existing image format; +# GHCR signatures and attestations are published by the separate attestation jobs. runs: using: composite steps: @@ -72,6 +75,7 @@ runs: platforms: ${{ inputs.platforms }} push: true tags: ${{ inputs.tags }} + build-args: ${{ inputs.build-args }} provenance: false sbom: false @@ -177,5 +181,6 @@ runs: platforms: ${{ inputs.platforms }} push: true tags: ${{ inputs.tags }} + build-args: ${{ inputs.build-args }} provenance: false sbom: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 75d45ac5e46..5c4f55f2958 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,7 +141,7 @@ jobs: environment: dev secrets: inherit - # Dev: build all 3 images for ECR only (no GHCR, no ARM64) + # Dev publishes SHA artifacts first; only promote-images moves deploy aliases. build-dev: name: Build Dev ECR needs: [detect-version, migrate-dev] @@ -215,13 +215,25 @@ jobs: env: ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }} + - name: Reuse existing SHA artifact + id: artifact + env: + ECR_REPO: ${{ steps.ecr-repo.outputs.name }} + run: | + [[ "$GITHUB_SHA" =~ ^[a-f0-9]{40}$ ]] && test -n "$ECR_REPO" + result="$(aws ecr batch-get-image --repository-name "$ECR_REPO" --image-ids "imageTag=$GITHUB_SHA" --output json)" + jq -e 'all(.failures[]; .failureCode == "ImageNotFound")' <<< "$result" > /dev/null + echo "exists=$(jq -r '.images | length == 1' <<< "$result")" >> "$GITHUB_OUTPUT" + - name: Build and push + if: steps.artifact.outputs.exists != 'true' uses: ./.github/actions/docker-build with: provider: ${{ vars.CI_PROVIDER }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 - tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev + tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:${{ github.sha }} + build-args: COMMIT_SHA=${{ github.sha }} max-cache-size-mb: ${{ matrix.cache_mb }} # Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch. @@ -232,7 +244,9 @@ jobs: needs: [migrate-dev] if: github.event_name == 'push' && github.ref == 'refs/heads/dev' runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 15 + timeout-minutes: 45 + outputs: + deployment_version: ${{ steps.deploy.outputs.deploymentVersion }} steps: - name: Checkout code uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 @@ -257,6 +271,7 @@ jobs: run: bun install --frozen-lockfile --ignore-scripts - name: Deploy to Trigger.dev + id: deploy working-directory: ./apps/sim env: TRIGGER_ACCESS_TOKEN: ${{ secrets.DEV_TRIGGER_ACCESS_TOKEN }} @@ -266,7 +281,8 @@ jobs: echo "ERROR: DEV_TRIGGER_ACCESS_TOKEN and TRIGGER_PROJECT_ID repo secrets must both be set" >&2 exit 1 fi - bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim + [[ "$GITHUB_SHA" =~ ^[a-f0-9]{40}$ ]] + bunx trigger.dev@4.5.12 deploy --env preview --branch dev-sim --external-id "$GITHUB_SHA" # Main/staging: build AMD64 images and push sha-tagged images to ECR + GHCR. # Runs in parallel with tests — only immutable sha tags are pushed here, and @@ -384,95 +400,137 @@ jobs: echo "tags=${TAGS}" >> $GITHUB_OUTPUT + - name: Reuse existing SHA artifact + id: artifact + if: matrix.ecr_repo_secret != '' + env: + ECR_REPO: ${{ steps.ecr-repo.outputs.name }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + GHCR_IMAGE: ${{ matrix.ghcr_image }} + run: | + [[ "$GITHUB_SHA" =~ ^[a-f0-9]{40}$ ]] && test -n "$ECR_REPO" + result="$(aws ecr batch-get-image --repository-name "$ECR_REPO" --image-ids "imageTag=$GITHUB_SHA" --output json)" + jq -e 'all(.failures[]; .failureCode == "ImageNotFound")' <<< "$result" > /dev/null + exists="$(jq -r '.images | length == 1' <<< "$result")" + echo "exists=$exists" >> "$GITHUB_OUTPUT" + if [ "$exists" = true ] && [ "$GITHUB_REF_NAME" = main ]; then + digest="$(jq -r '.images[0].imageId.imageDigest' <<< "$result")" + source="${ECR_REGISTRY}/${ECR_REPO}@${digest}" + docker pull "$source" + docker tag "$source" "${GHCR_IMAGE}:${GITHUB_SHA}-amd64" + docker push "${GHCR_IMAGE}:${GITHUB_SHA}-amd64" + fi + - name: Build and push images - if: steps.meta.outputs.skip != 'true' + if: steps.meta.outputs.skip != 'true' && steps.artifact.outputs.exists != 'true' uses: ./.github/actions/docker-build with: provider: ${{ vars.CI_PROVIDER }} file: ${{ matrix.dockerfile }} platforms: linux/amd64 tags: ${{ steps.meta.outputs.tags }} + build-args: COMMIT_SHA=${{ github.sha }} max-cache-size-mb: ${{ matrix.cache_mb }} - # Promote the sha-tagged ECR images to the deploy tags once tests and - # migrations pass. Pushing the ECR latest/staging tag is what triggers - # CodePipeline, so this seconds-long manifest retag is the deploy gate — - # the image builds themselves run in parallel with the tests. A single job - # (not a matrix) so all four sha manifests are verified before any tag - # moves; a missing image can't produce a partial mixed-version deploy. + # Keep the existing ECR -> CodePipeline path; verify both release halves. promote-images: - name: Promote Images - needs: [migrate, build-amd64] - # Explicit results: see migrate's comment. + name: Promote and Verify Release + needs: [migrate, build-amd64, migrate-dev, build-dev, deploy-trigger-dev] if: >- - !cancelled() && - needs.migrate.result == 'success' && - needs.build-amd64.result == 'success' && - github.event_name == 'push' && - (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') + !cancelled() && github.event_name == 'push' && + ((github.ref == 'refs/heads/dev' && + needs.migrate-dev.result == 'success' && needs.build-dev.result == 'success' && + needs.deploy-trigger-dev.result == 'success') || + ((github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging') && + needs.migrate.result == 'success' && needs.build-amd64.result == 'success')) runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 10 + timeout-minutes: 130 permissions: contents: read + checks: read id-token: write + outputs: + decision: ${{ steps.guard.outputs.decision }} + complete: ${{ steps.verify.outputs.complete }} + env: + GH_TOKEN: ${{ github.token }} steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 - with: - role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} - aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || secrets.STAGING_AWS_REGION }} + - name: Checkout code + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@d539f0932e70871a027e9d5a9d8fc38589180a64 # v2 + - name: Setup Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.1 - # Deploy-tag moves must be monotonic: a re-run of an old run must never - # retag latest/staging back to stale code. A superseded first-attempt - # run still promotes — the ci- concurrency group executes runs - # serially in commit order, so an ancestor of head is a forward deploy. - - name: Guard against stale promotion - id: guard + - name: Wait for matching worker deployment env: - GH_TOKEN: ${{ github.token }} - run: | - STATUS="$(gh api "repos/${{ github.repository }}/compare/${{ github.sha }}...${GITHUB_REF_NAME}" --jq '.status' || echo "unknown")" - if [ "$STATUS" = "identical" ] || { [ "$STATUS" = "ahead" ] && [ "${{ github.run_attempt }}" = "1" ]; }; then - echo "fresh=true" >> $GITHUB_OUTPUT - else - echo "::warning::Skipping promotion of ${{ github.sha }} (branch compare: ${STATUS}, attempt ${{ github.run_attempt }}). Moving the deploy tags here could deploy stale code; push a revert commit to roll back instead." - echo "fresh=false" >> $GITHUB_OUTPUT - fi + TRIGGER_PROJECT_ID: ${{ secrets.TRIGGER_PROJECT_ID }} + TRIGGER_DEPLOYMENT_VERSION: ${{ needs.deploy-trigger-dev.outputs.deployment_version }} + run: bun scripts/verify-release.ts worker + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6 + with: + role-to-assume: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ROLE_TO_ASSUME || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_ROLE_TO_ASSUME || secrets.STAGING_AWS_ROLE_TO_ASSUME }} + aws-region: ${{ github.ref == 'refs/heads/main' && secrets.AWS_REGION || github.ref == 'refs/heads/dev' && secrets.DEV_AWS_REGION || secrets.STAGING_AWS_REGION }} + role-duration-seconds: 5400 - - name: Promote images to deploy tags - if: steps.guard.outputs.fresh == 'true' + - name: Preflight all image artifacts + id: images env: + ECR_APP: ${{ secrets.ECR_APP }} ECR_REPOS: >- ${{ secrets.ECR_APP }} ${{ secrets.ECR_MIGRATIONS }} ${{ secrets.ECR_REALTIME }} ${{ secrets.ECR_PII }} + run: bun scripts/verify-release.ts images + + - name: Guard against stale promotion + id: guard + run: bun scripts/verify-release.ts freshness + + - name: Promote changed images to deploy tags + if: steps.guard.outputs.decision == 'promote' + env: + IMAGES: ${{ steps.images.outputs.images }} run: | - REGISTRY="${{ steps.login-ecr.outputs.registry }}" + tag="$GITHUB_REF_NAME" + [ "$tag" != main ] || tag=latest + while IFS= read -r entry; do + repo="$(jq -r '.repository' <<< "$entry")" + jq -jr '.manifest' <<< "$entry" > "$RUNNER_TEMP/release-manifest.json" + aws ecr put-image --repository-name "$repo" --image-tag "$tag" \ + --image-digest "$(jq -r '.sourceDigest' <<< "$entry")" \ + --image-manifest "file://$RUNNER_TEMP/release-manifest.json" \ + --image-manifest-media-type "$(jq -r '.mediaType' <<< "$entry")" > /dev/null + echo "Promoted $repo:$tag" >> "$GITHUB_STEP_SUMMARY" + done < <(jq -c '.[] | select(.promote)' <<< "$IMAGES") + + - name: Verify app deployment + id: verify + if: steps.guard.outputs.decision == 'promote' + env: + APP_DIGEST: ${{ steps.images.outputs.app_digest }} + APP_PLATFORM_DIGEST: ${{ steps.images.outputs.app_platform_digest }} + run: bun scripts/verify-release.ts app - if [ "${{ github.ref }}" = "refs/heads/main" ]; then - ECR_TAG="latest" + - name: Record release outcome + if: always() + env: + DECISION: ${{ steps.guard.outputs.decision }} + COMPLETE: ${{ steps.verify.outputs.complete }} + RESULT: ${{ job.status }} + run: | + if [ "$DECISION" = skip ]; then + outcome=superseded + elif [ "$RESULT" = success ] && [ "$COMPLETE" = true ]; then + outcome=complete else - ECR_TAG="staging" + outcome=incomplete fi - - # Verify every sha image exists before moving any deploy tag, so a - # missing/expired image aborts the whole promotion up front. - for repo in $ECR_REPOS; do - echo "🔍 Verifying ${repo}:${{ github.sha }}" - docker buildx imagetools inspect "${REGISTRY}/${repo}:${{ github.sha }}" > /dev/null - done - - for repo in $ECR_REPOS; do - echo "🚀 Promoting ${repo}:${{ github.sha }} to ${ECR_TAG}" - docker buildx imagetools create \ - -t "${REGISTRY}/${repo}:${ECR_TAG}" \ - "${REGISTRY}/${repo}:${{ github.sha }}" - done + echo "Release $GITHUB_SHA ($GITHUB_REF_NAME): **$outcome**" >> "$GITHUB_STEP_SUMMARY" # Build ARM64 images for GHCR (main branch only, runs in parallel with # tests). Pushes only the immutable sha tag — latest-arm64/version-arm64 @@ -534,6 +592,7 @@ jobs: file: ${{ matrix.dockerfile }} platforms: linux/arm64 tags: ${{ matrix.image }}:${{ github.sha }}-arm64 + build-args: COMMIT_SHA=${{ github.sha }} max-cache-size-mb: ${{ matrix.cache_mb }} # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags) @@ -548,6 +607,8 @@ jobs: if: >- !cancelled() && needs.promote-images.result == 'success' && + needs.promote-images.outputs.decision == 'promote' && + needs.promote-images.outputs.complete == 'true' && needs.build-ghcr-arm64.result == 'success' && needs.detect-version.result == 'success' && github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -583,12 +644,12 @@ jobs: env: GH_TOKEN: ${{ github.token }} run: | - STATUS="$(gh api "repos/${{ github.repository }}/compare/${{ github.sha }}...${GITHUB_REF_NAME}" --jq '.status' || echo "unknown")" - if [ "$STATUS" = "identical" ] || { [ "$STATUS" = "ahead" ] && [ "${{ github.run_attempt }}" = "1" ]; }; then - echo "fresh=true" >> $GITHUB_OUTPUT + HEAD_SHA="$(gh api "repos/${GITHUB_REPOSITORY}/git/ref/heads/${GITHUB_REF_NAME}" --jq '.object.sha')" + if [ "$HEAD_SHA" = "$GITHUB_SHA" ]; then + echo "fresh=true" >> "$GITHUB_OUTPUT" else - echo "::warning::Publishing immutable tags for ${{ github.sha }} but skipping the latest tags (branch compare: ${STATUS}, attempt ${{ github.run_attempt }})." - echo "fresh=false" >> $GITHUB_OUTPUT + echo "::warning::Skipping latest tags for superseded commit $GITHUB_SHA." + echo "fresh=false" >> "$GITHUB_OUTPUT" fi - name: Publish tags and manifests @@ -921,6 +982,8 @@ jobs: if: >- !cancelled() && needs.promote-images.result == 'success' && + needs.promote-images.outputs.decision == 'promote' && + needs.promote-images.outputs.complete == 'true' && needs.check-docs-changes.result == 'success' && needs.check-docs-changes.outputs.docs_changed == 'true' uses: ./.github/workflows/docs-embeddings.yml diff --git a/apps/sim/lib/core/config/trigger-runtime.test.ts b/apps/sim/lib/core/config/trigger-runtime.test.ts index bb905e0667f..db59edaae25 100644 --- a/apps/sim/lib/core/config/trigger-runtime.test.ts +++ b/apps/sim/lib/core/config/trigger-runtime.test.ts @@ -24,6 +24,7 @@ describe('trigger runtime detection', () => { }) afterEach(() => { + vi.unstubAllEnvs() mockTaskContext.isInsideTask = false resetInsideTriggerRunForTests() }) @@ -48,6 +49,23 @@ describe('trigger runtime detection', () => { expect(isInsideTriggerRun()).toBe(true) }) + it('pins outgoing work to the executing deployment on repeated initialization', () => { + vi.stubEnv('TRIGGER_VERSION', 'older-version') + markInsideTriggerRun('20260909.44') + markInsideTriggerRun('20260909.44') + expect(process.env.TRIGGER_VERSION).toBe('20260909.44') + expect(isInsideTriggerRun()).toBe(true) + }) + + it('preserves local configuration without a deployment version', () => { + vi.stubEnv('TRIGGER_VERSION', undefined) + markInsideTriggerRun() + expect(process.env.TRIGGER_VERSION).toBeUndefined() + vi.stubEnv('TRIGGER_VERSION', 'local-override') + markInsideTriggerRun() + expect(process.env.TRIGGER_VERSION).toBe('local-override') + }) + it('keeps the marker on globalThis so a duplicated bundle still sees it', () => { markInsideTriggerRun() const carrier = globalThis as Record diff --git a/apps/sim/lib/core/config/trigger-runtime.ts b/apps/sim/lib/core/config/trigger-runtime.ts index edac6f20238..5b3ced3054d 100644 --- a/apps/sim/lib/core/config/trigger-runtime.ts +++ b/apps/sim/lib/core/config/trigger-runtime.ts @@ -21,8 +21,10 @@ interface TriggerRunCarrier { * * @see https://trigger.dev/docs/config/config-file#lifecycle-functions */ -export function markInsideTriggerRun(): void { +export function markInsideTriggerRun(deploymentVersion?: string): void { ;(globalThis as TriggerRunCarrier)[INSIDE_TRIGGER_RUN] = true + /** Plain trigger/batchTrigger calls must stay on this worker's deployment too. */ + if (deploymentVersion) process.env.TRIGGER_VERSION = deploymentVersion } /** diff --git a/apps/sim/trigger.config.ts b/apps/sim/trigger.config.ts index 6a218121ee5..2199c48d2e2 100644 --- a/apps/sim/trigger.config.ts +++ b/apps/sim/trigger.config.ts @@ -110,8 +110,8 @@ export default defineConfig({ * * @see https://trigger.dev/docs/config/config-file#lifecycle-functions */ - init: async () => { - markInsideTriggerRun() + init: async ({ ctx }) => { + markInsideTriggerRun(ctx.deployment?.version) const { warmRedisConnection } = await import('./lib/core/config/redis') await warmRedisConnection() }, diff --git a/docker/app.Dockerfile b/docker/app.Dockerfile index 125c53a3cb8..6eb45beee2d 100644 --- a/docker/app.Dockerfile +++ b/docker/app.Dockerfile @@ -131,6 +131,9 @@ RUN bun build apps/sim/bootstrap.ts --target=bun --outfile=apps/sim/bootstrap.js FROM base AS runner WORKDIR /app +ARG COMMIT_SHA="" +ENV COMMIT_SHA=${COMMIT_SHA} + # Node.js 24, Python, ffmpeg, etc. are already installed in base stage ENV NODE_ENV=production diff --git a/scripts/verify-release.test.ts b/scripts/verify-release.test.ts new file mode 100644 index 00000000000..c95a04c7123 --- /dev/null +++ b/scripts/verify-release.test.ts @@ -0,0 +1,243 @@ +import { afterEach, describe, expect, it, vi } from 'vitest' +import { + appReady, + imagePromotion, + matchingExecution, + platformDigest, + promotionDecision, + waitFor, + workerCheck, +} from './verify-release' + +const SHA = 'a'.repeat(40) +const DIGEST = `sha256:${'b'.repeat(64)}` +const INDEX = `sha256:${'c'.repeat(64)}` +const OTHER = `sha256:${'d'.repeat(64)}` +const check = { + id: 1, + name: 'Trigger.dev deployment (proj_sim:staging)', + head_sha: SHA, + app: { slug: 'trigger-dev-app' }, + status: 'completed', + conclusion: 'success', + details_url: 'https://cloud.trigger.dev/deployment', +} +const execution = { + pipelineExecutionId: 'execution-1', + status: 'Succeeded', + startTime: '2026-09-09T12:00:00Z', + sourceRevisions: [{ actionName: 'ECR_Source', revisionId: INDEX }], +} + +afterEach(() => vi.useRealTimers()) + +describe('release prerequisites', () => { + it('accepts only the exact worker identity from the trusted GitHub app', () => { + expect(workerCheck([check], SHA, 'proj_sim', 'staging')).toEqual(check) + for (const unrelated of [ + { ...check, head_sha: 'e'.repeat(40) }, + { ...check, app: { slug: 'another-app' } }, + { ...check, name: 'Trigger.dev deployment (proj_other:staging)' }, + { ...check, name: 'Trigger.dev deployment (proj_sim:prod)' }, + ]) { + expect(workerCheck([unrelated], SHA, 'proj_sim', 'staging')).toBeUndefined() + } + }) + + it('does not let an older success hide a pending or failed worker retry', () => { + expect( + workerCheck([check, { ...check, id: 2, status: 'in_progress' }], SHA, 'proj_sim', 'staging') + ).toBeUndefined() + for (const conclusion of ['failure', 'cancelled', 'timed_out']) { + expect(() => + workerCheck([check, { ...check, id: 2, conclusion }], SHA, 'proj_sim', 'staging') + ).toThrow(conclusion) + } + }) + + it('bounds absent worker checks and propagates API errors without promotion', async () => { + vi.useFakeTimers() + const pending = waitFor(async () => workerCheck([], SHA, 'proj_sim', 'staging'), 45 * 60_000) + const rejected = expect(pending).rejects.toThrow('incomplete') + await vi.advanceTimersByTimeAsync(45 * 60_000) + await rejected + await expect( + waitFor(async () => { + throw new Error('permission denied') + }, 1000) + ).rejects.toThrow('permission denied') + }) + + it('allows only the exact branch head, including reruns', () => { + expect(promotionDecision(SHA, SHA)).toBe('promote') + expect(promotionDecision(SHA, 'e'.repeat(40))).toBe('skip') + expect(() => promotionDecision(SHA, '')).toThrow('full commit SHA') + }) +}) + +describe('image identity', () => { + const manifest = { config: { digest: OTHER } } + const index = { + manifests: [{ digest: DIGEST, platform: { os: 'linux', architecture: 'amd64' } }], + } + const source = { + imageId: { imageDigest: DIGEST }, + imageManifest: JSON.stringify(manifest), + imageManifestMediaType: 'application/vnd.oci.image.manifest.v1+json', + } + + it('reuses identical content without confusing index and running-image digests', () => { + expect(imagePromotion(source, source)).toMatchObject({ promote: false, deployDigest: DIGEST }) + expect( + imagePromotion(source, { + ...source, + imageId: { imageDigest: INDEX }, + imageManifest: JSON.stringify(index), + }) + ).toMatchObject({ promote: false, deployDigest: INDEX, platformDigest: DIGEST }) + expect(imagePromotion(source)).toMatchObject({ promote: true, deployDigest: DIGEST }) + expect(imagePromotion(source, { ...source, imageId: { imageDigest: OTHER } }).promote).toBe( + true + ) + }) + + it('rejects ambiguous or missing amd64 manifests', () => { + expect(platformDigest(INDEX, index)).toBe(DIGEST) + expect(() => platformDigest(INDEX, { manifests: [] })).toThrow('exactly one') + expect(() => + platformDigest(INDEX, { manifests: [...index.manifests, ...index.manifests] }) + ).toThrow('exactly one') + }) +}) + +/** Small provider-response fixtures exercise the correlation, without live AWS or secrets. */ +function awsFixture( + options: { + pipelineStatus?: string + deploymentStatus?: string + rollback?: boolean + primaryDeployment?: string + digest?: string + sourceDigest?: string + } = {} +) { + const responses: Record = { + 'codepipeline list-pipeline-executions': { + pipelineExecutionSummaries: [ + { + ...execution, + status: options.pipelineStatus ?? 'Succeeded', + sourceRevisions: [ + { actionName: 'ECR_Source', revisionId: options.sourceDigest ?? INDEX }, + ], + }, + ], + }, + 'codepipeline list-action-executions': { + actionExecutionDetails: [ + { + actionName: 'Deploy_to_ECS', + startTime: execution.startTime, + status: 'Succeeded', + output: { executionResult: { externalExecutionId: 'd-release' } }, + }, + ], + }, + 'deploy get-deployment': { + status: options.deploymentStatus ?? 'Succeeded', + rollbackInfo: options.rollback ? { rollbackDeploymentId: 'd-rollback' } : {}, + }, + 'ecs describe-task-sets': { + failures: [], + taskSets: [ + { + status: 'PRIMARY', + externalId: options.primaryDeployment ?? 'd-release', + taskDefinition: 'taskdef:2', + stabilityStatus: 'STEADY_STATE', + computedDesiredCount: 1, + runningCount: 1, + pendingCount: 0, + }, + ], + }, + 'ecs list-tasks': { taskArns: ['task'] }, + 'ecs describe-tasks': { + failures: [], + tasks: [ + { + taskDefinitionArn: 'taskdef:2', + lastStatus: 'RUNNING', + containers: [ + { name: 'app', lastStatus: 'RUNNING', imageDigest: options.digest ?? DIGEST }, + ], + }, + ], + }, + } + return async (_command: string, args: string[]): Promise => { + const key = args.slice(0, 2).join(' ') + if (!(key in responses)) throw new Error(`Unexpected metadata read: ${key}`) + return responses[key] as T + } +} + +describe('app completion', () => { + const verify = (read: ReturnType) => + appReady('pipeline', 'cluster', 'service', INDEX, DIGEST, read) + + it('verifies an existing successful deployment again without starting another one', async () => { + const read = awsFixture() + const expected = { executionId: 'execution-1', deploymentId: 'd-release' } + expect(await verify(read)).toEqual(expected) + expect(await verify(read)).toEqual(expected) + }) + + it('does not report success for pending, failed, rolled-back or mismatched deployments', async () => { + for (const options of [ + { pipelineStatus: 'InProgress' }, + { sourceDigest: OTHER }, + { digest: OTHER }, + { primaryDeployment: 'd-old' }, + ]) + expect(await verify(awsFixture(options))).toBeUndefined() + for (const options of [ + { pipelineStatus: 'Failed' }, + { pipelineStatus: 'Superseded' }, + { deploymentStatus: 'Failed' }, + { rollback: true }, + ]) + await expect(verify(awsFixture(options))).rejects.toThrow() + }) + + it('selects the latest matching attempt and accepts it after operator recovery', async () => { + const retry = { + ...execution, + pipelineExecutionId: 'execution-2', + startTime: '2026-09-09T13:00:00Z', + } + expect(() => matchingExecution([execution, { ...retry, status: 'Failed' }], INDEX)).toThrow( + 'execution-2' + ) + expect(matchingExecution([retry, execution], INDEX)).toEqual(retry) + expect(() => + matchingExecution( + [ + execution, + { + ...retry, + sourceRevisions: [{ actionName: 'ECR_Source', revisionId: OTHER }], + }, + ], + INDEX + ) + ).toThrow('superseded by another image') + vi.useFakeTimers() + let read = awsFixture({ pipelineStatus: 'InProgress' }) + const recovering = waitFor(() => verify(read), 75 * 60_000) + await vi.advanceTimersByTimeAsync(15_000) + read = awsFixture() + await vi.advanceTimersByTimeAsync(15_000) + await expect(recovering).resolves.toMatchObject({ deploymentId: 'd-release' }) + }) +}) diff --git a/scripts/verify-release.ts b/scripts/verify-release.ts new file mode 100644 index 00000000000..9aac98aa984 --- /dev/null +++ b/scripts/verify-release.ts @@ -0,0 +1,469 @@ +#!/usr/bin/env bun +/** + * Read-only release gates. No dependencies/install or cloud writes are needed. + * Usage: bun scripts/verify-release.ts worker|images|freshness|app + * Inputs are the CI environment; outputs/diagnostics go to the Actions summary. + * Recovery and required permissions are documented in simstudioai/infra's README. + */ +import { execFile } from 'node:child_process' +import { appendFileSync } from 'node:fs' +import { promisify } from 'node:util' +import { interruptibleSleep } from '../packages/utils/src/helpers' + +const exec = promisify(execFile) +const DIGEST = /^sha256:[a-f0-9]{64}$/ +type ReadJson = (command: string, args: string[], signal?: AbortSignal) => Promise + +const readJson: ReadJson = async ( + command: string, + args: string[], + signal?: AbortSignal +): Promise => { + /** Never echo subprocess output: these gates only expose selected metadata. */ + const { stdout } = await exec(command, args, { + timeout: 60_000, + signal, + maxBuffer: 8 * 1024 * 1024, + env: { ...process.env, AWS_PAGER: '', GH_PROMPT_DISABLED: '1' }, + }).catch(() => { + signal?.throwIfAborted() + throw new Error(`${command} ${args.slice(0, 2).join(' ')} failed; check metadata permissions`) + }) + return JSON.parse(stdout) as T +} + +interface CheckRun { + id: number + name: string + head_sha: string + app: { slug: string } + status: string + conclusion: string | null + details_url: string +} + +/** Select the newest attempt, never an earlier green check for the same SHA. */ +export function workerCheck(checks: CheckRun[], sha: string, project: string, environment: string) { + const check = checks + .filter( + (item) => + item.head_sha === sha && + item.app.slug === 'trigger-dev-app' && + item.name === `Trigger.dev deployment (${project}:${environment})` + ) + .sort((a, b) => b.id - a.id)[0] + if (check?.status === 'completed' && check.conclusion !== 'success') { + throw new Error(`Trigger check ${check.id} ended ${check.conclusion}`) + } + return check?.status === 'completed' && check.conclusion === 'success' ? check : undefined +} + +export function promotionDecision(sha: string, head: string): 'promote' | 'skip' { + if (!/^[a-f0-9]{40}$/.test(sha) || !/^[a-f0-9]{40}$/.test(head)) { + throw new Error('A full commit SHA is required') + } + return sha === head ? 'promote' : 'skip' +} + +/** Undefined means pending; API errors and terminal failures propagate immediately. */ +export async function waitFor( + probe: (signal: AbortSignal) => Promise, + timeout: number +) { + const controller = new AbortController() + const timer = setTimeout(() => { + controller.abort(new Error('Release verification timed out; release is incomplete')) + }, timeout) + try { + while (true) { + controller.signal.throwIfAborted() + const result = await probe(controller.signal) + controller.signal.throwIfAborted() + if (result !== undefined) return result + await interruptibleSleep(15_000, controller.signal) + } + } finally { + clearTimeout(timer) + } +} + +interface Manifest { + mediaType?: string + config?: { digest: string } + manifests?: { digest: string; platform?: { os: string; architecture: string } }[] +} + +export function platformDigest(sourceDigest: string, manifest: Manifest): string { + if (!DIGEST.test(sourceDigest)) throw new Error('Invalid registry digest') + if (!manifest.manifests) { + if (!manifest.config || !DIGEST.test(manifest.config.digest)) { + throw new Error('Expected an image manifest or an amd64 image index') + } + return sourceDigest + } + const candidates = manifest.manifests.filter( + (entry) => entry.platform?.os === 'linux' && entry.platform.architecture === 'amd64' + ) + if (candidates.length !== 1 || !DIGEST.test(candidates[0].digest)) { + throw new Error('Image index must contain exactly one linux/amd64 image') + } + return candidates[0].digest +} + +interface Image { + imageId: { imageDigest: string } + imageManifest: string + imageManifestMediaType: string +} + +async function image(repository: string, tag: string, read: ReadJson) { + const result = await read<{ images: Image[]; failures: { failureCode: string }[] }>('aws', [ + 'ecr', + 'batch-get-image', + '--repository-name', + repository, + '--image-ids', + `imageTag=${tag}`, + '--output', + 'json', + ]) + if (result.failures.some((failure) => failure.failureCode !== 'ImageNotFound')) { + throw new Error(`Cannot inspect image ${repository}:${tag}`) + } + if (result.images.length > 1) throw new Error(`Ambiguous image ${repository}:${tag}`) + return result.images[0] +} + +/** Compare deployable content, including legacy aliases wrapped in a single-image index. */ +export function imagePromotion(source: Image, current?: Image) { + const digest = platformDigest(source.imageId.imageDigest, JSON.parse(source.imageManifest)) + const unchanged = + current && + platformDigest(current.imageId.imageDigest, JSON.parse(current.imageManifest)) === digest + return { + sourceDigest: source.imageId.imageDigest, + platformDigest: digest, + deployDigest: unchanged ? current.imageId.imageDigest : source.imageId.imageDigest, + promote: !unchanged, + manifest: source.imageManifest, + mediaType: source.imageManifestMediaType, + } +} + +interface Execution { + pipelineExecutionId: string + status: string + startTime: string + sourceRevisions?: { actionName: string; revisionId: string }[] +} + +/** Most recent matching attempt wins, including failed retries and infra-only releases. */ +export function matchingExecution(executions: Execution[], digest: string) { + const execution = executions + .filter((item) => + item.sourceRevisions?.some( + (source) => source.actionName === 'ECR_Source' && source.revisionId === digest + ) + ) + .sort((a, b) => Date.parse(b.startTime) - Date.parse(a.startTime))[0] + if ( + execution && + executions.some( + (item) => + Date.parse(item.startTime) > Date.parse(execution.startTime) && + item.sourceRevisions?.some( + (source) => source.actionName === 'ECR_Source' && source.revisionId !== digest + ) + ) + ) + throw new Error(`CodePipeline ${execution.pipelineExecutionId} was superseded by another image`) + if (execution && !['InProgress', 'Succeeded'].includes(execution.status)) { + throw new Error(`CodePipeline ${execution.pipelineExecutionId} ended ${execution.status}`) + } + return execution +} + +interface ActionExecution { + actionName: string + startTime: string + status: string + output?: { executionResult?: { externalExecutionId?: string } } +} + +interface TaskSet { + status: string + externalId: string + taskDefinition: string + stabilityStatus: string + computedDesiredCount: number + runningCount: number + pendingCount: number +} + +interface Task { + taskDefinitionArn: string + lastStatus: string + containers: { name: string; imageDigest: string; lastStatus: string }[] +} + +/** Correlate the source revision, deployment, PRIMARY task set and running image. */ +export async function appReady( + pipeline: string, + cluster: string, + service: string, + sourceDigest: string, + expectedPlatformDigest: string, + read: ReadJson = readJson, + report: (message: string) => void = () => {} +) { + const { pipelineExecutionSummaries } = await read<{ pipelineExecutionSummaries: Execution[] }>( + 'aws', + ['codepipeline', 'list-pipeline-executions', '--pipeline-name', pipeline, '--output', 'json'] + ) + const execution = matchingExecution(pipelineExecutionSummaries, sourceDigest) + if (!execution) return undefined + report(`CodePipeline execution \`${execution.pipelineExecutionId}\`: ${execution.status}.`) + const { actionExecutionDetails } = await read<{ actionExecutionDetails: ActionExecution[] }>( + 'aws', + [ + 'codepipeline', + 'list-action-executions', + '--pipeline-name', + pipeline, + '--max-items', + '100', + '--filter', + `pipelineExecutionId=${execution.pipelineExecutionId}`, + '--output', + 'json', + ] + ) + const action = actionExecutionDetails + .filter((item) => item.actionName === 'Deploy_to_ECS') + .sort((a, b) => Date.parse(b.startTime) - Date.parse(a.startTime))[0] + const deploymentId = action?.output?.executionResult?.externalExecutionId + if (!deploymentId) return undefined + report(`CodeDeploy deployment: \`${deploymentId}\`.`) + const deployment = await read<{ + status: string + rollbackInfo?: { rollbackDeploymentId?: string } + }>('aws', [ + 'deploy', + 'get-deployment', + '--deployment-id', + deploymentId, + '--query', + 'deploymentInfo.{status:status,rollbackInfo:rollbackInfo}', + '--output', + 'json', + ]) + if ( + ['Failed', 'Stopped'].includes(deployment.status) || + deployment.rollbackInfo?.rollbackDeploymentId + ) { + throw new Error(`CodeDeploy ${deploymentId} failed or rolled back`) + } + if ( + execution.status !== 'Succeeded' || + action.status !== 'Succeeded' || + deployment.status !== 'Succeeded' + ) { + return undefined + } + const { taskSets, failures } = await read<{ taskSets: TaskSet[]; failures: unknown[] }>('aws', [ + 'ecs', + 'describe-task-sets', + '--cluster', + cluster, + '--service', + service, + '--output', + 'json', + ]) + if (failures?.length) throw new Error('Could not describe app task sets') + const primary = taskSets.filter((set) => set.status === 'PRIMARY') + if (primary.length !== 1 || primary[0].externalId !== deploymentId) return undefined + const set = primary[0] + if ( + set.stabilityStatus !== 'STEADY_STATE' || + set.computedDesiredCount < 1 || + set.runningCount < set.computedDesiredCount || + set.pendingCount !== 0 + ) + return undefined + const { taskArns } = await read<{ taskArns: string[] }>('aws', [ + 'ecs', + 'list-tasks', + '--cluster', + cluster, + '--service-name', + service, + '--desired-status', + 'RUNNING', + '--output', + 'json', + ]) + const tasks: Task[] = [] + for (let offset = 0; offset < taskArns.length; offset += 100) { + const response = await read<{ tasks: Task[]; failures: unknown[] }>('aws', [ + 'ecs', + 'describe-tasks', + '--cluster', + cluster, + '--tasks', + ...taskArns.slice(offset, offset + 100), + '--query', + '{failures:failures,tasks:tasks[].{taskDefinitionArn:taskDefinitionArn,lastStatus:lastStatus,containers:containers[].{name:name,imageDigest:imageDigest,lastStatus:lastStatus}}}', + '--output', + 'json', + ]) + if (response.failures?.length) return undefined + tasks.push(...response.tasks) + } + const running = tasks.filter((task) => task.taskDefinitionArn === set.taskDefinition) + if ( + running.length < set.computedDesiredCount || + !running.every( + (task) => + task.lastStatus === 'RUNNING' && + task.containers.some( + (container) => + container.name === 'app' && + container.lastStatus === 'RUNNING' && + container.imageDigest === expectedPlatformDigest + ) + ) + ) + return undefined + return { executionId: execution.pipelineExecutionId, deploymentId } +} + +function required(name: string) { + const value = process.env[name] + if (!value || /[\r\n]/.test(value)) throw new Error(`Missing or invalid ${name}`) + return value +} + +function output(name: string, value: string) { + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${name}=${value}\n`) +} + +function summary(message: string) { + process.stdout.write(`${message}\n`) + if (process.env.GITHUB_STEP_SUMMARY) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, `${message}\n\n`) +} + +async function main() { + const mode = process.argv[2] + const sha = required('GITHUB_SHA') + promotionDecision(sha, sha) + const branch = required('GITHUB_REF_NAME') + const environment = { main: 'production', staging: 'staging', dev: 'dev' }[branch] + if (!environment) throw new Error(`Unsupported release branch: ${branch}`) + summary(`Release \`${sha}\` / **${environment}** — ${mode}`) + const repo = required('GITHUB_REPOSITORY') + if (mode === 'freshness') { + const { object } = await readJson<{ object: { sha: string } }>('gh', [ + 'api', + `repos/${repo}/git/ref/heads/${branch}`, + ]) + const decision = promotionDecision(sha, object.sha) + output('decision', decision) + summary( + decision === 'skip' + ? 'Superseded: aliases will not move.' + : 'Current branch head: promotion allowed.' + ) + } else if (mode === 'worker') { + if (environment === 'dev') { + const version = required('TRIGGER_DEPLOYMENT_VERSION') + summary( + `Trigger preview/dev-sim version: \`${version}\` (successful CLI deployment in this run).` + ) + return + } + const project = required('TRIGGER_PROJECT_ID') + const check = await waitFor(async (signal) => { + const pages = await readJson<{ check_runs: CheckRun[] }[]>( + 'gh', + [ + 'api', + '--paginate', + '--slurp', + `repos/${repo}/commits/${sha}/check-runs?filter=all&per_page=100`, + ], + signal + ) + return workerCheck( + pages.flatMap((page) => page.check_runs), + sha, + project, + environment === 'production' ? 'prod' : 'staging' + ) + }, 45 * 60_000) + summary(`Trigger check **${check.id}** succeeded: ${check.details_url}`) + } else if (mode === 'images') { + const repositories = required('ECR_REPOS').trim().split(/\s+/) + if (repositories.length !== 4 || new Set(repositories).size !== 4) { + throw new Error('All four distinct ECR repositories are required before promotion') + } + const tag = environment === 'production' ? 'latest' : environment + const images = await Promise.all( + repositories.map(async (repository) => { + const [source, current] = await Promise.all([ + image(repository, sha, readJson), + image(repository, tag, readJson), + ]) + if (!source) throw new Error(`Missing SHA image: ${repository}:${sha}`) + return { repository, ...imagePromotion(source, current) } + }) + ) + output('images', JSON.stringify(images)) + const app = images.find((entry) => entry.repository === required('ECR_APP')) + if (!app) throw new Error('App repository is missing from image preflight') + output('app_digest', app.deployDigest) + output('app_platform_digest', app.platformDigest) + for (const item of images) + summary( + `\`${item.repository}\`: ${item.promote ? 'promote' : 'reuse'} \`${item.deployDigest}\` (amd64 \`${item.platformDigest}\`).` + ) + } else if (mode === 'app') { + const source = required('APP_DIGEST') + const platform = required('APP_PLATFORM_DIGEST') + if (!DIGEST.test(source) || !DIGEST.test(platform)) + throw new Error('Invalid expected app digests') + const prefix = `sim-${environment}-${required('AWS_REGION')}` + const reported = new Set() + const report = (message: string) => { + if (!reported.has(message)) summary(message) + reported.add(message) + } + const result = await waitFor( + (signal) => + appReady( + `${prefix}-app-deployment`, + `${prefix}-cluster`, + `${prefix}-app`, + source, + platform, + (command, args) => readJson(command, args, signal), + report + ), + 75 * 60_000 + ) + summary( + `App deployment verified: pipeline execution \`${result.executionId}\`, CodeDeploy \`${result.deploymentId}\`.` + ) + output('complete', 'true') + } else { + throw new Error('Usage: verify-release.ts worker|images|freshness|app') + } +} + +if (import.meta.main) { + main().catch((error: unknown) => { + summary(`**Incomplete release:** ${String(error)}`) + process.exitCode = 1 + }) +}