diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8360f92b..ddfd06cc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -236,6 +236,15 @@ jobs: - name: Board y catalogo deben contar lo mismo run: python3 .harness/scripts/check-gap-registry.py + # GT-435 — hasta el 2026-08-02 este repositorio publicaba TRES Dockerfiles y CERO + # imagenes: ningun workflow los construia, mientras los charts referenciaban + # `evolith-tracker-api:0.0.1`, `...-web:0.0.1` y `evolith-tracker-gateway:local`, este + # ultimo un tag que ningun registro puede servir. El camino de despliegue no estaba sin + # probar: estaba sin cablear. Nada habia tirado nunca de esas imagenes, asi que ningun + # CI en verde podia revelarlo. + - name: Los charts piden imagenes que este repositorio produce + run: node .harness/scripts/check-deployable-images.mjs + # El paso anterior contrasta ademas el REGISTRO, la tercera superficie, anadida el # 2026-08-01: board y catalogo llevaban sincronizados desde que existe el guard y # nadie miraba el registro, que comparte el mismo espacio de ids. De los 34 ids diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml new file mode 100644 index 00000000..ada78131 --- /dev/null +++ b/.github/workflows/images.yml @@ -0,0 +1,90 @@ +# Build and publish the Tracker's service images to GHCR. +# +# WHY THIS FILE DID NOT EXIST UNTIL NOW, WHICH IS THE POINT. The Tracker shipped three +# Dockerfiles and ZERO workflows that build or push them, while its Helm charts referenced +# `ghcr.io/beyondnetcode/evolith-tracker-api:0.0.1`, `…-web:0.0.1` and — worse — +# `evolith-tracker-gateway:local`, a tag no registry can ever serve. So the deployment path was +# not merely untested, it was not wired: every chart pointed at an image nothing produced. +# +# Found on 2026-08-02 while preflighting `GT-435` (road to production) from the Core repository, +# without a cluster and without a server. Nothing had ever pulled these images, so no green CI +# could have revealed it. +# +# WHAT IT PUBLISHES. `:latest` and `:` on every merge to `main` and `develop`, mirroring the +# Core's `ci-cd.yml` so both repositories deploy the same way. `latest` makes a chart deployable +# out of the box; `` is what a production deploy must pin, because `latest` cannot be rolled +# back and `GT-448` requires a rollback that has actually been exercised. +# +# AUTHENTICATION. The built-in `GITHUB_TOKEN` with `packages: write`. No extra secret, so this is +# safe to merge and works from the first run. + +name: Images (build & push to GHCR) + +on: + push: + branches: [main, develop] + # Manual runs are for rebuilding a tag without a code change (a base-image CVE, say). + workflow_dispatch: + +permissions: + contents: read + packages: write + +concurrency: + group: images-${{ github.ref }} + cancel-in-progress: false + +jobs: + build-push: + name: ${{ matrix.image }} + runs-on: ubuntu-latest + timeout-minutes: 45 + + strategy: + # One failing image must not hide the state of the other two: a partial publish is a + # deployment where some services are new and some are not, and that is worth seeing whole. + fail-fast: false + matrix: + include: + # Every context is `src/` — the Nx workspace root that holds `apps/`. Each Dockerfile + # documents this in its own header, and the COPY paths are relative to it. + - image: evolith-tracker-api + dockerfile: src/apps/tracker-api/Dockerfile + - image: evolith-tracker-gateway + dockerfile: src/apps/tracker-gateway/Dockerfile + - image: evolith-tracker-web + dockerfile: src/apps/tracker-web/Dockerfile + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push ${{ matrix.image }} + uses: docker/build-push-action@v6 + with: + context: src + file: ${{ matrix.dockerfile }} + push: true + tags: | + ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:latest + ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${{ github.sha }} + cache-from: type=gha,scope=${{ matrix.image }} + cache-to: type=gha,mode=max,scope=${{ matrix.image }} + + # A push that exits zero is not proof the image is pullable: the tag has to exist in the + # registry afterwards. This is the difference between "the job was green" and "the artifact + # is there", and it is exactly the confusion that let three charts point at nothing. + - name: The pushed tag exists in the registry + run: | + image="ghcr.io/${{ github.repository_owner }}/${{ matrix.image }}:${{ github.sha }}" + docker buildx imagetools inspect "$image" > /dev/null + echo "verified pullable: $image" diff --git a/.harness/scripts/check-deployable-images.mjs b/.harness/scripts/check-deployable-images.mjs new file mode 100644 index 00000000..f3c8c869 --- /dev/null +++ b/.harness/scripts/check-deployable-images.mjs @@ -0,0 +1,256 @@ +#!/usr/bin/env node +/** + * GT-435 — every image a Helm chart asks for must be one this repository actually publishes. + * + * PORTED FROM THE CORE, where the same check found the same class of defect on the same day. Here + * it is sharper: the Tracker had three Dockerfiles and NO workflow that built or pushed them, so + * `evolith-tracker-api:0.0.1`, `…-web:0.0.1` and `evolith-tracker-gateway:local` all named images + * that nothing produced — the last one a tag no registry can ever serve. + * + * WHY, AND IT IS NOT HYPOTHETICAL. Nothing has ever run in production, so nothing has ever pulled + * these images. Measured on 2026-08-02, all three Core charts named a tag that **no workflow in + * this repository produces**: `evolith-core-api:0.0.2`, `evolith-mcp:1.1.0` and + * `evolith-agent-runtime:0.1.0`, while `ci-cd.yml` publishes only `:latest` and `:` and + * `docker-images.yml`, which would publish semver, has never run. A `helm install` with default + * values would therefore have met `ImagePullBackOff` on every service — on day one, at the worst + * possible moment, and for a reason nobody could have diagnosed from a green CI. + * + * The whole point is that this is checkable WITHOUT a cluster, a registry credential or a server. + * The day the VPS exists is not the day to discover that the charts point at nothing. + * + * WHAT IT COMPARES. The image `repository` + `tag` each chart declares, against the set of + * `:` pairs the workflows in `.github/workflows` build and push. Third-party images + * (`postgres`, `openpolicyagent/opa`) are not ours to publish and are allowlisted BY NAME with a + * reason, never by a wildcard — a pattern that swallowed our own images would defeat the check. + * + * WHAT IT DOES NOT DO. It does not contact a registry. A guard that needs credentials runs in one + * job and rots everywhere else, and the failure it exists to catch is a mismatch between two files + * that are both right here. + */ + +import { readFileSync, readdirSync, existsSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { join } from 'node:path'; +import process from 'node:process'; + +const ROOT = process.cwd(); +const HELM_ROOT = 'product/infra/helm'; +const WORKFLOWS = '.github/workflows'; + +/** + * Images this repository does not build. Named individually: a prefix rule such as "anything + * without our org" would also excuse a typo in one of our own repositories. + */ +const THIRD_PARTY = new Map([ + ['postgres', 'Upstream database image, pulled from Docker Hub.'], + ['openpolicyagent/opa', 'Upstream OPA sidecar; the policy bundle we build is mounted into it.'], + ['busybox', 'Upstream init/utility image.'], +]); + +/** Minimal `key: value` reader — enough for `image:` blocks, and no YAML dependency. */ +function readImageBlocks(text) { + const found = []; + const lines = text.split('\n'); + for (let i = 0; i < lines.length; i += 1) { + const m = /^(\s*)(\w[\w-]*)?:?\s*$/.exec(lines[i]); + if (!m) continue; + const indent = m[1].length; + let repository; + let tag; + for (let j = i + 1; j < lines.length; j += 1) { + // Comments and blank lines are SKIPPED, not treated as the end of the block. Stopping at + // the first `#` made the parser report "declares no tag" for a block that documents its + // own tag choice — a guard defeated by the explanation of the thing it checks. + if (/^\s*(#|$)/.test(lines[j])) continue; + const child = /^(\s*)([\w-]+):\s*(.*)$/.exec(lines[j]); + if (!child) break; + if (child[1].length <= indent) break; + const value = child[3].trim().replace(/^["']|["']$/g, ''); + if (child[2] === 'repository') repository = value; + if (child[2] === 'tag') tag = value; + } + if (repository) found.push({ repository, tag: tag ?? '' }); + } + return found; +} + +/** + * Every `:` any workflow pushes. + * + * The image name is usually `${{ matrix.image }}` rather than a literal, so the matrix has to be + * resolved or this function finds nothing — and a guard that discovers zero published images would + * report every chart as broken, which is right today by accident and wrong the moment somebody + * fixes the tags. Its denominator is printed and asserted for exactly that reason. + */ +function publishedImages() { + const published = new Set(); + if (!existsSync(join(ROOT, WORKFLOWS))) return published; + + for (const file of readdirSync(join(ROOT, WORKFLOWS)).filter((f) => /\.ya?ml$/.test(f))) { + // COMMENTS ARE STRIPPED FIRST, and this is not cosmetic: a workflow that DOCUMENTS a broken + // image reference — `# the chart asked for evolith-tracker-api:0.0.1` — would otherwise be + // read as a declaration that it publishes it, and the guard would bless the very tag the + // comment exists to warn about. Found by a negative test, not by the happy path. + const raw = readFileSync(join(ROOT, WORKFLOWS, file), 'utf8') + .split('\n') + .map((line) => line.replace(/(^|\s)#.*$/, '$1')) + .join('\n'); + + // Expressions are collapsed to space-free tokens FIRST. `${{ github.repository_owner }}` + // contains spaces, so any `\S`-based pattern silently matches nothing — which is how a guard + // ends up discovering zero published images and blaming every chart. + const text = raw + .replace(/\$\{\{\s*matrix\.(?:image|name)\s*\}\}/g, '@IMAGE@') + .replace(/\$\{\{\s*github\.sha\s*\}\}/g, '@SHA@') + .replace(/\$\{\{\s*github\.repository_owner\s*\}\}/g, '@OWNER@') + .replace(/\$\{\{[^}]*\}\}/g, '@EXPR@'); + + // Literal `image:`/`name:` entries of the build matrix, which the tag lines interpolate. + const matrixImages = [ + ...text.matchAll(/^\s*-?\s*(?:image|name):\s*([a-z0-9][\w.-]*)\s*$/gm), + ] + .map((m) => m[1]) + .filter((n) => n.includes('-')); + + const expand = (name) => (name === '@IMAGE@' ? matrixImages : name.includes('@') ? [] : [name]); + + for (const m of text.matchAll(/ghcr\.io\/[\w.@-]+\/([\w.@-]+):([\w.@-]+)/g)) { + const tag = m[2] === '@SHA@' ? '' : m[2] === '@EXPR@' ? '' : m[2]; + for (const name of expand(m[1])) published.add(`${name}:${tag}`); + } + + // docker/metadata-action drives tags from `images:` plus a `tags:` pattern list. + for (const m of text.matchAll(/images:\s*ghcr\.io\/[\w.@-]+\/([\w.@-]+)/g)) { + for (const name of expand(m[1])) { + if (/type=semver/.test(text)) published.add(`${name}:`); + if (/type=raw,value=latest/.test(text)) published.add(`${name}:latest`); + } + } + } + return published; +} + +function normaliseTag(tag, workflowText) { + if (tag.includes('${{')) { + if (/github\.sha/.test(tag)) return ''; + if (/steps\.meta/.test(tag) && /type=semver/.test(workflowText)) return ''; + return ''; + } + return tag; +} + +/** Git tags in this checkout, which is what a semver image build is driven from. */ +function gitTags() { + try { + return new Set(execFileSync('git', ['tag', '--list'], { encoding: 'utf8' }).split('\n').map((t) => t.trim())); + } catch { + return new Set(); + } +} + +/** + * Whether some workflow could produce this exact image:tag. + * + * A `` publisher is driven by a GIT TAG, so declaring one is not enough: the tag has to + * exist. `evolith-agent-runtime:0.1.0` was requested by a chart while no `v0.1.0` tag has ever + * existed in this repository, which makes that image unproducible by any current path — a hard + * defect, and one provable without touching a registry. + */ +function producibility(image, tag, published, tags) { + if (published.has(`${image}:${tag}`)) return { ok: true, how: `published directly as :${tag}` }; + if (/^\d+\.\d+\.\d+/.test(tag) && published.has(`${image}:`)) { + if (tags.has(`v${tag}`) || tags.has(tag)) { + return { ok: true, how: `semver build from git tag v${tag}`, needsRegistryCheck: true }; + } + return { + ok: false, + why: + `a semver publisher exists but it is driven by a git tag, and neither \`v${tag}\` nor ` + + `\`${tag}\` is a tag in this repository. No current path can produce this image.`, + }; + } + return { + ok: false, + why: + `no workflow publishes it. Published for this image: ` + + `${[...published].filter((p) => p.startsWith(`${image}:`)).join(', ') || '(nothing)'}`, + }; +} + +function main() { + const published = publishedImages(); + const tags = gitTags(); + const problems = []; + const unverifiable = []; + let checked = 0; + + const charts = existsSync(join(ROOT, HELM_ROOT)) + ? readdirSync(join(ROOT, HELM_ROOT)).filter((d) => existsSync(join(ROOT, HELM_ROOT, d, 'values.yaml'))) + : []; + + for (const chart of charts) { + const values = readFileSync(join(ROOT, HELM_ROOT, chart, 'values.yaml'), 'utf8'); + for (const { repository, tag } of readImageBlocks(values)) { + const bare = repository.replace(/^ghcr\.io\/[^/]+\//, '').replace(/^docker\.io\//, ''); + if (THIRD_PARTY.has(bare) || THIRD_PARTY.has(repository)) continue; + checked += 1; + + if (!tag) { + problems.push(`${chart}: \`${repository}\` declares no tag — a floating image is not a deployment`); + continue; + } + // A tag a registry can never serve. `:local` only exists on the machine that built it. + if (/^(local|dev|latest-local)$/.test(tag)) { + problems.push( + `${chart}: \`${repository}:${tag}\` is a LOCAL-ONLY tag. No registry can serve it, so this ` + + `chart cannot deploy anywhere but the machine that built the image.`, + ); + continue; + } + const verdict = producibility(bare, tag, published, tags); + if (!verdict.ok) { + problems.push(`${chart}: \`${repository}:${tag}\` — ${verdict.why}`); + } else if (verdict.needsRegistryCheck) { + // The boundary of a static check, stated instead of hidden: that a build COULD produce + // this image is provable here; that it DID is a fact about a registry, and a guard that + // needed a credential to say so would run in one job and rot everywhere else. + unverifiable.push(`${chart}: \`${repository}:${tag}\` — ${verdict.how}. Whether that build ever RAN is not knowable from the repository.`); + } + } + } + + console.log( + `check-deployable-images: ${checked} chart image(s) checked across ${charts.length} chart(s); ` + + `${published.size} image:tag pair(s) discovered in ${WORKFLOWS}.`, + ); + + // Zero scanned is the shape every vacuous guard takes: both trees read, nothing contrasted, + // green tick. Inlined rather than imported because this repository has no shared coverage + // helper — the rule matters more than where it lives. + if (checked === 0) { + console.error( + `\n❌ Zero first-party chart images scanned under ${HELM_ROOT}/*/values.yaml. ` + + 'A check that inspected nothing did not run.\n', + ); + process.exit(1); + } + + if (problems.length > 0) { + console.error('\n❌ Charts request images this repository does not publish:\n'); + for (const p of problems) console.error(` ✖ ${p}`); + console.error( + '\nNothing has ever run in production, so nothing has ever pulled these images and no green\n' + + 'CI could have revealed this. The day the server exists is not the day to discover that the\n' + + 'charts point at nothing: fix the tag, or publish the tag.\n', + ); + process.exit(1); + } + + if (unverifiable.length > 0) { + console.log('\n Producible, but not verifiable from here:'); + for (const u of unverifiable) console.log(` · ${u}`); + } + console.log('\n✅ Every chart image is one this repository can produce.'); +} + +main(); diff --git a/product/infra/helm/README.md b/product/infra/helm/README.md index 4f04c57c..e4b6485c 100644 --- a/product/infra/helm/README.md +++ b/product/infra/helm/README.md @@ -91,6 +91,37 @@ up at the next evaluation rather than never. **What the chart never does:** generate, default or version key material. Every seed comes from the deployment's own secret store. +## Images and where they come from (GT-435) + +Until 2026-08-02 this repository shipped **three Dockerfiles and no image CD**. Nothing built or +pushed them, while the charts referenced `evolith-tracker-api:0.0.1`, `evolith-tracker-web:0.0.1` +and `evolith-tracker-gateway:local` — the last a tag no registry can ever serve. The deployment +path was not untested; it was **not wired**. No green CI could have shown it, because nothing had +ever pulled these images. + +`.github/workflows/images.yml` publishes all three to GHCR on every merge to `main` and `develop`: + +| image | Dockerfile | context | +|---|---|---| +| `ghcr.io/beyondnetcode/evolith-tracker-api` | `src/apps/tracker-api/Dockerfile` | `src/` | +| `ghcr.io/beyondnetcode/evolith-tracker-gateway` | `src/apps/tracker-gateway/Dockerfile` | `src/` | +| `ghcr.io/beyondnetcode/evolith-tracker-web` | `src/apps/tracker-web/Dockerfile` | `src/` | + +Tags are `:latest` and `:`. Authentication is the built-in `GITHUB_TOKEN` with +`packages: write` — no extra secret, so it works from the first run. + +The workflow does not trust its own green tick: after pushing, it runs +`docker buildx imagetools inspect` on the `:` tag. A push step exiting zero is not proof the +image is pullable, and that gap between "the job was green" and "the artifact is there" is exactly +what let three charts point at nothing. + +**Production must pin the sha.** `--set image.tag=`. `latest` makes a chart deployable out of +the box and cannot be rolled back to a previous build, and `GT-448` requires a rollback that has +actually been exercised. + +`check-deployable-images.mjs` runs in CI and fails when a chart names an image no workflow +publishes, a local-only tag, or a semver with no git tag behind it. + ## Notes - **Connection string.** The .NET config system cannot concatenate a password diff --git a/product/infra/helm/evolith-tracker-api/values.yaml b/product/infra/helm/evolith-tracker-api/values.yaml index 3d719531..e9c5d14f 100644 --- a/product/infra/helm/evolith-tracker-api/values.yaml +++ b/product/infra/helm/evolith-tracker-api/values.yaml @@ -3,8 +3,15 @@ replicaCount: 2 image: repository: ghcr.io/beyondnetcode/evolith-tracker-api pullPolicy: IfNotPresent - tag: "0.0.1" - + # GT-435 — the tag must be one CI actually publishes, and until 2026-08-02 nothing published + # these images at all: the Tracker shipped three Dockerfiles and ZERO workflows that built or + # pushed them, so every chart pointed at something that did not exist. `images.yml` now pushes + # `:latest` and `:` on every merge to main and develop. + # + # PRODUCTION MUST OVERRIDE THIS with `--set image.tag=`. `latest` cannot be rolled back to + # a previous build, and GT-448 requires a rollback that has actually been exercised, which a + # floating tag makes impossible to demonstrate. + tag: latest service: type: ClusterIP port: 80 diff --git a/product/infra/helm/evolith-tracker-gateway/values.yaml b/product/infra/helm/evolith-tracker-gateway/values.yaml index 69397afd..3d3859c2 100644 --- a/product/infra/helm/evolith-tracker-gateway/values.yaml +++ b/product/infra/helm/evolith-tracker-gateway/values.yaml @@ -1,7 +1,15 @@ # Default (production-leaning) values for the tracker-gateway. image: - repository: evolith-tracker-gateway - tag: local + repository: ghcr.io/beyondnetcode/evolith-tracker-gateway + # GT-435 — the tag must be one CI actually publishes, and until 2026-08-02 nothing published + # these images at all: the Tracker shipped three Dockerfiles and ZERO workflows that built or + # pushed them, so every chart pointed at something that did not exist. `images.yml` now pushes + # `:latest` and `:` on every merge to main and develop. + # + # PRODUCTION MUST OVERRIDE THIS with `--set image.tag=`. `latest` cannot be rolled back to + # a previous build, and GT-448 requires a rollback that has actually been exercised, which a + # floating tag makes impossible to demonstrate. + tag: latest pullPolicy: IfNotPresent replicaCount: 1 diff --git a/product/infra/helm/evolith-tracker-web/values.yaml b/product/infra/helm/evolith-tracker-web/values.yaml index 8ab19809..3eafd469 100644 --- a/product/infra/helm/evolith-tracker-web/values.yaml +++ b/product/infra/helm/evolith-tracker-web/values.yaml @@ -3,8 +3,15 @@ replicaCount: 2 image: repository: ghcr.io/beyondnetcode/evolith-tracker-web pullPolicy: IfNotPresent - tag: "0.0.1" - + # GT-435 — the tag must be one CI actually publishes, and until 2026-08-02 nothing published + # these images at all: the Tracker shipped three Dockerfiles and ZERO workflows that built or + # pushed them, so every chart pointed at something that did not exist. `images.yml` now pushes + # `:latest` and `:` on every merge to main and develop. + # + # PRODUCTION MUST OVERRIDE THIS with `--set image.tag=`. `latest` cannot be rolled back to + # a previous build, and GT-448 requires a rollback that has actually been exercised, which a + # floating tag makes impossible to demonstrate. + tag: latest service: type: ClusterIP port: 80