From f170303ce0ac53375a68b2d04c2673e0f2651f50 Mon Sep 17 00:00:00 2001 From: Pranay Prakash Date: Thu, 3 Sep 2026 16:58:48 -0700 Subject: [PATCH] Keep one failed publish from taking down the release, and verify what actually reached npm Publishing 5.0.0-beta.48 shipped 7 of 21 packages on the first attempt and 5 more on a rerun; 4 are still not on npm. Two things compounded: - @changesets/cli 2.29.8 crashes on an E401 whose JSON has no `detail` field, which is the shape pnpm 11 produces when its OIDC token exchange fails and it falls back to the (empty) .npmrc token. The TypeError killed the process with other publishes mid-flight, and hid pnpm's own warning that says why OIDC was skipped. Patch the CLI (pnpm patchedDependencies) to treat a missing `detail` as a plain publish failure for that one package and to print pnpm's message, so the rest of the release proceeds and the log names the cause. - Nothing compared the manifests on the commit with the registry, so a half-published release looked like any other red job. Add scripts/check-published.mjs and run it after the publish step regardless of that step's outcome. Also stop a broken Slack token from failing an otherwise successful release; that is what turned the Aug 26 and Aug 31 release jobs red. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/release.yml | 16 +++ patches/@changesets__cli@2.29.8.patch | 22 ++++ pnpm-lock.yaml | 7 +- pnpm-workspace.yaml | 3 + scripts/check-published.mjs | 157 ++++++++++++++++++++++++++ 5 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 patches/@changesets__cli@2.29.8.patch create mode 100644 scripts/check-published.mjs diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7bcfa0fd1d..1ff53a102d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -72,6 +72,17 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # `changeset publish` runs one `pnpm publish` per package and one failure + # leaves the release half-shipped, with `workflow` possibly pointing at a + # `@workflow/core` that is not on npm yet. 5.0.0-beta.48 shipped 7 of 21 + # packages on its first attempt, and the failure that stopped it crashed + # the publish loop before naming the packages it took down. Cross-check + # every publishable manifest on this commit against the registry, whatever + # the publish step's outcome, so the job says exactly what is missing. + - name: Verify every package version on this commit is on npm + if: ${{ always() && steps.changesets.outcome != 'skipped' && steps.changesets.outcome != 'cancelled' }} + run: node scripts/check-published.mjs + - name: Create GitHub Release if: steps.changesets.outputs.published == 'true' env: @@ -111,8 +122,13 @@ jobs: --prerelease=$PRERELEASE fi + # A broken Slack token must not turn a successful release red: the Aug 26 + # and Aug 31 releases published every package and still failed here + # (`token_expired`, then `invalid_auth`), which taught people to read a + # red Release job as noise. Surface it as a warning instead. - name: Post release notes to Slack if: steps.changesets.outputs.published == 'true' + continue-on-error: true env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} SLACK_RELEASE_CHANNEL_ID: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }} diff --git a/patches/@changesets__cli@2.29.8.patch b/patches/@changesets__cli@2.29.8.patch new file mode 100644 index 0000000000..28c342163f --- /dev/null +++ b/patches/@changesets__cli@2.29.8.patch @@ -0,0 +1,22 @@ +diff --git a/dist/changesets-cli.cjs.js b/dist/changesets-cli.cjs.js +index 0f03e31068a5a7f77ef03ee97684d7f4d5136367..b2bba73c0f10ddbf702fdd2d036277273e28d828 100644 +--- a/dist/changesets-cli.cjs.js ++++ b/dist/changesets-cli.cjs.js +@@ -783,7 +783,7 @@ async function internalPublish(packageJson, opts, twoFactorState) { + let json = getLastJsonObjectFromString(stderr.toString()) || getLastJsonObjectFromString(stdout.toString()); + if (json !== null && json !== void 0 && json.error) { + // The first case is no 2fa provided, the second is when the 2fa is wrong (timeout or wrong words) +- if ((json.error.code === "EOTP" || json.error.code === "E401" && json.error.detail.includes("--otp=")) && !ciInfo.isCI) { ++ if ((json.error.code === "EOTP" || json.error.code === "E401" && typeof json.error.detail === "string" && json.error.detail.includes("--otp=")) && !ciInfo.isCI) { + if (twoFactorState.token !== null) { + // the current otp code must be invalid since it errored + twoFactorState.token = null; +@@ -792,7 +792,7 @@ async function internalPublish(packageJson, opts, twoFactorState) { + twoFactorState.isRequired = Promise.resolve(true); + return internalPublish(packageJson, opts, twoFactorState); + } +- logger.error(`an error occurred while publishing ${packageJson.name}: ${json.error.code}`, json.error.summary, json.error.detail ? "\n" + json.error.detail : ""); ++ logger.error(`an error occurred while publishing ${packageJson.name}: ${json.error.code}`, json.error.summary ?? json.error.message ?? "", json.error.detail ? "\n" + json.error.detail : ""); + } + logger.error(stderr.toString() || stdout.toString()); + return { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b98f4765a2..6abbd33d5d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,6 +4,9 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +patchedDependencies: + '@changesets/cli@2.29.8': 17871fd218965b410e1e3adad11e877d99c222c3cefb03773e353c2bb1c87d15 + catalogs: default: '@biomejs/biome': @@ -81,7 +84,7 @@ importers: version: 0.5.2 '@changesets/cli': specifier: ^2.29.8 - version: 2.29.8(@types/node@24.6.2) + version: 2.29.8(patch_hash=17871fd218965b410e1e3adad11e877d99c222c3cefb03773e353c2bb1c87d15)(@types/node@24.6.2) '@vitest/coverage-v8': specifier: 'catalog:' version: 4.1.10(vitest@4.1.10) @@ -18746,7 +18749,7 @@ snapshots: transitivePeerDependencies: - encoding - '@changesets/cli@2.29.8(@types/node@24.6.2)': + '@changesets/cli@2.29.8(patch_hash=17871fd218965b410e1e3adad11e877d99c222c3cefb03773e353c2bb1c87d15)(@types/node@24.6.2)': dependencies: '@changesets/apply-release-plan': 7.0.14 '@changesets/assemble-release-plan': 6.0.9 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 37bf7c0164..cfc2bb308b 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -87,3 +87,6 @@ allowBuilds: protobufjs: false sharp: false ssh2: false + +patchedDependencies: + '@changesets/cli@2.29.8': patches/@changesets__cli@2.29.8.patch diff --git a/scripts/check-published.mjs b/scripts/check-published.mjs new file mode 100644 index 0000000000..6a03c650ad --- /dev/null +++ b/scripts/check-published.mjs @@ -0,0 +1,157 @@ +#!/usr/bin/env node +/** + * Fails when a publishable package version on this commit is not on npm. + * + * `changeset publish` runs one `pnpm publish` per package, concurrently, and + * reports the release as a whole. When one of those publishes fails the + * release is left half-shipped: some versions are live, others are not, and + * `workflow` can point at an `@workflow/core` that does not exist yet. The + * 5.0.0-beta.48 release shipped 7 of 21 packages on its first attempt, and + * the failure that stopped it (an E401 whose JSON had no `detail` field) + * crashed the publish loop before it printed which packages were affected. + * + * This script is the check the Release job runs after publishing, whatever + * the publish step's outcome: every non-private package that changesets does + * not ignore must have its manifest version in the registry's version list, + * and the branch's dist-tag (the pre-release tag from `.changeset/pre.json` + * while in pre mode, `latest` otherwise) must point at it. The invariant + * holds on every commit of a release branch, not only right after a publish, + * so a gap keeps failing the job until it is closed. + * + * The registry can lag a publish by a few seconds, so missing versions are + * re-checked a few times before they are reported. + * + * Usage: node scripts/check-published.mjs [--tag ] + */ + +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const REGISTRY = 'https://registry.npmjs.org'; +const ATTEMPTS = Number(process.env.CHECK_PUBLISHED_ATTEMPTS ?? 6); +const DELAY_MS = Number(process.env.CHECK_PUBLISHED_DELAY_MS ?? 10_000); + +function readJson(path) { + return JSON.parse(readFileSync(path, 'utf8')); +} + +function distTagForBranch() { + const argIndex = process.argv.indexOf('--tag'); + if (argIndex !== -1 && process.argv[argIndex + 1]) { + return process.argv[argIndex + 1]; + } + const prePath = join(root, '.changeset', 'pre.json'); + if (existsSync(prePath)) { + const pre = readJson(prePath); + if (pre.mode === 'pre' && pre.tag) return pre.tag; + } + return 'latest'; +} + +function publishablePackages() { + const { ignore = [] } = readJson(join(root, '.changeset', 'config.json')); + const ignored = new Set(ignore.filter((name) => !name.includes('*'))); + const ignoredPatterns = ignore + .filter((name) => name.includes('*')) + .map( + (glob) => new RegExp(`^${glob.split('*').map(escapeRegExp).join('.*')}$`) + ); + const packagesDir = join(root, 'packages'); + const result = []; + for (const dir of readdirSync(packagesDir)) { + const manifestPath = join(packagesDir, dir, 'package.json'); + if (!existsSync(manifestPath)) continue; + const manifest = readJson(manifestPath); + if (!manifest.name || !manifest.version || manifest.private) continue; + if (ignored.has(manifest.name)) continue; + if (ignoredPatterns.some((re) => re.test(manifest.name))) continue; + result.push({ name: manifest.name, version: manifest.version }); + } + return result.sort((a, b) => a.name.localeCompare(b.name)); +} + +function escapeRegExp(s) { + return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +async function registryState(name) { + const url = `${REGISTRY}/${name.replace('/', '%2F')}`; + const response = await fetch(url, { + headers: { accept: 'application/json', 'cache-control': 'no-cache' }, + }); + if (response.status === 404) return { versions: {}, distTags: {} }; + if (!response.ok) { + throw new Error( + `GET ${url} failed: ${response.status} ${response.statusText}` + ); + } + const doc = await response.json(); + return { versions: doc.versions ?? {}, distTags: doc['dist-tags'] ?? {} }; +} + +async function check(pkg, tag) { + const { versions, distTags } = await registryState(pkg.name); + const visible = Object.hasOwn(versions, pkg.version); + const tagged = distTags[tag] === pkg.version; + return { ...pkg, visible, tagged, actualTag: distTags[tag] }; +} + +const tag = distTagForBranch(); +const packages = publishablePackages(); +let results = await Promise.all(packages.map((pkg) => check(pkg, tag))); + +for (let attempt = 1; attempt < ATTEMPTS; attempt++) { + const pending = results.filter((r) => !(r.visible && r.tagged)); + if (pending.length === 0) break; + console.log( + `${pending.length} package(s) not yet visible with the "${tag}" tag; re-checking in ${DELAY_MS / 1000}s (attempt ${attempt + 1}/${ATTEMPTS})...` + ); + await new Promise((resolveDelay) => setTimeout(resolveDelay, DELAY_MS)); + const rechecked = await Promise.all(pending.map((r) => check(r, tag))); + results = results.map((r) => rechecked.find((n) => n.name === r.name) ?? r); +} + +const width = Math.max(...results.map((r) => r.name.length)); +for (const r of results) { + const status = r.visible + ? r.tagged + ? 'ok' + : `visible, but "${tag}" is ${r.actualTag ?? 'unset'}` + : 'NOT ON NPM'; + console.log(`${r.name.padEnd(width)} ${r.version.padEnd(14)} ${status}`); +} + +const missing = results.filter((r) => !r.visible); +const untagged = results.filter((r) => r.visible && !r.tagged); +if (missing.length === 0 && untagged.length === 0) { + console.log( + `\nāœ“ all ${results.length} publishable packages are on npm under the "${tag}" tag.` + ); + process.exit(0); +} + +console.error(''); +if (missing.length > 0) { + console.error( + `āœ— ${missing.length} package version(s) on this commit are not on npm:` + ); + for (const r of missing) console.error(` ${r.name}@${r.version}`); + console.error( + '\n Read the publish step above for each one. If `pnpm publish` reported the\n' + + ' version as "previously staged", it is waiting for a maintainer to approve it in\n' + + " the package's Staged Packages tab on npmjs.com. Otherwise re-run the Release\n" + + ' workflow (workflow_dispatch); `changeset publish` skips versions already live.' + ); +} +if (untagged.length > 0) { + console.error( + `āœ— ${untagged.length} package(s) are on npm but the "${tag}" dist-tag does not point at them:` + ); + for (const r of untagged) + console.error( + ` ${r.name}@${r.version} (tag is ${r.actualTag ?? 'unset'})` + ); +} +process.exit(1);