From f234ded2e88fa8947407ab7e15fa277364c21754 Mon Sep 17 00:00:00 2001 From: Tyler Dixon Date: Wed, 5 Aug 2026 12:09:01 -0700 Subject: [PATCH] ci: add a manual firestore flake probe for #776 Every measurement of the #776 flake so far has been local, where the failure is a plain waitFor timeout with no gRPC error. In CI it arrives alongside a gRPC framing desync (RESOURCE_EXHAUSTED: Received message larger than max), which raises the possibility that the local repro and the CI failure are not the same bug. That matters, because the @grpc/grpc-js override proposed as the fix was measured only against the local one. This runs the firestore suite N times per arm, across both Node versions and both grpc-js versions, under CI conditions, so the comparison happens where the failure actually occurs. Notes on the design: - workflow_dispatch only. It never runs on a push, a PR or a schedule, so it costs nothing until someone asks for it. - A fresh emulator per iteration, matching how npm test runs in CI. Reusing one emulator across iterations would measure something else. - Failures are classified, not counted. Only the #776 assertion signature counts toward the rate; emulator start failures are reported separately, because folding them in previously inflated a local rate estimate by roughly 50%. - The job reports rather than fails. A red run here means the probe broke, not that the flake reproduced. - Inputs reach the script through env rather than interpolation, and iterations is validated before it reaches the loop. Classifier dry-run against synthetic logs covering pass, flake, flake-with-gRPC-error and infra-failure returns the expected counts and excludes infra failures from the rate. zizmor 1.25.2 reports no findings beyond the cache-poisoning rule CI suppresses. Refs #776 --- .github/workflows/flake-probe.yaml | 187 +++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 .github/workflows/flake-probe.yaml diff --git a/.github/workflows/flake-probe.yaml b/.github/workflows/flake-probe.yaml new file mode 100644 index 00000000..1ce542e8 --- /dev/null +++ b/.github/workflows/flake-probe.yaml @@ -0,0 +1,187 @@ +# Measures the `test/firestore.test.tsx` flake rate (#776) in CI rather than locally. +# +# Every measurement of this flake so far has been on a laptop, where the failure looks +# like a plain `waitFor` timeout. In CI it comes with a gRPC framing desync +# (`RESOURCE_EXHAUSTED: Received message larger than max`), which may mean the two are +# not the same bug. This runs both `@grpc/grpc-js` arms on both Node versions under CI +# conditions so the comparison is made where the failure actually happens. +# +# Manual only. It never runs on a push, a PR or a schedule, so it costs nothing until +# someone asks for it. +name: Firestore flake probe + +on: + workflow_dispatch: + inputs: + iterations: + description: "Test runs per arm (each is a full emulator start/stop, roughly 25s)" + required: false + default: "20" + node_versions: + description: "JSON array of Node majors to probe" + required: false + default: '["22", "24"]' + arms: + description: "JSON array of grpc-js arms: baseline, override, or both" + required: false + default: '["baseline", "override"]' + +# Least privilege. This workflow reads the repo and writes nothing back. +permissions: + contents: read + +jobs: + probe: + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + matrix: + node: ${{ fromJSON(inputs.node_versions) }} + arm: ${{ fromJSON(inputs.arms) }} + fail-fast: false + name: Probe Node ${{ matrix.node }} / ${{ matrix.arm }} + steps: + - name: Checkout + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + persist-credentials: false + + - name: Setup node + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: ${{ matrix.node }} + check-latest: true + cache: 'npm' + + - name: Setup Java + uses: actions/setup-java@c1e323688fd81a25caa38c78aa6df2d33d3e20d9 # v4.8.0 + with: + distribution: 'temurin' + java-version: '21' + + - name: Firebase emulator cache + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: ~/.cache/firebase/emulators + key: firebase_emulators + + - name: Install deps + run: npm ci + + # `npm pkg set` mangles keys containing a slash, so edit package.json directly. + # `npm install` (not `npm ci`) is required here because applying an override + # necessarily changes the lockfile. + - name: Apply the grpc-js override + if: ${{ matrix.arm == 'override' }} + run: | + node -e ' + const fs = require("fs"); + const pkg = JSON.parse(fs.readFileSync("package.json", "utf8")); + pkg.overrides = { ...pkg.overrides, "@grpc/grpc-js": "^1.14.0" }; + fs.writeFileSync("package.json", JSON.stringify(pkg, null, 2) + "\n"); + ' + npm install --no-audit --no-fund + + - name: Record the resolved grpc-js version + run: | + RESOLVED="$(node -p "require('@grpc/grpc-js/package.json').version")" + echo "resolved_grpc=$RESOLVED" >> "$GITHUB_ENV" + echo "Resolved @grpc/grpc-js: $RESOLVED" + + # Inputs and matrix values are passed through `env` rather than interpolated into + # the script body, so nothing from the dispatch form can be executed as shell. + - name: Run the probe + env: + ITERATIONS: ${{ inputs.iterations }} + ARM: ${{ matrix.arm }} + NODE_MAJOR: ${{ matrix.node }} + run: | + set -uo pipefail + + # Guard against a non-numeric or absurd `iterations` before it reaches the loop. + case "$ITERATIONS" in + ''|*[!0-9]*) echo "iterations must be a positive integer, got '$ITERATIONS'"; exit 1 ;; + esac + if [ "$ITERATIONS" -lt 1 ] || [ "$ITERATIONS" -gt 200 ]; then + echo "iterations must be between 1 and 200, got '$ITERATIONS'" + exit 1 + fi + + pass=0 + flake=0 + infra=0 + grpc_err=0 + mkdir -p probe-logs + + for i in $(seq 1 "$ITERATIONS"); do + log="probe-logs/run-$i.log" + + # A fresh emulator per iteration, matching how `npm test` runs in CI. Reusing + # one emulator across iterations would measure a different thing. + set +e + npx firebase emulators:exec --only firestore --project=rxfire-525a3 \ + "npx vitest run firestore" > "$log" 2>&1 + rc=$? + set -e + + if grep -q "RESOURCE_EXHAUSTED: Received message larger than max" "$log"; then + grpc_err=$((grpc_err + 1)) + fi + + if [ "$rc" -eq 0 ]; then + pass=$((pass + 1)) + echo "run $i: PASS" + elif grep -q "expected 'loading' to deeply equal 'success'" "$log"; then + # The #776 signature specifically, rather than "the job went red". + flake=$((flake + 1)) + echo "run $i: FLAKE (rc=$rc)" + else + # Emulator start failures and the like. Counted separately because folding + # them in previously inflated a local flake-rate estimate by ~50%. + infra=$((infra + 1)) + echo "run $i: INFRA FAILURE (rc=$rc), excluded from the rate" + tail -20 "$log" + fi + done + + counted=$((pass + flake)) + if [ "$counted" -gt 0 ]; then + rate="$(node -e "process.stdout.write(((${flake}/${counted})*100).toFixed(1))")" + else + rate="n/a" + fi + + { + echo "### Node ${NODE_MAJOR} / ${ARM} (@grpc/grpc-js ${resolved_grpc})" + echo "" + echo "| Outcome | Count |" + echo "| --- | --- |" + echo "| Pass | ${pass} |" + echo "| Flake (#776 signature) | ${flake} |" + echo "| Infra failure (excluded) | ${infra} |" + echo "| Runs showing RESOURCE_EXHAUSTED | ${grpc_err} |" + echo "" + echo "**Flake rate: ${rate}% of ${counted} counted runs.**" + echo "" + if [ "$infra" -gt 0 ]; then + echo "> ${infra} run(s) failed for reasons other than the #776 assertion and are excluded from the rate." + echo "" + fi + } >> "$GITHUB_STEP_SUMMARY" + + echo "arm=${ARM} node=${NODE_MAJOR} pass=${pass} flake=${flake} infra=${infra} grpc_err=${grpc_err} rate=${rate}%" + + # The probe reports; it does not fail. A red job here would mean the probe + # broke, not that the flake reproduced. + if [ "$counted" -eq 0 ]; then + echo "Every run failed for infrastructure reasons; the probe measured nothing." + exit 1 + fi + + - name: Upload probe logs + if: ${{ always() }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: probe-logs-node${{ matrix.node }}-${{ matrix.arm }} + path: probe-logs/ + retention-days: 7