From 70b00c65718ba480a117bcca18b7dc8b3c23e148 Mon Sep 17 00:00:00 2001 From: tiXor-code Date: Mon, 7 Sep 2026 23:24:19 +0300 Subject: [PATCH 1/2] publish when the live outage state moves, not only at 03:10 The site is static and was rebuilt once a night, so a newly announced outage - or a revised restore estimate - could sit invisible for up to 24h. A visitor said exactly that on 2026-08-31: "Nu este actualizat la data de 31.08.2026". The data was already here; only the publish lagged. The scrape now dispatches Nightly publish when the LIVE state changes. Nightly keeps its 03:10 schedule for the historical rebuild; this only adds intraday publishes, and nightly's own `concurrency: nightly` group serialises overlaps. Gated on the canonical hash over page A's parsed records, not the raw bytes: - a revised `remediere_raw` DOES trigger, which matters because the restore time is the number people are actually asking for; - reordered rows and markup churn do NOT (verified: whitespace and attribute churn yields an identical hash); - the affected-street list is not in the key tuple, so a change only to which streets a PT lists will not publish. Accepted and documented. Page A lists only currently-active outages - records vanish on resolve - so a change to it is by definition a change to the live state. MEASURED, and it corrects the estimate this was designed against: across the last 12 snapshots the live hash changed on 11 of 11 transitions. The filter is therefore barely tighter than "any data change"; it earns its place as protection against cosmetic churn, not as a way to cut build count. Expect roughly 8 publishes/day, bounded by how often GitHub actually runs the schedule - measured at ~8/day despite the */15 cron, because GitHub throttles frequent schedules. Vercel Hobby runs one build at a time at ~6 min, so those serialise comfortably. The detector fails OPEN: any read, git or parse error reports changed=true. A broken detector should degrade to publishing too often, never to going silently stale - which is the failure this whole change exists to fix. Co-Authored-By: Claude Code --- .github/workflows/scrape.yml | 24 ++++++++++ scripts/live_changed.py | 89 ++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 scripts/live_changed.py diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index 54f286c..1b8762d 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -7,6 +7,7 @@ on: permissions: contents: write + actions: write # dispatch Nightly publish when the live state moves concurrency: group: scrape @@ -22,15 +23,38 @@ jobs: - name: Scrape run: python3 scraper/scrape.py + # Must run BEFORE the commit: it diffs the working tree against HEAD. + - name: Did the live outage state change? + id: live + run: python3 scripts/live_changed.py + - name: Commit if changed + id: commit run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add data/ if git diff --cached --quiet; then echo "no changes" + echo "committed=false" >> "$GITHUB_OUTPUT" else git commit -m "snapshot $(date -u +'%Y-%m-%dT%H:%M:%SZ')" git pull --rebase origin main git push + echo "committed=true" >> "$GITHUB_OUTPUT" fi + + # The site is static and was rebuilt only at 03:10, so a new outage - or a + # revised restore estimate - could sit invisible for up to 24h. A visitor + # reported exactly that on 2026-08-31. Publishing here closes that gap to + # roughly the scrape interval. + # + # Gated on the live-state hash, not the raw bytes, so markup churn cannot + # trigger a rebuild. Nightly publish keeps its own 03:10 schedule for the + # historical rebuild; this only adds intraday publishes. Its `concurrency: + # nightly` group serialises overlapping runs. + - name: Publish now if the live state moved + if: steps.commit.outputs.committed == 'true' && steps.live.outputs.changed == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run nightly.yml -R ${{ github.repository }} diff --git a/scripts/live_changed.py b/scripts/live_changed.py new file mode 100644 index 0000000..ee1cb93 --- /dev/null +++ b/scripts/live_changed.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Did this scrape change the LIVE outage state, or only the raw bytes? + +The site is rebuilt nightly, so a newly announced outage - or a revised +restore estimate - could sit invisible for up to 24h. A visitor said exactly +that on 2026-08-31: "Nu este actualizat la data de 31.08.2026". The scrape +itself runs far more often, so the data is already there; only the publish +lags. + +Publishing on every byte change would be wasteful and, on Vercel Hobby (one +concurrent build, ~6 min each), self-queueing. So publish only when the state +a reader would actually see has changed. + +"Live state" = the canonical hash over page A's parsed records +(pipeline.parse.content_hash). That hash is built from sorted key tuples, so: + + - a revised `remediere_raw` (the estimated restore time) DOES trigger, which + matters because that is the number people are asking for; + - reordered rows, whitespace and markup churn do NOT; + - the affected-street list is NOT in the key tuple, so a change only to which + streets a PT lists will not trigger a publish. Accepted: the PT identity, + cause, restore estimate and block count are what the live band renders. + +Page A only ever lists CURRENTLY ACTIVE outages - records vanish when resolved +- so a change to it is by definition a change to the live state. + +Compares the working tree against HEAD, so it must run BEFORE the commit step. + +Exits 0 always. Prints `changed=true|false`, and writes the same to +$GITHUB_OUTPUT when present. Any failure is reported as changed=true: a broken +detector should degrade to publishing too often, never to going silently stale. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +PAGE_A = REPO / "data" / "functionare.html" + +sys.path.insert(0, str(REPO)) + + +def emit(changed: bool, reason: str) -> None: + value = "true" if changed else "false" + print(f"changed={value} ({reason})") + out = os.environ.get("GITHUB_OUTPUT") + if out: + with open(out, "a", encoding="utf-8") as fh: + fh.write(f"changed={value}\n") + sys.exit(0) + + +def live_hash(html: bytes) -> str: + from pipeline.parse import content_hash, parse_page + + return content_hash(parse_page(html)) + + +def main() -> None: + try: + current = PAGE_A.read_bytes() + except OSError as exc: + emit(True, f"cannot read working-tree page A ({exc}) - failing open") + + try: + previous = subprocess.run( + ["git", "show", f"HEAD:data/{PAGE_A.name}"], + cwd=REPO, + capture_output=True, + check=True, + ).stdout + except subprocess.CalledProcessError: + emit(True, "no previous page A at HEAD - first run, failing open") + + try: + before = live_hash(previous) + after = live_hash(current) + except Exception as exc: # noqa: BLE001 - a parser change must not go silent + emit(True, f"parse failed ({type(exc).__name__}: {exc}) - failing open") + + emit(before != after, f"{before[:12]} -> {after[:12]}") + + +if __name__ == "__main__": + main() From 4de85e91895d804535558267ca852cfdeed6b596 Mon Sep 17 00:00:00 2001 From: tiXor-code Date: Mon, 7 Sep 2026 23:30:41 +0300 Subject: [PATCH 2/2] fix: EmptyState is a state, not a parse failure Caught in review, and it defeated the throttle this PR exists to add. pipeline/parse.py:179 raises EmptyState for CMTEB's routine "nu exista inregistrari" banner; backfill.py:65 already handles it separately from ParseFailure for exactly this reason. live_hash() wrapped parsing in a broad `except Exception`, so EmptyState fell into the fail-open branch and the detector reported changed=true on EVERY scrape for as long as the city had no active outages - publishing constantly at precisely the times there was nothing to publish. Reproduced against a synthetic banner page before fixing: two consecutive empty scrapes reported changed=true; they now compare equal, while empty->outage and outage->empty both still trigger. No empty snapshot exists in the last 400 commits, so this had not bitten yet - it would have arrived quietly in summer. Second finding, also real and worse than a missed publish: this step runs BEFORE "Commit if changed", so an uncaught exception would fail the step, skip the commit and LOSE the scraped snapshot. Now belt and braces - main() catches everything, and the step carries continue-on-error so even a hard crash cannot stop the commit. With continue-on-error the step's outputs can be unset, so the publish condition is now `!= 'false'` rather than `== 'true'`: only an explicit "no change" suppresses a publish, keeping the fail-open intent intact when the detector itself dies. 4 regression tests pin all of it. 49 passed, 15 skipped. Co-Authored-By: Claude Code --- .github/workflows/scrape.yml | 6 +++- scripts/live_changed.py | 29 +++++++++++++++++--- tests/test_live_changed.py | 53 ++++++++++++++++++++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 tests/test_live_changed.py diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index 1b8762d..0f6e1f7 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -24,8 +24,12 @@ jobs: run: python3 scraper/scrape.py # Must run BEFORE the commit: it diffs the working tree against HEAD. + # continue-on-error is the structural guarantee behind the script's own + # fail-open: even a hard crash here must never stop the commit below and + # lose the scraped snapshot. Losing data is worse than a missed publish. - name: Did the live outage state change? id: live + continue-on-error: true run: python3 scripts/live_changed.py - name: Commit if changed @@ -54,7 +58,7 @@ jobs: # historical rebuild; this only adds intraday publishes. Its `concurrency: # nightly` group serialises overlapping runs. - name: Publish now if the live state moved - if: steps.commit.outputs.committed == 'true' && steps.live.outputs.changed == 'true' + if: steps.commit.outputs.committed == 'true' && steps.live.outputs.changed != 'false' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: gh workflow run nightly.yml -R ${{ github.repository }} diff --git a/scripts/live_changed.py b/scripts/live_changed.py index ee1cb93..ca88f1e 100644 --- a/scripts/live_changed.py +++ b/scripts/live_changed.py @@ -55,9 +55,22 @@ def emit(changed: bool, reason: str) -> None: def live_hash(html: bytes) -> str: - from pipeline.parse import content_hash, parse_page + """Canonical hash of the live records, with "no outages" as a real state. + + `EmptyState` is NOT an error: parse.py raises it for CMTEB's routine + "nu exista inregistrari" banner, and backfill.py already handles it + separately from ParseFailure. Folding it into the generic failure path + would make every scrape look changed for as long as the city has no active + outages - publishing constantly at exactly the times there is nothing to + publish. Zero records is a legitimate hash, so two empty scrapes compare + equal while empty->outage and outage->empty both still trigger. + """ + from pipeline.parse import EmptyState, content_hash, parse_page - return content_hash(parse_page(html)) + try: + return content_hash(parse_page(html)) + except EmptyState: + return content_hash([]) def main() -> None: @@ -79,11 +92,19 @@ def main() -> None: try: before = live_hash(previous) after = live_hash(current) - except Exception as exc: # noqa: BLE001 - a parser change must not go silent + except Exception as exc: # noqa: BLE001 - a real ParseFailure must not go silent emit(True, f"parse failed ({type(exc).__name__}: {exc}) - failing open") emit(before != after, f"{before[:12]} -> {after[:12]}") if __name__ == "__main__": - main() + # This step runs BEFORE "Commit if changed". An uncaught exception here + # would fail the step and skip the commit, losing the scraped snapshot + # entirely - far worse than a missed publish. Nothing escapes. + try: + main() + except SystemExit: + raise + except BaseException as exc: # noqa: BLE001 + emit(True, f"detector crashed ({type(exc).__name__}: {exc}) - failing open") diff --git a/tests/test_live_changed.py b/tests/test_live_changed.py new file mode 100644 index 0000000..1aefbfd --- /dev/null +++ b/tests/test_live_changed.py @@ -0,0 +1,53 @@ +"""The publish trigger must not fire while the city simply has no outages. + +Regression for a bug caught in review on PR #4: `live_hash` wrapped parsing in a +broad `except Exception`, which swallowed `EmptyState`. `EmptyState` is not a +failure - parse.py raises it for CMTEB's routine "nu exista inregistrari" +banner - so the detector fell into its fail-open branch and reported +`changed=true` on EVERY scrape for as long as the outage list stayed empty, +publishing constantly at exactly the times there was nothing to publish. +""" + +import importlib.util +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent + +_spec = importlib.util.spec_from_file_location( + "live_changed", REPO / "scripts" / "live_changed.py" +) +live_changed = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(live_changed) +live_hash = live_changed.live_hash + +EMPTY = ( + b"
" + b"Nu exista inregistrari pentru criteriile selectate." + b"
" +) + + +def _real_page() -> bytes: + return (REPO / "data" / "functionare.html").read_bytes() + + +def test_two_empty_snapshots_hash_equal(): + """Nothing happening, twice, is not a change.""" + assert live_hash(EMPTY) == live_hash(EMPTY) + + +def test_empty_is_not_conflated_with_a_real_page(): + """empty -> outage must still publish.""" + assert live_hash(EMPTY) != live_hash(_real_page()) + + +def test_identical_real_pages_hash_equal(): + page = _real_page() + assert live_hash(page) == live_hash(page) + + +def test_markup_churn_does_not_change_the_hash(): + """Whitespace and attribute noise must not trigger a rebuild.""" + page = _real_page() + noisy = page.replace(b"