diff --git a/.github/workflows/scrape.yml b/.github/workflows/scrape.yml index 54f286c..0f6e1f7 100644 --- a/.github/workflows/scrape.yml +++ b/.github/workflows/scrape.yml @@ -7,6 +7,7 @@ on: permissions: contents: write + actions: write # dispatch Nightly publish when the live state moves concurrency: group: scrape @@ -22,15 +23,42 @@ jobs: - name: Scrape run: python3 scraper/scrape.py + # Must run BEFORE the commit: it diffs the working tree against HEAD. + # continue-on-error is the structural guarantee behind the script's own + # fail-open: even a hard crash here must never stop the commit below and + # lose the scraped snapshot. Losing data is worse than a missed publish. + - name: Did the live outage state change? + id: live + continue-on-error: true + run: python3 scripts/live_changed.py + - name: Commit if changed + id: commit run: | git config user.name "github-actions[bot]" git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add data/ if git diff --cached --quiet; then echo "no changes" + echo "committed=false" >> "$GITHUB_OUTPUT" else git commit -m "snapshot $(date -u +'%Y-%m-%dT%H:%M:%SZ')" git pull --rebase origin main git push + echo "committed=true" >> "$GITHUB_OUTPUT" fi + + # The site is static and was rebuilt only at 03:10, so a new outage - or a + # revised restore estimate - could sit invisible for up to 24h. A visitor + # reported exactly that on 2026-08-31. Publishing here closes that gap to + # roughly the scrape interval. + # + # Gated on the live-state hash, not the raw bytes, so markup churn cannot + # trigger a rebuild. Nightly publish keeps its own 03:10 schedule for the + # historical rebuild; this only adds intraday publishes. Its `concurrency: + # nightly` group serialises overlapping runs. + - name: Publish now if the live state moved + if: steps.commit.outputs.committed == 'true' && steps.live.outputs.changed != 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run nightly.yml -R ${{ github.repository }} diff --git a/scripts/live_changed.py b/scripts/live_changed.py new file mode 100644 index 0000000..ca88f1e --- /dev/null +++ b/scripts/live_changed.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Did this scrape change the LIVE outage state, or only the raw bytes? + +The site is rebuilt nightly, so a newly announced outage - or a revised +restore estimate - could sit invisible for up to 24h. A visitor said exactly +that on 2026-08-31: "Nu este actualizat la data de 31.08.2026". The scrape +itself runs far more often, so the data is already there; only the publish +lags. + +Publishing on every byte change would be wasteful and, on Vercel Hobby (one +concurrent build, ~6 min each), self-queueing. So publish only when the state +a reader would actually see has changed. + +"Live state" = the canonical hash over page A's parsed records +(pipeline.parse.content_hash). That hash is built from sorted key tuples, so: + + - a revised `remediere_raw` (the estimated restore time) DOES trigger, which + matters because that is the number people are asking for; + - reordered rows, whitespace and markup churn do NOT; + - the affected-street list is NOT in the key tuple, so a change only to which + streets a PT lists will not trigger a publish. Accepted: the PT identity, + cause, restore estimate and block count are what the live band renders. + +Page A only ever lists CURRENTLY ACTIVE outages - records vanish when resolved +- so a change to it is by definition a change to the live state. + +Compares the working tree against HEAD, so it must run BEFORE the commit step. + +Exits 0 always. Prints `changed=true|false`, and writes the same to +$GITHUB_OUTPUT when present. Any failure is reported as changed=true: a broken +detector should degrade to publishing too often, never to going silently stale. +""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +PAGE_A = REPO / "data" / "functionare.html" + +sys.path.insert(0, str(REPO)) + + +def emit(changed: bool, reason: str) -> None: + value = "true" if changed else "false" + print(f"changed={value} ({reason})") + out = os.environ.get("GITHUB_OUTPUT") + if out: + with open(out, "a", encoding="utf-8") as fh: + fh.write(f"changed={value}\n") + sys.exit(0) + + +def live_hash(html: bytes) -> str: + """Canonical hash of the live records, with "no outages" as a real state. + + `EmptyState` is NOT an error: parse.py raises it for CMTEB's routine + "nu exista inregistrari" banner, and backfill.py already handles it + separately from ParseFailure. Folding it into the generic failure path + would make every scrape look changed for as long as the city has no active + outages - publishing constantly at exactly the times there is nothing to + publish. Zero records is a legitimate hash, so two empty scrapes compare + equal while empty->outage and outage->empty both still trigger. + """ + from pipeline.parse import EmptyState, content_hash, parse_page + + try: + return content_hash(parse_page(html)) + except EmptyState: + return content_hash([]) + + +def main() -> None: + try: + current = PAGE_A.read_bytes() + except OSError as exc: + emit(True, f"cannot read working-tree page A ({exc}) - failing open") + + try: + previous = subprocess.run( + ["git", "show", f"HEAD:data/{PAGE_A.name}"], + cwd=REPO, + capture_output=True, + check=True, + ).stdout + except subprocess.CalledProcessError: + emit(True, "no previous page A at HEAD - first run, failing open") + + try: + before = live_hash(previous) + after = live_hash(current) + except Exception as exc: # noqa: BLE001 - a real ParseFailure must not go silent + emit(True, f"parse failed ({type(exc).__name__}: {exc}) - failing open") + + emit(before != after, f"{before[:12]} -> {after[:12]}") + + +if __name__ == "__main__": + # This step runs BEFORE "Commit if changed". An uncaught exception here + # would fail the step and skip the commit, losing the scraped snapshot + # entirely - far worse than a missed publish. Nothing escapes. + try: + main() + except SystemExit: + raise + except BaseException as exc: # noqa: BLE001 + emit(True, f"detector crashed ({type(exc).__name__}: {exc}) - failing open") diff --git a/tests/test_live_changed.py b/tests/test_live_changed.py new file mode 100644 index 0000000..1aefbfd --- /dev/null +++ b/tests/test_live_changed.py @@ -0,0 +1,53 @@ +"""The publish trigger must not fire while the city simply has no outages. + +Regression for a bug caught in review on PR #4: `live_hash` wrapped parsing in a +broad `except Exception`, which swallowed `EmptyState`. `EmptyState` is not a +failure - parse.py raises it for CMTEB's routine "nu exista inregistrari" +banner - so the detector fell into its fail-open branch and reported +`changed=true` on EVERY scrape for as long as the outage list stayed empty, +publishing constantly at exactly the times there was nothing to publish. +""" + +import importlib.util +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent + +_spec = importlib.util.spec_from_file_location( + "live_changed", REPO / "scripts" / "live_changed.py" +) +live_changed = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(live_changed) +live_hash = live_changed.live_hash + +EMPTY = ( + b"
" + b"Nu exista inregistrari pentru criteriile selectate." + b"
" +) + + +def _real_page() -> bytes: + return (REPO / "data" / "functionare.html").read_bytes() + + +def test_two_empty_snapshots_hash_equal(): + """Nothing happening, twice, is not a change.""" + assert live_hash(EMPTY) == live_hash(EMPTY) + + +def test_empty_is_not_conflated_with_a_real_page(): + """empty -> outage must still publish.""" + assert live_hash(EMPTY) != live_hash(_real_page()) + + +def test_identical_real_pages_hash_equal(): + page = _real_page() + assert live_hash(page) == live_hash(page) + + +def test_markup_churn_does_not_change_the_hash(): + """Whitespace and attribute noise must not trigger a rebuild.""" + page = _real_page() + noisy = page.replace(b"