render.py

#!/usr/bin/env python3
"""Render charts, LIFETIME.md, data/lifetime.json and the README table from data/ledger.csv.

Output is a pure function of the ledger plus the current UTC month, so re-running
without ledger changes produces byte-identical files (no daily churn commits).
"""

from __future__ import annotations

import json
import re
import sys
from pathlib import Path

import ledger as L

ROOT = Path(__file__).parent
CHARTS = ROOT / "charts"
README = ROOT / "README.md"
LIFETIME_MD = ROOT / "LIFETIME.md"
LIFETIME_JSON = ROOT / "data" / "lifetime.json"
START, END = "<!-- stats:start -->", "<!-- stats:end -->"
REPO = "https://github.com/TimeToBuildBob/stats"

BLUE, ORANGE = "#2a78d6", "#eb6834"
INK, INK2, GRID = "#0b0b0b", "#52514e", "#e4e3df"

CAVEATS = [
    (
        "**Month-frozen.** A month is recomputed daily until 3 days after it ends, then frozen and never "
        "rewritten. Lifetime = sum of all months (frozen months + the open current month)."
    ),
    (
        "**Public and private are separate scopes.** Never add `brain_commits` to "
        "`commits_public_default_branch`, and never add private numbers to public ones."
    ),
    (
        "**Commit search sees default branches only.** Squash merges collapse a PR's commits into one, "
        "so `commits_public_default_branch` undercounts commits actually written."
    ),
    (
        "**Search counts reflect visibility at compute time.** A repo that goes private or is deleted "
        "later does not change frozen months."
    ),
    (
        "**Private metrics are lower bounds.** `brain_commits` starts 2025-08 (earlier Bob work was committed "
        "under Erik's identity); `sessions` starts 2026-06 (no reliable records before) and is approximate."
    ),
    'Months and date ranges are UTC. "As of" = the date the metric\'s value last changed.',
]


def fmt(n: int | None) -> str:
    return "—" if n is None else f"{n:,}"


def summarize(led: L.Ledger, now_month: str) -> dict:
    metrics = {}
    for metric, spec in L.METRICS.items():
        rows = led.months(metric)
        if not rows:
            continue
        frozen = [r for r in rows if r.frozen]
        cur = led.get(metric, now_month)
        metrics[metric] = {
            "label": spec["label"],
            "scope": spec["scope"],
            "definition": spec["definition"],
            "lifetime": sum(r.value for r in rows),
            "lifetime_frozen": sum(r.value for r in frozen),
            "first_month": rows[0].month,
            "frozen_through": frozen[-1].month if frozen else None,
            "current_month": now_month,
            "current_month_value": cur.value if cur else None,
            "as_of": max(r.computed_at for r in rows)[:10],
            "monthly": {r.month: r.value for r in rows},
            "open_months": [r.month for r in rows if not r.frozen],
        }
    as_of = max((m["as_of"] for m in metrics.values()), default=None)
    return {
        "as_of": as_of,
        "source": f"{REPO}/blob/master/data/ledger.csv",
        "method": "month-frozen ledger: months recompute until month_end+3d, then never change; lifetime = sum of months",
        "metrics": metrics,
    }


def readme_block(s: dict) -> str:
    lines = [
        START,
        "| Metric | Lifetime | Current month (partial) | As of | Scope |",
        "|---|---:|---:|---|---|",
    ]
    for name, m in s["metrics"].items():
        lines.append(
            f"| {m['label']} (`{name}`) | **{fmt(m['lifetime'])}** | "
            f"{fmt(m['current_month_value'])} ({m['current_month']}) | {m['as_of']} | {m['scope']} |"
        )
    lines += ["", "Definitions and caveats: [LIFETIME.md](LIFETIME.md). Public and private rows are never summed.", END]
    return "\n".join(lines)


def lifetime_md(s: dict) -> str:
    ms = s["metrics"]
    out = [
        "# Bob's lifetime numbers",
        "",
        (
            "> **Quote these; don't re-estimate.** Generated by `render.py` from [`data/ledger.csv`](data/ledger.csv), "
            "a month-frozen ledger. Do not edit by hand."
        ),
        "",
    ]
    if "prs_merged_public" in ms:
        m = ms["prs_merged_public"]
        out += [
            "How to cite:",
            "",
            f"> As of {m['as_of']}: {fmt(m['lifetime'])} merged public PRs ({REPO})",
            "",
        ]
    out += [
        "## Lifetime totals",
        "",
        "| Metric | Lifetime | Frozen total (through) | Current month (partial) | As of | Scope | Since |",
        "|---|---:|---:|---:|---|---|---|",
    ]
    for name, m in ms.items():
        out.append(
            f"| {m['label']} (`{name}`) | **{fmt(m['lifetime'])}** | {fmt(m['lifetime_frozen'])} "
            f"({m['frozen_through'] or '—'}) | {fmt(m['current_month_value'])} ({m['current_month']}) | "
            f"{m['as_of']} | {m['scope']} | {m['first_month']} |"
        )
    out += ["", "## Definitions", ""]
    for name, m in ms.items():
        out.append(f"- **`{name}`** ({m['scope']}, since {m['first_month']}): {m['definition']}")
    out += ["", "The exact query or command for every value is stored per row in the ledger's `query` column.", ""]
    out += ["## Caveats", ""] + [f"- {c}" for c in CAVEATS] + [""]

    months = sorted({mo for m in ms.values() for mo in m["monthly"]}, reverse=True)
    names = list(ms)
    out += [
        "## Monthly values",
        "",
        "`*` = open month (still recomputed daily); all other values are frozen.",
        "",
        "| Month | " + " | ".join(f"`{n}`" for n in names) + " |",
        "|---|" + "---:|" * len(names),
    ]
    for mo in months:
        cells = []
        for n in names:
            v = ms[n]["monthly"].get(mo)
            cells.append(fmt(v) + ("*" if mo in ms[n]["open_months"] else "") if v is not None else "")
        out.append(f"| {mo} | " + " | ".join(cells) + " |")
    return "\n".join(out) + "\n"


def update_readme(block: str, path: Path = README) -> None:
    text = path.read_text()
    pattern = re.compile(re.escape(START) + ".*?" + re.escape(END), re.DOTALL)
    if not pattern.search(text):
        raise SystemExit(f"README markers {START} / {END} not found")
    new = pattern.sub(lambda _: block, text)
    if new != text:
        path.write_text(new)


def write_if_changed(path: Path, content: str) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    if not path.exists() or path.read_text() != content:
        path.write_text(content)


# --- charts -----------------------------------------------------------------


def _plt():
    import matplotlib

    matplotlib.use("Agg")
    import matplotlib.pyplot as plt

    plt.rcParams.update(
        {
            "svg.hashsalt": "bob-stats",
            "svg.fonttype": "none",
            "font.family": "DejaVu Sans",
            "font.size": 9,
            "axes.edgecolor": GRID,
            "axes.labelcolor": INK2,
            "axes.titlecolor": INK,
            "xtick.color": INK2,
            "ytick.color": INK2,
            "axes.spines.top": False,
            "axes.spines.right": False,
            "axes.spines.left": False,
            "axes.grid": True,
            "axes.grid.axis": "y",
            "grid.color": GRID,
            "grid.linewidth": 0.6,
            "axes.axisbelow": True,
            "figure.facecolor": "white",
            "axes.facecolor": "white",
            "legend.frameon": False,
        }
    )
    return plt


def _save(plt, fig, name: str) -> None:
    CHARTS.mkdir(exist_ok=True)
    fig.savefig(CHARTS / name, format="svg", metadata={"Date": None}, bbox_inches="tight")
    plt.close(fig)


def _xticks(ax, months: list[str]) -> None:
    idx = [i for i, m in enumerate(months) if m.endswith(("-01", "-04", "-07", "-10"))]
    ax.set_xticks(idx, [months[i] for i in idx])
    ax.tick_params(axis="x", length=0)
    ax.set_xlim(-0.7, len(months) - 0.3)


def _title(ax, title: str, subtitle: str) -> None:
    ax.set_title(title, loc="left", fontsize=12, fontweight="bold", pad=22)
    ax.text(0, 1.03, subtitle, transform=ax.transAxes, color=INK2, fontsize=8.5, va="bottom")


def render_charts(s: dict) -> None:
    from matplotlib.ticker import FuncFormatter

    plt = _plt()
    ms = s["metrics"]
    thousands = FuncFormatter(lambda v, _: f"{v:,.0f}")

    if "prs_merged_public" in ms:
        m = ms["prs_merged_public"]
        months = list(m["monthly"])
        vals = list(m["monthly"].values())
        fig, ax = plt.subplots(figsize=(9, 3.4))
        bars = ax.bar(range(len(months)), vals, width=0.78, color=BLUE, edgecolor="white", linewidth=1)
        for b, mo in zip(bars, months):
            if mo in m["open_months"]:
                b.set_alpha(0.45)
                b.set_hatch("///")
        peak = max(range(len(vals)), key=vals.__getitem__)
        for i in {peak, len(vals) - 1}:
            label = f"{vals[i]:,}" + (" (open)" if months[i] in m["open_months"] else "")
            ax.annotate(label, (i, vals[i]), xytext=(0, 3), textcoords="offset points", ha="center", color=INK, fontsize=8)
        _xticks(ax, months)
        ax.yaxis.set_major_formatter(thousands)
        _title(ax, "Merged public PRs per month", f"TimeToBuildBob · hatched = open month, still recomputed · as of {m['as_of']}")
        _save(plt, fig, "prs_merged_monthly.svg")

    series = [(n, c) for n, c in (("prs_merged_public", BLUE), ("prs_opened_public", ORANGE)) if n in ms]
    if series:
        fig, ax = plt.subplots(figsize=(9, 3.4))
        months = list(ms[series[0][0]]["monthly"])
        for name, color in series:
            vals = [ms[name]["monthly"].get(mo, 0) for mo in months]
            cum, total = [], 0
            for v in vals:
                total += v
                cum.append(total)
            ax.plot(range(len(months)), cum, color=color, linewidth=2, label=ms[name]["label"])
            ax.annotate(
                f"{cum[-1]:,}",
                (len(months) - 1, cum[-1]),
                xytext=(6, 0),
                textcoords="offset points",
                va="center",
                color=INK,
                fontsize=8.5,
            )
        _xticks(ax, months)
        ax.set_xlim(-0.7, len(months) + 1.2)
        ax.yaxis.set_major_formatter(thousands)
        ax.legend(loc="upper left")
        _title(ax, "Lifetime public PRs (cumulative)", f"TimeToBuildBob · sum of monthly ledger rows · as of {s['as_of']}")
        _save(plt, fig, "lifetime_prs.svg")

    pub = ms.get("commits_public_default_branch")
    if pub:
        months = list(pub["monthly"])
        brain = ms.get("brain_commits")
        fig, (a1, a2) = plt.subplots(2, 1, figsize=(9, 5.4), sharex=True, gridspec_kw={"hspace": 0.55})
        panels = [
            (a1, pub, BLUE, "Public commits, default branches (GitHub commit search)"),
            (a2, brain, ORANGE, "Brain repo commits (private repo, Bob identities only)"),
        ]
        for ax, m, color, title in panels:
            ax.set_title(title, loc="left", fontsize=10, fontweight="bold")
            ax.yaxis.set_major_formatter(thousands)
            if not m:
                ax.text(0.5, 0.5, "no data yet", transform=ax.transAxes, ha="center", color=INK2)
                continue
            xs = [i for i, mo in enumerate(months) if mo in m["monthly"]]
            ys = [m["monthly"][months[i]] for i in xs]
            ax.plot(xs, ys, color=color, linewidth=2, marker="o", markersize=3.5)
            ax.set_ylim(bottom=0)
        _xticks(a2, months)
        fig.suptitle(
            "Commits per month — two separate scopes, never summed (last point = open month)", x=0.125, ha="left", fontsize=12, fontweight="bold", color=INK
        )
        _save(plt, fig, "commits.svg")


def main() -> int:
    led = L.Ledger.load()
    s = summarize(led, L.month_of(L.utcnow()))
    write_if_changed(LIFETIME_JSON, json.dumps(s, indent=2, sort_keys=False) + "\n")
    write_if_changed(LIFETIME_MD, lifetime_md(s))
    update_readme(readme_block(s))
    render_charts(s)
    return 0


if __name__ == "__main__":
    sys.exit(main())