test: focus PR benchmarks on client_java and measure label lookups

.github/workflows/lint.yml

@@ -34,3 +34,9 @@ jobs:
           GITHUB_HEAD_REF: ${{ github.head_ref }}
           PR_HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
         run: mise run lint
+
+      - name: Test benchmark tooling
+        run: |
+          for test_file in .mise/tasks/test_*.py; do
+            python3 -B "$test_file"
+          done

.github/workflows/pr-benchmarks.yml

@@ -24,9 +24,9 @@ jobs:
       matrix:
         include:
           - topic: counter
-            pattern: CounterBenchmark
+            pattern: 'CounterBenchmark[.]prometheus.*'
           - topic: histogram
-            pattern: HistogramBenchmark
+            pattern: 'HistogramBenchmark[.]prometheus.*'
           - topic: exposition
             pattern: HistogramTextFormatBenchmark|TextFormatUtilBenchmark
     permissions:

.mise/tasks/generate_benchmark_summary.py

@@ -470,6 +470,60 @@ def generate_comparison_section(
     return md
 
 
+def allocation_score(result: dict) -> float | None:
+    """Read normalized GC allocation, accepting zero but not missing/invalid data."""
+    metric = result.get("secondaryMetrics", {}).get("gc.alloc.rate.norm", {})
+    if metric.get("scoreUnit") != "B/op":
+        return None
+    try:
+        score = float(metric.get("score"))
+    except (TypeError, ValueError):
+        return None
+    return score if math.isfinite(score) and score >= 0 else None
+
+
+def generate_allocation_section(results: list, baseline_results: list) -> list[str]:
+    """Show allocation separately from the throughput/latency regression verdict."""
+    baseline_by_name = {b.get("benchmark", ""): b for b in baseline_results}
+    rows = []
+    for head in sorted(results, key=lambda b: b.get("benchmark", "")):
+        name = head.get("benchmark", "")
+        baseline = baseline_by_name.get(name, {})
+        head_score = allocation_score(head)
+        base_score = allocation_score(baseline)
+        if head_score is None and base_score is None:
+            continue
+        head_text = "—" if head_score is None else f"{head_score:.3f}"
+        base_text = "—" if base_score is None else f"{base_score:.3f}"
+        change = "—"
+        if (
+            head_score is not None
+            and base_score is not None
+            and comparable_metadata(head, baseline)
+        ):
+            change = f"{head_score - base_score:+.3f}"
+        rows.append(
+            f"| {short_benchmark_name(name)} | {head_text} | {base_text} | {change} |"
+        )
+    if not rows:
+        return []
+    return [
+        "## Allocation per operation",
+        "",
+        "JMH GC profiler `gc.alloc.rate.norm`, in bytes per benchmark operation (lower is better).",
+        (
+            "Delta is PR minus base, shown only for matching benchmark configurations. "
+            "Values are descriptive, not statistical regression verdicts; "
+            "— means unavailable or not comparable. Each benchmark defines its own operation."
+        ),
+        "",
+        "| Benchmark | PR B/op | Base B/op | Delta B/op |",
+        "|:----------|--------:|----------:|-----------:|",
+        *rows,
+        "",
+    ]
+
+
 def generate_markdown(
     results: list,
     commit_sha: str,
@@ -488,7 +542,7 @@ def generate_markdown(
     first = results[0] if results else {}
     jdk_version = first.get("jdkVersion", "unknown")
     vm_name = first.get("vmName", "unknown")
-    threads = first.get("threads", "?")
+    threads = "/".join(sorted({str(b.get("threads", "?")) for b in results})) or "?"
     forks = first.get("forks", "?")
     warmup_iters = first.get("warmupIterations", "?")
     measure_iters = first.get("measurementIterations", "?")
@@ -619,9 +673,11 @@ def generate_markdown(
             )
         md.append("")
 
+    md.extend(generate_allocation_section(results, baseline_results or []))
+
     md.append("### Raw Results")
     md.append("")
-    md.append("```")
+    md.append("```text")
     md.append(
         f"{'Benchmark':<50} {'Mode':>6} {'Cnt':>4} {'Score':>14} {'Error':>12}  Units"
     )
@@ -680,8 +736,8 @@ def generate_markdown(
     md.append("| Benchmark | Description |")
     md.append("|:----------|:------------|")
     md.append(
-        "| **CounterBenchmark** | Counter increment performance: "
-        "Prometheus, OpenTelemetry, simpleclient, Codahale |"
+        "| **CounterBenchmark** | Counter updates and label-value lookup "
+        "(selected methods only) |"
     )
     md.append(
         "| **HistogramBenchmark** | Histogram observation performance "

.mise/tasks/test_generate-benchmark-summary.py

@@ -7,7 +7,9 @@
     sys.path.insert(0, here)
 
 from generate_benchmark_summary import (
+    allocation_score,
     comparison_status,
+    generate_allocation_section,
     generate_markdown,
 )
 
@@ -45,6 +47,63 @@ def result(
     }
 
 
+def with_allocation(benchmark, score, unit="B/op"):
+    benchmark["secondaryMetrics"] = {
+        "gc.alloc.rate.norm": {"score": score, "scoreUnit": unit}
+    }
+    return benchmark
+
+
+class TestAllocationSummary(unittest.TestCase):
+    def test_zero_is_valid_but_missing_invalid_and_wrong_units_are_not(self):
+        self.assertEqual(allocation_score(with_allocation(result(), 0)), 0)
+        self.assertIsNone(allocation_score(result()))
+        self.assertIsNone(allocation_score(with_allocation(result(), 1, "MB/sec")))
+        for score in (None, "NaN", float("inf"), -1, "not a number"):
+            self.assertIsNone(allocation_score(with_allocation(result(), score)))
+
+    def test_allocation_delta_is_absolute_and_lower_is_better(self):
+        head = with_allocation(result(), 0)
+        base = with_allocation(result(), 16)
+        section = "\n".join(generate_allocation_section([head], [base]))
+        self.assertIn("| 0.000 | 16.000 | -16.000 |", section)
+        self.assertIn("not statistical regression verdicts", section)
+
+    def test_missing_or_incomparable_base_has_no_delta(self):
+        head = with_allocation(result(), 16)
+        for base in ([], [result()], [with_allocation(result(threads=1), 32)]):
+            section = "\n".join(generate_allocation_section([head], base))
+            self.assertTrue(section.rstrip().endswith("| — |"))
+        self.assertIn(
+            "| 16.000 | — | — |",
+            "\n".join(generate_allocation_section([head], [])),
+        )
+
+    def test_missing_head_allocation_is_not_reported_as_zero(self):
+        section = "\n".join(
+            generate_allocation_section([result()], [with_allocation(result(), 16)])
+        )
+        self.assertIn("| — | 16.000 | — |", section)
+
+    def test_no_gc_data_omits_allocation_section(self):
+        self.assertEqual(generate_allocation_section([result()], []), [])
+
+    def test_markdown_includes_head_only_allocations_and_mixed_threads(self):
+        base = result()
+        head = with_allocation(result(name="CounterBenchmark.newLookup", threads=1), 24)
+        markdown = generate_markdown(
+            [base, head],
+            "head",
+            "prometheus/client_java",
+            [base],
+            "base",
+            "prometheus/client_java",
+        )
+        self.assertIn("## Allocation per operation", markdown)
+        self.assertIn("| CounterBenchmark.newLookup | 24.000 | — | — |", markdown)
+        self.assertIn("1/4 threads", markdown)
+
+
 class TestBenchmarkComparison(unittest.TestCase):
     def test_meaningful_improvement_requires_threshold_and_separation(self):
         self.assertEqual(

.mise/tasks/test_pr-benchmark-selection.py

@@ -0,0 +1,79 @@
+import re
+import unittest
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+WORKFLOW = (ROOT / ".github/workflows/pr-benchmarks.yml").read_text()
+PATTERNS = {
+    topic: pattern.strip("'\"")
+    for topic, pattern in re.findall(r"- topic: (\w+)\s+pattern: ([^\n]+)", WORKFLOW)
+}
+PACKAGE = "io.prometheus.metrics.benchmarks."
+
+
+class TestPrBenchmarkSelection(unittest.TestCase):
+    def test_only_client_java_counter_and_histogram_methods_are_selected(self):
+        for topic, class_name in (
+            ("counter", "CounterBenchmark"),
+            ("histogram", "HistogramBenchmark"),
+        ):
+            source = (
+                ROOT
+                / "benchmarks/src/main/java/io/prometheus/metrics/benchmarks"
+                / f"{class_name}.java"
+            ).read_text()
+            methods = re.findall(
+                r"@Benchmark\s+@Threads\(\d+\)\s+public \S+ (\w+)\(", source
+            )
+            self.assertTrue(methods)
+            for method in methods:
+                with self.subTest(method=method):
+                    selected = re.search(
+                        PATTERNS[topic], PACKAGE + class_name + "." + method
+                    )
+                    self.assertEqual(
+                        selected is not None, method.startswith("prometheus")
+                    )
+
+    def test_external_systems_including_future_bound_instruments_are_excluded(self):
+        for class_name in ("CounterBenchmark", "HistogramBenchmark"):
+            for method in (
+                "openTelemetryAdd",
+                "openTelemetryBoundInc",
+                "openTelemetryBoundClassic",
+                "codahaleIncNoLabels",
+                "simpleclientAdd",
+                "simpleclient",
+            ):
+                name = PACKAGE + class_name + "." + method
+                self.assertFalse(
+                    any(re.search(p, name) for p in PATTERNS.values()), name
+                )
+
+    def test_all_lookup_and_cached_variants_are_selected(self):
+        for method in (
+            "prometheusLabelValuesInc",
+            "prometheusLabelValuesIncSingleThread",
+            "prometheusCachedLabelValuesInc",
+            "prometheusCachedLabelValuesIncSingleThread",
+        ):
+            self.assertRegex(
+                PACKAGE + "CounterBenchmark." + method, PATTERNS["counter"]
+            )
+
+    def test_exposition_keeps_openmetrics_and_prometheus_formats(self):
+        for class_name in ("HistogramTextFormatBenchmark", "TextFormatUtilBenchmark"):
+            for method in ("openMetricsWriteToNull", "prometheusWriteToNull"):
+                self.assertRegex(
+                    PACKAGE + class_name + "." + method, PATTERNS["exposition"]
+                )
+
+    def test_base_and_head_use_the_same_selection(self):
+        self.assertIn("JMH_PATTERN: ${{ matrix.pattern }}", WORKFLOW)
+        self.assertEqual(
+            WORKFLOW.count("JMH_ARGS: -f 3 -wi 3 -i 5 ${{ env.JMH_PATTERN }}"), 2
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

benchmarks/README.md

@@ -55,6 +55,35 @@ JMH parameter reference:
 
 ## Results
 
+### Pull request benchmarks
+
+The `benchmark` label runs the PR head and base on the same runner for each topic.
+PR runs select only client_java counter and histogram methods (`prometheus*`), plus
+the exposition benchmarks. OpenTelemetry, Codahale, and legacy simpleclient methods
+are excluded from PR runs, but remain available in the full/local and nightly suites.
+OpenMetrics exposition remains included: it is a client_java output format.
+
+The `CounterBenchmark.prometheusLabelValuesInc*` methods repeatedly look up an
+existing label combination and increment it. The matching
+`prometheusCachedLabelValuesInc*` methods increment a cached data point instead.
+Both have one-thread and four-thread variants sharing a counter. Each invocation
+performs one metric update, so throughput and GC profiler allocation in B/op are
+per update, unlike older benchmarks that batch updates in a loop.
+
+Run just the lookup and cached variants with allocation profiling:
+
+```shell
+./mvnw -pl benchmarks -am package -DskipTests
+java -jar benchmarks/target/benchmarks.jar \
+  'CounterBenchmark[.]prometheus(Cached)?LabelValuesInc.*' \
+  -f 3 -wi 3 -i 5 -prof gc
+```
+
+The PR report shows allocation separately from throughput. Allocation deltas are
+descriptive, not statistical verdicts, and require matching base/head configurations.
+New benchmarks initially have head-only results. To evaluate a production change,
+run the same benchmark source and JVM configuration against both implementations.
+
 See Javadoc of the benchmark classes:
 
 - [CounterBenchmark](https://github.com/prometheus/client_java/blob/main/benchmarks/src/main/java/io/prometheus/metrics/benchmarks/CounterBenchmark.java)

benchmarks/src/main/java/io/prometheus/metrics/benchmarks/CounterBenchmark.java

@@ -55,6 +55,48 @@ public PrometheusCounter() {
     }
   }
 
+  /** Pre-populated labels so lookup benchmarks measure hits, not data point creation. */
+  @State(Scope.Benchmark)
+  public static class PrometheusLabelLookup {
+    final Counter counter =
+        Counter.builder().name("lookup_test").labelNames("path", "status").build();
+    final String path = "/";
+    final String status = "200";
+    final CounterDataPoint cached = counter.labelValues(path, status);
+  }
+
+  // Each invocation performs one increment, so GC profiler B/op is per metric update. Keep the
+  // same work and shared counter in the lookup and cached variants; only label resolution differs.
+  @Benchmark
+  @Threads(1)
+  public CounterDataPoint prometheusLabelValuesIncSingleThread(PrometheusLabelLookup state) {
+    CounterDataPoint dataPoint = state.counter.labelValues(state.path, state.status);
+    dataPoint.inc();
+    return dataPoint;
+  }
+
+  @Benchmark
+  @Threads(4)
+  public CounterDataPoint prometheusLabelValuesInc(PrometheusLabelLookup state) {
+    CounterDataPoint dataPoint = state.counter.labelValues(state.path, state.status);
+    dataPoint.inc();
+    return dataPoint;
+  }
+
+  @Benchmark
+  @Threads(1)
+  public CounterDataPoint prometheusCachedLabelValuesIncSingleThread(PrometheusLabelLookup state) {
+    state.cached.inc();
+    return state.cached;
+  }
+
+  @Benchmark
+  @Threads(4)
+  public CounterDataPoint prometheusCachedLabelValuesInc(PrometheusLabelLookup state) {
+    state.cached.inc();
+    return state.cached;
+  }
+
   @State(Scope.Benchmark)
   public static class SimpleclientCounter {