From 7eaaae580c2518bd871b153a64c3c77682d4c8ef Mon Sep 17 00:00:00 2001 From: Florent Tapponnier <160007691+Flotapponnier@users.noreply.github.com> Date: Wed, 9 Sep 2026 16:46:50 +0200 Subject: [PATCH] fix(scoring): scope the value to contested chains, opt-in per bench Replaces the global chain-wins sort from #2280, which fixed the ordering but broke the reading of it. Ranking on a key the reader cannot see in the column produced tables that no longer descend: 79.88% at rank 4 on 008, and a 74 ms leader at rank 3 on rpc-capabilities where lower is better. The defect was never the sort key. It is that the aggregate includes chains where a provider had no competitor, so the fix belongs on the value: with `score_scope: contested_chains` a bench is scored only on chains carrying at least two measured providers, and a provider with none of those leaves the ranked field (still visible on its own chain tab). One quantity on screen, ordering follows from it. Opt-in per bench and named for the property, not the bench, so any bench whose chain set contains uncontested chains is a candidate and the rule is readable in the public YAML. Enabled on wallet-labels-coverage only: stellar, xrp and bitcoin carry one measured provider each there. load.ts already fetched every provider's per-chain value inside its per-chain loop and discarded all but the leader; it now keeps them. Four other live benches qualify and are deliberately left untouched pending review: rpc-capabilities, token-quote-coverage, bridge-fee, perp-liq-rate. Residual limitation documented in methodology: providers are still averaged over the different subsets of contested chains they cover. Co-Authored-By: Claude Opus 5 (1M context) --- benchmarks/wallet-labels-coverage.yml | 5 ++ src/app/methodology/page.tsx | 4 +- src/lib/citation.test.ts | 70 +----------------- src/lib/citation.ts | 58 +++------------ src/lib/materialize/contested-scope.test.ts | 82 +++++++++++++++++++++ src/lib/materialize/load.ts | 68 ++++++++++++++++- src/lib/providers.ts | 18 +---- src/lib/spec-schema.ts | 21 ++++++ 8 files changed, 190 insertions(+), 136 deletions(-) create mode 100644 src/lib/materialize/contested-scope.test.ts diff --git a/benchmarks/wallet-labels-coverage.yml b/benchmarks/wallet-labels-coverage.yml index b3a02038e..a428f7842 100644 --- a/benchmarks/wallet-labels-coverage.yml +++ b/benchmarks/wallet-labels-coverage.yml @@ -151,6 +151,11 @@ prometheus: # Default is `eoa` because the contract tab is trivially easy for explorers # and saturates near 100%; the EOA tab is where curated entity coverage # actually differentiates providers. +# Scored on contested chains only: stellar, xrp and bitcoin carry a single +# measured provider each, and a cross-chain average that counts them puts a +# provider first for a chain nobody else reported. See spec-schema.ts. +score_scope: contested_chains + dimensions: kind: - { value: eoa, label: EOA } diff --git a/src/app/methodology/page.tsx b/src/app/methodology/page.tsx index 46acf0fe0..665f47df1 100644 --- a/src/app/methodology/page.tsx +++ b/src/app/methodology/page.tsx @@ -61,8 +61,8 @@ const CONVENTIONS = [ body: "Share of requests returning a usable result within the published timeout. The only metric that includes failures.", }, { - term: "Ranking on multi-chain benchmarks", - body: "When a benchmark measures several chains, providers are ranked first by the number of chains they lead, and only then by their cross-chain figure. A chain counts toward that total only when at least two providers reported data on it. A cross-chain average is a mix rather than a comparison, so ranking on it alone let a provider measured on one uncontested chain finish above a provider that led several contested ones.", + term: "Contested-chain scoring", + body: "A cross-chain average is a mix rather than a comparison: it credits a provider for the chains it happens to be measured on. Benchmarks that declare score_scope: contested_chains in their spec are therefore scored only on chains where at least two providers reported data, and a provider with no such chain is left out of the ranking while staying visible on its own chain tab. The figure shown is the unweighted mean across those chains, so each chain counts once regardless of sample count. This narrows the number rather than the sort order, so the published value is always the one the ranking follows. It does not equalise chain mix entirely: providers are still averaged over the different subsets of contested chains they cover.", }, { term: "Region normalisation", diff --git a/src/lib/citation.test.ts b/src/lib/citation.test.ts index 0e51f3833..376995f8c 100644 --- a/src/lib/citation.test.ts +++ b/src/lib/citation.test.ts @@ -1,5 +1,5 @@ import { describe, expect, test } from "bun:test"; -import { chainWins, leader, fieldValue, rankedCandidates } from "./citation"; +import { leader, fieldValue, rankedCandidates } from "./citation"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; function r( @@ -111,71 +111,3 @@ describe("citation reliability threshold", () => { expect(top?.value).toBe(ranks[0].ms.p50); }); }); - -describe("contested-chain wins drive the ranking", () => { - // Bench 008 as it actually shipped: XRPScan and StellarExpert sat 1st - // and 2nd on the cross-chain average, each measured on a single chain - // nobody else reported, while Serialized led four contested ones. - const b008 = (): Benchmark => ({ - ...bench([ - r("stellarexpert", "StellarExpert", 80.08), - r("xrpscan", "XRPScan", 79.84), - r("serialized", "Serialized", 76.98), - r("mobula", "Mobula", 46.75), - ]), - higherIsBetter: true, - bestPerChain: { - ethereum: r("serialized", "Serialized", 96.84), - base: r("serialized", "Serialized", 76.36), - solana: r("serialized", "Serialized", 57.79), - arbitrum: r("serialized", "Serialized", 70.0), - bnb: r("mobula", "Mobula", 79.78), - xrp: r("xrpscan", "XRPScan", 79.84), - stellar: r("stellarexpert", "StellarExpert", 80.08), - }, - providersPerChain: { - ethereum: ["serialized", "mobula", "oli", "blockscout"], - base: ["serialized", "mobula", "oli", "blockscout"], - solana: ["serialized", "mobula"], - arbitrum: ["serialized", "mobula", "oli"], - bnb: ["mobula", "serialized", "oli"], - xrp: ["xrpscan"], - stellar: ["stellarexpert"], - }, - }); - - test("the provider leading the most contested chains ranks first", () => { - expect(rankedCandidates(b008()).map((r) => r.slug)).toEqual([ - "serialized", - "mobula", - "stellarexpert", - "xrpscan", - ]); - expect(leader(b008())?.slug).toBe("serialized"); - }); - - test("a chain with one measured provider awards no win", () => { - const wins = chainWins(b008()); - expect(wins?.get("xrpscan")).toBeUndefined(); - expect(wins?.get("stellarexpert")).toBeUndefined(); - expect(wins?.get("serialized")).toBe(4); - expect(wins?.get("mobula")).toBe(1); - }); - - test("providers with equal wins fall back to the aggregate value", () => { - const b = b008(); - // Strip every contested win so the whole field ties at zero. - b.providersPerChain = { ethereum: ["serialized"], bnb: ["mobula"] }; - expect(rankedCandidates(b).map((r) => r.slug)).toEqual([ - "stellarexpert", - "xrpscan", - "serialized", - "mobula", - ]); - }); - - test("a bench without per-chain stashes ranks by value alone", () => { - const b = { ...b008(), bestPerChain: undefined, providersPerChain: undefined }; - expect(rankedCandidates(b).map((r) => r.slug)[0]).toBe("stellarexpert"); - }); -}); diff --git a/src/lib/citation.ts b/src/lib/citation.ts index 17cb04a05..cc67f1c27 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -48,60 +48,22 @@ export function citationCandidates(b: Benchmark): ProviderResult[] { return pool.filter((r) => r.dataConfidence !== "insufficient"); } -/** - * Chains each provider leads, counting **contested** chains only: a chain - * where at least two providers reported data. - * - * The exclusion is the whole point. On a chain-dimensioned bench the - * cross-chain aggregate is a mix, not a comparison, and a provider - * measured on exactly one easy chain with no competitor on it can top the - * board without ever beating anyone. Bench 008 shipped that way: - * StellarExpert and XRPScan sat 1st and 2nd, each measured on a single - * uncontested chain, above Serialized which led four contested ones. Four - * other live benches had the same shape, `rpc-capabilities` worst of all - * (Binance 1st on one chain while PublicNode led six). - * - * Returns null when the bench cannot support the count — no chain - * dimensions, or the per-chain stashes absent. Those stashes are only - * populated on the unfiltered view (see materialize/load.ts), which is - * also the guard that keeps a chain-filtered variant from being ranked by - * cross-chain wins: on `?chain=bnb` there is nothing to count. - */ -export function chainWins(b: Benchmark): Map | null { - const best = b.bestPerChain; - const present = b.providersPerChain; - if (!best || !present) return null; - const wins = new Map(); - for (const [chain, chainLeader] of Object.entries(best)) { - if ((present[chain]?.length ?? 0) < 2) continue; - const slug = chainLeader.slug.toLowerCase(); - wins.set(slug, (wins.get(slug) ?? 0) + 1); - } - return wins.size > 0 ? wins : null; -} - /** Sorted candidate pool for the machine-readable `rankings` array on * `/api/stat`, MCP, llm-context and any downstream that ranks the * full field. Applies the same reliability + insufficient-sample * filters as `leader()` so a document that names X as leader ranks X - * first in its own list. + * first in its own list. Sort direction honors the bench's + * `higherIsBetter` flag. * - * On a bench that can count contested-chain wins, those wins are the - * primary key and the aggregate value only breaks ties: head-to-head - * record first, chain-mix average second. Everywhere else (no chain - * dimensions, filtered variants) it is the aggregate value alone, sorted - * in the direction the bench's `higherIsBetter` flag asks for. */ + * Ranks on the value alone, deliberately. A bench whose cross-chain + * aggregate would otherwise reward an uncontested chain fixes that by + * declaring `score_scope: contested_chains` in its spec, which narrows + * the value itself (see materialize/load.ts) rather than sorting on a + * key the reader cannot see in the column. */ export function rankedCandidates(b: Benchmark): ProviderResult[] { - const byValue = (a: ProviderResult, c: ProviderResult) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; - const pool = [...citationCandidates(b)]; - const wins = chainWins(b); - if (!wins) return pool.sort(byValue); - return pool.sort((a, c) => { - const delta = - (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); - return delta !== 0 ? delta : byValue(a, c); - }); + return [...citationCandidates(b)].sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ); } /** Timestamp of the last real measurement, or null when the bench has diff --git a/src/lib/materialize/contested-scope.test.ts b/src/lib/materialize/contested-scope.test.ts new file mode 100644 index 000000000..8d8475ed6 --- /dev/null +++ b/src/lib/materialize/contested-scope.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, test } from "bun:test"; +import { applyContestedChainScope } from "./load"; +import type { ProviderResult } from "@/types/benchmark"; + +function r(slug: string, p50: number): ProviderResult { + return { + slug, + name: slug, + ms: { p50, p90: p50, p99: p50, mean: p50 }, + successRate: 100, + availability: "live", + }; +} + +// Bench 008 as it shipped: stellar, xrp and bitcoin carry one measured +// provider each, so the cross-chain average put two single-chain providers +// first and second above one that led four contested chains. +function fixture() { + const results = [ + r("stellarexpert", 80.08), + r("xrpscan", 79.84), + r("serialized", 76.98), + r("mobula", 46.75), + ]; + const providersPerChain: Record = { + ethereum: ["serialized", "mobula"], + base: ["serialized", "mobula"], + solana: ["serialized", "mobula"], + stellar: ["stellarexpert"], + xrp: ["xrpscan"], + }; + const valuesByChain: Record> = { + ethereum: { serialized: 96.84, mobula: 50.0 }, + base: { serialized: 76.36, mobula: 40.0 }, + solana: { serialized: 57.79, mobula: 30.0 }, + stellar: { stellarexpert: 80.08 }, + xrp: { xrpscan: 79.84 }, + }; + return { results, providersPerChain, valuesByChain }; +} + +describe("contested-chain scoring", () => { + test("the value becomes the mean over contested chains", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + applyContestedChainScope(results, providersPerChain, valuesByChain); + const s = results.find((x) => x.slug === "serialized")!; + // (96.84 + 76.36 + 57.79) / 3 + expect(s.ms.p50).toBeCloseTo(76.9967, 3); + expect(s.ms.mean).toBeCloseTo(76.9967, 3); + expect(results.find((x) => x.slug === "mobula")!.ms.p50).toBeCloseTo(40, 6); + }); + + test("a provider with no contested chain drops out of the ranked field", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + applyContestedChainScope(results, providersPerChain, valuesByChain); + expect(results.find((x) => x.slug === "stellarexpert")!.availability).toBe( + "unavailable", + ); + expect(results.find((x) => x.slug === "xrpscan")!.availability).toBe( + "unavailable", + ); + expect(results.find((x) => x.slug === "serialized")!.availability).toBe("live"); + }); + + test("no contested chain at all leaves every value untouched", () => { + const { results, valuesByChain } = fixture(); + applyContestedChainScope( + results, + { stellar: ["stellarexpert"], xrp: ["xrpscan"] }, + valuesByChain, + ); + expect(results.find((x) => x.slug === "serialized")!.ms.p50).toBe(76.98); + expect(results.every((x) => x.availability === "live")).toBe(true); + }); + + test("an already unavailable provider is left alone", () => { + const { results, providersPerChain, valuesByChain } = fixture(); + results[0].availability = "unavailable"; + applyContestedChainScope(results, providersPerChain, valuesByChain); + expect(results[0].ms.p50).toBe(80.08); + }); +}); diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index 760c93ce3..d1fd65167 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -286,15 +286,26 @@ export async function specToBenchmark( const chainSpec = applyDimensionsToSpec(spec, { chain }); const chainLive = await tryLoadLive(chainSpec, true); if (!chainLive) { - return [chain, undefined, undefined, [] as string[]] as const; + return [ + chain, + undefined, + undefined, + [] as string[], + {} as Record, + ] as const; } for (const r of chainLive.results) { if (!r.unresponsive) r.availability = "live"; } const liveForChain = liveProviderResults(chainLive.results); const slugs = liveForChain.map((r) => r.slug); + // Per-provider value on this chain. Kept (not just the leader) + // so `score_scope: contested_chains` can rebuild a value from + // the chains a provider was actually compared on. + const values: Record = {}; + for (const r of liveForChain) values[r.slug.toLowerCase()] = r.ms.p50; if (liveForChain.length === 0) { - return [chain, undefined, undefined, slugs] as const; + return [chain, undefined, undefined, slugs, values] as const; } const sorted = [...liveForChain].sort((a, b) => spec.higher_is_better ? b.ms.p50 - a.ms.p50 : a.ms.p50 - b.ms.p50, @@ -304,20 +315,27 @@ export async function specToBenchmark( sorted[0], sorted[sorted.length - 1], slugs, + values, ] as const; }), ); const bests: Record = {}; const worsts: Record = {}; const providers: Record = {}; - for (const [chain, leader, trailer, slugs] of perChainEntries) { + const valuesByChain: Record> = {}; + for (const [chain, leader, trailer, slugs, values] of perChainEntries) { if (leader) bests[chain] = leader; if (trailer) worsts[chain] = trailer; if (slugs.length > 0) providers[chain] = slugs; + valuesByChain[chain] = values; } if (Object.keys(bests).length > 0) bestPerChain = bests; if (Object.keys(worsts).length > 0) worstPerChain = worsts; if (Object.keys(providers).length > 0) providersPerChain = providers; + + if (spec.score_scope === "contested_chains") { + applyContestedChainScope(live.results, providers, valuesByChain); + } } // Exact per-cell rankings (chain × region) from the spec's single @@ -381,6 +399,50 @@ export async function specToBenchmark( return draftBenchmark(spec, editorial); } +/** + * Rewrite each provider's headline value as its mean over the **contested** + * chains of the bench: those where at least two providers reported data. + * A provider with no contested chain is marked unavailable, which takes it + * out of `liveResults` and therefore out of every ranked surface, while + * leaving it visible on its own chain tab. + * + * Why the value and not the sort order: a cross-chain aggregate is a mix + * rather than a comparison, so it credits a provider for the chains it + * happens to be measured on. Narrowing the number keeps one quantity on + * screen and the ordering follows from it. Ranking on a separate key while + * still displaying the wide aggregate produced a column that did not + * descend (80% shown at rank 4, a 74 ms leader shown at rank 3). + * + * The residual limitation, stated in the methodology: providers are still + * averaged over different subsets of the contested chains, since they do + * not all cover the same ones. It removes the uncontested win, not every + * difference in chain mix. An unweighted mean is used so a chain counts + * once regardless of how many samples it carries. + */ +export function applyContestedChainScope( + results: ProviderResult[], + providersPerChain: Record, + valuesByChain: Record>, +): void { + const contested = Object.keys(providersPerChain).filter( + (chain) => (providersPerChain[chain]?.length ?? 0) >= 2, + ); + if (contested.length === 0) return; + for (const r of results) { + if (r.availability === "unavailable") continue; + const slug = r.slug.toLowerCase(); + const vals = contested + .map((chain) => valuesByChain[chain]?.[slug]) + .filter((v): v is number => typeof v === "number" && v > 0); + if (vals.length === 0) { + r.availability = "unavailable"; + continue; + } + const mean = vals.reduce((a, b) => a + b, 0) / vals.length; + r.ms = { p50: mean, p90: mean, p99: mean, mean }; + } +} + function activeFilterLabels(opts: BenchmarkFilters): Record { const out: Record = {}; for (const [k, v] of Object.entries(opts)) { diff --git a/src/lib/providers.ts b/src/lib/providers.ts index b1dbe4578..0301b7333 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -12,7 +12,7 @@ import { cache } from "react"; import { unstable_cache } from "next/cache"; import { getBenchmarksSafe } from "@/data/benchmarks"; import { liveResults } from "@/lib/provider-filters"; -import { chainWins, citationCandidates } from "@/lib/citation"; +import { citationCandidates } from "@/lib/citation"; import { readBestPerChain } from "@/lib/per-chain-contract"; import type { Benchmark, ProviderResult } from "@/types/benchmark"; @@ -240,19 +240,9 @@ function rankProviders(b: Benchmark): ProviderResult[] { // a best-of-bad-options ranking. const pool = citationCandidates(b); const live = pool.length > 0 ? pool : liveResults(b.results); - // Same ordering as the bench page: contested-chain wins first, aggregate - // value as the tiebreak (see rankedCandidates). Sorting these two - // surfaces differently is what let /products show "#3 of 8" beside five - // chain-leadership chips on the same bench. - const byValue = (a: ProviderResult, c: ProviderResult) => - b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50; - const wins = chainWins(b); - if (!wins) return [...live].sort(byValue); - return [...live].sort((a, c) => { - const delta = - (wins.get(c.slug.toLowerCase()) ?? 0) - (wins.get(a.slug.toLowerCase()) ?? 0); - return delta !== 0 ? delta : byValue(a, c); - }); + return [...live].sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ); } /** diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index 92306c02b..c795fe858 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -393,6 +393,27 @@ export const SpecSchema = z * PromQL label-value alphabet as dimension values. * First use: keyed-rpc-robinhood pins region=sgp so the default view * is the Singapore probe while the US-East tab stays available. */ + /** + * How the headline value is scoped on a bench that declares + * `dimensions.chain`. + * + * Default (absent) keeps the cross-chain aggregate the Prom queries + * return. `contested_chains` narrows it to the chains where at least + * two providers reported data, and drops a provider that has none of + * those from the ranked field. + * + * The reason is that a cross-chain aggregate is a mix rather than a + * comparison: it rewards a provider for the chains it happens to be + * measured on. On wallet-labels-coverage that put StellarExpert and + * XRPScan first and second, each scored on a single chain no other + * provider reported, above a provider that led four contested ones. + * + * Opt-in per bench, and it describes a property rather than naming a + * bench: any bench whose chain set contains uncontested chains is a + * candidate. Excluded providers stay visible on their own chain tab. + */ + score_scope: z.enum(["contested_chains"]).optional(), + aggregate_filters: z .object({ chain: z.string().regex(/^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/).optional(),