diff --git a/.github/workflows/benchmark-policy.yml b/.github/workflows/benchmark-policy.yml index 7d472b7..82a3abc 100644 --- a/.github/workflows/benchmark-policy.yml +++ b/.github/workflows/benchmark-policy.yml @@ -11,16 +11,12 @@ permissions: issues: read concurrency: - group: >- - benchmark-policy-${{ github.event.pull_request.number }}-${{ ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'benchmarks: skip' || github.event.action == 'edited' && !github.event.changes.base) && github.run_id || 'current' }} + group: benchmark-policy-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: policy: - name: >- - ${{ ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'benchmarks: skip' || github.event.action == 'edited' && !github.event.changes.base) && 'Ignored policy event' || 'Benchmark Policy' }} - if: >- - ${{ (!(github.event.action == 'labeled' || github.event.action == 'unlabeled') || github.event.label.name == 'benchmarks: skip') && (github.event.action != 'edited' || github.event.changes.base) }} + name: Benchmark Policy runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 00244db..7575ec8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -13,8 +13,7 @@ permissions: issues: read concurrency: - group: >- - benchmark-${{ github.event.pull_request.number || github.ref }}-${{ ((github.event.action == 'labeled' || github.event.action == 'unlabeled') && github.event.label.name != 'benchmarks: skip' || github.event.action == 'edited' && !github.event.changes.base) && github.run_id || 'current' }} + group: benchmark-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true env: @@ -22,8 +21,7 @@ env: jobs: eligibility: - if: >- - github.event_name == 'pull_request' && (!(github.event.action == 'labeled' || github.event.action == 'unlabeled') || github.event.label.name == 'benchmarks: skip') && (github.event.action != 'edited' || github.event.changes.base) + if: github.event_name == 'pull_request' runs-on: ubuntu-latest timeout-minutes: 10 outputs: @@ -256,8 +254,8 @@ jobs: # Provide one required check that also reports failed, skipped, or cancelled stages. benchmark-status: - name: ${{ needs.eligibility.result != 'skipped' && 'Benchmark Status' || 'Ignored benchmark event' }} - if: ${{ always() && github.event_name == 'pull_request' && needs.eligibility.result != 'skipped' }} + name: Benchmark Status + if: ${{ always() && github.event_name == 'pull_request' }} needs: [eligibility, configure, producer, prepare, measure, collect] runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/docs/README.md b/docs/README.md index f23d62b..b1ffccc 100644 --- a/docs/README.md +++ b/docs/README.md @@ -96,6 +96,7 @@ The preview URL then returns 404. ## CI Rules * `benchmarks: skip` skips measurement and result publication. Dependabot applies it automatically. +* Required checks evaluate the current PR policy on every subscribed event. Label and description edits restart the benchmark workflow. Measurements run unless `benchmarks: skip` is present. * Each of the 36 measurement jobs runs all three strategies sequentially with a fresh server process for each. Retrying a job repeats its three strategies. * Preview builds run independently of benchmarks. Draft and Dependabot PRs have no preview. * Production uses published history. Previews include matching pending results when available. diff --git a/src/main.ts b/src/main.ts index 6f98ed2..391f50e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -4,7 +4,7 @@ import '@fontsource/ibm-plex-sans/600.css' import '@fontsource/ibm-plex-mono/400.css' import './style.css' import { Chart, registerables } from 'chart.js' -import { createIcons, Download, Link, ArrowUpRight, AlertTriangle, RotateCcw, ChevronRight } from 'lucide' +import { createIcons, Download, Link, ArrowUpRight, AlertTriangle, RotateCcw, ChevronRight, ArrowUp, ArrowDown, ArrowUpDown } from 'lucide' import { loadDataset } from './data' import { catalog } from '../shared/catalog.ts' import { runSelectionSchema } from '../shared/results.ts' @@ -15,7 +15,7 @@ Chart.defaults.font.family = 'IBM Plex Sans' Chart.defaults.color = '#626b66' const palette = ['#65736a', '#93663c', '#2f66cb', '#e06b2f', '#968526', '#148c87', '#a34764', '#7159b5', '#438333', '#bf4961', '#427992'] -const icons = { Download, Link, ArrowUpRight, AlertTriangle, RotateCcw, ChevronRight } +const icons = { Download, Link, ArrowUpRight, AlertTriangle, RotateCcw, ChevronRight, ArrowUp, ArrowDown, ArrowUpDown } const strategyDescriptions: Record = { reload: 'Restore the sandbox for each request.', reuse: 'Keep the sandbox as-is across requests.', @@ -56,14 +56,49 @@ function renderDashboard(data: Dataset) { let range = params.get('range') === '7' ? 7 : 14 const selectedRuntimes = new Set((params.has('runtimes') ? params.get('runtimes')!.split(',') : defaultRuntimes).filter(id => data.runtimes.some(runtime => runtime.id === id))) const selectedPlatforms = new Set((params.has('platforms') ? params.get('platforms')!.split(',') : [defaultPlatform]).filter(id => data.platforms.some(platform => platform.id === id)) as PlatformId[]) + const hardwareColors = (values: (string | null | undefined)[], tints: string[]) => new Map( + [...new Set(values.filter((value): value is string => typeof value === 'string' && value.length > 0))].sort() + .map((value, index) => [value, tints[index % tints.length]!]), + ) + const cpuColors = hardwareColors(data.runs.flatMap(run => run.runners.map(runner => runner.cpu.model)), ['#e3edf9', '#e6f1e7', '#dcf0ef']) + const skuColors = hardwareColors(data.runs.flatMap(run => run.runners.map(runner => runner.sku)), ['#f8ead5', '#f2e5ef', '#f8e3df']) let selectedRunId = data.runs.find(run => run.id === params.get('run'))?.id ?? data.runs.at(-1)?.id ?? '' let chart: Chart<'line'> | undefined + let tooltipMode = 'ranked' + let snapshotMode = 'ranked' + let snapshotSort: { platformId: string, bestFirst: boolean } | undefined + let comparisonSort: { platformId: string, bestFirst: boolean } | undefined + try { + if (localStorage.getItem('benchmark-tooltip-mode') === 'grouped') tooltipMode = 'grouped' + } catch {} + let renderTooltipContents = () => {} + let tooltipHideTimer: ReturnType | undefined + const dummyOrder: Record = { + dummy: 0, + 'hyperlight-dummy': 1, + 'hyperlight-wasm-dummy': 2, + 'hyperlight-wasm-pulley-dummy': 3, + 'wasmtime-dummy': 4, + 'wasmtime-aot-dummy': 5, + 'wasmtime-pulley-dummy': 6, + } const runtimeGroups = [ - { label: 'Hyperlight JS', runtimes: data.runtimes.filter(runtime => runtime.id === 'hyperlight-js') }, - { label: 'Hyperlight Wasm', runtimes: data.runtimes.filter(runtime => runtime.id.startsWith('hyperlight-wasm-') && !runtime.id.endsWith('-dummy')) }, - { label: 'Wasmtime', runtimes: data.runtimes.filter(runtime => runtime.id.startsWith('wasmtime-') && !runtime.id.endsWith('-dummy')) }, - { label: 'Dummy', runtimes: data.runtimes.filter(runtime => runtime.id === 'dummy' || runtime.id.endsWith('-dummy')) }, + { label: 'Hyperlight JS', shape: 'circle', runtimes: data.runtimes.filter(runtime => runtime.id === 'hyperlight-js') }, + { label: 'Hyperlight Wasm', shape: 'square', runtimes: data.runtimes.filter(runtime => runtime.id.startsWith('hyperlight-wasm-') && !runtime.id.endsWith('-dummy')).sort((first, second) => Number(first.id.includes('-pulley')) - Number(second.id.includes('-pulley'))) }, + { label: 'Wasmtime', shape: 'diamond', runtimes: data.runtimes.filter(runtime => runtime.id.startsWith('wasmtime-') && !runtime.id.endsWith('-dummy')) }, + { label: 'Dummy', shape: 'triangle', runtimes: data.runtimes.filter(runtime => runtime.id === 'dummy' || runtime.id.endsWith('-dummy')).sort((first, second) => (dummyOrder[first.id] ?? Number.MAX_SAFE_INTEGER) - (dummyOrder[second.id] ?? Number.MAX_SAFE_INTEGER)) }, ] + const runtimeMarker = (runtimeId: string, markerColor: string) => { + const shape = runtimeGroups.find(group => group.runtimes.some(runtime => runtime.id === runtimeId))?.shape ?? 'square' + return `` + } + const shortRuntimeName = (runtimeId: string) => { + if (runtimeId === 'hyperlight-js') return 'QuickJS' + const name = runtimeId === 'dummy' ? 'native' : runtimeId.endsWith('-dummy') + ? runtimeId.slice(0, -6) : runtimeId.replace(/^(hyperlight-wasm|wasmtime)-/, '') + const words: Record = { native: 'Native', hyperlight: 'Hyperlight', wasm: 'Wasm', wasmtime: 'Wasmtime', aot: 'AOT', jco: 'JCO', qjs: 'QuickJS', pulley: 'Pulley' } + return name.split('-').map(word => words[word] ?? word).join(' ') + } app.innerHTML = `