diff --git a/Cargo.lock b/Cargo.lock index 4d3e63b..6d10817 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -114,17 +114,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "async-trait" -version = "0.1.89" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "atomic-waker" version = "1.1.2" @@ -1300,7 +1289,6 @@ dependencies = [ name = "http-bench" version = "0.1.0" dependencies = [ - "async-trait", "clap", "http-body-util", "hyper", diff --git a/README.md b/README.md index 1ece193..c202a95 100644 --- a/README.md +++ b/README.md @@ -36,4 +36,17 @@ Add `--smoke` to verify every configuration with one measured request each. ## Documentation * [Development guide](docs/README.md): setup, benchmark commands, CI, and deployment. -* [Result format](docs/results.md): measurements, history, and validation. \ No newline at end of file +* [Result format](docs/results.md): measurements, history, and validation. + +## Limitations and future work + +This benchmark is intended to model a service that selects and executes customer code at request time to handle each incoming request. + +The current implementation has some limitations: + +* Each runtime configuration uses one fixed guest. A representative workload would provide multiple guests and select one based on the incoming request. +* Guest selection is not dynamic. Hyperlight JS uses one fixed source string. Hyperlight Wasm restores a snapshot prepared for one fixed module. Wasmtime compiles or deserializes one fixed component when each worker context is created. Each Restore then creates a fresh store and instance from that resident component, which explains the similar Wasmtime JIT and AOT performance. + +Real services may have more customer code than fits in memory. They load artifacts from disk and evict inactive code from memory. + +Future work should select guest code from each request and allow every worker to execute any selected guest. It should not require every artifact to remain resident. This would let the benchmark model caching strategies, such as keeping the top N customer artifacts in memory while loading others on demand, and produce more representative results. \ No newline at end of file diff --git a/crates/server/Cargo.toml b/crates/server/Cargo.toml index 0eabb2c..e3c21a4 100644 --- a/crates/server/Cargo.toml +++ b/crates/server/Cargo.toml @@ -11,7 +11,6 @@ pulley = ["hyperlight-wasm/pulley"] time_phases = [] [dependencies] -async-trait = "0.1" clap = { version = "4.5.40", features = ["derive"] } http-body-util = "0.1.3" hyper = { version = "1.6.0", features = ["http1", "server"] } diff --git a/crates/server/src/handlers/hyperlight_dummy.rs b/crates/server/src/handlers/hyperlight_dummy.rs index 5ff5f30..d28dfce 100644 --- a/crates/server/src/handlers/hyperlight_dummy.rs +++ b/crates/server/src/handlers/hyperlight_dummy.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use hyperlight_host::MultiUseSandbox; +use hyperlight_host::sandbox::snapshot::Snapshot; use sandbox_observer::observer::CpuTimeObserver; use crate::{DEFAULT_REQUEST_BODY, SandboxReuseStrategy}; @@ -11,9 +12,20 @@ use super::Handler; /// Useful for seeing maximal performance of the server. pub struct HyperlightDummyHandler; +enum DummyLifecycle { + Renew, + Restore(Arc), + Reuse, +} + +pub struct HyperlightDummyContext { + sandbox: MultiUseSandbox, + lifecycle: DummyLifecycle, +} + impl Handler for HyperlightDummyHandler { type Config = (); - type Context = MultiUseSandbox; + type Context = HyperlightDummyContext; type WorkerState = Vec; fn prepare_worker(_: Self::Config, _: Option>) -> Self::WorkerState { @@ -22,25 +34,38 @@ impl Handler for HyperlightDummyHandler { fn new_context( worker: &Self::WorkerState, - _strategy: SandboxReuseStrategy, + strategy: SandboxReuseStrategy, ) -> Self::Context { let guest_binary = hyperlight_host::GuestBinary::Buffer(worker.clone()); - hyperlight_host::sandbox::UninitializedSandbox::new(guest_binary, None) + let mut sandbox = hyperlight_host::sandbox::UninitializedSandbox::new(guest_binary, None) .unwrap() .evolve() - .unwrap() + .unwrap(); + let lifecycle = match strategy { + SandboxReuseStrategy::New => DummyLifecycle::Renew, + SandboxReuseStrategy::Reload => DummyLifecycle::Restore( + sandbox.snapshot().expect("Failed to snapshot the dummy sandbox"), + ), + SandboxReuseStrategy::Reuse => DummyLifecycle::Reuse, + }; + HyperlightDummyContext { sandbox, lifecycle } } fn load(ctx: Self::Context) -> Self::Context { ctx } - fn unload(ctx: Self::Context) -> Self::Context { + fn unload(mut ctx: Self::Context) -> Self::Context { + let DummyLifecycle::Restore(snapshot) = &ctx.lifecycle else { + panic!("Only Restore may unload a dummy sandbox"); + }; + ctx.sandbox.restore(snapshot.clone()) + .expect("Failed to restore the dummy sandbox snapshot"); ctx } fn handle_request(ctx: &mut Self::Context) -> String { - let res: String = ctx + let res: String = ctx.sandbox .call("handle_request", DEFAULT_REQUEST_BODY.to_string()) .unwrap(); res diff --git a/crates/server/src/handlers/hyperlight_js.rs b/crates/server/src/handlers/hyperlight_js.rs index cd1fb64..22c362f 100644 --- a/crates/server/src/handlers/hyperlight_js.rs +++ b/crates/server/src/handlers/hyperlight_js.rs @@ -9,9 +9,14 @@ pub struct HyperlightJSHandler; pub struct HyperlightJSWorkerState { observer: Option>, } + +enum JSState { + Unloaded(hyperlight_js::JSSandbox), + Loaded(hyperlight_js::LoadedJSSandbox), +} + pub struct HyperlightJSContext { - sandbox: Option, - loaded_sandbox: Option, + state: JSState, observer: Option>, } @@ -36,53 +41,55 @@ impl Handler for HyperlightJSHandler { type Context = HyperlightJSContext; type WorkerState = HyperlightJSWorkerState; - fn prepare_worker(_: Self::Config, observer: Option>) -> Self::WorkerState { + fn prepare_worker( + _: Self::Config, + observer: Option>, + ) -> Self::WorkerState { HyperlightJSWorkerState { observer: observer } } - fn new_context( - worker: &Self::WorkerState, - _strategy: SandboxReuseStrategy, - ) -> Self::Context { + fn new_context(worker: &Self::WorkerState, _strategy: SandboxReuseStrategy) -> Self::Context { let js = SandboxBuilder::new() .build() .unwrap() .load_runtime() .unwrap(); - let observer = worker.observer.clone(); - HyperlightJSContext { - sandbox: Some(js), - loaded_sandbox: None, - observer: observer, + state: JSState::Unloaded(js), + observer: worker.observer.clone(), } } - fn load(mut ctx: Self::Context) -> Self::Context { - let mut js = ctx.sandbox.take().unwrap(); + fn load(ctx: Self::Context) -> Self::Context { + let JSState::Unloaded(mut js) = ctx.state else { + panic!("JS load requires an unloaded sandbox"); + }; + // Restore models a different customer's handler on each request, so registration belongs here. js.add_handler("handler".to_string(), HANDLER.to_string().into()) .unwrap(); let loaded = js.get_loaded_sandbox().unwrap(); HyperlightJSContext { - sandbox: None, - loaded_sandbox: Some(loaded), + state: JSState::Loaded(loaded), observer: ctx.observer, } } - fn unload(mut ctx: Self::Context) -> Self::Context { - let loaded = ctx.loaded_sandbox.take().unwrap(); + fn unload(ctx: Self::Context) -> Self::Context { + let JSState::Loaded(loaded) = ctx.state else { + panic!("JS unload requires a loaded sandbox"); + }; let js = loaded.unload().unwrap(); HyperlightJSContext { - sandbox: Some(js), - loaded_sandbox: None, + state: JSState::Unloaded(js), observer: ctx.observer, } } fn handle_request(ctx: &mut Self::Context) -> String { - let loaded = ctx.loaded_sandbox.as_mut().unwrap(); + let JSState::Loaded(loaded) = &mut ctx.state else { + panic!("JS handle_request requires a loaded sandbox"); + }; let interrupt_handle = loaded.interrupt_handle(); if let Some(obs) = &ctx.observer { diff --git a/crates/server/src/handlers/hyperlight_wasm.rs b/crates/server/src/handlers/hyperlight_wasm.rs index 949cb67..9db82cb 100644 --- a/crates/server/src/handlers/hyperlight_wasm.rs +++ b/crates/server/src/handlers/hyperlight_wasm.rs @@ -1,5 +1,5 @@ use super::{Handler, SandboxMemory}; -use crate::SandboxReuseStrategy; +use crate::{DEFAULT_REQUEST_URI, SandboxReuseStrategy}; use bindings::hyperlight::bench::handler_interface::Request; use hyperlight_host::sandbox::snapshot::Snapshot; @@ -54,11 +54,23 @@ impl bindings::hyperlight::bench::HandlerWorldImports>>, + }, + Loaded(bindings::HandlerWorldSandbox), +} + +enum WasmLifecycle { + Renew, + Restore(Arc), + Reuse(Arc), +} + pub struct HyperlightWASMContext { - unloaded: Option, - loaded_snapshot: Arc, - wrapped: Option>, - rt: Option>>>, + state: WasmState, + lifecycle: WasmLifecycle, aot: Arc, observer: Option>, } @@ -111,61 +123,72 @@ impl Handler for HyperlightWASMHandler { let rt = bindings::register_host_functions(&mut sb, state).unwrap(); let sb = sb.load_runtime().unwrap(); - let mut loaded_sb = unsafe { - sb.load_module_by_mapping(worker.aot.base, worker.aot.len).unwrap() + let prepare_snapshot = |sandbox: WasmSandbox, lifecycle: fn(Arc) -> WasmLifecycle| { + let mut loaded = unsafe { + sandbox.load_module_by_mapping(worker.aot.base, worker.aot.len) + .expect("Failed to load the Wasm module for snapshot preparation") + }; + let snapshot = loaded.snapshot().expect("Failed to snapshot the loaded Wasm module"); + let sandbox = loaded.unload_module().expect("Failed to unload the Wasm module after snapshot preparation"); + (sandbox, lifecycle(snapshot)) + }; + let (sb, lifecycle) = match strategy { + SandboxReuseStrategy::New => (sb, WasmLifecycle::Renew), + SandboxReuseStrategy::Reload => prepare_snapshot(sb, WasmLifecycle::Restore), + SandboxReuseStrategy::Reuse => prepare_snapshot(sb, WasmLifecycle::Reuse), }; - let loaded_snapshot = loaded_sb.snapshot().unwrap(); - let sb = loaded_sb.unload_module().unwrap(); HyperlightWASMContext { - unloaded: Some(sb), - loaded_snapshot, - wrapped: None, - rt: Some(rt), + state: WasmState::Unloaded { sandbox: sb, resources: rt }, + lifecycle, aot: worker.aot.clone(), observer: observer, } } fn load(mut ctx: Self::Context) -> Self::Context { - let wasm_sandbox = ctx.unloaded.take().unwrap(); - let sb = wasm_sandbox - .load_from_snapshot(ctx.loaded_snapshot.clone()) - .unwrap(); - - let wrapped = bindings::HandlerWorldSandbox { - sb, - rt: ctx.rt.unwrap(), + let WasmState::Unloaded { sandbox, resources } = ctx.state else { + panic!("Wasm load requires an unloaded sandbox"); + }; + let sb = match &ctx.lifecycle { + WasmLifecycle::Renew => { + // The context retains the AOT mapping for the loaded module's lifetime. + unsafe { + sandbox.load_module_by_mapping(ctx.aot.base, ctx.aot.len) + .expect("Failed to load the Wasm module for Renew") + } + } + WasmLifecycle::Restore(snapshot) | WasmLifecycle::Reuse(snapshot) => sandbox + .load_from_snapshot(snapshot.clone()) + .expect("Failed to restore the loaded Wasm module snapshot"), }; - HyperlightWASMContext { - unloaded: None, - wrapped: Some(wrapped), - loaded_snapshot: ctx.loaded_snapshot, - rt: None, - aot: ctx.aot, - observer: ctx.observer, - } + ctx.state = WasmState::Loaded(bindings::HandlerWorldSandbox { + sb, + rt: resources, + }); + ctx } fn unload(mut ctx: Self::Context) -> Self::Context { - let wrapped = ctx.wrapped.take().unwrap(); - let unloaded = wrapped.sb.unload_module().unwrap(); + let WasmLifecycle::Restore(_) = &ctx.lifecycle else { + panic!("Only Restore may unload a Wasm module"); + }; + let WasmState::Loaded(wrapped) = ctx.state else { + panic!("Wasm unload requires a loaded module"); + }; + let unloaded = wrapped.sb.unload_module().expect("Failed to unload the Wasm module"); - HyperlightWASMContext { - unloaded: Some(unloaded), - wrapped: None, - loaded_snapshot: ctx.loaded_snapshot, - rt: Some(wrapped.rt), - aot: ctx.aot, - observer: ctx.observer, - } + ctx.state = WasmState::Unloaded { sandbox: unloaded, resources: wrapped.rt }; + ctx } fn handle_request(ctx: &mut Self::Context) -> String { use bindings::hyperlight::bench::HandlerInterface; - let world_sb = ctx.wrapped.as_ref().unwrap(); + let WasmState::Loaded(world_sb) = &mut ctx.state else { + panic!("Wasm handle_request requires a loaded module"); + }; let handle = world_sb.sb.interrupt_handle().unwrap(); if let Some(obs) = &ctx.observer { @@ -173,11 +196,11 @@ impl Handler for HyperlightWASMHandler { } let handler = bindings::hyperlight::bench::HandlerWorldExports::handler_interface( - ctx.wrapped.as_mut().unwrap(), + world_sb, ); let request = Request { - uri: "/default.html".to_string(), + uri: DEFAULT_REQUEST_URI.to_string(), }; let response = handler.handleevent(request).unwrap(); @@ -187,6 +210,6 @@ impl Handler for HyperlightWASMHandler { obs.stop_timeout(&handle); } - format!("{{\"uri\":\"{}\"}}", uri).to_string() + format!("{{\"uri\":\"{}\"}}", uri) } } diff --git a/crates/server/src/handlers/wasmtime.rs b/crates/server/src/handlers/wasmtime.rs index f09d65c..33e42fa 100644 --- a/crates/server/src/handlers/wasmtime.rs +++ b/crates/server/src/handlers/wasmtime.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use sandbox_observer::observer::CpuTimeObserver; use super::Handler; -use crate::{DEFAULT_REQUEST_BODY, SandboxReuseStrategy}; +use crate::{DEFAULT_REQUEST_URI, SandboxReuseStrategy}; mod bindings { wasmtime::component::bindgen!(in "../../js/src/wit"); @@ -27,7 +27,6 @@ pub struct WasmtimeContext { engine: wasmtime::Engine, component: wasmtime::component::Component, linker: wasmtime::component::Linker<()>, - // Below are populated by `load` and `unload` store: Option>, instance: Option, } @@ -49,10 +48,7 @@ impl Handler for WasmtimeHandler { } } - fn new_context( - worker: &Self::WorkerState, - _strategy: SandboxReuseStrategy, - ) -> Self::Context { + fn new_context(worker: &Self::WorkerState, _strategy: SandboxReuseStrategy) -> Self::Context { let mut config = wasmtime::Config::new(); match worker.source { ComponentSource::Wasm(_) => {} @@ -92,9 +88,9 @@ impl Handler for WasmtimeHandler { fn load(mut ctx: Self::Context) -> Self::Context { let mut store = wasmtime::Store::new(&ctx.engine, ()); - let bindings = + let instance = bindings::HandlerWorld::instantiate(&mut store, &ctx.component, &ctx.linker).unwrap(); - let _ = ctx.instance.insert(bindings); + let _ = ctx.instance.insert(instance); let _ = ctx.store.insert(store); ctx } @@ -106,25 +102,18 @@ impl Handler for WasmtimeHandler { } fn handle_request(ctx: &mut Self::Context) -> String { - let bindings = ctx.instance.as_ref().unwrap(); + let instance = ctx.instance.as_ref().unwrap(); - let parsed: serde_json::Value = serde_json::from_str(DEFAULT_REQUEST_BODY) - .unwrap_or_else(|_| serde_json::json!({"uri": "/default.html"})); - - let uri = parsed["uri"].as_str().unwrap().to_string(); - - let request = bindings::exports::hyperlight::bench::handler_interface::Request { uri }; + let request = bindings::exports::hyperlight::bench::handler_interface::Request { + uri: DEFAULT_REQUEST_URI.to_string(), + }; // Call the WIT handler - let handler = bindings.hyperlight_bench_handler_interface(); + let handler = instance.hyperlight_bench_handler_interface(); let result = handler .call_handleevent(ctx.store.as_mut().unwrap(), &request) .unwrap(); - // Return the result as JSON - serde_json::json!({ - "uri": result.uri - }) - .to_string() + format!("{{\"uri\":\"{}\"}}", result.uri) } } diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 2d05227..84f30b5 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -12,7 +12,6 @@ use std::time::Duration; extern crate alloc; -use async_trait::async_trait; use clap::Parser; use clap::ValueEnum; use http_body_util::Full; @@ -34,6 +33,7 @@ use handlers::Handler; mod memory_monitor; +const DEFAULT_REQUEST_URI: &str = "/index.html"; const DEFAULT_REQUEST_BODY: &str = r#"{"uri": "/index.html"}"#; #[cfg(feature = "time_phases")] @@ -52,17 +52,10 @@ struct JobRequest { reqnum: usize, } -#[async_trait] -trait SandboxPoolTrait: Send + Sync { - /// Send a job to the pool and await the result. - async fn execute(&self) -> String; -} - /// Pool of sandbox worker threads for handling requests. -struct SandboxPool { +struct SandboxPool { senders: Vec>, counter: AtomicUsize, - _marker: std::marker::PhantomData, } #[derive(Copy, Clone)] @@ -71,9 +64,9 @@ struct ObserverConfig { check_in: Duration, } -impl SandboxPool { +impl SandboxPool { /// Worker thread for `New` strategy: creates a new sandbox for each request. - fn worker_new( + fn worker_new( mut rx: mpsc::UnboundedReceiver, i: usize, worker: &H::WorkerState, @@ -114,7 +107,7 @@ impl SandboxPool { } /// Worker thread for `Reload` strategy: reuses the sandbox, but reloads/unloads for each request. - fn worker_reload( + fn worker_reload( mut rx: mpsc::UnboundedReceiver, i: usize, worker: &H::WorkerState, @@ -155,7 +148,7 @@ impl SandboxPool { } /// Worker thread for `Reuse` strategy: reuses the same sandbox and handler for all requests. - fn worker_reuse( + fn worker_reuse( mut rx: mpsc::UnboundedReceiver, i: usize, worker: &H::WorkerState, @@ -195,12 +188,12 @@ impl SandboxPool { } /// Create a new sandbox pool with the given number of workers, mode, and handler. - fn start( + fn start( pool_size: usize, mode: SandboxReuseStrategy, config: Option, handler_config: H::Config, - ) -> Arc { + ) -> Arc { assert!(pool_size > 0, "Pool size must be positive"); let mut senders = Vec::with_capacity(pool_size); let (ready_tx, ready_rx) = std::sync::mpsc::channel(); @@ -221,9 +214,9 @@ impl SandboxPool { let worker = H::prepare_worker(handler_config, obs); eprintln!("Worker thread {} preparation completed", i); match mode { - SandboxReuseStrategy::New => Self::worker_new(rx, i, &worker, ready), - SandboxReuseStrategy::Reload => Self::worker_reload(rx, i, &worker, ready), - SandboxReuseStrategy::Reuse => Self::worker_reuse(rx, i, &worker, ready), + SandboxReuseStrategy::New => Self::worker_new::(rx, i, &worker, ready), + SandboxReuseStrategy::Reload => Self::worker_reload::(rx, i, &worker, ready), + SandboxReuseStrategy::Reuse => Self::worker_reuse::(rx, i, &worker, ready), } })); @@ -258,13 +251,9 @@ impl SandboxPool { Arc::new(Self { senders, counter: AtomicUsize::new(0), - _marker: std::marker::PhantomData, }) } -} -#[async_trait] -impl SandboxPoolTrait for SandboxPool { /// Send a job to the pool and await the result. async fn execute(&self) -> String { #[cfg(feature = "time_phases")] @@ -293,7 +282,7 @@ impl SandboxPoolTrait for SandboxPool { /// Main HTTP handler: runs the JS handler in a sandbox and returns the response. async fn handler( - pool: Arc, + pool: Arc, ) -> Result>, Infallible> { let res = pool.execute().await; // make sure the handler ran @@ -352,14 +341,14 @@ impl Runtime { pool_size: usize, strategy: SandboxReuseStrategy, observer: Option, - ) -> Arc { + ) -> Arc { use handlers::{ComponentSource::*, *}; let wasmtime = |source| { - SandboxPool::::start(pool_size, strategy, observer, source) + SandboxPool::start::(pool_size, strategy, observer, source) }; let hyperlight_wasm = |artifact, memory| { - SandboxPool::::start( + SandboxPool::start::( pool_size, strategy, observer, @@ -384,13 +373,13 @@ impl Runtime { Self::HyperlightWASMPulleyQjs => hyperlight_wasm(QJS_PULLEY, QJS_MEMORY), Self::HyperlightWASMPulleyDummy => hyperlight_wasm(RUST_PULLEY, RUST_MEMORY), Self::HyperlightJS => { - SandboxPool::::start(pool_size, strategy, observer, ()) + SandboxPool::start::(pool_size, strategy, observer, ()) } Self::HyperlightDummy => { - SandboxPool::::start(pool_size, strategy, observer, ()) + SandboxPool::start::(pool_size, strategy, observer, ()) } Self::Dummy => { - SandboxPool::::start(pool_size, strategy, observer, ()) + SandboxPool::start::(pool_size, strategy, observer, ()) } } } diff --git a/crates/server/src/memory_monitor.rs b/crates/server/src/memory_monitor.rs index 90741d9..30a7285 100644 --- a/crates/server/src/memory_monitor.rs +++ b/crates/server/src/memory_monitor.rs @@ -3,7 +3,7 @@ use serde::Serialize; use std::fs; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use sysinfo::{Pid, ProcessesToUpdate, System}; +use sysinfo::{Pid, ProcessRefreshKind, ProcessesToUpdate, System}; use tokio::signal; use crate::{Runtime, SandboxReuseStrategy}; @@ -16,9 +16,9 @@ struct MemoryUsageEntry { value: u64, } -/// Monitors and logs memory usage every second +/// Samples peak resident memory every 500 ms. pub(crate) async fn monitor_memory_usage(peak_memory: Arc) { - let mut system = System::new_all(); + let mut system = System::new(); let current_pid = Pid::from_u32(std::process::id()); let mut interval = tokio::time::interval(std::time::Duration::from_millis(500)); @@ -26,7 +26,11 @@ pub(crate) async fn monitor_memory_usage(peak_memory: Arc) { loop { interval.tick().await; - system.refresh_all(); + system.refresh_processes_specifics( + ProcessesToUpdate::Some(&[current_pid]), + true, + ProcessRefreshKind::new().with_memory(), + ); if let Some(process) = system.process(current_pid) { peak_resident_memory_usage_bytes = @@ -73,7 +77,11 @@ pub(crate) async fn setup_signal_handler( } let current_pid = Pid::from_u32(std::process::id()); let mut system = System::new(); - system.refresh_processes(ProcessesToUpdate::Some(&[current_pid]), true); + system.refresh_processes_specifics( + ProcessesToUpdate::Some(&[current_pid]), + true, + ProcessRefreshKind::new().with_memory(), + ); let final_bytes = system.process(current_pid).expect("Missing benchmark process").memory(); let peak_bytes = peak_memory.load(Ordering::Relaxed).max(final_bytes); diff --git a/docs/results.md b/docs/results.md index e7c1855..20a2d26 100644 --- a/docs/results.md +++ b/docs/results.md @@ -103,7 +103,7 @@ npm run validate -- --policy policy.json run.json ``` The command accepts multiple files and exits with status 1 on validation failure. -It partitions the supplied bundles into compatible dashboard histories and +It partitions the supplied bundles by benchmark version and validates each group. ## Dashboard Selection Archives @@ -187,9 +187,10 @@ It copies the bundle unchanged and marks its pending status in serving metadata. Production indexes have no preview metadata. The production serving copy contains only published bundles. Each preview has its own `data/index.json` and `data/runs/`. -The dashboard defaults to the pending run's compatible history. Older histories -remain selectable. The header indicates when pending results belong to another -history. +The dashboard defaults to the pending run's benchmark version. Older versions +are accessible through `?history=`. A comma-separated list, such as +`?history=2,3`, compares several versions. The header indicates when pending +results belong to another history. ## Publication Records @@ -267,19 +268,20 @@ operations or remove PR previews. Approved benchmark-skip PRs do not invoke prom ## History Compatibility -The index retains every published run. Publication and Pages partition runs by -workload ID, version, settings, metric definitions, and runtime definitions. +The index retains every published run. Each benchmark version defines one chart +history. Changes to the workload ID or settings require a version bump. Changes to metric units, directions, method versions, guest engines, execution -modes, or the set of metrics or runtimes start a separate group. Display labels +modes, or the set of metrics or runtimes also require a version bump. Display labels and catalog ordering do not affect grouping. -The dashboard compares one group at a time. It defaults to the group containing -the newest run. The Benchmark history selector exposes older groups. Shared -links identify a group through `history=.` using a member -run. Downloads contain only the selected group's visible runs. +The dashboard defaults to the group containing the newest run. Older groups are +accessible through the `history=` query parameter, for example +`?history=2`. Comma-separated versions combine their runs for an explicit +comparison. Shared links preserve the selected groups. Downloads contain only +the selected groups' visible runs. Duplicate run identities and mixed local, synthetic, or published sources fail -validation. Each group retains strict comparison checks. No archived bundles +validation. Conflicting definitions within a version also fail validation. No archived bundles are rewritten when a new group is published. The snapshot table shows successful results for the selected metric. Runner diff --git a/scripts/publication.test.ts b/scripts/publication.test.ts index 1a14f59..7e5f0a9 100644 --- a/scripts/publication.test.ts +++ b/scripts/publication.test.ts @@ -211,9 +211,10 @@ test('benchmark definition changes retain selectable histories and pending previ next.run.createdAt = `2026-09-0${number + 1}T00:00:00Z` next.run.pullRequest!.number = 7 + number next.run.workflow.url = `https://github.com/${repository}/actions/runs/${next.run.id}/attempts/1` - if (change === 'version') next.benchmark.version++ if (change === 'settings') next.benchmark.settings.durationSeconds = 120 if (change === 'metric') next.catalog.metrics[0]!.methodVersion++ + if (change !== 'version') assert.throws(() => groupHistories([...bundles, next]), /Bump the benchmark version/) + next.benchmark.version++ const policy = { ...publicationPolicy, catalog: next.catalog, benchmark: next.benchmark } assert.throws(() => validateHistory([first, next])) storeRun(store, next, policy) @@ -229,7 +230,7 @@ test('benchmark definition changes retain selectable histories and pending previ reordered.run.id = 'reordered' reordered.catalog.metrics.reverse() reordered.catalog.runtimes.reverse() - assert.equal(groupHistories([first, reordered]).length, 1) + assert.deepEqual(groupHistories([first, reordered]), [[first, reordered]]) assert.throws(() => groupHistories([first, first]), /Duplicate run/) assert.throws(() => groupHistories([first, { ...reordered, source: 'local' }]), /sources differ/) @@ -251,21 +252,31 @@ test('benchmark definition changes retain selectable histories and pending previ const url = new URL('https://fixture.test/data/index.json') const latest = await loadHistory(url) assert.equal(latest!.histories!.length, 4) + assert.equal(latest!.historyId, String(bundles.at(-1)!.benchmark.version)) assert.deepEqual(latest!.runs.map(run => run.id), [runKey(bundles.at(-1)!)]) assert.equal(latest!.runs[0]!.message, 'Improve benchmarks (#10)') assert.equal(latest!.runs[0]!.commit, 'ccccccc') assert.equal(latest!.runs[0]!.commitUrl, `https://github.com/${repository}/commit/${'c'.repeat(40)}`) assert.deepEqual(latest!.runs[0]!.bundle, bundles.at(-1)) - const older = await loadHistory(url, runKey(first)) + const older = await loadHistory(url, String(first.benchmark.version)) + assert.equal(older!.historyId, String(first.benchmark.version)) assert.deepEqual(older!.runs.map(run => run.id), [runKey(first)]) assert.equal(older!.runs[0]!.message, 'Benchmark results (#7)') - assert.equal((await loadHistory(url, 'missing'))!.historyId, latest!.historyId) + const compared = await loadHistory(url, `${first.benchmark.version},${bundles.at(-1)!.benchmark.version}`) + assert.equal(compared!.historyId, `${first.benchmark.version},${bundles.at(-1)!.benchmark.version}`) + assert.deepEqual(compared!.runs.map(run => run.id), [runKey(first), runKey(bundles.at(-1)!)]) + await assert.rejects(loadHistory(url, 'missing'), /Benchmark version not found/) + await assert.rejects(loadHistory(url, `${first.benchmark.version},missing`), /Benchmark version not found/) + await assert.rejects(loadHistory(url, `${first.benchmark.version},${first.benchmark.version}`), /Invalid benchmark version selection/) + assert.deepEqual((await loadHistory(url, undefined, runKey(first)))!.runs.map(run => run.id), [runKey(first)]) + assert.equal((await loadHistory(url, String(first.benchmark.version), runKey(bundles.at(-1)!)))!.historyId, String(first.benchmark.version)) const previewUrl = new URL('https://fixture.test/previews/pr-7/data/index.json') const preview = await loadHistory(previewUrl) assert.equal(preview!.histories!.length, 5) + assert.equal(preview!.historyId, '99') assert.deepEqual(preview!.runs.map(run => run.id), [runKey(pending)]) assert.equal(preview!.runs[0]!.commit, pending.run.commit.sha.slice(0, 7)) - assert.deepEqual((await loadHistory(previewUrl, runKey(first)))!.runs.map(run => run.id), [runKey(first)]) + assert.deepEqual((await loadHistory(previewUrl, String(first.benchmark.version)))!.runs.map(run => run.id), [runKey(first)]) }) test('local result storage and promotion', async context => { diff --git a/shared/catalog.ts b/shared/catalog.ts index b6de5b5..88a2726 100644 --- a/shared/catalog.ts +++ b/shared/catalog.ts @@ -47,7 +47,7 @@ export const metrics: Metric[] = [ export const strategies: Strategy[] = ['reload', 'reuse', 'new'] export const catalog = { runtimes, platforms, metrics } export const benchmark = { - id: 'http-redirect', version: 2, + id: 'http-redirect', version: 3, settings: { durationSeconds: 60, concurrency: 50, poolSize: 4, workerThreads: 'available-parallelism', workerReadiness: 'all-workers', sandboxTimeoutMs: 1000, timeoutCheckIntervalMs: 10, clientTimeoutSeconds: null, diff --git a/shared/results.ts b/shared/results.ts index 45f726b..deec568 100644 --- a/shared/results.ts +++ b/shared/results.ts @@ -276,7 +276,13 @@ export function groupHistories(bundles: RunBundle[]): RunBundle[][] { groups.set(definition, group) } const histories = [...groups.values()].sort((first, second) => second.at(-1)!.run.createdAt.localeCompare(first.at(-1)!.run.createdAt) || runKey(second.at(-1)!).localeCompare(runKey(first.at(-1)!))) - for (const history of histories) validateHistory(history) + const versions = new Set() + for (const history of histories) { + const version = history[0]!.benchmark.version + if (versions.has(version)) throw new Error(`Incompatible definitions for benchmark version ${version}. Bump the benchmark version.`) + versions.add(version) + validateHistory(history) + } return histories } diff --git a/src/data.ts b/src/data.ts index fd66a55..335d5f6 100644 --- a/src/data.ts +++ b/src/data.ts @@ -37,6 +37,36 @@ export interface Dataset { historyId?: string } +function datasetFromHistories(histories: RunBundle[][]): Dataset { + const datasets = histories.map(datasetFromBundles) + if (datasets.length === 1) return datasets[0]! + + const bundles = histories.flat().sort((first, second) => first.run.createdAt.localeCompare(second.run.createdAt) || runKey(first).localeCompare(runKey(second))) + const runtimes = new Map() + const platforms = new Map() + const metrics = new Map() + for (const bundle of bundles) { + for (const runtime of bundle.catalog.runtimes) runtimes.set(runtime.id, runtime) + for (const platform of bundle.catalog.platforms) platforms.set(platform.id, platform) + for (const metric of bundle.catalog.metrics) { + const previous = metrics.get(metric.id) + if (previous && (previous.unit !== metric.unit || previous.direction !== metric.direction)) { + throw new Error(`Benchmark versions use incompatible definitions for metric ${metric.id}`) + } + metrics.set(metric.id, metric) + } + } + return { + schemaVersion: 1, + source: datasets[0]!.source, + runtimes: [...runtimes.values()], + platforms: [...platforms.values()], + metrics: [...metrics.values()], + runs: datasets.flatMap(dataset => dataset.runs).sort((first, second) => first.date.localeCompare(second.date) || first.id.localeCompare(second.id)), + measurements: datasets.flatMap(dataset => dataset.measurements), + } +} + export async function loadDataset(): Promise { if (new URLSearchParams(location.search).get('demo') === '1') { const { createMockBundles } = await import('./mock-data') @@ -44,10 +74,10 @@ export async function loadDataset(): Promise { } const indexUrl = new URL(import.meta.env.VITE_HISTORY_URL || 'https://raw.githubusercontent.com/hyperlight-dev/hyperlight-bench/data/index.json', location.href) const params = new URLSearchParams(location.search) - return loadHistory(indexUrl, params.get('history') ?? params.get('run') ?? undefined) + return loadHistory(indexUrl, params.get('history') ?? undefined, params.get('run') ?? undefined) } -export async function loadHistory(indexUrl: URL, selectedHistory?: string): Promise { +export async function loadHistory(indexUrl: URL, selectedHistory?: string, selectedRun?: string): Promise { async function readJson(url: URL): Promise { const response = await fetch(url, { cache: 'no-cache', signal: AbortSignal.timeout(30000) }) if (!response.ok) throw new Error(`Unable to load ${url}: HTTP ${response.status}`) @@ -74,8 +104,17 @@ export async function loadHistory(indexUrl: URL, selectedHistory?: string): Prom const groups = groupHistories(bundles) const containing = (key: string | undefined) => groups.find(group => group.some(bundle => runKey(bundle) === key)) const pending = index.preview?.pending - const selected = containing(selectedHistory) ?? containing(pending ? `${pending.id}.${pending.attempt}` : undefined) ?? groups[0]! - const dataset = datasetFromBundles(selected) + const requestedVersions = selectedHistory?.split(',') + if (requestedVersions?.some(version => !version) || requestedVersions && new Set(requestedVersions).size !== requestedVersions.length) { + throw new Error(`Invalid benchmark version selection: ${selectedHistory}`) + } + const requested = requestedVersions?.map(version => { + const group = groups.find(candidate => String(candidate[0]!.benchmark.version) === version) + if (!group) throw new Error(`Benchmark version not found: ${version}`) + return group + }) + const selected = requested ?? [containing(selectedRun) ?? containing(pending ? `${pending.id}.${pending.attempt}` : undefined) ?? groups[0]!] + const dataset = datasetFromHistories(selected) for (const run of dataset.runs) { const display = index.runs.find(entry => `${entry.id}.${entry.attempt}` === run.id)?.displayCommit if (display) { @@ -87,13 +126,13 @@ export async function loadHistory(indexUrl: URL, selectedHistory?: string): Prom return { ...dataset, preview: index.preview, - historyId: runKey(selected[0]!), + historyId: selected.map(group => group[0]!.benchmark.version).join(','), histories: groups.map((group, index) => { const latest = group.at(-1)! const settings = latest.benchmark.settings const load = settings.requestCount ? `${settings.requestCount} request` : `${settings.durationSeconds}s` return { - id: runKey(group[0]!), + id: String(group[0]!.benchmark.version), label: `${latest.benchmark.id} v${latest.benchmark.version} / ${load} / ${settings.concurrency} connections / ${latest.run.createdAt.slice(0, 10)} / ${index + 1}`, } }), diff --git a/src/main.ts b/src/main.ts index 391f50e..6e001dc 100644 --- a/src/main.ts +++ b/src/main.ts @@ -21,6 +21,19 @@ const strategyDescriptions: Record = { reuse: 'Keep the sandbox as-is across requests.', new: 'Create a new sandbox for each request.', } +const lifecycleDetails = ` +
+ Runtime lifecycle details +
+ + + + + + + +
RuntimeVariantRestoreReuseRenew
Hyperlight JSQuickJSRegister the JavaScript handler, call it, then restore to the handler-free runtime snapshot.Call the registered handler.Create a sandbox, load JavaScript, register and call the handler, then destroy the sandbox.
Hyperlight WasmJCO, QuickJS, PulleyRestore to the loaded-module snapshot, call the component, then unload the module for the next restore.Call the loaded component.Create a sandbox, load the Wasm runtime, map and call the module, then destroy the sandbox.
WasmtimeJITInstantiate and call the resident compiled component in a fresh store, then drop the instance and store.Call the instantiated component.Create an engine and linker, compile, instantiate, and call the component, then drop the context.
AOT, PulleyInstantiate and call the resident deserialized component in a fresh store, then drop the instance and store.Call the instantiated component.Create an engine and linker, deserialize, instantiate, and call the component, then drop the context.
DummyNativeHandle the request in the host process.Handle the request in the host process.Handle the request in the host process.
HyperlightCall the guest, then restore to the sandbox snapshot.Call the guest.Create a sandbox, call the guest, then destroy the sandbox.
Wasm variantsUse the matching Hyperlight Wasm or Wasmtime lifecycle above.
+
` const app = document.querySelector('#app')! const escapeHtml = (value: string) => value.replace(/[&<>"']/g, character => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[character]!) @@ -109,14 +122,13 @@ function renderDashboard(data: Dataset) { ${data.preview ? `

${escapeHtml(previewLabel(data))}

` : ''}