Improve benchmark lifecycle and result history

README.md

@@ -36,4 +36,17 @@ Add `--smoke` to verify every configuration with one measured request each.
 ## Documentation
 
 * [Development guide](docs/README.md): setup, benchmark commands, CI, and deployment.
-* [Result format](docs/results.md): measurements, history, and validation.
+* [Result format](docs/results.md): measurements, history, and validation.
+
+## Limitations and future work
+
+This benchmark is intended to model a service that selects and executes customer code at request time to handle each incoming request.
+
+The current implementation has some limitations:
+
+* Each runtime configuration uses one fixed guest. A representative workload would provide multiple guests and select one based on the incoming request.
+* Guest selection is not dynamic. Hyperlight JS uses one fixed source string. Hyperlight Wasm restores a snapshot prepared for one fixed module. Wasmtime compiles or deserializes one fixed component when each worker context is created. Each Restore then creates a fresh store and instance from that resident component, which explains the similar Wasmtime JIT and AOT performance.
+
+Real services may have more customer code than fits in memory. They load artifacts from disk and evict inactive code from memory.
+
+Future work should select guest code from each request and allow every worker to execute any selected guest. It should not require every artifact to remain resident. This would let the benchmark model caching strategies, such as keeping the top N customer artifacts in memory while loading others on demand, and produce more representative results.

crates/server/Cargo.toml

@@ -11,7 +11,6 @@ pulley = ["hyperlight-wasm/pulley"]
 time_phases = []
 
 [dependencies]
-async-trait = "0.1"
 clap = { version = "4.5.40", features = ["derive"] }
 http-body-util = "0.1.3"
 hyper = { version = "1.6.0", features = ["http1", "server"] }

crates/server/src/handlers/hyperlight_dummy.rs

@@ -1,6 +1,7 @@
 use std::sync::Arc;
 
 use hyperlight_host::MultiUseSandbox;
+use hyperlight_host::sandbox::snapshot::Snapshot;
 use sandbox_observer::observer::CpuTimeObserver;
 
 use crate::{DEFAULT_REQUEST_BODY, SandboxReuseStrategy};
@@ -11,9 +12,20 @@ use super::Handler;
 /// Useful for seeing maximal performance of the server.
 pub struct HyperlightDummyHandler;
 
+enum DummyLifecycle {
+    Renew,
+    Restore(Arc<Snapshot>),
+    Reuse,
+}
+
+pub struct HyperlightDummyContext {
+    sandbox: MultiUseSandbox,
+    lifecycle: DummyLifecycle,
+}
+
 impl Handler for HyperlightDummyHandler {
     type Config = ();
-    type Context = MultiUseSandbox;
+    type Context = HyperlightDummyContext;
     type WorkerState = Vec<u8>;
 
     fn prepare_worker(_: Self::Config, _: Option<Arc<CpuTimeObserver>>) -> Self::WorkerState {
@@ -22,25 +34,38 @@ impl Handler for HyperlightDummyHandler {
 
     fn new_context(
         worker: &Self::WorkerState,
-        _strategy: SandboxReuseStrategy,
+        strategy: SandboxReuseStrategy,
     ) -> Self::Context {
         let guest_binary = hyperlight_host::GuestBinary::Buffer(worker.clone());
-        hyperlight_host::sandbox::UninitializedSandbox::new(guest_binary, None)
+        let mut sandbox = hyperlight_host::sandbox::UninitializedSandbox::new(guest_binary, None)
             .unwrap()
             .evolve()
-            .unwrap()
+            .unwrap();
+        let lifecycle = match strategy {
+            SandboxReuseStrategy::New => DummyLifecycle::Renew,
+            SandboxReuseStrategy::Reload => DummyLifecycle::Restore(
+                sandbox.snapshot().expect("Failed to snapshot the dummy sandbox"),
+            ),
+            SandboxReuseStrategy::Reuse => DummyLifecycle::Reuse,
+        };
+        HyperlightDummyContext { sandbox, lifecycle }
     }
 
     fn load(ctx: Self::Context) -> Self::Context {
         ctx
     }
 
-    fn unload(ctx: Self::Context) -> Self::Context {
+    fn unload(mut ctx: Self::Context) -> Self::Context {
+        let DummyLifecycle::Restore(snapshot) = &ctx.lifecycle else {
+            panic!("Only Restore may unload a dummy sandbox");
+        };
+        ctx.sandbox.restore(snapshot.clone())
+            .expect("Failed to restore the dummy sandbox snapshot");
         ctx
     }
 
     fn handle_request(ctx: &mut Self::Context) -> String {
-        let res: String = ctx
+        let res: String = ctx.sandbox
             .call("handle_request", DEFAULT_REQUEST_BODY.to_string())
             .unwrap();
         res

crates/server/src/handlers/hyperlight_js.rs

@@ -9,9 +9,14 @@ pub struct HyperlightJSHandler;
 pub struct HyperlightJSWorkerState {
     observer: Option<Arc<CpuTimeObserver>>,
 }
+
+enum JSState {
+    Unloaded(hyperlight_js::JSSandbox),
+    Loaded(hyperlight_js::LoadedJSSandbox),
+}
+
 pub struct HyperlightJSContext {
-    sandbox: Option<hyperlight_js::JSSandbox>,
-    loaded_sandbox: Option<hyperlight_js::LoadedJSSandbox>,
+    state: JSState,
     observer: Option<Arc<CpuTimeObserver>>,
 }
 
@@ -36,53 +41,55 @@ impl Handler for HyperlightJSHandler {
     type Context = HyperlightJSContext;
     type WorkerState = HyperlightJSWorkerState;
 
-    fn prepare_worker(_: Self::Config, observer: Option<Arc<CpuTimeObserver>>) -> Self::WorkerState {
+    fn prepare_worker(
+        _: Self::Config,
+        observer: Option<Arc<CpuTimeObserver>>,
+    ) -> Self::WorkerState {
         HyperlightJSWorkerState { observer: observer }
     }
 
-    fn new_context(
-        worker: &Self::WorkerState,
-        _strategy: SandboxReuseStrategy,
-    ) -> Self::Context {
+    fn new_context(worker: &Self::WorkerState, _strategy: SandboxReuseStrategy) -> Self::Context {
         let js = SandboxBuilder::new()
             .build()
             .unwrap()
             .load_runtime()
             .unwrap();
 
-        let observer = worker.observer.clone();
-
         HyperlightJSContext {
-            sandbox: Some(js),
-            loaded_sandbox: None,
-            observer: observer,
+            state: JSState::Unloaded(js),
+            observer: worker.observer.clone(),
         }
     }
 
-    fn load(mut ctx: Self::Context) -> Self::Context {
-        let mut js = ctx.sandbox.take().unwrap();
+    fn load(ctx: Self::Context) -> Self::Context {
+        let JSState::Unloaded(mut js) = ctx.state else {
+            panic!("JS load requires an unloaded sandbox");
+        };
+        // Restore models a different customer's handler on each request, so registration belongs here.
         js.add_handler("handler".to_string(), HANDLER.to_string().into())
             .unwrap();
         let loaded = js.get_loaded_sandbox().unwrap();
         HyperlightJSContext {
-            sandbox: None,
-            loaded_sandbox: Some(loaded),
+            state: JSState::Loaded(loaded),
             observer: ctx.observer,
         }
     }
 
-    fn unload(mut ctx: Self::Context) -> Self::Context {
-        let loaded = ctx.loaded_sandbox.take().unwrap();
+    fn unload(ctx: Self::Context) -> Self::Context {
+        let JSState::Loaded(loaded) = ctx.state else {
+            panic!("JS unload requires a loaded sandbox");
+        };
         let js = loaded.unload().unwrap();
         HyperlightJSContext {
-            sandbox: Some(js),
-            loaded_sandbox: None,
+            state: JSState::Unloaded(js),
             observer: ctx.observer,
         }
     }
 
     fn handle_request(ctx: &mut Self::Context) -> String {
-        let loaded = ctx.loaded_sandbox.as_mut().unwrap();
+        let JSState::Loaded(loaded) = &mut ctx.state else {
+            panic!("JS handle_request requires a loaded sandbox");
+        };
         let interrupt_handle = loaded.interrupt_handle();
 
         if let Some(obs) = &ctx.observer {

crates/server/src/handlers/hyperlight_wasm.rs

@@ -1,5 +1,5 @@
 use super::{Handler, SandboxMemory};
-use crate::SandboxReuseStrategy;
+use crate::{DEFAULT_REQUEST_URI, SandboxReuseStrategy};
 
 use bindings::hyperlight::bench::handler_interface::Request;
 use hyperlight_host::sandbox::snapshot::Snapshot;
@@ -54,11 +54,23 @@ impl bindings::hyperlight::bench::HandlerWorldImports<hyperlight_common::compone
 {
 }
 
+enum WasmState {
+    Unloaded {
+        sandbox: WasmSandbox,
+        resources: Arc<std::sync::Mutex<bindings::HandlerWorldResources<MyState>>>,
+    },
+    Loaded(bindings::HandlerWorldSandbox<MyState, LoadedWasmSandbox>),
+}
+
+enum WasmLifecycle {
+    Renew,
+    Restore(Arc<Snapshot>),
+    Reuse(Arc<Snapshot>),
+}
+
 pub struct HyperlightWASMContext {
-    unloaded: Option<WasmSandbox>,
-    loaded_snapshot: Arc<Snapshot>,
-    wrapped: Option<bindings::HandlerWorldSandbox<MyState, LoadedWasmSandbox>>,
-    rt: Option<std::sync::Arc<std::sync::Mutex<bindings::HandlerWorldResources<MyState>>>>,
+    state: WasmState,
+    lifecycle: WasmLifecycle,
     aot: Arc<PreparedAot>,
     observer: Option<Arc<CpuTimeObserver>>,
 }
@@ -111,73 +123,84 @@ impl Handler for HyperlightWASMHandler {
         let rt = bindings::register_host_functions(&mut sb, state).unwrap();
         let sb = sb.load_runtime().unwrap();
 
-        let mut loaded_sb = unsafe {
-            sb.load_module_by_mapping(worker.aot.base, worker.aot.len).unwrap()
+        let prepare_snapshot = |sandbox: WasmSandbox, lifecycle: fn(Arc<Snapshot>) -> WasmLifecycle| {
+            let mut loaded = unsafe {
+                sandbox.load_module_by_mapping(worker.aot.base, worker.aot.len)
+                    .expect("Failed to load the Wasm module for snapshot preparation")
+            };
+            let snapshot = loaded.snapshot().expect("Failed to snapshot the loaded Wasm module");
+            let sandbox = loaded.unload_module().expect("Failed to unload the Wasm module after snapshot preparation");
+            (sandbox, lifecycle(snapshot))
+        };
+        let (sb, lifecycle) = match strategy {
+            SandboxReuseStrategy::New => (sb, WasmLifecycle::Renew),
+            SandboxReuseStrategy::Reload => prepare_snapshot(sb, WasmLifecycle::Restore),
+            SandboxReuseStrategy::Reuse => prepare_snapshot(sb, WasmLifecycle::Reuse),
         };
-        let loaded_snapshot = loaded_sb.snapshot().unwrap();
-        let sb = loaded_sb.unload_module().unwrap();
 
         HyperlightWASMContext {
-            unloaded: Some(sb),
-            loaded_snapshot,
-            wrapped: None,
-            rt: Some(rt),
+            state: WasmState::Unloaded { sandbox: sb, resources: rt },
+            lifecycle,
             aot: worker.aot.clone(),
             observer: observer,
         }
     }
 
     fn load(mut ctx: Self::Context) -> Self::Context {
-        let wasm_sandbox = ctx.unloaded.take().unwrap();
-        let sb = wasm_sandbox
-            .load_from_snapshot(ctx.loaded_snapshot.clone())
-            .unwrap();
-
-        let wrapped = bindings::HandlerWorldSandbox {
-            sb,
-            rt: ctx.rt.unwrap(),
+        let WasmState::Unloaded { sandbox, resources } = ctx.state else {
+            panic!("Wasm load requires an unloaded sandbox");
+        };
+        let sb = match &ctx.lifecycle {
+            WasmLifecycle::Renew => {
+                // The context retains the AOT mapping for the loaded module's lifetime.
+                unsafe {
+                    sandbox.load_module_by_mapping(ctx.aot.base, ctx.aot.len)
+                        .expect("Failed to load the Wasm module for Renew")
+                }
+            }
+            WasmLifecycle::Restore(snapshot) | WasmLifecycle::Reuse(snapshot) => sandbox
+                .load_from_snapshot(snapshot.clone())
+                .expect("Failed to restore the loaded Wasm module snapshot"),
         };
 
-        HyperlightWASMContext {
-            unloaded: None,
-            wrapped: Some(wrapped),
-            loaded_snapshot: ctx.loaded_snapshot,
-            rt: None,
-            aot: ctx.aot,
-            observer: ctx.observer,
-        }
+        ctx.state = WasmState::Loaded(bindings::HandlerWorldSandbox {
+            sb,
+            rt: resources,
+        });
+        ctx
     }
 
     fn unload(mut ctx: Self::Context) -> Self::Context {
-        let wrapped = ctx.wrapped.take().unwrap();
-        let unloaded = wrapped.sb.unload_module().unwrap();
+        let WasmLifecycle::Restore(_) = &ctx.lifecycle else {
+            panic!("Only Restore may unload a Wasm module");
+        };
+        let WasmState::Loaded(wrapped) = ctx.state else {
+            panic!("Wasm unload requires a loaded module");
+        };
+        let unloaded = wrapped.sb.unload_module().expect("Failed to unload the Wasm module");
 
-        HyperlightWASMContext {
-            unloaded: Some(unloaded),
-            wrapped: None,
-            loaded_snapshot: ctx.loaded_snapshot,
-            rt: Some(wrapped.rt),
-            aot: ctx.aot,
-            observer: ctx.observer,
-        }
+        ctx.state = WasmState::Unloaded { sandbox: unloaded, resources: wrapped.rt };
+        ctx
     }
 
     fn handle_request(ctx: &mut Self::Context) -> String {
         use bindings::hyperlight::bench::HandlerInterface;
 
-        let world_sb = ctx.wrapped.as_ref().unwrap();
+        let WasmState::Loaded(world_sb) = &mut ctx.state else {
+            panic!("Wasm handle_request requires a loaded module");
+        };
         let handle = world_sb.sb.interrupt_handle().unwrap();
 
         if let Some(obs) = &ctx.observer {
             obs.start_timeout(&handle);
         }
 
         let handler = bindings::hyperlight::bench::HandlerWorldExports::handler_interface(
-            ctx.wrapped.as_mut().unwrap(),
+            world_sb,
         );
 
         let request = Request {
-            uri: "/default.html".to_string(),
+            uri: DEFAULT_REQUEST_URI.to_string(),
         };
 
         let response = handler.handleevent(request).unwrap();
@@ -187,6 +210,6 @@ impl Handler for HyperlightWASMHandler {
             obs.stop_timeout(&handle);
         }
 
-        format!("{{\"uri\":\"{}\"}}", uri).to_string()
+        format!("{{\"uri\":\"{}\"}}", uri)
     }
 }