diff --git a/Cargo.lock b/Cargo.lock index 9d5960f..cb35539 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -41,6 +50,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -106,6 +121,12 @@ dependencies = [ "rustversion", ] +[[package]] +name = "assert_matches" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" + [[package]] name = "async-channel" version = "2.5.0" @@ -346,6 +367,12 @@ version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.4.0" @@ -568,6 +595,42 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools 0.13.0", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "tokio", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools 0.13.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -577,6 +640,16 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + [[package]] name = "crossbeam-epoch" version = "0.9.20" @@ -1442,6 +1515,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -1999,6 +2081,12 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "openssl-probe" version = "0.2.1" @@ -2105,12 +2193,14 @@ version = "0.38.0" dependencies = [ "anyhow", "arc-swap", + "assert_matches", "axum", "axum-prometheus", "backon", "chrono", "ciborium", "clap", + "criterion", "crossbeam-skiplist", "dashmap", "either", @@ -2188,6 +2278,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "parking" version = "2.2.1" @@ -2255,6 +2355,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "portable-atomic" version = "1.14.0" @@ -2596,6 +2724,26 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -3538,6 +3686,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tinyvec" version = "1.12.0" diff --git a/default.nix b/default.nix index ef1e7ae..043db99 100644 --- a/default.nix +++ b/default.nix @@ -72,6 +72,8 @@ let # This is needed for the libsqlite3-sys crate to find the correct sqlite3. export PKG_CONFIG_PATH="${pkgs.sqlite.dev}/lib/pkgconfig''${PKG_CONFIG_PATH:+:''${PKG_CONFIG_PATH}}" ''; + # Fixes: libstdc++.so.6: cannot open shared object file: No such file or directory + LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath [ pkgs.stdenv.cc.cc.lib ]; }; }; in diff --git a/justfile b/justfile index 9b9dd67..4bd4099 100644 --- a/justfile +++ b/justfile @@ -75,6 +75,16 @@ nix-test-integration *TEST_ARGS: nix-build timeout --signal term --kill-after {{pytest_timeout_kill_seconds}} {{pytest_timeout_seconds}} pytest --color=yes {{TEST_ARGS}} +# Benchmark query cost and render a graph. +[group('bench')] +bench-chunks-select: + #!/usr/bin/env bash + set -euo pipefail + cargo bench --bench chunks_select + + source "./libs/opsqueue_python/.setup_local_venv.sh" + python opsqueue/benches/plot_chunks_select.py + # Run all linters, fast and slow [group('lint')] lint: (lint-light "--all-files") lint-heavy diff --git a/libs/opsqueue_python/.setup_local_venv.sh b/libs/opsqueue_python/.setup_local_venv.sh index 7794c4f..afa395d 100644 --- a/libs/opsqueue_python/.setup_local_venv.sh +++ b/libs/opsqueue_python/.setup_local_venv.sh @@ -1,15 +1,19 @@ +relative_dir_path="$(dirname "${BASH_SOURCE[0]}")" +absolute_dir_path="$(cd "$relative_dir_path" && pwd)" + python_version_hash=$(python --version --version | sha1sum | cut -c1-40) +venv_path="$absolute_dir_path/.venv-$python_version_hash" # Then, we (create iff necessary) and activate an (empty!) virtual env # so Maturin doesn't complain when running `maturin develop`. -if [ ! -d ".venv-$python_version_hash" ]; then +if [ ! -d "$venv_path" ]; then echo "Creating an empty Python virtualenv to be able to run 'maturin develop', for Python version '$(python --version --version)' (hash $python_version_hash)..." - python -m venv ".venv-$python_version_hash" + python -m venv "$venv_path" echo "Done!" fi -source ".venv-$python_version_hash/bin/activate" +source "$venv_path/bin/activate" # Ensure `pytest` is available in the venv; # If we were to use `pytest` from Nix, it would not see the locally built python package! -uv pip install -r pyproject.toml --all-extras --quiet +uv pip install -r "$absolute_dir_path/pyproject.toml" --all-extras --quiet diff --git a/libs/opsqueue_python/pyproject.toml b/libs/opsqueue_python/pyproject.toml index c30ad39..6120c62 100644 --- a/libs/opsqueue_python/pyproject.toml +++ b/libs/opsqueue_python/pyproject.toml @@ -45,6 +45,7 @@ test = [ "pytest-xdist==3.8.0", "multiprocess==0.70.19", "pytest-timeout==2.4.0", + "matplotlib>=3.11.1", ] [tool.pytest.ini_options] diff --git a/opsqueue/Cargo.toml b/opsqueue/Cargo.toml index 310eb64..0bbcecc 100644 --- a/opsqueue/Cargo.toml +++ b/opsqueue/Cargo.toml @@ -79,7 +79,14 @@ workspace-hack.workspace = true workspace = true [dev-dependencies] +criterion.workspace = true insta.workspace = true +assert_matches.workspace = true + +[[bench]] +name = "chunks_select" +harness = false +required-features = ["server-logic"] [features] # Dependencies only in use by the server-logic: diff --git a/opsqueue/benches/chunks_select.rs b/opsqueue/benches/chunks_select.rs new file mode 100644 index 0000000..c36f97c --- /dev/null +++ b/opsqueue/benches/chunks_select.rs @@ -0,0 +1,318 @@ +/// For each shape: +/// For each amount of chunks: +/// Seed (all chunks) or extend (additional chunks) the database +/// FOR EACH strategy: +/// Create new dispatcher. +/// Run benchmark +/// Calculate stats +/// Write result +use opsqueue::common::StrategicMetadataMap; +use opsqueue::common::chunk::{ChunkId, ChunkSize}; +use opsqueue::common::submission::db::insert_submission_from_chunks; +use opsqueue::consumer::dispatcher::Dispatcher; +use opsqueue::consumer::strategy::Strategy; +use opsqueue::db::{self}; +use std::io::Write; +use std::num::NonZero; +use std::path::PathBuf; +use std::time::{Duration, Instant}; + +// Human-readable name for an OpsQueue strategy. +type StrategyName = &'static str; + +// Shape of the data we are inserting. +#[derive(Debug, Eq, Ord, PartialEq, PartialOrd)] +enum Shape { + FewSubmissionsManyChunks, + ManySubmissionsFewChunks, + Realistic, +} + +const CHUNKS_PER_METADATA_VALUE: u64 = CHUNKS_PER_SUBMISSION * SUBMISSIONS_PER_METADATA_VALUE; +const CHUNKS_PER_SUBMISSION: u64 = 1024; +// Increase in the total amount of chunks for the `Shape::Realistic` strategy. +const CHUNKS_STEP: usize = 500_000; +const MAX_CHUNKS: u64 = 4_500_001; +const METADATA_VALUES: u64 = 100_000; +const SUBMISSIONS: u64 = 300_000; +// The amount of samples to collect per iteration of the +// shape/amount_of_chunks/strategy main loop. Note that each "sample" itself +// requires a number of reservations to be performed (see `bench_strategy`). +const SAMPLES_PER_STAT: usize = 20; +const SUBMISSIONS_PER_METADATA_VALUE: u64 = SUBMISSIONS / METADATA_VALUES; +// The amount of reservations to collect a single sample. +// Note that the time between these is NOT INDEPENDENT: +// the earlier reservations affect the later ones. +const RESERVATIONS_IN_SAMPLE: usize = 30; +// The amount of reservations to perform before we begin collecting a sample. +const RESERVATIONS_IN_WARMUP: usize = 5; + +#[derive(Debug)] +pub struct BenchStats { + pub median: f64, // sorted_samples[len(samples) / 2] + pub p10: f64, // sorted_samples[len(samples) * 0.1] + pub p90: f64, // sorted_samples[len(samples) * 0.9] +} + +#[allow(clippy::missing_panics_doc)] +impl BenchStats { + #[must_use] + /// Benchmark stats from reservation durations. For code simplicity, and + /// given it's only a benchmark, not library code, we intentionally avoid + /// the unnecessary error handling. + pub fn new(runs: Vec>) -> Self { + // Calculate the median for each individual run. + let mut run_medians: Vec = runs + .into_iter() + .map(|mut run_samples| { + run_samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + run_samples[run_samples.len() / 2] + }) + .collect(); + // Sort the N medians to find p10 etc. of the medians. + run_medians.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let len = run_medians.len(); + BenchStats { + p10: run_medians[len * 10 / 100], + median: run_medians[len / 2], + p90: run_medians[len * 90 / 100], + } + } +} + +/// Maximum amount of chunks for the bench run. +/// +/// This is based on the shape of the data, as that can dramatically affect +/// run-time, and for some shapes we can push the total chunks higher without +/// waiting too long. +fn max_chunks_by_shape(shape: &Shape) -> Vec { + let mut vector: Vec = vec![ + 100, 500, 1_000, 2_000, 4_000, 6_000, 8_000, 10_000, 15_000, 20_000, 30_000, 50_000, + 75_000, 100_000, 200_000, 500_000, 1_000_000, + ]; + if shape == &Shape::Realistic { + let max = MAX_CHUNKS.min(METADATA_VALUES * CHUNKS_PER_METADATA_VALUE); + vector.extend((*vector.iter().max().unwrap()..max).step_by(CHUNKS_STEP)); + } + vector +} + +/// All the strategies we are benchmarking. +fn strategies() -> [(StrategyName, Strategy); 2] { + [ + ("Random", Strategy::Random), + ( + "PreferDistinct(metadata_value, Oldest)", + Strategy::PreferDistinct { + meta_key: "metadata_value".to_string(), + underlying: Box::new(Strategy::Oldest), + }, + ), + ] +} + +/// (total metadata values, `submissions_per_metadata_value`, chunks per +/// submission) for a shape and total chunks. Note we don't always quite reach +/// `total_chunks` due to integer division, we return the closest layout of +/// chunks respecting the given shape <= `total_chunks`. +fn layout(shape: &Shape, total_chunks: u64) -> (u64, u64, u64) { + match shape { + Shape::ManySubmissionsFewChunks => (total_chunks, 1, 1), + Shape::FewSubmissionsManyChunks => { + let submissions = 10.min(total_chunks.max(1)); + (submissions, 1, total_chunks.div_ceil(submissions)) + } + Shape::Realistic => { + let metadata_values = total_chunks.div_ceil(CHUNKS_PER_METADATA_VALUE); + ( + metadata_values, + SUBMISSIONS_PER_METADATA_VALUE, + CHUNKS_PER_SUBMISSION, + ) + } + } +} + +/// Either seeds a fresh DB with all chunks or extend the existing DB with additional chunks. +async fn seed_or_extend( + shape: &Shape, + total_chunks: u64, + current_db_pools: Option, // What was returned by `layout` on the previous call to `seed_or_extend`. + current_layout: Option<(u64, u64, u64)>, + db_path: &PathBuf, +) -> (db::DBPools, Option<(u64, u64, u64)>) { + // Determine the layout of the data we want to insert. + let next_layout + @ (total_metadata_values, submissions_per_metadata_value, chunks_per_submission) = + layout(shape, total_chunks); + // Check if we are seeding the database from scratch, or extending. We can + // only extend if the shape of the data is compatible with the current + // inserted data. + let (we_are_extending, current_metadata_values) = if let Some(( + current_metadata_values_, + current_submissions_per_metadata_value, + current_chunks_per_submission, + )) = current_layout + && current_submissions_per_metadata_value == submissions_per_metadata_value + && current_chunks_per_submission == chunks_per_submission + { + (true, current_metadata_values_) + } else { + (false, 0) + }; + // Time to acquire the DB connection. Either an existing connection, or set up from scratch. + println!(); // Visually separate each run. + let db_pools = if we_are_extending { + println!("Extending database with shape={shape:?} to total_chunks={total_chunks}:"); + current_db_pools.unwrap() + } else { + println!("Seeding database with shape={shape:?} and total_chunks={total_chunks}:"); + drop(current_db_pools); + std::fs::remove_file(db_path).ok(); + std::fs::remove_file(db_path.with_extension("sqlite-wal")).ok(); + std::fs::remove_file(db_path.with_extension("sqlite-shm")).ok(); + db::open_and_setup(db_path.to_str().unwrap(), NonZero::new(16).unwrap()).await + }; + println!( + " Metadata values: {total_metadata_values:<30}\n Submissions per metadata value: {submissions_per_metadata_value}\n Chunks per submission: {chunks_per_submission}" + ); + let mut conn = db_pools.writer_conn().await.unwrap(); + for metadata_value in current_metadata_values..total_metadata_values { + for _ in 0..submissions_per_metadata_value { + let mut metadata = StrategicMetadataMap::default(); + metadata.insert( + "metadata_value".to_string(), + i64::try_from(metadata_value).unwrap(), + ); + let chunks = vec![Some(b"x".to_vec()); usize::try_from(chunks_per_submission).unwrap()]; + insert_submission_from_chunks( + None, + chunks, + None, + metadata, + ChunkSize::default(), + &mut conn, + ) + .await + .unwrap(); + } + } + (db_pools, Some(next_layout)) +} + +/// Runs the selection query and fetches just the first chunk. +async fn fetch_and_reserve_chunk( + db_pools: &db::DBPools, + strategy: Strategy, + dispatcher: &Dispatcher, +) -> ChunkId { + use tokio::sync::mpsc::unbounded_channel; + let (notifier, _) = unbounded_channel(); + let reserved = dispatcher + .fetch_and_reserve_chunks(db_pools.reader_pool(), strategy, 1, ¬ifier) + .await + .unwrap(); + if let [(chunk, _)] = reserved.as_slice() { + ChunkId::from((chunk.submission_id, chunk.chunk_index)) + } else { + panic!( + "Yowza! Expected exactly 1 chunk, but got {}", + reserved.len() + ); + } +} + +/// Executes reservations for a given DB state and returns `BenchStats`. +async fn bench_strategy( + db_pools: &db::DBPools, + strategy: &Strategy, + dispatcher: &Dispatcher, +) -> BenchStats { + let mut all_reservation_durations: Vec> = Vec::with_capacity(SAMPLES_PER_STAT); + let mut chunks_reserved = Vec::with_capacity(RESERVATIONS_IN_WARMUP + RESERVATIONS_IN_SAMPLE); + // For each of the samples we want to collect. + for _ in 0..SAMPLES_PER_STAT { + let mut sample_reservation_durations = Vec::with_capacity(RESERVATIONS_IN_SAMPLE); + // For each sample, reserve N chunks (some are warmups). + for i in 0..(RESERVATIONS_IN_WARMUP + RESERVATIONS_IN_SAMPLE) { + let start = Instant::now(); + let chunk_id = fetch_and_reserve_chunk(db_pools, strategy.clone(), dispatcher).await; + if i >= RESERVATIONS_IN_WARMUP { + sample_reservation_durations.push(start.elapsed().as_secs_f64() * 1e6); + } + chunks_reserved.push(chunk_id); + } + all_reservation_durations.push(sample_reservation_durations); + // Reset the queue state for the next run + let mut conn = db_pools.writer_conn().await.unwrap(); + for chunk_id in chunks_reserved.drain(..) { + dispatcher + .finish_reservation(&mut conn, chunk_id, false) + .await; + } + } + BenchStats::new(all_reservation_durations) +} + +fn main() { + let runtime = tokio::runtime::Runtime::new().unwrap(); + let csv_path = PathBuf::from("benches/chunks_select_bench.csv"); + if let Some(parent) = csv_path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + let mut csv = std::fs::File::create(&csv_path) + .unwrap_or_else(|e| panic!("Failed to create CSV at {}: {}", csv_path.display(), e)); + writeln!(csv, "shape,strategy,backlog_size,p10_us,median_us,p90_us").unwrap(); + // Pretty header. + println!( + "{:<30} {:<35} {:<12} {:<10} {:<15}", + "SHAPE", "STRATEGY", "SIZE", "MEDIAN", "P10 / P90" + ); + println!("{}", "-".repeat(105)); + let db_path = std::env::temp_dir().join("opsqueue_bench.sqlite"); + for shape in &[ + Shape::FewSubmissionsManyChunks, + Shape::ManySubmissionsFewChunks, + Shape::Realistic, + ] { + // Track the current database state. + let mut db_pools: Option = None; + let mut current_layout: Option<(u64, u64, u64)> = None; + for size in max_chunks_by_shape(shape) { + // Either seed a new database or extend an existing one. + let returned_pools; + (returned_pools, current_layout) = runtime.block_on(seed_or_extend( + shape, + size, + db_pools, + current_layout, + &db_path, + )); + db_pools = Some(returned_pools); + // Then for each strategy: hit the bench! + for (strategy_label, strategy) in strategies() { + let dispatcher = Dispatcher::new(Duration::from_mins(1_000_000)); + let stats = runtime.block_on(bench_strategy( + db_pools.as_ref().unwrap(), + &strategy, + &dispatcher, + )); + let bounds = format!("{:.1} / {:.1}", stats.p10, stats.p90); + println!( + "{:<30} {:<38} {:<12} {:<10.1} {:<15}", + format!("{shape:?}"), + strategy_label, + size, + stats.median, + bounds + ); + writeln!( + csv, + "{shape:?},\"{strategy_label}\",{size},{:.1},{:.1},{:.1}", + stats.p10, stats.median, stats.p90 + ) + .unwrap(); + } + } + } +} diff --git a/opsqueue/benches/chunks_select_bench.svg b/opsqueue/benches/chunks_select_bench.svg new file mode 100644 index 0000000..754380d --- /dev/null +++ b/opsqueue/benches/chunks_select_bench.svg @@ -0,0 +1,5777 @@ + + + + + + + + 1980-01-01T00:00:00+00:00 + image/svg+xml + + + Matplotlib v3.11.1, https://matplotlib.org/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/opsqueue/benches/plot_chunks_select.py b/opsqueue/benches/plot_chunks_select.py new file mode 100644 index 0000000..ba0631f --- /dev/null +++ b/opsqueue/benches/plot_chunks_select.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Plot the chunks_select benchmark: one subplot per backlog shape, a line per strategy. + +Reads chunks_select_bench.csv (written by the bench) and writes a SVG next to +this script so it can be committed alongside the code it measures. + +""" + +import csv +import matplotlib.pyplot as plt # noqa: E402 +from collections import defaultdict + +CSV_PATH = "opsqueue/benches/chunks_select_bench.csv" +OUT_PATH = "opsqueue/benches/chunks_select_bench.svg" + + +def main() -> None: + # {shape: {strategy: {backlog_size: (p10_us, median_us, p90_us)}}} + series = defaultdict(lambda: defaultdict(dict)) + shapes: list[str] = [] + max_x = 0 + + print(f"Reading CSV: {CSV_PATH}") + with open(CSV_PATH) as f: + for row in csv.DictReader(f): + shape = row["shape"] + size = int(row["backlog_size"]) + max_x = max(max_x, size) + series[shape][row["strategy"]][size] = ( + float(row["p10_us"]), + float(row["median_us"]), + float(row["p90_us"]), + ) + if shape not in shapes: + shapes.append(shape) + + fig, axes = plt.subplots(1, len(shapes), figsize=(7.5 * len(shapes), 5.5)) + fig.suptitle("Reservation duration (no network in path)") + + for ax, shape in zip(axes.flatten(), shapes): + for strategy, points in sorted(series[shape].items()): + xs = sorted(points) + p10s, medians, p90s = zip(*[points[x] for x in xs]) + (line,) = ax.plot(xs, medians, marker="o", label=strategy) + ax.fill_between(xs, p10s, p90s, color=line.get_color(), alpha=0.2) + + ax.set_xscale("log") + ax.set_yscale("log") + ax.set_xlim(right=max_x) + ax.set_ylim(top=1_000_000) + ax.set_title(shape) + ax.set_xlabel("Backlog size (total chunks)") + ax.set_ylabel("Latency to select first chunk (us) [Median ± P10/P90]") + ax.grid(True, which="both", linestyle=":", alpha=0.5) + ax.legend(fontsize=9) + + fig.tight_layout() + fig.savefig(OUT_PATH, dpi=150, bbox_inches="tight") + print(f"Wrote {OUT_PATH}") + + +if __name__ == "__main__": + main()