diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99ee1761..007467ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,14 +30,21 @@ jobs: run: cmake -S . -B build-bench -DLOGIT_BENCH_ENABLE=ON -DLOGIT_BENCH_WITH_SPDLOG=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DLOGIT_WITH_SYSLOG=ON -DLOGIT_WITH_WIN_EVENT_LOG=OFF - name: Build benchmarks # if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/stable') }} - run: cmake --build build-bench --target logit_bench logit_bench_flush_test logit_public_macro_bench logit_hotpath_bench logit_hotpath_bench_legacy benchmark_validation_test + run: cmake --build build-bench --target logit_bench logit_bench_flush_test logit_public_macro_bench logit_public_macro_formatted_bench logit_hotpath_bench logit_hotpath_bench_legacy benchmark_validation_test - name: Run spdlog async flush regression run: ./build-bench/logit_bench_flush_test - name: Run public macro benchmark smoke env: LOGIT_PUBLIC_BENCH_TOTAL: 2000 LOGIT_PUBLIC_BENCH_PRODUCERS: 4 + LOGIT_PUBLIC_BENCH_WARMUP: 200 run: ./build-bench/logit_public_macro_bench + - name: Run formatted public macro benchmark smoke + env: + LOGIT_PUBLIC_BENCH_TOTAL: 2000 + LOGIT_PUBLIC_BENCH_PRODUCERS: 4 + LOGIT_PUBLIC_BENCH_WARMUP: 200 + run: ./build-bench/logit_public_macro_formatted_bench - name: Run benchmark validation tests run: ./build-bench/benchmark_validation_test - name: Run logger hot-path A/B smoke diff --git a/bench/BenchmarkMetadata.hpp b/bench/BenchmarkMetadata.hpp new file mode 100644 index 00000000..494520c7 --- /dev/null +++ b/bench/BenchmarkMetadata.hpp @@ -0,0 +1,234 @@ +#pragma once + +#ifndef LOGIT_CPP_HEADER_BENCH_BENCHMARK_METADATA_HPP_INCLUDED +#define LOGIT_CPP_HEADER_BENCH_BENCHMARK_METADATA_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace logit_bench { + +struct BenchmarkMetadata { + std::string source_commit; + std::string compiler; + std::string compiler_version; + std::string toolchain; + std::string cxx_standard; + std::string platform; + std::string build_type; + std::string architecture; + std::string machine_id; + std::string cpu_model; + std::string queue_capacity; + std::string queue_policy; + std::string latency_completion; + std::string flush_barrier; +}; + +inline std::string benchmark_env(const char* name, const char* fallback) { + if (const char* value = std::getenv(name)) { + if (*value != '\0') return value; + } + return fallback; +} + +inline std::string benchmark_label(const char* name, const char* fallback) { + std::string value = benchmark_env(name, fallback); + for (char& character : value) { + if (character == ' ' || character == '\t' || + character == '\r' || character == '\n') { + character = '_'; + } + } + return value; +} + +inline std::string benchmark_compiler() { +#if defined(_MSC_VER) + return "MSVC"; +#elif defined(__clang__) + return "Clang"; +#elif defined(__GNUC__) + return "GCC"; +#else + return "unknown"; +#endif +} + +inline std::string benchmark_compiler_version() { +#if defined(_MSC_VER) + return std::to_string(_MSC_VER); +#elif defined(__clang__) + return std::to_string(__clang_major__) + "." + + std::to_string(__clang_minor__) + "." + + std::to_string(__clang_patchlevel__); +#elif defined(__GNUC__) + return std::to_string(__GNUC__) + "." + + std::to_string(__GNUC_MINOR__) + "." + + std::to_string(__GNUC_PATCHLEVEL__); +#else + return "unknown"; +#endif +} + +inline std::string benchmark_source_commit() { + if (const char* value = std::getenv("LOGIT_BENCH_COMMIT")) { + if (*value != '\0') return value; + } + if (const char* value = std::getenv("GITHUB_SHA")) { + if (*value != '\0') return value; + } + return "unknown"; +} + +inline std::string benchmark_cxx_standard() { +#if defined(_MSVC_LANG) + return std::to_string(_MSVC_LANG); +#else + return std::to_string(__cplusplus); +#endif +} + +inline std::string benchmark_platform() { +#if defined(__EMSCRIPTEN__) + return "emscripten"; +#elif defined(_WIN32) + return "windows"; +#elif defined(__APPLE__) + return "macos"; +#elif defined(__linux__) + return "linux"; +#else + return "unknown"; +#endif +} + +inline std::string benchmark_architecture() { +#if defined(__EMSCRIPTEN__) && defined(__wasm32__) + return "wasm32"; +#elif defined(_M_X64) || defined(__x86_64__) + return "x86_64"; +#elif defined(_M_IX86) || defined(__i386__) + return "x86"; +#elif defined(_M_ARM64) || defined(__aarch64__) + return "arm64"; +#elif defined(_M_ARM) || defined(__arm__) + return "arm"; +#else + return "unknown"; +#endif +} + +inline std::string benchmark_build_type() { + if (const char* value = std::getenv("LOGIT_BENCH_BUILD_TYPE")) { + if (*value != '\0') return value; + } +#ifdef LOGIT_BENCH_BUILD_TYPE + return LOGIT_BENCH_BUILD_TYPE; +#else + return "unknown"; +#endif +} + +inline BenchmarkMetadata make_benchmark_metadata( + std::string queue_capacity, + std::string queue_policy, + std::string latency_completion, + std::string flush_barrier) { + const std::string compiler = benchmark_compiler(); + const std::string compiler_version = benchmark_compiler_version(); + std::string toolchain = compiler + "-" + compiler_version; + toolchain = benchmark_label("LOGIT_BENCH_TOOLCHAIN", toolchain.c_str()); + return BenchmarkMetadata{ + benchmark_source_commit(), + compiler, + compiler_version, + std::move(toolchain), + benchmark_cxx_standard(), + benchmark_platform(), + benchmark_build_type(), + benchmark_architecture(), + benchmark_label("LOGIT_BENCH_MACHINE_ID", "unknown"), + benchmark_label("LOGIT_BENCH_CPU_MODEL", "unknown"), + std::move(queue_capacity), + std::move(queue_policy), + std::move(latency_completion), + std::move(flush_barrier)}; +} + +inline bool benchmark_value_unknown(const std::string& value) { + return value.empty() || value == "unknown"; +} + +inline void validate_comparable_metadata(const BenchmarkMetadata& metadata, + bool require_comparable) { + if (!require_comparable) return; + + struct MetadataField { + const char* name; + const std::string* value; + bool allow_not_applicable; + }; + const MetadataField values[] = { + {"source_commit", &metadata.source_commit, false}, + {"compiler", &metadata.compiler, false}, + {"compiler_version", &metadata.compiler_version, false}, + {"toolchain", &metadata.toolchain, false}, + {"cxx_standard", &metadata.cxx_standard, false}, + {"platform", &metadata.platform, false}, + {"build_type", &metadata.build_type, false}, + {"architecture", &metadata.architecture, false}, + {"machine_id", &metadata.machine_id, false}, + {"cpu_model", &metadata.cpu_model, false}, + {"queue_capacity", &metadata.queue_capacity, true}, + {"queue_policy", &metadata.queue_policy, true}, + {"latency_completion", &metadata.latency_completion, false}, + {"flush_barrier", &metadata.flush_barrier, false}}; + + for (const auto& value : values) { + if (benchmark_value_unknown(*value.value) || + (!value.allow_not_applicable && *value.value == "not-applicable")) { + throw std::runtime_error( + std::string("Comparable benchmark requires metadata field '") + + value.name + "'; set the corresponding LOGIT_BENCH_* value or disable " + "LOGIT_BENCH_REQUIRE_COMPARABLE"); + } + } +} + +inline void validate_comparable_metadata(const BenchmarkMetadata& metadata) { + const char* required = std::getenv("LOGIT_BENCH_REQUIRE_COMPARABLE"); + validate_comparable_metadata( + metadata, required && std::string(required) == "1"); +} + +inline void print_benchmark_metadata(std::ostream& out, + const BenchmarkMetadata& metadata, + std::size_t total_messages, + std::size_t warmup_messages) { + out << "benchmark-fixture version=1" + << " source_commit=" << metadata.source_commit + << " compiler=" << metadata.compiler + << " compiler_version=" << metadata.compiler_version + << " toolchain=" << metadata.toolchain + << " cxx_standard=" << metadata.cxx_standard + << " platform=" << metadata.platform + << " build_type=" << metadata.build_type + << " architecture=" << metadata.architecture + << " machine_id=" << metadata.machine_id + << " cpu_model=" << metadata.cpu_model + << " queue_capacity=" << metadata.queue_capacity + << " queue_policy=" << metadata.queue_policy + << " latency_completion=" << metadata.latency_completion + << " flush_barrier=" << metadata.flush_barrier + << " total=" << total_messages + << " warmup=" << warmup_messages << '\n'; +} + +} // namespace logit_bench + +#endif // LOGIT_CPP_HEADER_BENCH_BENCHMARK_METADATA_HPP_INCLUDED diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index be08aa9f..a0728789 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -10,6 +10,7 @@ endif() add_executable(logit_bench ${LOGIT_BENCH_SOURCES}) target_include_directories(logit_bench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_definitions(logit_bench PRIVATE LOGIT_BENCH_BUILD_TYPE=\"$\") target_compile_features(logit_bench PRIVATE cxx_std_17) @@ -27,11 +28,23 @@ target_link_libraries(logit_bench PRIVATE log-it-cpp::log-it-cpp) add_executable(logit_public_macro_bench public_macro_bench.cpp) target_compile_features(logit_public_macro_bench PRIVATE cxx_std_17) +target_compile_definitions(logit_public_macro_bench PRIVATE LOGIT_BENCH_BUILD_TYPE=\"$\") target_link_libraries(logit_public_macro_bench PRIVATE log-it-cpp::log-it-cpp) set_target_properties(logit_public_macro_bench PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} ) +add_executable(logit_public_macro_formatted_bench public_macro_bench.cpp) +target_compile_features(logit_public_macro_formatted_bench PRIVATE cxx_std_17) +target_compile_definitions(logit_public_macro_formatted_bench PRIVATE + LOGIT_PUBLIC_BENCH_FORMATTED=1 + LOGIT_BENCH_BUILD_TYPE=\"$\" +) +target_link_libraries(logit_public_macro_formatted_bench PRIVATE log-it-cpp::log-it-cpp) +set_target_properties(logit_public_macro_formatted_bench PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} +) + add_executable(logit_hotpath_bench logger_hotpath_bench.cpp) target_compile_features(logit_hotpath_bench PRIVATE cxx_std_17) target_link_libraries(logit_hotpath_bench PRIVATE log-it-cpp::log-it-cpp) diff --git a/bench/benchmark_validation_test.cpp b/bench/benchmark_validation_test.cpp index 2d19642f..e84e1c48 100644 --- a/bench/benchmark_validation_test.cpp +++ b/bench/benchmark_validation_test.cpp @@ -1,4 +1,5 @@ #include "BenchmarkValidation.hpp" +#include "BenchmarkMetadata.hpp" #include #include @@ -26,5 +27,31 @@ int main() { validate_latency_csv_header(std::string(latency_csv_header()) + "\r"); validate_latency_csv_header(latency_csv_header()); + + const auto comparable = make_benchmark_metadata( + "8192", "block", "sink-entry", "all-prior-work-drained"); + auto complete = comparable; + complete.source_commit = "test-commit"; + complete.compiler = "test-compiler"; + complete.compiler_version = "1"; + complete.toolchain = "test-toolchain"; + complete.cxx_standard = "201703"; + complete.platform = "test-platform"; + complete.build_type = "Release"; + complete.architecture = "test-architecture"; + complete.machine_id = "test-machine"; + complete.cpu_model = "test-cpu"; + validate_comparable_metadata(complete, true); + + auto incomplete = complete; + incomplete.source_commit = "unknown"; + bool rejected_unknown_metadata = false; + try { + validate_comparable_metadata(incomplete, true); + } catch (const std::runtime_error&) { + rejected_unknown_metadata = true; + } + if (!rejected_unknown_metadata) return 3; + return 0; } diff --git a/bench/logit_bench.cpp b/bench/logit_bench.cpp index 19e5bd95..fdd820f0 100644 --- a/bench/logit_bench.cpp +++ b/bench/logit_bench.cpp @@ -21,6 +21,7 @@ #include "LatencyRecorder.hpp" #include "BenchmarkValidation.hpp" +#include "BenchmarkMetadata.hpp" #include "Scenario.hpp" #include "adapters/LogItAdapter.hpp" @@ -394,6 +395,14 @@ int main() { const BenchFilter filter = load_filter(); + const auto metadata = make_benchmark_metadata( + std::to_string(queue_capacity), + "block", + "sink-entry", + "all-prior-work-drained"); + validate_comparable_metadata(metadata); + print_benchmark_metadata(std::cout, metadata, total_messages, warmup_messages); + LOGIT_SET_MAX_QUEUE(queue_capacity); LOGIT_SET_QUEUE_POLICY(LOGIT_QUEUE_BLOCK); diff --git a/bench/public_macro_bench.cpp b/bench/public_macro_bench.cpp index 3c9b2b74..73a33361 100644 --- a/bench/public_macro_bench.cpp +++ b/bench/public_macro_bench.cpp @@ -2,15 +2,19 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include "BenchmarkMetadata.hpp" + namespace { class CountingLogger final : public logit::ILogger { @@ -27,6 +31,7 @@ class CountingLogger final : public logit::ILogger { } void wait() override {} std::size_t count() const { return m_count.load(std::memory_order_relaxed); } + void reset() { m_count.store(0, std::memory_order_relaxed); } private: std::atomic m_count{0}; @@ -40,6 +45,22 @@ class PassthroughFormatter final : public logit::ILogFormatter { bool is_passthrough() const noexcept override { return true; } }; +std::unique_ptr make_formatter() { +#ifdef LOGIT_PUBLIC_BENCH_FORMATTED + return std::make_unique("[%l] %v"); +#else + return std::make_unique(); +#endif +} + +const char* benchmark_mode() { +#ifdef LOGIT_PUBLIC_BENCH_FORMATTED + return "formatted"; +#else + return "passthrough"; +#endif +} + std::size_t env_size(const char* name, std::size_t fallback) { if (const char* value = std::getenv(name)) { try { return static_cast(std::stoull(value)); } @@ -48,38 +69,77 @@ std::size_t env_size(const char* name, std::size_t fallback) { return fallback; } -} // namespace +std::chrono::nanoseconds run_workload(std::size_t producers, std::size_t total) { + std::mutex start_mx; + std::condition_variable start_cv; + std::condition_variable ready_cv; + bool start_flag = false; + std::size_t ready = 0; -int main() { - const std::size_t producers = env_size("LOGIT_PUBLIC_BENCH_PRODUCERS", 4); - const std::size_t total = env_size("LOGIT_PUBLIC_BENCH_TOTAL", 20000); - if (producers == 0 || total == 0) return 2; - - auto sink = std::make_unique(); - auto* sink_ptr = sink.get(); - logit::Logger::get_instance().add_logger( - std::move(sink), std::make_unique()); - - const auto start = std::chrono::steady_clock::now(); std::vector workers; workers.reserve(producers); for (std::size_t producer = 0; producer < producers; ++producer) { - workers.emplace_back([producer, producers, total]() { + workers.emplace_back([&, producer]() { const std::size_t begin = (total * producer) / producers; const std::size_t end = (total * (producer + 1)) / producers; + { + std::unique_lock lock(start_mx); + ++ready; + if (ready == producers) ready_cv.notify_one(); + start_cv.wait(lock, [&] { return start_flag; }); + } for (std::size_t i = begin; i < end; ++i) { LOGIT_INFO("public macro message", i); } }); } + + std::chrono::steady_clock::time_point start; + { + std::unique_lock lock(start_mx); + ready_cv.wait(lock, [&] { return ready == producers; }); + start = std::chrono::steady_clock::now(); + start_flag = true; + } + start_cv.notify_all(); + for (auto& worker : workers) worker.join(); logit::Logger::get_instance().wait(); - const auto elapsed = std::chrono::duration_cast( - std::chrono::steady_clock::now() - start).count(); + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); +} + +} // namespace + +int main() { + const std::size_t producers = env_size("LOGIT_PUBLIC_BENCH_PRODUCERS", 4); + const std::size_t total = env_size("LOGIT_PUBLIC_BENCH_TOTAL", 20000); + const std::size_t warmup = env_size("LOGIT_PUBLIC_BENCH_WARMUP", 0); + if (producers == 0 || total == 0) return 2; + + const auto metadata = logit_bench::make_benchmark_metadata( + "not-applicable", + "not-applicable", + "backend-count", + "logger-wait"); + logit_bench::validate_comparable_metadata(metadata); + logit_bench::print_benchmark_metadata(std::cout, metadata, total, warmup); + + auto sink = std::make_unique(); + auto* sink_ptr = sink.get(); + logit::Logger::get_instance().add_logger( + std::move(sink), make_formatter()); + + if (warmup > 0) { + run_workload(producers, warmup); + sink_ptr->reset(); + } + const auto elapsed = run_workload(producers, total).count(); if (sink_ptr->count() != total) return 1; const double throughput = static_cast(total) * 1e9 / static_cast(elapsed); - std::cout << "public-macro producers=" << producers + std::cout << "public-macro mode=" << benchmark_mode() + << " producers=" << producers << " total=" << total << " elapsed_ns=" << elapsed << " throughput=" << throughput << " msg/s\n"; diff --git a/bench/results/benchmark-fixture-v1.json b/bench/results/benchmark-fixture-v1.json new file mode 100644 index 00000000..b77a9684 --- /dev/null +++ b/bench/results/benchmark-fixture-v1.json @@ -0,0 +1,59 @@ +{ + "fixture_version": 1, + "name": "log-it-cpp-release-benchmark", + "metadata_required": [ + "source_commit", + "compiler", + "compiler_version", + "toolchain", + "cxx_standard", + "platform", + "build_type", + "architecture", + "machine_id", + "cpu_model", + "queue_capacity", + "queue_policy", + "latency_completion", + "flush_barrier" + ], + "metadata_must_match": [ + "compiler", + "compiler_version", + "toolchain", + "cxx_standard", + "platform", + "build_type", + "architecture", + "machine_id", + "cpu_model", + "queue_capacity", + "queue_policy", + "latency_completion", + "flush_barrier" + ], + "latency_workload": { + "total_messages": 200000, + "warmup_messages": 4096, + "producer_counts": [1, 4, 16, 32], + "message_bytes": [40, 200, 1024], + "async_modes": [false, true], + "sinks": ["null", "file"], + "queue_policy": "block", + "latency_completion": "sink-entry", + "flush_barrier": "all-prior-work-drained" + }, + "public_macro_workload": { + "total_messages": 20000, + "producer_counts": [1, 4, 16, 32], + "queue_capacity": "not-applicable", + "queue_policy": "not-applicable", + "latency_completion": "backend-count", + "flush_barrier": "logger-wait", + "scenarios": [ + "public macro record construction and dispatch with passthrough formatter", + "public macro record construction, dispatch, and SimpleLogFormatter formatting" + ] + }, + "measurement_rule": "All required metadata fields must be present. Compare runs only when metadata_must_match fields match; source_commit is provenance and may differ between before/after runs. Set LOGIT_BENCH_REQUIRE_COMPARABLE=1 for publication runs so unknown metadata is rejected; CI runs are regression smoke, not universal performance claims." +} diff --git a/docs/adr/0005-benchmark-methodology.md b/docs/adr/0005-benchmark-methodology.md index 72051642..ffa343e2 100644 --- a/docs/adr/0005-benchmark-methodology.md +++ b/docs/adr/0005-benchmark-methodology.md @@ -15,10 +15,20 @@ publication machine. Keep separate scenarios for prepared-record dispatch, the real public `LOGIT_INFO(...)` path, formatting, and external-library comparisons. The public macro smoke benchmark may use a passthrough formatter when it is -explicitly documented as record-construction/dispatch coverage. Report -absolute timings only with compiler, platform, commit, queue, producer, and -flush settings; treat CI runs as regression smoke unless the environment is -fixed. +explicitly documented as record-construction/dispatch coverage. Producer +scaling measurements create all workers before the timed interval and release +them through a shared start barrier; an optional warmup is separate from the +measured run. Report absolute timings only with compiler, platform, commit, +build type, architecture, machine/CPU identity, queue settings, and separate +latency-completion and flush-barrier semantics. Treat CI runs as regression +smoke unless the environment is fixed. + +The fixture distinguishes required provenance from comparison identity: +`source_commit` must be present and is expected to differ for before/after +measurements, while the fields listed in `metadata_must_match` must match. The +`LOGIT_BENCH_REQUIRE_COMPARABLE=1` mode checks metadata completeness and known +values; it does not enforce the canonical fixture workload unless a separate +workload-validation mode is requested. ## Consequences diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 7053d86e..79cc468c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -51,8 +51,9 @@ The repository includes a **legacy** comparison snapshot from 2025-12-05 in This fixture predates the current benchmark dependency metadata and does not record the spdlog version, compiler/toolchain, or harness commit. Treat it as legacy context rather than a reproducible current performance claim. New -published figures must include the dependency versions, toolchain, harness -commit, queue capacity, and flush semantics used for the run. +published figures must include the fixture metadata, dependency versions, +toolchain, harness commit, build and hardware identity, queue settings, and +separate latency-completion and flush-barrier semantics used for the run. ## Harness details @@ -85,15 +86,42 @@ The prepared-message/direct-dispatch pipeline and a true public macro benchmark calls `LOGIT_INFO(...)` are separate scenarios with different work contracts; their results must not be presented as one number. -`logit_public_macro_bench` is the focused public-API smoke benchmark. It invokes -`LOGIT_INFO(...)` from multiple producer threads and therefore includes argument -name parsing, `args_array` construction, and dispatch. Its passthrough formatter -intentionally bypasses formatter work, so this is a public macro -record-construction + dispatch benchmark rather than a formatting benchmark. -Configure it -with `LOGIT_PUBLIC_BENCH_TOTAL` and `LOGIT_PUBLIC_BENCH_PRODUCERS`; its throughput -is reported separately from `latency.csv` and is intended for before/after -hot-path experiments on identical hardware. +`logit_public_macro_bench` and `logit_public_macro_formatted_bench` are focused +public-API smoke benchmarks. Both invoke `LOGIT_INFO(...)` from multiple +producer threads and therefore include argument-name parsing, `args_array` +construction, and dispatch. Producers are created before the timed interval, +wait on a ready barrier, and are released together; thread creation and the +startup skew are not part of the producer-scaling measurement. The optional +`LOGIT_PUBLIC_BENCH_WARMUP` run is executed before the measured workload. +The first target uses a passthrough formatter and intentionally bypasses +formatter work; the second uses `SimpleLogFormatter` and includes the formatter +path. Their throughputs are separate scenarios and must not be presented as one +number. Configure either with `LOGIT_PUBLIC_BENCH_TOTAL`, +`LOGIT_PUBLIC_BENCH_PRODUCERS`, and `LOGIT_PUBLIC_BENCH_WARMUP`; results are +reported separately from `latency.csv` and are intended for before/after +experiments on identical hardware. Both targets print the same fixture metadata +line as `logit_bench`, using `not-applicable` for queue settings and explicit +`backend-count` / `logger-wait` completion semantics. + +The checked-in [`benchmark-fixture-v1.json`](https://github.com/LimiNode/log-it-cpp/blob/main/bench/results/benchmark-fixture-v1.json) +defines the required metadata and workload contract. All publication-capable +benchmark binaries print a versioned `benchmark-fixture` metadata line for +each run, including source commit, compiler/version, toolchain, C++ standard, +platform, build type, architecture, machine identity, CPU model, queue +settings, latency completion, and flush barrier. The commit defaults to +`LOGIT_BENCH_COMMIT` or `GITHUB_SHA`; machine identity and CPU model can be +provided through `LOGIT_BENCH_MACHINE_ID` and `LOGIT_BENCH_CPU_MODEL`. +For a comparable/publication run, set `LOGIT_BENCH_REQUIRE_COMPARABLE=1` and +provide all required metadata; public-macro runs may explicitly use +`not-applicable` queue settings. Smoke runs may leave unavailable values as +`unknown`. The fixture separates metadata that must be present from metadata +that must match between runs: `source_commit` is required provenance and is +expected to differ in before/after comparisons, while the fields listed in +`metadata_must_match` (compiler, toolchain, platform, build, hardware, queue, +and completion semantics) must be identical. `LOGIT_BENCH_REQUIRE_COMPARABLE=1` +checks metadata completeness and known values; it does not enforce the +canonical fixture workload values such as total messages, warmup, or producer +matrix. `logit_hotpath_bench` and `logit_hotpath_bench_legacy` provide a controlled A/B measurement for the registry read path. Both run the same prepared `LogRecord` @@ -114,6 +142,8 @@ explicit concurrency contract rather than infer one from a benchmark sink. The flush regression target uses an intentionally delayed asynchronous sink and asserts that `flush()` does not return before every queued message has reached -that sink. `benchmark_validation_test` covers the comparative-protocol guardrails +that sink. The fixture records latency completion and the flush barrier as +separate semantics: the latency benchmark completes at sink entry, while the +flush barrier drains all prior work. `benchmark_validation_test` covers the comparative-protocol guardrails (`queue_capacity=0` and legacy CSV schema rejection) without relying on packages installed on the host. diff --git a/docs/future-plans.md b/docs/future-plans.md index b1b895e6..2a18a83d 100644 --- a/docs/future-plans.md +++ b/docs/future-plans.md @@ -53,10 +53,10 @@ Legend: follow-up, not part of the first configuration API. - [ ] **Extended filtering** — evaluate source/file, message, tag/MDC, and range filters; define their cost and ordering before adding public API. -- [ ] **Benchmark follow-up** — add a genuinely formatted public-macro scenario, - a versioned fixture containing compiler/toolchain/commit/queue/flush metadata, - and a broader 1/4/16/32-producer matrix. Keep publication numbers tied to a - fixed machine and toolchain. +- [x] **Benchmark follow-up** — formatted and passthrough public-macro scenarios, + a versioned fixture contract with compiler/toolchain/commit/queue/flush + metadata, and a 1/4/16/32-producer matrix are now covered. Keep publication + numbers tied to a fixed machine and toolchain. - [ ] **Concurrency fast-path research** — only after documenting a formal thread-safety capability for formatters/backends. Do not remove `exec_mx` based on benchmark results alone.