From 008e923057c11f3bc9fc56f222185cc78ac73347 Mon Sep 17 00:00:00 2001 From: Aster Seker Date: Mon, 14 Sep 2026 01:04:40 +0300 Subject: [PATCH 1/3] perf(bench): separate formatted public macro scenario Add a distinct SimpleLogFormatter benchmark alongside the passthrough public macro smoke. Record versioned fixture metadata for latency runs, keep the producer matrix explicit, and execute both public scenarios in CI without changing production logger locking. --- .github/workflows/ci.yml | 7 ++- bench/CMakeLists.txt | 8 +++ bench/logit_bench.cpp | 76 +++++++++++++++++++++++++ bench/public_macro_bench.cpp | 21 ++++++- bench/results/benchmark-fixture-v1.json | 33 +++++++++++ docs/benchmarks.md | 26 ++++++--- docs/future-plans.md | 8 +-- 7 files changed, 163 insertions(+), 16 deletions(-) create mode 100644 bench/results/benchmark-fixture-v1.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99ee1761..84d5cb64 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: run: cmake -S . -B build-bench -DLOGIT_BENCH_ENABLE=ON -DLOGIT_BENCH_WITH_SPDLOG=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DLOGIT_WITH_SYSLOG=ON -DLOGIT_WITH_WIN_EVENT_LOG=OFF - name: Build benchmarks # if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/stable') }} - run: cmake --build build-bench --target logit_bench logit_bench_flush_test logit_public_macro_bench logit_hotpath_bench logit_hotpath_bench_legacy benchmark_validation_test + run: cmake --build build-bench --target logit_bench logit_bench_flush_test logit_public_macro_bench logit_public_macro_formatted_bench logit_hotpath_bench logit_hotpath_bench_legacy benchmark_validation_test - name: Run spdlog async flush regression run: ./build-bench/logit_bench_flush_test - name: Run public macro benchmark smoke @@ -38,6 +38,11 @@ jobs: LOGIT_PUBLIC_BENCH_TOTAL: 2000 LOGIT_PUBLIC_BENCH_PRODUCERS: 4 run: ./build-bench/logit_public_macro_bench + - name: Run formatted public macro benchmark smoke + env: + LOGIT_PUBLIC_BENCH_TOTAL: 2000 + LOGIT_PUBLIC_BENCH_PRODUCERS: 4 + run: ./build-bench/logit_public_macro_formatted_bench - name: Run benchmark validation tests run: ./build-bench/benchmark_validation_test - name: Run logger hot-path A/B smoke diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index be08aa9f..81f67cc6 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -32,6 +32,14 @@ set_target_properties(logit_public_macro_bench PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} ) +add_executable(logit_public_macro_formatted_bench public_macro_bench.cpp) +target_compile_features(logit_public_macro_formatted_bench PRIVATE cxx_std_17) +target_compile_definitions(logit_public_macro_formatted_bench PRIVATE LOGIT_PUBLIC_BENCH_FORMATTED=1) +target_link_libraries(logit_public_macro_formatted_bench PRIVATE log-it-cpp::log-it-cpp) +set_target_properties(logit_public_macro_formatted_bench PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} +) + add_executable(logit_hotpath_bench logger_hotpath_bench.cpp) target_compile_features(logit_hotpath_bench PRIVATE cxx_std_17) target_link_libraries(logit_hotpath_bench PRIVATE log-it-cpp::log-it-cpp) diff --git a/bench/logit_bench.cpp b/bench/logit_bench.cpp index 19e5bd95..20f25968 100644 --- a/bench/logit_bench.cpp +++ b/bench/logit_bench.cpp @@ -51,6 +51,80 @@ std::size_t get_env_size_t(const char* name, std::size_t def) { return def; } +std::string compiler_name() { +#if defined(_MSC_VER) + return "MSVC"; +#elif defined(__clang__) + return "Clang"; +#elif defined(__GNUC__) + return "GCC"; +#else + return "unknown"; +#endif +} + +std::string compiler_version() { +#if defined(_MSC_VER) + return std::to_string(_MSC_VER); +#elif defined(__clang__) + return std::to_string(__clang_major__) + "." + + std::to_string(__clang_minor__) + "." + + std::to_string(__clang_patchlevel__); +#elif defined(__GNUC__) + return std::to_string(__GNUC__) + "." + + std::to_string(__GNUC_MINOR__) + "." + + std::to_string(__GNUC_PATCHLEVEL__); +#else + return "unknown"; +#endif +} + +std::string benchmark_commit() { + if (const char* value = std::getenv("LOGIT_BENCH_COMMIT")) { + return value; + } + if (const char* value = std::getenv("GITHUB_SHA")) { + return value; + } + return "unknown"; +} + +long cxx_standard() { +#if defined(_MSVC_LANG) + return _MSVC_LANG; +#else + return __cplusplus; +#endif +} + +void print_fixture_metadata(std::size_t queue_capacity, + std::size_t total_messages, + std::size_t warmup_messages) { + std::cout << "benchmark-fixture version=1" + << " source_commit=" << benchmark_commit() + << " compiler=" << compiler_name() + << " compiler_version=" << compiler_version() + << " toolchain=" << compiler_name() << "-" << compiler_version() + << " cxx_standard=" << cxx_standard() + << " platform=" << +#if defined(_WIN32) + "windows" +#elif defined(__APPLE__) + "macos" +#elif defined(__linux__) + "linux" +#elif defined(__EMSCRIPTEN__) + "emscripten" +#else + "unknown" +#endif + << " queue_capacity=" << queue_capacity + << " queue_policy=block" + << " flush_semantics=adapter.flush-drain-to-sink-entry" + << " total=" << total_messages + << " warmup=" << warmup_messages << '\n'; +} + struct BenchFilter { std::optional library; std::optional async; @@ -394,6 +468,8 @@ int main() { const BenchFilter filter = load_filter(); + print_fixture_metadata(queue_capacity, total_messages, warmup_messages); + LOGIT_SET_MAX_QUEUE(queue_capacity); LOGIT_SET_QUEUE_POLICY(LOGIT_QUEUE_BLOCK); diff --git a/bench/public_macro_bench.cpp b/bench/public_macro_bench.cpp index 3c9b2b74..09c30134 100644 --- a/bench/public_macro_bench.cpp +++ b/bench/public_macro_bench.cpp @@ -40,6 +40,22 @@ class PassthroughFormatter final : public logit::ILogFormatter { bool is_passthrough() const noexcept override { return true; } }; +std::unique_ptr make_formatter() { +#ifdef LOGIT_PUBLIC_BENCH_FORMATTED + return std::make_unique("[%l] %v"); +#else + return std::make_unique(); +#endif +} + +const char* benchmark_mode() { +#ifdef LOGIT_PUBLIC_BENCH_FORMATTED + return "formatted"; +#else + return "passthrough"; +#endif +} + std::size_t env_size(const char* name, std::size_t fallback) { if (const char* value = std::getenv(name)) { try { return static_cast(std::stoull(value)); } @@ -58,7 +74,7 @@ int main() { auto sink = std::make_unique(); auto* sink_ptr = sink.get(); logit::Logger::get_instance().add_logger( - std::move(sink), std::make_unique()); + std::move(sink), make_formatter()); const auto start = std::chrono::steady_clock::now(); std::vector workers; @@ -79,7 +95,8 @@ int main() { if (sink_ptr->count() != total) return 1; const double throughput = static_cast(total) * 1e9 / static_cast(elapsed); - std::cout << "public-macro producers=" << producers + std::cout << "public-macro mode=" << benchmark_mode() + << " producers=" << producers << " total=" << total << " elapsed_ns=" << elapsed << " throughput=" << throughput << " msg/s\n"; diff --git a/bench/results/benchmark-fixture-v1.json b/bench/results/benchmark-fixture-v1.json new file mode 100644 index 00000000..5cbe03df --- /dev/null +++ b/bench/results/benchmark-fixture-v1.json @@ -0,0 +1,33 @@ +{ + "fixture_version": 1, + "name": "log-it-cpp-release-benchmark", + "metadata_required": [ + "source_commit", + "compiler", + "toolchain", + "cxx_standard", + "platform", + "queue_capacity", + "queue_policy", + "flush_semantics" + ], + "latency_workload": { + "total_messages": 200000, + "warmup_messages": 4096, + "producer_counts": [1, 4, 16, 32], + "message_bytes": [40, 200, 1024], + "async_modes": [false, true], + "sinks": ["null", "file"], + "queue_policy": "block", + "flush_semantics": "adapter.flush drains the worker pipeline through sink entry" + }, + "public_macro_workload": { + "total_messages": 20000, + "producer_counts": [1, 4, 16, 32], + "scenarios": [ + "public macro record construction and dispatch with passthrough formatter", + "public macro record construction, dispatch, and SimpleLogFormatter formatting" + ] + }, + "measurement_rule": "Compare runs only when all required metadata fields match; CI runs are regression smoke, not universal performance claims." +} diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 7053d86e..1675a7f5 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -85,15 +85,23 @@ The prepared-message/direct-dispatch pipeline and a true public macro benchmark calls `LOGIT_INFO(...)` are separate scenarios with different work contracts; their results must not be presented as one number. -`logit_public_macro_bench` is the focused public-API smoke benchmark. It invokes -`LOGIT_INFO(...)` from multiple producer threads and therefore includes argument -name parsing, `args_array` construction, and dispatch. Its passthrough formatter -intentionally bypasses formatter work, so this is a public macro -record-construction + dispatch benchmark rather than a formatting benchmark. -Configure it -with `LOGIT_PUBLIC_BENCH_TOTAL` and `LOGIT_PUBLIC_BENCH_PRODUCERS`; its throughput -is reported separately from `latency.csv` and is intended for before/after -hot-path experiments on identical hardware. +`logit_public_macro_bench` and `logit_public_macro_formatted_bench` are focused +public-API smoke benchmarks. Both invoke `LOGIT_INFO(...)` from multiple +producer threads and therefore include argument-name parsing, `args_array` +construction, and dispatch. The first uses a passthrough formatter and +intentionally bypasses formatter work; the second uses `SimpleLogFormatter` and +includes the formatter path. Their throughputs are separate scenarios and must +not be presented as one number. Configure either with +`LOGIT_PUBLIC_BENCH_TOTAL` and `LOGIT_PUBLIC_BENCH_PRODUCERS`; results are +reported separately from `latency.csv` and are intended for before/after +experiments on identical hardware. + +The checked-in [`benchmark-fixture-v1.json`](https://github.com/LimiNode/log-it-cpp/blob/main/bench/results/benchmark-fixture-v1.json) +defines the required metadata and workload contract. `logit_bench` prints a +versioned `benchmark-fixture` metadata line for each run, including compiler, +platform, source commit (when `LOGIT_BENCH_COMMIT` or `GITHUB_SHA` is set), queue +capacity, and flush semantics. Compare measurements only when those metadata +fields match. `logit_hotpath_bench` and `logit_hotpath_bench_legacy` provide a controlled A/B measurement for the registry read path. Both run the same prepared `LogRecord` diff --git a/docs/future-plans.md b/docs/future-plans.md index b1b895e6..2a18a83d 100644 --- a/docs/future-plans.md +++ b/docs/future-plans.md @@ -53,10 +53,10 @@ Legend: follow-up, not part of the first configuration API. - [ ] **Extended filtering** — evaluate source/file, message, tag/MDC, and range filters; define their cost and ordering before adding public API. -- [ ] **Benchmark follow-up** — add a genuinely formatted public-macro scenario, - a versioned fixture containing compiler/toolchain/commit/queue/flush metadata, - and a broader 1/4/16/32-producer matrix. Keep publication numbers tied to a - fixed machine and toolchain. +- [x] **Benchmark follow-up** — formatted and passthrough public-macro scenarios, + a versioned fixture contract with compiler/toolchain/commit/queue/flush + metadata, and a 1/4/16/32-producer matrix are now covered. Keep publication + numbers tied to a fixed machine and toolchain. - [ ] **Concurrency fast-path research** — only after documenting a formal thread-safety capability for formatters/backends. Do not remove `exec_mx` based on benchmark results alone. From 83f113b4853ca12b6723d927b0287315b21ae7ac Mon Sep 17 00:00:00 2001 From: Aster Seker Date: Mon, 14 Sep 2026 01:47:32 +0300 Subject: [PATCH 2/3] fix(bench): make public runs reproducible Synchronize public-macro producers with a ready/start barrier and optional warmup, and share versioned metadata across benchmark targets. Require explicit build and hardware identity for comparable runs, update the fixture and methodology docs, and exercise metadata validation in CI benchmark smoke tests. --- .github/workflows/ci.yml | 2 + bench/BenchmarkMetadata.hpp | 234 ++++++++++++++++++++++++ bench/CMakeLists.txt | 7 +- bench/benchmark_validation_test.cpp | 27 +++ bench/logit_bench.cpp | 83 +-------- bench/public_macro_bench.cpp | 73 ++++++-- bench/results/benchmark-fixture-v1.json | 17 +- docs/adr/0005-benchmark-methodology.md | 11 +- docs/benchmarks.md | 43 +++-- 9 files changed, 385 insertions(+), 112 deletions(-) create mode 100644 bench/BenchmarkMetadata.hpp diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84d5cb64..007467ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,11 +37,13 @@ jobs: env: LOGIT_PUBLIC_BENCH_TOTAL: 2000 LOGIT_PUBLIC_BENCH_PRODUCERS: 4 + LOGIT_PUBLIC_BENCH_WARMUP: 200 run: ./build-bench/logit_public_macro_bench - name: Run formatted public macro benchmark smoke env: LOGIT_PUBLIC_BENCH_TOTAL: 2000 LOGIT_PUBLIC_BENCH_PRODUCERS: 4 + LOGIT_PUBLIC_BENCH_WARMUP: 200 run: ./build-bench/logit_public_macro_formatted_bench - name: Run benchmark validation tests run: ./build-bench/benchmark_validation_test diff --git a/bench/BenchmarkMetadata.hpp b/bench/BenchmarkMetadata.hpp new file mode 100644 index 00000000..494520c7 --- /dev/null +++ b/bench/BenchmarkMetadata.hpp @@ -0,0 +1,234 @@ +#pragma once + +#ifndef LOGIT_CPP_HEADER_BENCH_BENCHMARK_METADATA_HPP_INCLUDED +#define LOGIT_CPP_HEADER_BENCH_BENCHMARK_METADATA_HPP_INCLUDED + +#include +#include +#include +#include +#include +#include + +namespace logit_bench { + +struct BenchmarkMetadata { + std::string source_commit; + std::string compiler; + std::string compiler_version; + std::string toolchain; + std::string cxx_standard; + std::string platform; + std::string build_type; + std::string architecture; + std::string machine_id; + std::string cpu_model; + std::string queue_capacity; + std::string queue_policy; + std::string latency_completion; + std::string flush_barrier; +}; + +inline std::string benchmark_env(const char* name, const char* fallback) { + if (const char* value = std::getenv(name)) { + if (*value != '\0') return value; + } + return fallback; +} + +inline std::string benchmark_label(const char* name, const char* fallback) { + std::string value = benchmark_env(name, fallback); + for (char& character : value) { + if (character == ' ' || character == '\t' || + character == '\r' || character == '\n') { + character = '_'; + } + } + return value; +} + +inline std::string benchmark_compiler() { +#if defined(_MSC_VER) + return "MSVC"; +#elif defined(__clang__) + return "Clang"; +#elif defined(__GNUC__) + return "GCC"; +#else + return "unknown"; +#endif +} + +inline std::string benchmark_compiler_version() { +#if defined(_MSC_VER) + return std::to_string(_MSC_VER); +#elif defined(__clang__) + return std::to_string(__clang_major__) + "." + + std::to_string(__clang_minor__) + "." + + std::to_string(__clang_patchlevel__); +#elif defined(__GNUC__) + return std::to_string(__GNUC__) + "." + + std::to_string(__GNUC_MINOR__) + "." + + std::to_string(__GNUC_PATCHLEVEL__); +#else + return "unknown"; +#endif +} + +inline std::string benchmark_source_commit() { + if (const char* value = std::getenv("LOGIT_BENCH_COMMIT")) { + if (*value != '\0') return value; + } + if (const char* value = std::getenv("GITHUB_SHA")) { + if (*value != '\0') return value; + } + return "unknown"; +} + +inline std::string benchmark_cxx_standard() { +#if defined(_MSVC_LANG) + return std::to_string(_MSVC_LANG); +#else + return std::to_string(__cplusplus); +#endif +} + +inline std::string benchmark_platform() { +#if defined(__EMSCRIPTEN__) + return "emscripten"; +#elif defined(_WIN32) + return "windows"; +#elif defined(__APPLE__) + return "macos"; +#elif defined(__linux__) + return "linux"; +#else + return "unknown"; +#endif +} + +inline std::string benchmark_architecture() { +#if defined(__EMSCRIPTEN__) && defined(__wasm32__) + return "wasm32"; +#elif defined(_M_X64) || defined(__x86_64__) + return "x86_64"; +#elif defined(_M_IX86) || defined(__i386__) + return "x86"; +#elif defined(_M_ARM64) || defined(__aarch64__) + return "arm64"; +#elif defined(_M_ARM) || defined(__arm__) + return "arm"; +#else + return "unknown"; +#endif +} + +inline std::string benchmark_build_type() { + if (const char* value = std::getenv("LOGIT_BENCH_BUILD_TYPE")) { + if (*value != '\0') return value; + } +#ifdef LOGIT_BENCH_BUILD_TYPE + return LOGIT_BENCH_BUILD_TYPE; +#else + return "unknown"; +#endif +} + +inline BenchmarkMetadata make_benchmark_metadata( + std::string queue_capacity, + std::string queue_policy, + std::string latency_completion, + std::string flush_barrier) { + const std::string compiler = benchmark_compiler(); + const std::string compiler_version = benchmark_compiler_version(); + std::string toolchain = compiler + "-" + compiler_version; + toolchain = benchmark_label("LOGIT_BENCH_TOOLCHAIN", toolchain.c_str()); + return BenchmarkMetadata{ + benchmark_source_commit(), + compiler, + compiler_version, + std::move(toolchain), + benchmark_cxx_standard(), + benchmark_platform(), + benchmark_build_type(), + benchmark_architecture(), + benchmark_label("LOGIT_BENCH_MACHINE_ID", "unknown"), + benchmark_label("LOGIT_BENCH_CPU_MODEL", "unknown"), + std::move(queue_capacity), + std::move(queue_policy), + std::move(latency_completion), + std::move(flush_barrier)}; +} + +inline bool benchmark_value_unknown(const std::string& value) { + return value.empty() || value == "unknown"; +} + +inline void validate_comparable_metadata(const BenchmarkMetadata& metadata, + bool require_comparable) { + if (!require_comparable) return; + + struct MetadataField { + const char* name; + const std::string* value; + bool allow_not_applicable; + }; + const MetadataField values[] = { + {"source_commit", &metadata.source_commit, false}, + {"compiler", &metadata.compiler, false}, + {"compiler_version", &metadata.compiler_version, false}, + {"toolchain", &metadata.toolchain, false}, + {"cxx_standard", &metadata.cxx_standard, false}, + {"platform", &metadata.platform, false}, + {"build_type", &metadata.build_type, false}, + {"architecture", &metadata.architecture, false}, + {"machine_id", &metadata.machine_id, false}, + {"cpu_model", &metadata.cpu_model, false}, + {"queue_capacity", &metadata.queue_capacity, true}, + {"queue_policy", &metadata.queue_policy, true}, + {"latency_completion", &metadata.latency_completion, false}, + {"flush_barrier", &metadata.flush_barrier, false}}; + + for (const auto& value : values) { + if (benchmark_value_unknown(*value.value) || + (!value.allow_not_applicable && *value.value == "not-applicable")) { + throw std::runtime_error( + std::string("Comparable benchmark requires metadata field '") + + value.name + "'; set the corresponding LOGIT_BENCH_* value or disable " + "LOGIT_BENCH_REQUIRE_COMPARABLE"); + } + } +} + +inline void validate_comparable_metadata(const BenchmarkMetadata& metadata) { + const char* required = std::getenv("LOGIT_BENCH_REQUIRE_COMPARABLE"); + validate_comparable_metadata( + metadata, required && std::string(required) == "1"); +} + +inline void print_benchmark_metadata(std::ostream& out, + const BenchmarkMetadata& metadata, + std::size_t total_messages, + std::size_t warmup_messages) { + out << "benchmark-fixture version=1" + << " source_commit=" << metadata.source_commit + << " compiler=" << metadata.compiler + << " compiler_version=" << metadata.compiler_version + << " toolchain=" << metadata.toolchain + << " cxx_standard=" << metadata.cxx_standard + << " platform=" << metadata.platform + << " build_type=" << metadata.build_type + << " architecture=" << metadata.architecture + << " machine_id=" << metadata.machine_id + << " cpu_model=" << metadata.cpu_model + << " queue_capacity=" << metadata.queue_capacity + << " queue_policy=" << metadata.queue_policy + << " latency_completion=" << metadata.latency_completion + << " flush_barrier=" << metadata.flush_barrier + << " total=" << total_messages + << " warmup=" << warmup_messages << '\n'; +} + +} // namespace logit_bench + +#endif // LOGIT_CPP_HEADER_BENCH_BENCHMARK_METADATA_HPP_INCLUDED diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 81f67cc6..a0728789 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -10,6 +10,7 @@ endif() add_executable(logit_bench ${LOGIT_BENCH_SOURCES}) target_include_directories(logit_bench PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_definitions(logit_bench PRIVATE LOGIT_BENCH_BUILD_TYPE=\"$\") target_compile_features(logit_bench PRIVATE cxx_std_17) @@ -27,6 +28,7 @@ target_link_libraries(logit_bench PRIVATE log-it-cpp::log-it-cpp) add_executable(logit_public_macro_bench public_macro_bench.cpp) target_compile_features(logit_public_macro_bench PRIVATE cxx_std_17) +target_compile_definitions(logit_public_macro_bench PRIVATE LOGIT_BENCH_BUILD_TYPE=\"$\") target_link_libraries(logit_public_macro_bench PRIVATE log-it-cpp::log-it-cpp) set_target_properties(logit_public_macro_bench PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} @@ -34,7 +36,10 @@ set_target_properties(logit_public_macro_bench PROPERTIES add_executable(logit_public_macro_formatted_bench public_macro_bench.cpp) target_compile_features(logit_public_macro_formatted_bench PRIVATE cxx_std_17) -target_compile_definitions(logit_public_macro_formatted_bench PRIVATE LOGIT_PUBLIC_BENCH_FORMATTED=1) +target_compile_definitions(logit_public_macro_formatted_bench PRIVATE + LOGIT_PUBLIC_BENCH_FORMATTED=1 + LOGIT_BENCH_BUILD_TYPE=\"$\" +) target_link_libraries(logit_public_macro_formatted_bench PRIVATE log-it-cpp::log-it-cpp) set_target_properties(logit_public_macro_formatted_bench PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} diff --git a/bench/benchmark_validation_test.cpp b/bench/benchmark_validation_test.cpp index 2d19642f..e84e1c48 100644 --- a/bench/benchmark_validation_test.cpp +++ b/bench/benchmark_validation_test.cpp @@ -1,4 +1,5 @@ #include "BenchmarkValidation.hpp" +#include "BenchmarkMetadata.hpp" #include #include @@ -26,5 +27,31 @@ int main() { validate_latency_csv_header(std::string(latency_csv_header()) + "\r"); validate_latency_csv_header(latency_csv_header()); + + const auto comparable = make_benchmark_metadata( + "8192", "block", "sink-entry", "all-prior-work-drained"); + auto complete = comparable; + complete.source_commit = "test-commit"; + complete.compiler = "test-compiler"; + complete.compiler_version = "1"; + complete.toolchain = "test-toolchain"; + complete.cxx_standard = "201703"; + complete.platform = "test-platform"; + complete.build_type = "Release"; + complete.architecture = "test-architecture"; + complete.machine_id = "test-machine"; + complete.cpu_model = "test-cpu"; + validate_comparable_metadata(complete, true); + + auto incomplete = complete; + incomplete.source_commit = "unknown"; + bool rejected_unknown_metadata = false; + try { + validate_comparable_metadata(incomplete, true); + } catch (const std::runtime_error&) { + rejected_unknown_metadata = true; + } + if (!rejected_unknown_metadata) return 3; + return 0; } diff --git a/bench/logit_bench.cpp b/bench/logit_bench.cpp index 20f25968..fdd820f0 100644 --- a/bench/logit_bench.cpp +++ b/bench/logit_bench.cpp @@ -21,6 +21,7 @@ #include "LatencyRecorder.hpp" #include "BenchmarkValidation.hpp" +#include "BenchmarkMetadata.hpp" #include "Scenario.hpp" #include "adapters/LogItAdapter.hpp" @@ -51,80 +52,6 @@ std::size_t get_env_size_t(const char* name, std::size_t def) { return def; } -std::string compiler_name() { -#if defined(_MSC_VER) - return "MSVC"; -#elif defined(__clang__) - return "Clang"; -#elif defined(__GNUC__) - return "GCC"; -#else - return "unknown"; -#endif -} - -std::string compiler_version() { -#if defined(_MSC_VER) - return std::to_string(_MSC_VER); -#elif defined(__clang__) - return std::to_string(__clang_major__) + "." + - std::to_string(__clang_minor__) + "." + - std::to_string(__clang_patchlevel__); -#elif defined(__GNUC__) - return std::to_string(__GNUC__) + "." + - std::to_string(__GNUC_MINOR__) + "." + - std::to_string(__GNUC_PATCHLEVEL__); -#else - return "unknown"; -#endif -} - -std::string benchmark_commit() { - if (const char* value = std::getenv("LOGIT_BENCH_COMMIT")) { - return value; - } - if (const char* value = std::getenv("GITHUB_SHA")) { - return value; - } - return "unknown"; -} - -long cxx_standard() { -#if defined(_MSVC_LANG) - return _MSVC_LANG; -#else - return __cplusplus; -#endif -} - -void print_fixture_metadata(std::size_t queue_capacity, - std::size_t total_messages, - std::size_t warmup_messages) { - std::cout << "benchmark-fixture version=1" - << " source_commit=" << benchmark_commit() - << " compiler=" << compiler_name() - << " compiler_version=" << compiler_version() - << " toolchain=" << compiler_name() << "-" << compiler_version() - << " cxx_standard=" << cxx_standard() - << " platform=" << -#if defined(_WIN32) - "windows" -#elif defined(__APPLE__) - "macos" -#elif defined(__linux__) - "linux" -#elif defined(__EMSCRIPTEN__) - "emscripten" -#else - "unknown" -#endif - << " queue_capacity=" << queue_capacity - << " queue_policy=block" - << " flush_semantics=adapter.flush-drain-to-sink-entry" - << " total=" << total_messages - << " warmup=" << warmup_messages << '\n'; -} - struct BenchFilter { std::optional library; std::optional async; @@ -468,7 +395,13 @@ int main() { const BenchFilter filter = load_filter(); - print_fixture_metadata(queue_capacity, total_messages, warmup_messages); + const auto metadata = make_benchmark_metadata( + std::to_string(queue_capacity), + "block", + "sink-entry", + "all-prior-work-drained"); + validate_comparable_metadata(metadata); + print_benchmark_metadata(std::cout, metadata, total_messages, warmup_messages); LOGIT_SET_MAX_QUEUE(queue_capacity); LOGIT_SET_QUEUE_POLICY(LOGIT_QUEUE_BLOCK); diff --git a/bench/public_macro_bench.cpp b/bench/public_macro_bench.cpp index 09c30134..73a33361 100644 --- a/bench/public_macro_bench.cpp +++ b/bench/public_macro_bench.cpp @@ -2,15 +2,19 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include +#include "BenchmarkMetadata.hpp" + namespace { class CountingLogger final : public logit::ILogger { @@ -27,6 +31,7 @@ class CountingLogger final : public logit::ILogger { } void wait() override {} std::size_t count() const { return m_count.load(std::memory_order_relaxed); } + void reset() { m_count.store(0, std::memory_order_relaxed); } private: std::atomic m_count{0}; @@ -64,34 +69,72 @@ std::size_t env_size(const char* name, std::size_t fallback) { return fallback; } -} // namespace - -int main() { - const std::size_t producers = env_size("LOGIT_PUBLIC_BENCH_PRODUCERS", 4); - const std::size_t total = env_size("LOGIT_PUBLIC_BENCH_TOTAL", 20000); - if (producers == 0 || total == 0) return 2; - - auto sink = std::make_unique(); - auto* sink_ptr = sink.get(); - logit::Logger::get_instance().add_logger( - std::move(sink), make_formatter()); +std::chrono::nanoseconds run_workload(std::size_t producers, std::size_t total) { + std::mutex start_mx; + std::condition_variable start_cv; + std::condition_variable ready_cv; + bool start_flag = false; + std::size_t ready = 0; - const auto start = std::chrono::steady_clock::now(); std::vector workers; workers.reserve(producers); for (std::size_t producer = 0; producer < producers; ++producer) { - workers.emplace_back([producer, producers, total]() { + workers.emplace_back([&, producer]() { const std::size_t begin = (total * producer) / producers; const std::size_t end = (total * (producer + 1)) / producers; + { + std::unique_lock lock(start_mx); + ++ready; + if (ready == producers) ready_cv.notify_one(); + start_cv.wait(lock, [&] { return start_flag; }); + } for (std::size_t i = begin; i < end; ++i) { LOGIT_INFO("public macro message", i); } }); } + + std::chrono::steady_clock::time_point start; + { + std::unique_lock lock(start_mx); + ready_cv.wait(lock, [&] { return ready == producers; }); + start = std::chrono::steady_clock::now(); + start_flag = true; + } + start_cv.notify_all(); + for (auto& worker : workers) worker.join(); logit::Logger::get_instance().wait(); - const auto elapsed = std::chrono::duration_cast( - std::chrono::steady_clock::now() - start).count(); + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); +} + +} // namespace + +int main() { + const std::size_t producers = env_size("LOGIT_PUBLIC_BENCH_PRODUCERS", 4); + const std::size_t total = env_size("LOGIT_PUBLIC_BENCH_TOTAL", 20000); + const std::size_t warmup = env_size("LOGIT_PUBLIC_BENCH_WARMUP", 0); + if (producers == 0 || total == 0) return 2; + + const auto metadata = logit_bench::make_benchmark_metadata( + "not-applicable", + "not-applicable", + "backend-count", + "logger-wait"); + logit_bench::validate_comparable_metadata(metadata); + logit_bench::print_benchmark_metadata(std::cout, metadata, total, warmup); + + auto sink = std::make_unique(); + auto* sink_ptr = sink.get(); + logit::Logger::get_instance().add_logger( + std::move(sink), make_formatter()); + + if (warmup > 0) { + run_workload(producers, warmup); + sink_ptr->reset(); + } + const auto elapsed = run_workload(producers, total).count(); if (sink_ptr->count() != total) return 1; const double throughput = static_cast(total) * 1e9 / static_cast(elapsed); diff --git a/bench/results/benchmark-fixture-v1.json b/bench/results/benchmark-fixture-v1.json index 5cbe03df..79ab315b 100644 --- a/bench/results/benchmark-fixture-v1.json +++ b/bench/results/benchmark-fixture-v1.json @@ -4,12 +4,18 @@ "metadata_required": [ "source_commit", "compiler", + "compiler_version", "toolchain", "cxx_standard", "platform", + "build_type", + "architecture", + "machine_id", + "cpu_model", "queue_capacity", "queue_policy", - "flush_semantics" + "latency_completion", + "flush_barrier" ], "latency_workload": { "total_messages": 200000, @@ -19,15 +25,20 @@ "async_modes": [false, true], "sinks": ["null", "file"], "queue_policy": "block", - "flush_semantics": "adapter.flush drains the worker pipeline through sink entry" + "latency_completion": "sink-entry", + "flush_barrier": "all-prior-work-drained" }, "public_macro_workload": { "total_messages": 20000, "producer_counts": [1, 4, 16, 32], + "queue_capacity": "not-applicable", + "queue_policy": "not-applicable", + "latency_completion": "backend-count", + "flush_barrier": "logger-wait", "scenarios": [ "public macro record construction and dispatch with passthrough formatter", "public macro record construction, dispatch, and SimpleLogFormatter formatting" ] }, - "measurement_rule": "Compare runs only when all required metadata fields match; CI runs are regression smoke, not universal performance claims." + "measurement_rule": "Compare runs only when all required metadata fields match; set LOGIT_BENCH_REQUIRE_COMPARABLE=1 for publication runs so unknown commit or machine metadata is rejected; CI runs are regression smoke, not universal performance claims." } diff --git a/docs/adr/0005-benchmark-methodology.md b/docs/adr/0005-benchmark-methodology.md index 72051642..7016ba79 100644 --- a/docs/adr/0005-benchmark-methodology.md +++ b/docs/adr/0005-benchmark-methodology.md @@ -15,10 +15,13 @@ publication machine. Keep separate scenarios for prepared-record dispatch, the real public `LOGIT_INFO(...)` path, formatting, and external-library comparisons. The public macro smoke benchmark may use a passthrough formatter when it is -explicitly documented as record-construction/dispatch coverage. Report -absolute timings only with compiler, platform, commit, queue, producer, and -flush settings; treat CI runs as regression smoke unless the environment is -fixed. +explicitly documented as record-construction/dispatch coverage. Producer +scaling measurements create all workers before the timed interval and release +them through a shared start barrier; an optional warmup is separate from the +measured run. Report absolute timings only with compiler, platform, commit, +build type, architecture, machine/CPU identity, queue settings, and separate +latency-completion and flush-barrier semantics. Treat CI runs as regression +smoke unless the environment is fixed. ## Consequences diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 1675a7f5..e122d329 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -51,8 +51,9 @@ The repository includes a **legacy** comparison snapshot from 2025-12-05 in This fixture predates the current benchmark dependency metadata and does not record the spdlog version, compiler/toolchain, or harness commit. Treat it as legacy context rather than a reproducible current performance claim. New -published figures must include the dependency versions, toolchain, harness -commit, queue capacity, and flush semantics used for the run. +published figures must include the fixture metadata, dependency versions, +toolchain, harness commit, build and hardware identity, queue settings, and +separate latency-completion and flush-barrier semantics used for the run. ## Harness details @@ -88,20 +89,32 @@ their results must not be presented as one number. `logit_public_macro_bench` and `logit_public_macro_formatted_bench` are focused public-API smoke benchmarks. Both invoke `LOGIT_INFO(...)` from multiple producer threads and therefore include argument-name parsing, `args_array` -construction, and dispatch. The first uses a passthrough formatter and -intentionally bypasses formatter work; the second uses `SimpleLogFormatter` and -includes the formatter path. Their throughputs are separate scenarios and must -not be presented as one number. Configure either with -`LOGIT_PUBLIC_BENCH_TOTAL` and `LOGIT_PUBLIC_BENCH_PRODUCERS`; results are +construction, and dispatch. Producers are created before the timed interval, +wait on a ready barrier, and are released together; thread creation and the +startup skew are not part of the producer-scaling measurement. The optional +`LOGIT_PUBLIC_BENCH_WARMUP` run is executed before the measured workload. +The first target uses a passthrough formatter and intentionally bypasses +formatter work; the second uses `SimpleLogFormatter` and includes the formatter +path. Their throughputs are separate scenarios and must not be presented as one +number. Configure either with `LOGIT_PUBLIC_BENCH_TOTAL`, +`LOGIT_PUBLIC_BENCH_PRODUCERS`, and `LOGIT_PUBLIC_BENCH_WARMUP`; results are reported separately from `latency.csv` and are intended for before/after -experiments on identical hardware. +experiments on identical hardware. Both targets print the same fixture metadata +line as `logit_bench`, using `not-applicable` for queue settings and explicit +`backend-count` / `logger-wait` completion semantics. The checked-in [`benchmark-fixture-v1.json`](https://github.com/LimiNode/log-it-cpp/blob/main/bench/results/benchmark-fixture-v1.json) -defines the required metadata and workload contract. `logit_bench` prints a -versioned `benchmark-fixture` metadata line for each run, including compiler, -platform, source commit (when `LOGIT_BENCH_COMMIT` or `GITHUB_SHA` is set), queue -capacity, and flush semantics. Compare measurements only when those metadata -fields match. +defines the required metadata and workload contract. All publication-capable +benchmark binaries print a versioned `benchmark-fixture` metadata line for +each run, including source commit, compiler/version, toolchain, C++ standard, +platform, build type, architecture, machine identity, CPU model, queue +settings, latency completion, and flush barrier. The commit defaults to +`LOGIT_BENCH_COMMIT` or `GITHUB_SHA`; machine identity and CPU model can be +provided through `LOGIT_BENCH_MACHINE_ID` and `LOGIT_BENCH_CPU_MODEL`. +For a comparable/publication run, set `LOGIT_BENCH_REQUIRE_COMPARABLE=1` and +provide all required metadata; public-macro runs may explicitly use +`not-applicable` queue settings. Smoke runs may leave unavailable values as +`unknown`. Compare measurements only when all fixture metadata fields match. `logit_hotpath_bench` and `logit_hotpath_bench_legacy` provide a controlled A/B measurement for the registry read path. Both run the same prepared `LogRecord` @@ -122,6 +135,8 @@ explicit concurrency contract rather than infer one from a benchmark sink. The flush regression target uses an intentionally delayed asynchronous sink and asserts that `flush()` does not return before every queued message has reached -that sink. `benchmark_validation_test` covers the comparative-protocol guardrails +that sink. The fixture records latency completion and the flush barrier as +separate semantics: the latency benchmark completes at sink entry, while the +flush barrier drains all prior work. `benchmark_validation_test` covers the comparative-protocol guardrails (`queue_capacity=0` and legacy CSV schema rejection) without relying on packages installed on the host. From d85448e7b734e3d3608435c40f4ed4cb0232c480 Mon Sep 17 00:00:00 2001 From: Aster Seker Date: Mon, 14 Sep 2026 13:36:16 +0300 Subject: [PATCH 3/3] fix(bench): separate provenance from comparable metadata Require source commits for benchmark provenance without requiring them to match across before/after runs. Define the metadata_must_match fixture contract and document that comparable mode validates metadata completeness rather than canonical workload values. --- bench/results/benchmark-fixture-v1.json | 17 ++++++++++++++++- docs/adr/0005-benchmark-methodology.md | 7 +++++++ docs/benchmarks.md | 9 ++++++++- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/bench/results/benchmark-fixture-v1.json b/bench/results/benchmark-fixture-v1.json index 79ab315b..b77a9684 100644 --- a/bench/results/benchmark-fixture-v1.json +++ b/bench/results/benchmark-fixture-v1.json @@ -17,6 +17,21 @@ "latency_completion", "flush_barrier" ], + "metadata_must_match": [ + "compiler", + "compiler_version", + "toolchain", + "cxx_standard", + "platform", + "build_type", + "architecture", + "machine_id", + "cpu_model", + "queue_capacity", + "queue_policy", + "latency_completion", + "flush_barrier" + ], "latency_workload": { "total_messages": 200000, "warmup_messages": 4096, @@ -40,5 +55,5 @@ "public macro record construction, dispatch, and SimpleLogFormatter formatting" ] }, - "measurement_rule": "Compare runs only when all required metadata fields match; set LOGIT_BENCH_REQUIRE_COMPARABLE=1 for publication runs so unknown commit or machine metadata is rejected; CI runs are regression smoke, not universal performance claims." + "measurement_rule": "All required metadata fields must be present. Compare runs only when metadata_must_match fields match; source_commit is provenance and may differ between before/after runs. Set LOGIT_BENCH_REQUIRE_COMPARABLE=1 for publication runs so unknown metadata is rejected; CI runs are regression smoke, not universal performance claims." } diff --git a/docs/adr/0005-benchmark-methodology.md b/docs/adr/0005-benchmark-methodology.md index 7016ba79..ffa343e2 100644 --- a/docs/adr/0005-benchmark-methodology.md +++ b/docs/adr/0005-benchmark-methodology.md @@ -23,6 +23,13 @@ build type, architecture, machine/CPU identity, queue settings, and separate latency-completion and flush-barrier semantics. Treat CI runs as regression smoke unless the environment is fixed. +The fixture distinguishes required provenance from comparison identity: +`source_commit` must be present and is expected to differ for before/after +measurements, while the fields listed in `metadata_must_match` must match. The +`LOGIT_BENCH_REQUIRE_COMPARABLE=1` mode checks metadata completeness and known +values; it does not enforce the canonical fixture workload unless a separate +workload-validation mode is requested. + ## Consequences Benchmark documentation remains comparable and honest across changes. New diff --git a/docs/benchmarks.md b/docs/benchmarks.md index e122d329..79cc468c 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -114,7 +114,14 @@ provided through `LOGIT_BENCH_MACHINE_ID` and `LOGIT_BENCH_CPU_MODEL`. For a comparable/publication run, set `LOGIT_BENCH_REQUIRE_COMPARABLE=1` and provide all required metadata; public-macro runs may explicitly use `not-applicable` queue settings. Smoke runs may leave unavailable values as -`unknown`. Compare measurements only when all fixture metadata fields match. +`unknown`. The fixture separates metadata that must be present from metadata +that must match between runs: `source_commit` is required provenance and is +expected to differ in before/after comparisons, while the fields listed in +`metadata_must_match` (compiler, toolchain, platform, build, hardware, queue, +and completion semantics) must be identical. `LOGIT_BENCH_REQUIRE_COMPARABLE=1` +checks metadata completeness and known values; it does not enforce the +canonical fixture workload values such as total messages, warmup, or producer +matrix. `logit_hotpath_bench` and `logit_hotpath_bench_legacy` provide a controlled A/B measurement for the registry read path. Both run the same prepared `LogRecord`