From 4058b6e83cce1a1748d2c15ac89a1794d018047d Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 1 Sep 2026 11:07:00 +0800 Subject: [PATCH 01/47] io_pattern frame --- mooncake-store/include/io_pattern/analyzer.h | 14 ++ mooncake-store/include/io_pattern/collector.h | 17 ++ .../include/io_pattern/io_pattern.h | 8 + mooncake-store/include/io_pattern/ops.h | 44 ++++ .../include/io_pattern/policy_engine.h | 23 ++ mooncake-store/include/io_pattern/registry.h | 68 ++++++ mooncake-store/include/io_pattern/types.h | 216 ++++++++++++++++++ mooncake-store/tests/CMakeLists.txt | 1 + .../tests/io_pattern_framework_test.cpp | 59 +++++ 9 files changed, 450 insertions(+) create mode 100644 mooncake-store/include/io_pattern/analyzer.h create mode 100644 mooncake-store/include/io_pattern/collector.h create mode 100644 mooncake-store/include/io_pattern/io_pattern.h create mode 100644 mooncake-store/include/io_pattern/ops.h create mode 100644 mooncake-store/include/io_pattern/policy_engine.h create mode 100644 mooncake-store/include/io_pattern/registry.h create mode 100644 mooncake-store/include/io_pattern/types.h create mode 100644 mooncake-store/tests/io_pattern_framework_test.cpp diff --git a/mooncake-store/include/io_pattern/analyzer.h b/mooncake-store/include/io_pattern/analyzer.h new file mode 100644 index 0000000000..94efd1f71e --- /dev/null +++ b/mooncake-store/include/io_pattern/analyzer.h @@ -0,0 +1,14 @@ +#pragma once + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +class IoPatternAnalyzer { + public: + virtual ~IoPatternAnalyzer() = default; + + virtual PatternResult Analyze(const IoPatternSnapshot& snapshot) const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector.h b/mooncake-store/include/io_pattern/collector.h new file mode 100644 index 0000000000..94d1ccbbee --- /dev/null +++ b/mooncake-store/include/io_pattern/collector.h @@ -0,0 +1,17 @@ +#pragma once + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +class IoPatternCollector { + public: + virtual ~IoPatternCollector() = default; + + virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; + virtual void RecordAccess(const AccessRecord& record) = 0; + virtual void RecordStorageMetric(const StorageMetric& metric) = 0; + virtual IoPatternSnapshot GetSnapshot() const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h new file mode 100644 index 0000000000..99834c3561 --- /dev/null +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -0,0 +1,8 @@ +#pragma once + +#include "io_pattern/analyzer.h" +#include "io_pattern/collector.h" +#include "io_pattern/ops.h" +#include "io_pattern/policy_engine.h" +#include "io_pattern/registry.h" +#include "io_pattern/types.h" diff --git a/mooncake-store/include/io_pattern/ops.h b/mooncake-store/include/io_pattern/ops.h new file mode 100644 index 0000000000..5db67f2953 --- /dev/null +++ b/mooncake-store/include/io_pattern/ops.h @@ -0,0 +1,44 @@ +#pragma once + +#include + +#include "io_pattern/types.h" +#include "types.h" + +namespace mooncake::io_pattern { + +class EvictionOps { + public: + virtual ~EvictionOps() = default; + + virtual EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const = 0; +}; + +class PrefetchOps { + public: + virtual ~PrefetchOps() = default; + + virtual PrefetchPlan Evaluate(const PolicyContext& context, + const TraceHistory& trace) const = 0; +}; + +class AdmissionOps { + public: + virtual ~AdmissionOps() = default; + + virtual AdmissionResult Evaluate(const ObjectRef& object, + CacheTier target_tier, + const PolicyContext& context) const = 0; +}; + +// Data movement is a client-side seam. Keeping it separate prevents a +// SubMaster-side prefetch planner from depending on a concrete storage client. +class PrefetchExecutor { + public: + virtual ~PrefetchExecutor() = default; + + virtual ErrorCode Execute(const PrefetchPlan& plan) = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h new file mode 100644 index 0000000000..bf7e2adc19 --- /dev/null +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -0,0 +1,23 @@ +#pragma once + +#include + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +class PolicyEngine { + public: + virtual ~PolicyEngine() = default; + + virtual EvictionPlan PlanEviction(const PolicyContext& context, + CacheTier tier, + uint64_t target_bytes) const = 0; + virtual PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const = 0; + virtual AdmissionResult DecideAdmission( + const ObjectRef& object, CacheTier target_tier, + const PolicyContext& context) const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/registry.h b/mooncake-store/include/io_pattern/registry.h new file mode 100644 index 0000000000..e560967310 --- /dev/null +++ b/mooncake-store/include/io_pattern/registry.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "io_pattern/ops.h" + +namespace mooncake::io_pattern { + +template +class OpsRegistry { + public: + using Factory = std::function()>; + + bool Register(std::string name, Factory factory) { + if (name.empty() || !factory) { + return false; + } + std::unique_lock lock(mutex_); + return factories_.emplace(std::move(name), std::move(factory)).second; + } + + std::shared_ptr Create(std::string_view name) const { + Factory factory; + { + std::shared_lock lock(mutex_); + const auto it = factories_.find(std::string(name)); + if (it == factories_.end()) { + return nullptr; + } + factory = it->second; + } + return factory(); + } + + std::vector RegisteredNames() const { + std::vector names; + { + std::shared_lock lock(mutex_); + names.reserve(factories_.size()); + for (const auto& entry : factories_) { + names.push_back(entry.first); + } + } + std::sort(names.begin(), names.end()); + return names; + } + + private: + mutable std::shared_mutex mutex_; + std::unordered_map factories_; +}; + +struct PolicyOpsRegistries { + OpsRegistry eviction; + OpsRegistry prefetch; + OpsRegistry admission; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h new file mode 100644 index 0000000000..2a39ae0cc7 --- /dev/null +++ b/mooncake-store/include/io_pattern/types.h @@ -0,0 +1,216 @@ +#pragma once + +#include +#include +#include + +#include "tenant_id.h" + +namespace mooncake::io_pattern { + +enum class CacheTier : uint8_t { + kL0Hbm = 0, + kL1Host = 1, + kL2Segment = 2, + kL3NofSsd = 3, +}; + +using CacheTierMask = uint8_t; + +constexpr CacheTierMask CacheTierBit(CacheTier tier) { + return static_cast(1U << static_cast(tier)); +} + +enum class IoOperation : uint8_t { + kGet, + kPut, + kTierUp, + kTierDown, +}; + +enum class CacheLayout : uint8_t { + kUnknown, + kLayerFirst, + kPageFirst, + kPageFirstDirect, + kPageBased, + kHmaMultiGroup, +}; + +enum class StorageGcState : uint8_t { + kUnknown, + kIdle, + kRunning, + kStalled, +}; + +enum class WorkloadType : uint8_t { + kUnknown, + kCodeAgent, + kGenerativeRecommendation, + kMultiTurnConversation, + kMixed, +}; + +struct ObjectRef { + TenantId tenant_id; + std::string key; + + bool operator==(const ObjectRef&) const = default; +}; + +struct InferenceMetrics { + ObjectRef object; + std::string session_id; + CacheLayout layout{CacheLayout::kUnknown}; + uint32_t layout_group{0}; + uint32_t prefix_depth{0}; + uint32_t prefix_fanout{0}; + uint32_t match_length{0}; + uint32_t continuous_prefix_length{0}; + uint32_t token_count{0}; + float recompute_cost{0.0F}; + uint8_t request_priority{0}; +}; + +struct AccessRecord { + ObjectRef object; + uint64_t observed_at_ns{0}; + uint64_t block_size{0}; + uint64_t latency_us{0}; + CacheTier tier{CacheTier::kL2Segment}; + IoOperation operation{IoOperation::kGet}; + bool is_hit{false}; +}; + +struct StorageMetric { + std::string source_id; + uint64_t observed_at_ns{0}; + CacheTier tier{CacheTier::kL2Segment}; + StorageGcState gc_state{StorageGcState::kUnknown}; + uint64_t read_bandwidth_bytes_per_sec{0}; + uint64_t write_bandwidth_bytes_per_sec{0}; + uint64_t read_latency_us{0}; + uint64_t write_latency_us{0}; + uint64_t used_bytes{0}; + uint64_t capacity_bytes{0}; + uint64_t rpc_latency_us{0}; + float memory_used_ratio{0.0F}; +}; + +struct KeyMetrics { + ObjectRef object; + uint64_t last_access_time_ns{0}; + uint64_t access_count_window{0}; + uint64_t idle_time_us{0}; + uint64_t block_size{0}; + uint64_t transfer_eta_us{0}; + uint32_t token_count{0}; + uint32_t prefix_depth{0}; + uint32_t prefix_fanout{0}; + uint32_t match_length{0}; + uint32_t continuous_prefix_length{0}; + uint32_t other_replica_count{0}; + uint32_t write_batch_size{0}; + uint32_t write_frequency{0}; + uint64_t write_object_size{0}; + float recompute_cost{0.0F}; + float overwrite_ratio{0.0F}; + CacheTierMask replica_tiers{0}; + CacheLayout layout{CacheLayout::kUnknown}; + uint32_t layout_group{0}; + uint8_t request_priority{0}; + bool active{false}; + bool pinned{false}; + bool ssd_replica_exists{false}; + bool write_burst{false}; +}; + +struct IoPatternSnapshot { + uint64_t generated_at_ns{0}; + std::vector keys; + std::vector storage; +}; + +struct KeyPattern { + ObjectRef object; + float confidence{0.0F}; + float frequency_score{0.0F}; + float idle_score{0.0F}; + float prefix_score{0.0F}; + float recompute_score{0.0F}; + float transfer_roi{0.0F}; + bool migration_safe{false}; +}; + +struct PatternResult { + WorkloadType workload_type{WorkloadType::kUnknown}; + float workload_confidence{0.0F}; + std::vector keys; +}; + +struct PolicyContext { + IoPatternSnapshot snapshot; + PatternResult analysis; +}; + +struct TraceEvent { + ObjectRef object; + uint64_t observed_at_ns{0}; + uint32_t match_length{0}; + bool is_hit{false}; +}; + +struct TraceHistory { + std::vector events; +}; + +enum class PrefetchStrategy : uint8_t { + kBestEffort, + kTimeout, + kWaitComplete, +}; + +struct PrefetchCandidate { + ObjectRef object; + CacheTier source_tier{CacheTier::kL3NofSsd}; + CacheTier target_tier{CacheTier::kL2Segment}; + uint64_t bytes{0}; + float priority{0.0F}; + float confidence{0.0F}; +}; + +struct PrefetchPlan { + PrefetchStrategy strategy{PrefetchStrategy::kBestEffort}; + uint64_t timeout_us{0}; + std::vector candidates; +}; + +struct EvictionCandidate { + ObjectRef object; + uint64_t bytes{0}; + float score{0.0F}; +}; + +struct EvictionPlan { + CacheTier source_tier{CacheTier::kL0Hbm}; + uint64_t target_bytes{0}; + std::vector candidates; +}; + +enum class AdmissionDecision : uint8_t { + kAdmit, + kRejectFrequency, + kRejectWatermark, + kRejectPrefix, + kDefer, +}; + +struct AdmissionResult { + ObjectRef object; + CacheTier target_tier{CacheTier::kL2Segment}; + AdmissionDecision decision{AdmissionDecision::kDefer}; + float confidence{0.0F}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/tests/CMakeLists.txt b/mooncake-store/tests/CMakeLists.txt index 1969fe0f42..edf6ceb951 100644 --- a/mooncake-store/tests/CMakeLists.txt +++ b/mooncake-store/tests/CMakeLists.txt @@ -54,6 +54,7 @@ add_test( set_tests_properties(replica_selection_env_opt_in_test PROPERTIES ENVIRONMENT "MC_STORE_REPLICA_SCORING=1") add_store_test(eviction_strategy_test eviction_strategy_test.cpp) +add_store_test(io_pattern_framework_test io_pattern_framework_test.cpp) add_store_test(deadline_scheduler_test deadline_scheduler_test.cpp) add_store_test(kv_event_publisher_test kv_event_publisher_test.cpp) if(ENABLE_KV_EVENTS) diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp new file mode 100644 index 0000000000..cd74468a20 --- /dev/null +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -0,0 +1,59 @@ +#include "io_pattern/io_pattern.h" + +#include +#include + +#include + +namespace mooncake::io_pattern { +namespace { + +class TestEvictionOps final : public EvictionOps { + public: + EvictionPlan Evaluate(const PolicyContext&, CacheTier tier, + uint64_t target_bytes) const override { + return EvictionPlan{.source_tier = tier, + .target_bytes = target_bytes, + .candidates = {}}; + } +}; + +TEST(IoPatternFrameworkTest, PublicSeamsRemainAbstract) { + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); +} + +TEST(IoPatternFrameworkTest, CacheTierMaskRepresentsAllTiers) { + const CacheTierMask all_tiers = + CacheTierBit(CacheTier::kL0Hbm) | CacheTierBit(CacheTier::kL1Host) | + CacheTierBit(CacheTier::kL2Segment) | + CacheTierBit(CacheTier::kL3NofSsd); + + EXPECT_EQ(all_tiers, 0x0F); +} + +TEST(IoPatternFrameworkTest, RegistryCreatesTypedOpsAndRejectsDuplicates) { + OpsRegistry registry; + + EXPECT_TRUE(registry.Register( + "test", [] { return std::make_shared(); })); + EXPECT_FALSE(registry.Register( + "test", [] { return std::make_shared(); })); + EXPECT_FALSE(registry.Register("", {})); + EXPECT_EQ(registry.RegisteredNames(), std::vector{"test"}); + + const auto ops = registry.Create("test"); + ASSERT_NE(ops, nullptr); + const auto plan = ops->Evaluate({}, CacheTier::kL2Segment, 4096); + EXPECT_EQ(plan.source_tier, CacheTier::kL2Segment); + EXPECT_EQ(plan.target_bytes, 4096); + EXPECT_EQ(registry.Create("missing"), nullptr); +} + +} // namespace +} // namespace mooncake::io_pattern From cfec19f29f44f909d8f9b732bae66611ce741683 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 1 Sep 2026 11:52:28 +0800 Subject: [PATCH 02/47] io_pattern frame --- mooncake-store/include/admission_ops.h | 3 + mooncake-store/include/cache_view_manager.h | 3 + mooncake-store/include/cfm_client.h | 3 + mooncake-store/include/eviction_ops.h | 3 + mooncake-store/include/io_pattern.h | 3 + mooncake-store/include/io_pattern/analyzer.h | 5 + mooncake-store/include/io_pattern/client.h | 18 ++ mooncake-store/include/io_pattern/collector.h | 2 + .../include/io_pattern/io_pattern.h | 2 + mooncake-store/include/io_pattern/ops.h | 5 +- .../include/io_pattern/policy_engine.h | 48 +++++ mooncake-store/include/io_pattern/registry.h | 1 + mooncake-store/include/io_pattern/types.h | 35 +++- .../include/io_pattern/view_manager.h | 17 ++ mooncake-store/include/io_pattern_analyzer.h | 3 + mooncake-store/include/io_pattern_collector.h | 3 + mooncake-store/include/io_pattern_registry.h | 3 + mooncake-store/include/io_pattern_types.h | 3 + mooncake-store/include/policy_engine.h | 3 + mooncake-store/include/prefetch_ops.h | 3 + .../tests/io_pattern_framework_test.cpp | 187 ++++++++++++++++++ 21 files changed, 350 insertions(+), 3 deletions(-) create mode 100644 mooncake-store/include/admission_ops.h create mode 100644 mooncake-store/include/cache_view_manager.h create mode 100644 mooncake-store/include/cfm_client.h create mode 100644 mooncake-store/include/eviction_ops.h create mode 100644 mooncake-store/include/io_pattern.h create mode 100644 mooncake-store/include/io_pattern/client.h create mode 100644 mooncake-store/include/io_pattern/view_manager.h create mode 100644 mooncake-store/include/io_pattern_analyzer.h create mode 100644 mooncake-store/include/io_pattern_collector.h create mode 100644 mooncake-store/include/io_pattern_registry.h create mode 100644 mooncake-store/include/io_pattern_types.h create mode 100644 mooncake-store/include/policy_engine.h create mode 100644 mooncake-store/include/prefetch_ops.h diff --git a/mooncake-store/include/admission_ops.h b/mooncake-store/include/admission_ops.h new file mode 100644 index 0000000000..9ef1ade825 --- /dev/null +++ b/mooncake-store/include/admission_ops.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/ops.h" diff --git a/mooncake-store/include/cache_view_manager.h b/mooncake-store/include/cache_view_manager.h new file mode 100644 index 0000000000..0a96be6fec --- /dev/null +++ b/mooncake-store/include/cache_view_manager.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/view_manager.h" diff --git a/mooncake-store/include/cfm_client.h b/mooncake-store/include/cfm_client.h new file mode 100644 index 0000000000..c94a0b0929 --- /dev/null +++ b/mooncake-store/include/cfm_client.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/client.h" diff --git a/mooncake-store/include/eviction_ops.h b/mooncake-store/include/eviction_ops.h new file mode 100644 index 0000000000..9ef1ade825 --- /dev/null +++ b/mooncake-store/include/eviction_ops.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/ops.h" diff --git a/mooncake-store/include/io_pattern.h b/mooncake-store/include/io_pattern.h new file mode 100644 index 0000000000..b382ec908e --- /dev/null +++ b/mooncake-store/include/io_pattern.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/io_pattern.h" diff --git a/mooncake-store/include/io_pattern/analyzer.h b/mooncake-store/include/io_pattern/analyzer.h index 94efd1f71e..1b5e8b6682 100644 --- a/mooncake-store/include/io_pattern/analyzer.h +++ b/mooncake-store/include/io_pattern/analyzer.h @@ -4,11 +4,16 @@ namespace mooncake::io_pattern { +// Converts an immutable snapshot into workload and per-object features. class IoPatternAnalyzer { public: virtual ~IoPatternAnalyzer() = default; virtual PatternResult Analyze(const IoPatternSnapshot& snapshot) const = 0; + virtual WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const = 0; + virtual float CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const = 0; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/client.h b/mooncake-store/include/io_pattern/client.h new file mode 100644 index 0000000000..1c4e061c9d --- /dev/null +++ b/mooncake-store/include/io_pattern/client.h @@ -0,0 +1,18 @@ +#pragma once + +#include "../types.h" +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +// Adapter seam between an inference node and the remote Cache Flow Manager. +class CfmClient { + public: + virtual ~CfmClient() = default; + + virtual ErrorCode ReportSnapshot(const IoPatternSnapshot& snapshot) = 0; + virtual ErrorCode ReceivePolicy(const PolicyCommand& command) = 0; + virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector.h b/mooncake-store/include/io_pattern/collector.h index 94d1ccbbee..986a477b8f 100644 --- a/mooncake-store/include/io_pattern/collector.h +++ b/mooncake-store/include/io_pattern/collector.h @@ -4,10 +4,12 @@ namespace mooncake::io_pattern { +// Collects non-blocking, already-aggregated observations from data paths. class IoPatternCollector { public: virtual ~IoPatternCollector() = default; + // Implementations must not block the caller on RPC or storage I/O. virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; virtual void RecordAccess(const AccessRecord& record) = 0; virtual void RecordStorageMetric(const StorageMetric& metric) = 0; diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h index 99834c3561..4ffbcaa5f1 100644 --- a/mooncake-store/include/io_pattern/io_pattern.h +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -1,8 +1,10 @@ #pragma once #include "io_pattern/analyzer.h" +#include "io_pattern/client.h" #include "io_pattern/collector.h" #include "io_pattern/ops.h" #include "io_pattern/policy_engine.h" #include "io_pattern/registry.h" #include "io_pattern/types.h" +#include "io_pattern/view_manager.h" diff --git a/mooncake-store/include/io_pattern/ops.h b/mooncake-store/include/io_pattern/ops.h index 5db67f2953..37e6c64941 100644 --- a/mooncake-store/include/io_pattern/ops.h +++ b/mooncake-store/include/io_pattern/ops.h @@ -3,10 +3,11 @@ #include #include "io_pattern/types.h" -#include "types.h" +#include "../types.h" namespace mooncake::io_pattern { +// Produces an eviction plan; it does not move or delete data. class EvictionOps { public: virtual ~EvictionOps() = default; @@ -15,6 +16,7 @@ class EvictionOps { uint64_t target_bytes) const = 0; }; +// Produces a prefetch plan; execution belongs to PrefetchExecutor. class PrefetchOps { public: virtual ~PrefetchOps() = default; @@ -23,6 +25,7 @@ class PrefetchOps { const TraceHistory& trace) const = 0; }; +// Decides whether an object may enter a target tier. class AdmissionOps { public: virtual ~AdmissionOps() = default; diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h index bf7e2adc19..4d419b1a72 100644 --- a/mooncake-store/include/io_pattern/policy_engine.h +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -1,11 +1,15 @@ #pragma once #include +#include +#include +#include "io_pattern/ops.h" #include "io_pattern/types.h" namespace mooncake::io_pattern { +// Coordinates configured Ops implementations without owning data-path state. class PolicyEngine { public: virtual ~PolicyEngine() = default; @@ -20,4 +24,48 @@ class PolicyEngine { const PolicyContext& context) const = 0; }; +// A small composition adapter that wires selected Ops instances together. +// Missing optional Ops degrade to empty plans or a deferred admission result. +class ComposedPolicyEngine final : public PolicyEngine { + public: + ComposedPolicyEngine(std::shared_ptr eviction, + std::shared_ptr prefetch, + std::shared_ptr admission) + : eviction_(std::move(eviction)), + prefetch_(std::move(prefetch)), + admission_(std::move(admission)) {} + + EvictionPlan PlanEviction(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override { + if (!eviction_) { + return EvictionPlan{.source_tier = tier, + .target_bytes = target_bytes}; + } + return eviction_->Evaluate(context, tier, target_bytes); + } + + PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const override { + if (!prefetch_) { + return {}; + } + return prefetch_->Evaluate(context, trace); + } + + AdmissionResult DecideAdmission(const ObjectRef& object, + CacheTier target_tier, + const PolicyContext& context) const override { + if (!admission_) { + return AdmissionResult{.object = object, + .target_tier = target_tier}; + } + return admission_->Evaluate(object, target_tier, context); + } + + private: + std::shared_ptr eviction_; + std::shared_ptr prefetch_; + std::shared_ptr admission_; +}; + } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/registry.h b/mooncake-store/include/io_pattern/registry.h index e560967310..518a2ef3b8 100644 --- a/mooncake-store/include/io_pattern/registry.h +++ b/mooncake-store/include/io_pattern/registry.h @@ -20,6 +20,7 @@ class OpsRegistry { public: using Factory = std::function()>; + // Registration is thread-safe; duplicate names are rejected. bool Register(std::string name, Factory factory) { if (name.empty() || !factory) { return false; diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h index 2a39ae0cc7..1fb936e56d 100644 --- a/mooncake-store/include/io_pattern/types.h +++ b/mooncake-store/include/io_pattern/types.h @@ -2,6 +2,7 @@ #include #include +#include #include #include "tenant_id.h" @@ -66,7 +67,7 @@ struct InferenceMetrics { uint32_t layout_group{0}; uint32_t prefix_depth{0}; uint32_t prefix_fanout{0}; - uint32_t match_length{0}; + uint32_t match_length{0}; // tokens/blocks, as defined by the connector uint32_t continuous_prefix_length{0}; uint32_t token_count{0}; float recompute_cost{0.0F}; @@ -75,7 +76,7 @@ struct InferenceMetrics { struct AccessRecord { ObjectRef object; - uint64_t observed_at_ns{0}; + uint64_t observed_at_ns{0}; // monotonic nanoseconds uint64_t block_size{0}; uint64_t latency_us{0}; CacheTier tier{CacheTier::kL2Segment}; @@ -213,4 +214,34 @@ struct AdmissionResult { float confidence{0.0F}; }; +struct CacheViewEntry { + ObjectRef object; + CacheTier tier{CacheTier::kL2Segment}; + uint64_t bytes{0}; +}; + +struct CacheView { + uint64_t version{0}; + std::vector entries; +}; + +using KVMappingTable = std::vector; + +enum class CacheEventType : uint8_t { + kUnknown, + kInserted, + kRemoved, + kTierChanged, +}; + +struct CacheEvent { + CacheEventType type{CacheEventType::kUnknown}; + ObjectRef object; + CacheTier source_tier{CacheTier::kL2Segment}; + CacheTier target_tier{CacheTier::kL2Segment}; +}; + +using PolicyCommand = + std::variant; + } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/view_manager.h b/mooncake-store/include/io_pattern/view_manager.h new file mode 100644 index 0000000000..4ef636b71d --- /dev/null +++ b/mooncake-store/include/io_pattern/view_manager.h @@ -0,0 +1,17 @@ +#pragma once + +#include "io_pattern/types.h" + +namespace mooncake::io_pattern { + +// Owns the published cache view, not the storage operations that realize it. +class CacheViewManager { + public: + virtual ~CacheViewManager() = default; + + virtual CacheView ComputeView() const = 0; + virtual void PublishEvent(const CacheEvent& event) = 0; + virtual KVMappingTable GetGlobalMapping() const = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern_analyzer.h b/mooncake-store/include/io_pattern_analyzer.h new file mode 100644 index 0000000000..7579ac39c1 --- /dev/null +++ b/mooncake-store/include/io_pattern_analyzer.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/analyzer.h" diff --git a/mooncake-store/include/io_pattern_collector.h b/mooncake-store/include/io_pattern_collector.h new file mode 100644 index 0000000000..448be4a72f --- /dev/null +++ b/mooncake-store/include/io_pattern_collector.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/collector.h" diff --git a/mooncake-store/include/io_pattern_registry.h b/mooncake-store/include/io_pattern_registry.h new file mode 100644 index 0000000000..9b08e9df83 --- /dev/null +++ b/mooncake-store/include/io_pattern_registry.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/registry.h" diff --git a/mooncake-store/include/io_pattern_types.h b/mooncake-store/include/io_pattern_types.h new file mode 100644 index 0000000000..a8a6fb2610 --- /dev/null +++ b/mooncake-store/include/io_pattern_types.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/types.h" diff --git a/mooncake-store/include/policy_engine.h b/mooncake-store/include/policy_engine.h new file mode 100644 index 0000000000..9861c9c2fe --- /dev/null +++ b/mooncake-store/include/policy_engine.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/policy_engine.h" diff --git a/mooncake-store/include/prefetch_ops.h b/mooncake-store/include/prefetch_ops.h new file mode 100644 index 0000000000..9ef1ade825 --- /dev/null +++ b/mooncake-store/include/prefetch_ops.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/ops.h" diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index cd74468a20..e35e2c6dd7 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include @@ -18,9 +20,78 @@ class TestEvictionOps final : public EvictionOps { } }; +class TestCollector final : public IoPatternCollector { + public: + void ReportInferenceMetrics(const InferenceMetrics& metrics) override { + inference_metrics = metrics; + } + void RecordAccess(const AccessRecord& record) override { + access_record = record; + } + void RecordStorageMetric(const StorageMetric& metric) override { + storage_metric = metric; + } + IoPatternSnapshot GetSnapshot() const override { return snapshot; } + + InferenceMetrics inference_metrics; + AccessRecord access_record; + StorageMetric storage_metric; + IoPatternSnapshot snapshot; +}; + +class TestAnalyzer final : public IoPatternAnalyzer { + public: + PatternResult Analyze(const IoPatternSnapshot&) const override { + return result; + } + WorkloadType DetectWorkloadType( + const IoPatternSnapshot&) const override { + return result.workload_type; + } + float CalculateConfidence(const ObjectRef&, + const IoPatternSnapshot&) const override { + return result.workload_confidence; + } + + PatternResult result{.workload_type = WorkloadType::kMixed, + .workload_confidence = 0.75F}; +}; + +class TestPrefetchOps final : public PrefetchOps { + public: + PrefetchPlan Evaluate(const PolicyContext&, + const TraceHistory&) const override { + return plan; + } + + PrefetchPlan plan; +}; + +class TestAdmissionOps final : public AdmissionOps { + public: + AdmissionResult Evaluate(const ObjectRef& object, CacheTier tier, + const PolicyContext&) const override { + return AdmissionResult{.object = object, + .target_tier = tier, + .decision = AdmissionDecision::kAdmit}; + } +}; + +class TestPrefetchExecutor final : public PrefetchExecutor { + public: + ErrorCode Execute(const PrefetchPlan& value) override { + plan = value; + return ErrorCode::OK; + } + + PrefetchPlan plan; +}; + TEST(IoPatternFrameworkTest, PublicSeamsRemainAbstract) { static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); + static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); @@ -37,6 +108,90 @@ TEST(IoPatternFrameworkTest, CacheTierMaskRepresentsAllTiers) { EXPECT_EQ(all_tiers, 0x0F); } +TEST(IoPatternFrameworkTest, MetricsKeepTenantAndLayoutIdentity) { + InferenceMetrics metrics; + metrics.object = {TenantId("tenant-a"), "prefix/block-1"}; + metrics.layout = CacheLayout::kHmaMultiGroup; + metrics.layout_group = 3; + metrics.match_length = 512; + + IoPatternSnapshot snapshot; + KeyMetrics key_metrics; + key_metrics.object = metrics.object; + key_metrics.match_length = metrics.match_length; + key_metrics.layout = metrics.layout; + key_metrics.layout_group = metrics.layout_group; + snapshot.keys.push_back(key_metrics); + + ASSERT_EQ(snapshot.keys.size(), 1); + EXPECT_EQ(snapshot.keys.front().object.tenant_id.value(), "tenant-a"); + EXPECT_EQ(snapshot.keys.front().object.key, "prefix/block-1"); + EXPECT_EQ(snapshot.keys.front().layout, CacheLayout::kHmaMultiGroup); + EXPECT_EQ(snapshot.keys.front().layout_group, 3); + EXPECT_EQ(snapshot.keys.front().match_length, 512); +} + +TEST(IoPatternFrameworkTest, CollectorAndAnalyzerExposeValueFlow) { + TestCollector collector; + collector.inference_metrics.object = + {TenantId("tenant-a"), "prefix/block-1"}; + collector.snapshot.generated_at_ns = 42; + collector.snapshot.keys.push_back( + KeyMetrics{.object = collector.inference_metrics.object}); + + collector.ReportInferenceMetrics(collector.inference_metrics); + AccessRecord access_record; + access_record.object = collector.inference_metrics.object; + access_record.observed_at_ns = 43; + access_record.is_hit = true; + collector.RecordAccess(access_record); + collector.RecordStorageMetric(StorageMetric{.source_id = "segment-1"}); + + const auto snapshot = collector.GetSnapshot(); + ASSERT_EQ(snapshot.keys.size(), 1); + EXPECT_EQ(collector.inference_metrics.object.key, "prefix/block-1"); + EXPECT_TRUE(collector.access_record.is_hit); + EXPECT_EQ(collector.storage_metric.source_id, "segment-1"); + + TestAnalyzer analyzer; + const auto result = analyzer.Analyze(snapshot); + EXPECT_EQ(result.workload_type, WorkloadType::kMixed); + EXPECT_FLOAT_EQ(analyzer.CalculateConfidence({}, snapshot), 0.75F); + EXPECT_EQ(analyzer.DetectWorkloadType(snapshot), WorkloadType::kMixed); +} + +TEST(IoPatternFrameworkTest, PolicyContextCarriesRawAndDerivedViews) { + PolicyContext context; + context.snapshot.generated_at_ns = 123; + context.analysis.workload_type = WorkloadType::kMixed; + context.analysis.workload_confidence = 0.75F; + + EXPECT_EQ(context.snapshot.generated_at_ns, 123); + EXPECT_EQ(context.analysis.workload_type, WorkloadType::kMixed); + EXPECT_FLOAT_EQ(context.analysis.workload_confidence, 0.75F); +} + +TEST(IoPatternFrameworkTest, PolicyCommandAndViewRemainValueTypes) { + const ObjectRef object{TenantId("tenant-b"), "block"}; + PrefetchCandidate candidate; + candidate.object = object; + candidate.bytes = 4096; + PrefetchPlan prefetch_plan; + prefetch_plan.strategy = PrefetchStrategy::kTimeout; + prefetch_plan.timeout_us = 1000; + prefetch_plan.candidates.push_back(candidate); + const PolicyCommand command = prefetch_plan; + ASSERT_TRUE(std::holds_alternative(command)); + EXPECT_EQ(std::get(command).candidates.front().object, + object); + + CacheView view; + view.version = 7; + view.entries.push_back({object, CacheTier::kL1Host, 4096}); + EXPECT_EQ(view.entries.front().tier, CacheTier::kL1Host); + EXPECT_EQ(view.entries.front().bytes, 4096); +} + TEST(IoPatternFrameworkTest, RegistryCreatesTypedOpsAndRejectsDuplicates) { OpsRegistry registry; @@ -55,5 +210,37 @@ TEST(IoPatternFrameworkTest, RegistryCreatesTypedOpsAndRejectsDuplicates) { EXPECT_EQ(registry.Create("missing"), nullptr); } +TEST(IoPatternFrameworkTest, ComposedEngineDelegatesAndDegradesSafely) { + auto eviction = std::make_shared(); + auto prefetch = std::make_shared(); + auto admission = std::make_shared(); + prefetch->plan.strategy = PrefetchStrategy::kWaitComplete; + ComposedPolicyEngine engine(eviction, prefetch, admission); + + const auto delegated = + engine.PlanEviction({}, CacheTier::kL1Host, 2048); + EXPECT_EQ(delegated.source_tier, CacheTier::kL1Host); + EXPECT_EQ(delegated.target_bytes, 2048); + + const auto prefetch_plan = engine.PlanPrefetch({}, {}); + EXPECT_EQ(prefetch_plan.strategy, PrefetchStrategy::kWaitComplete); + + const ObjectRef object{TenantId("tenant-a"), "key"}; + const auto admitted = engine.DecideAdmission( + object, CacheTier::kL2Segment, {}); + EXPECT_EQ(admitted.object, object); + EXPECT_EQ(admitted.target_tier, CacheTier::kL2Segment); + EXPECT_EQ(admitted.decision, AdmissionDecision::kAdmit); + + ComposedPolicyEngine degraded(nullptr, nullptr, nullptr); + const auto deferred = + degraded.DecideAdmission(object, CacheTier::kL2Segment, {}); + EXPECT_EQ(deferred.decision, AdmissionDecision::kDefer); + + TestPrefetchExecutor executor; + EXPECT_EQ(executor.Execute(prefetch_plan), ErrorCode::OK); + EXPECT_EQ(executor.plan.strategy, PrefetchStrategy::kWaitComplete); +} + } // namespace } // namespace mooncake::io_pattern From 1a19ded3bd794f6ed6263d91772cd196fbe38ed2 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 2 Sep 2026 10:07:05 +0800 Subject: [PATCH 03/47] =?UTF-8?q?io=20pattern=E4=BB=A3=E7=A0=81=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E6=AD=A5=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/io_pattern_design.md | 1097 +++++++++++++++++ mooncake-store/include/cache_view_manager.h | 3 - mooncake-store/include/cfm_client_impl.h | 2 + mooncake-store/include/collector_impl.h | 2 + .../include/degrading_policy_engine.h | 2 + mooncake-store/include/feedback.h | 2 + .../include/io_pattern/cfm_channel.h | 20 + .../include/io_pattern/cfm_client_impl.h | 34 + .../include/io_pattern/cfm_ingress.h | 27 + .../include/io_pattern/cfm_protocol.h | 22 + mooncake-store/include/io_pattern/collector.h | 3 +- .../include/io_pattern/collector_impl.h | 64 + .../io_pattern/degrading_policy_engine.h | 43 + mooncake-store/include/io_pattern/feedback.h | 61 + .../include/io_pattern/io_pattern.h | 20 +- .../include/io_pattern/kmeans_analyzer.h | 28 + .../include/io_pattern/legacy_eviction_ops.h | 24 + .../include/io_pattern/observability.h | 39 + mooncake-store/include/io_pattern/ops.h | 4 + .../include/io_pattern/policy_engine.h | 281 +++++ .../include/io_pattern/policy_strategies.h | 74 ++ mooncake-store/include/io_pattern/reporter.h | 68 + .../include/io_pattern/resilient_analyzer.h | 40 + .../io_pattern/resilient_cfm_channel.h | 43 + .../include/io_pattern/rpc_transport.h | 126 ++ mooncake-store/include/io_pattern/runtime.h | 94 ++ .../io_pattern/sliding_window_analyzer.h | 50 + .../include/io_pattern/threshold_analyzer.h | 38 + .../include/io_pattern/tier_executor.h | 40 + mooncake-store/include/io_pattern/types.h | 28 + .../include/io_pattern/view_manager.h | 17 - mooncake-store/include/io_pattern_analyzer.h | 1 + mooncake-store/include/legacy_eviction_ops.h | 2 + mooncake-store/include/master_service.h | 9 + mooncake-store/include/observability.h | 2 + mooncake-store/include/policy_strategies.h | 3 + mooncake-store/include/reporter.h | 2 + mooncake-store/include/resilient_analyzer.h | 2 + .../include/resilient_cfm_channel.h | 2 + mooncake-store/include/rpc_transport.h | 2 + .../include/sliding_window_analyzer.h | 2 + mooncake-store/include/threshold_analyzer.h | 3 + mooncake-store/include/tier_executor.h | 2 + mooncake-store/src/CMakeLists.txt | 18 + .../src/io_pattern/cfm_client_impl.cpp | 31 + mooncake-store/src/io_pattern/cfm_ingress.cpp | 36 + .../src/io_pattern/cfm_protocol.cpp | 425 +++++++ .../src/io_pattern/collector_impl.cpp | 183 +++ .../io_pattern/degrading_policy_engine.cpp | 60 + mooncake-store/src/io_pattern/feedback.cpp | 58 + .../src/io_pattern/kmeans_analyzer.cpp | 161 +++ .../src/io_pattern/legacy_eviction_ops.cpp | 35 + .../src/io_pattern/observability.cpp | 63 + .../src/io_pattern/policy_strategies.cpp | 213 ++++ mooncake-store/src/io_pattern/reporter.cpp | 143 +++ .../src/io_pattern/resilient_analyzer.cpp | 67 + .../src/io_pattern/resilient_cfm_channel.cpp | 80 ++ .../src/io_pattern/rpc_transport.cpp | 111 ++ mooncake-store/src/io_pattern/runtime.cpp | 261 ++++ .../io_pattern/sliding_window_analyzer.cpp | 92 ++ .../src/io_pattern/threshold_analyzer.cpp | 134 ++ .../src/io_pattern/tier_executor.cpp | 35 + mooncake-store/src/master_service.cpp | 109 +- .../tests/io_pattern_framework_test.cpp | 957 +++++++++++++- mooncake-wheel/mooncake/io_pattern_bridge.py | 81 ++ .../mooncake/mooncake_connector_v1.py | 68 +- 66 files changed, 5822 insertions(+), 27 deletions(-) create mode 100644 docs/source/io_pattern_design.md delete mode 100644 mooncake-store/include/cache_view_manager.h create mode 100644 mooncake-store/include/cfm_client_impl.h create mode 100644 mooncake-store/include/collector_impl.h create mode 100644 mooncake-store/include/degrading_policy_engine.h create mode 100644 mooncake-store/include/feedback.h create mode 100644 mooncake-store/include/io_pattern/cfm_channel.h create mode 100644 mooncake-store/include/io_pattern/cfm_client_impl.h create mode 100644 mooncake-store/include/io_pattern/cfm_ingress.h create mode 100644 mooncake-store/include/io_pattern/cfm_protocol.h create mode 100644 mooncake-store/include/io_pattern/collector_impl.h create mode 100644 mooncake-store/include/io_pattern/degrading_policy_engine.h create mode 100644 mooncake-store/include/io_pattern/feedback.h create mode 100644 mooncake-store/include/io_pattern/kmeans_analyzer.h create mode 100644 mooncake-store/include/io_pattern/legacy_eviction_ops.h create mode 100644 mooncake-store/include/io_pattern/observability.h create mode 100644 mooncake-store/include/io_pattern/policy_strategies.h create mode 100644 mooncake-store/include/io_pattern/reporter.h create mode 100644 mooncake-store/include/io_pattern/resilient_analyzer.h create mode 100644 mooncake-store/include/io_pattern/resilient_cfm_channel.h create mode 100644 mooncake-store/include/io_pattern/rpc_transport.h create mode 100644 mooncake-store/include/io_pattern/runtime.h create mode 100644 mooncake-store/include/io_pattern/sliding_window_analyzer.h create mode 100644 mooncake-store/include/io_pattern/threshold_analyzer.h create mode 100644 mooncake-store/include/io_pattern/tier_executor.h delete mode 100644 mooncake-store/include/io_pattern/view_manager.h create mode 100644 mooncake-store/include/legacy_eviction_ops.h create mode 100644 mooncake-store/include/observability.h create mode 100644 mooncake-store/include/policy_strategies.h create mode 100644 mooncake-store/include/reporter.h create mode 100644 mooncake-store/include/resilient_analyzer.h create mode 100644 mooncake-store/include/resilient_cfm_channel.h create mode 100644 mooncake-store/include/rpc_transport.h create mode 100644 mooncake-store/include/sliding_window_analyzer.h create mode 100644 mooncake-store/include/threshold_analyzer.h create mode 100644 mooncake-store/include/tier_executor.h create mode 100644 mooncake-store/src/io_pattern/cfm_client_impl.cpp create mode 100644 mooncake-store/src/io_pattern/cfm_ingress.cpp create mode 100644 mooncake-store/src/io_pattern/cfm_protocol.cpp create mode 100644 mooncake-store/src/io_pattern/collector_impl.cpp create mode 100644 mooncake-store/src/io_pattern/degrading_policy_engine.cpp create mode 100644 mooncake-store/src/io_pattern/feedback.cpp create mode 100644 mooncake-store/src/io_pattern/kmeans_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/legacy_eviction_ops.cpp create mode 100644 mooncake-store/src/io_pattern/observability.cpp create mode 100644 mooncake-store/src/io_pattern/policy_strategies.cpp create mode 100644 mooncake-store/src/io_pattern/reporter.cpp create mode 100644 mooncake-store/src/io_pattern/resilient_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/resilient_cfm_channel.cpp create mode 100644 mooncake-store/src/io_pattern/rpc_transport.cpp create mode 100644 mooncake-store/src/io_pattern/runtime.cpp create mode 100644 mooncake-store/src/io_pattern/sliding_window_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/threshold_analyzer.cpp create mode 100644 mooncake-store/src/io_pattern/tier_executor.cpp create mode 100644 mooncake-wheel/mooncake/io_pattern_bridge.py diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md new file mode 100644 index 0000000000..1c9ab5818c --- /dev/null +++ b/docs/source/io_pattern_design.md @@ -0,0 +1,1097 @@ +--- +orphan: true +--- +## 5.4.1 设计概述 + +``` +flowchart TD + subgraph 推理框架层 ["推理框架层 (vLLM/SGLang)"] + MC["MooncakeConnector
(vLLM v1)"] + HC["HiCache Connector
(SGLang)"] + PC["Prefix Cache Manager"] + end + + subgraph CFM ["CFM (Cache Flow Manager)"] + COL["Collector
(数据采集)"] + ANA["Analyzer
(模式分析)"] + ENG["Policy Engine
(策略决策)"] + COL --> ANA --> ENG + EV["Eviction Ops"] + PF["Prefetch Ops"] + AD["Admission Ops"] + ENG --> EV & PF & AD + end + + MC -->|"CFM Client
(采集/策略/预取)"| COL + HC -->|"CFM Client"| COL + PC -->|"CFM Client"| COL + + subgraph CVM ["CVM (Cache View Manager)"] + VIEW["视图计算 / 发布 / 系统事件感知 / 全局 KV 映射表"] + end + + EV --> VIEW + PF --> VIEW + AD --> VIEW + + subgraph 存储层 ["存储层"] + L0["L0: HBM
(UB2PCIe/d2h)"] + L1["L1: Host DRAM/SSD
(计算节点本地内存/SSD(xds)"] + L2["L2: Segment DRAM
(池化内存 URMA mem)"] + L3["L3: Nof SSD
(SSU/远端池化 SSD)"] + end + + VIEW --> L0 + VIEW --> L2 + VIEW --> L3 + L0 <-.->|"tier down/up"| L1 + L1 <-.->|"tier down/up"| L2 + L2 <-.->|"offload/promotion"| L3 +``` + +**模块总体架构图** + +``` +classDiagram + class IoPatternCollector { + <> + +ReportInferenceMetrics(metrics) void + +RecordAccess(key, record) void + +RecordStorageMetric(metric) void + +GetSnapshot() IoPatternSnapshot + } + + class IoPatternAnalyzer { + <> + +AnalyzePattern(snapshot) PatternResult + +DetectWorkloadType(window) WorkloadType + +CalculateConfidence(key) float + } + + class PolicyEngine { + <> + +ExecutePolicy(context, tier, bytes, trace, admissions) PolicyResult + } + + class EvictionOps { + <> + +Evaluate(context, tier, bytes) EvictionPlan + } + + class PrefetchOps { + <> + +Evaluate(context, trace) PrefetchPlan + } + + class AdmissionOps { + <> + +Evaluate(object, tier, context) AdmissionResult + } + + class CfmClient { + +ReportInferenceMetrics(metrics) void + +ReceivePolicy指令() void + +ExecutePrefetch(candidates) void + } + + class ScoreBasedEviction { + +Evaluate(context, tier, bytes) EvictionPlan + } + + class TraceBasedPrefetch { + +Evaluate(context, trace) PrefetchPlan + } + + class PrefixMatchAdmission { + +Evaluate(object, tier, context) AdmissionResult + } + + PolicyEngine *-- EvictionOps : contains + PolicyEngine *-- PrefetchOps : contains + PolicyEngine *-- AdmissionOps : contains + IoPatternCollector --> IoPatternAnalyzer : reports + IoPatternAnalyzer --> PolicyEngine : analyzes + CfmClient --> IoPatternCollector : reports metrics + CfmClient --> PolicyEngine : receives policy + EvictionOps <|.. ScoreBasedEviction : implements + PrefetchOps <|.. TraceBasedPrefetch : implements + AdmissionOps <|.. PrefixMatchAdmission : implements +``` + +## 5.4.2 IO Pattern 三层架构 + +IO Pattern 模块采用**采集层 -> 分析层 -> 策略层**的三层架构。 + +### 5.4.2.1 采集层 (IO Pattern Collector) + +采集层负责从各数据源采集原始 IO 指标,采用异步上报机制避免阻塞数据路径。 + +**采集来源分三层:** + +| 采集层 | 数据源 | 采集指标 | 现有代码锚点 | +| ----------------- | ------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------------------------------- | +| 推理框架层 | vLLM MooncakeConnector / SGLang HiCache Connector | prefix match length, request priority, token 序列, recompute cost | `mooncake_connector_v1.py`, SGLang hicache connector | +| SuperCache SDK 层 | Client / Master / SubMaster | Get/Put/Tier 命中率, 访问时序, key 频率, 前缀树深度/fanout, 副本分布, 迁移 ETA, 写路径指标 (batch_size, overwrite_ratio) | `client_service.cpp`, `master_service.cpp`, `local_hot_cache.cpp` | +| 存储后端层 | SSD/Nof Segment | 读写带宽, 读写延迟, GC 状态, 盘内 Superblock 布局, 容量水位 | `client_metric.h:SsdMetric`, `allocation_strategy.h:SsdMetricsProvider` | + +**采集机制设计:** + +1. **轻量级埋点**:复用和扩展现有 `CountMinSketch`(频率统计)、`SsdMetric`(SSD 延迟/吞吐)、`storage_backend.h:last_access_ns_`(最后访问时间)等埋点,避免重复建设 +2. **异步上报**:CFM Client 定期异步上报指标至 SubMaster,采用 batch 聚合减少 RPC 开销。上报间隔自适应负载(低负载 100ms,高负载退避至 500ms-1s) +3. **全局聚合**:SubMaster 聚合各节点上报的指标,维护全局 token 指标流动视图 +4. **采样降级**:在高负载场景下支持采样率动态调整,优先保障数据路径性能 +5. **多租户隔离**:指标按 `TenantId` 分桶采集,避免高频租户淹没低频租户,沿用现有 `CountMinSketch` 的 `tenant_id.MakeScopedKey` 模式 + +固定 100ms 上报间隔在大规模集群下可能产生可观开销。采用自适应间隔: + +| 负载状态 | 上报间隔 | 触发条件 | +| -------- | -------- | ----------------------------------------- | +| 低负载 | 100ms | mem_used_ratio < 50% | +| 中负载 | 200ms | 50% <= mem_used_ratio < 80% | +| 高负载 | 500ms | mem_used_ratio >= 80% | +| 极高负载 | 1000ms | mem_used_ratio >= 95% 或 RPC 延迟 > 100ms | + +> +> 高负载时拉长间隔减少 RPC 开销,但保持最低 1s 上报频率确保策略时效性。量化估算:4000 节点集群,100ms 间隔下每秒 40000 RPC,单 RPC ~2KB,总带宽 ~80MB/s + +#### 5.4.2.1.1 指标采集与上报流程 + +``` +sequenceDiagram + participant INF as 推理框架 (vLLM/SGLang) + participant CFM as CFM Client + participant SUB as SubMaster + + INF->>CFM: 1. 请求完成/前缀匹配 + CFM->>SUB: 2. 批量上报指标 (InferenceMetrics) + CFM->>SUB: 3. SDK 层埋点 (AccessRecord) + CFM->>SUB: 4. 存储后端指标 (StorageMetric) + Note over SUB: 5. 全局聚合
IoPatternSnapshot +``` + +#### 5.4.2.1.2 指标分类 + +IO Pattern 采集指标分为六大类,对应分级缓存淘汰/准入/预取流程的采集指标定义: + +**时序指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| --------------------- | --------- | ------------------------ | ---------- | ----------------------------------- | +| `last_access_time` | timestamp | 最近一次访问时间 | SDK 层 | `storage_backend.h:last_access_ns_` | +| `access_count_window` | uint32 | 最近时间窗口内访问次数 | SDK 层 | `count_min_sketch.h:CountMinSketch` | +| `idle_time` | duration | = now - last_access_time | 分析层计算 | - | + +**价值指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| ---------------- | ------ | ------------------------------ | ---------- | ---------------- | +| `recompute_cost` | float | 重新计算时间 (token 数 / 时间) | 推理框架层 | connector 层估算 | +| `block_size` | uint64 | 数据大小 (bytes) | SDK 层 | object metadata | +| `token_count` | uint32 | token 数 | 推理框架层 | connector 层 | + +**结构指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| -------------------------- | ------ | ---------------------------- | ---------- | -------------------- | +| `prefix_depth` | uint32 | 前缀深度 (prefix tree level) | 推理框架层 | prefix cache manager | +| `prefix_fanout` | uint32 | 共享该前缀的请求/分支数量 | 推理框架层 | prefix cache manager | +| `match_length` | uint32 | 前缀匹配长度 | 推理框架层 | connector 层 | +| `continuous_prefix_length` | uint32 | 连续前缀长度 | 推理框架层 | connector 层 | + +**副本指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| --------------------- | -------- | -------------------------- | ---------- | -------------------------- | +| `replica_tiers` | bitmap | 当前在哪些层有副本 (L0-L3) | SDK 层 | `master_service.h:Replica` | +| `transfer_eta` | duration | 迁移路径预计耗时 | 分析层计算 | - | +| `ssd_replica_exists` | bool | SSD 层是否有副本 | SDK 层 | replica metadata | +| `other_replica_count` | uint32 | 其他层副本数 | SDK 层 | replica metadata | + +**状态指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| -------- | ---- | ------------ | -------- | ----------------------------- | +| `active` | bool | 是否正在使用 | SDK 层 | `local_hot_cache.h:ref_count` | +| `pinned` | bool | 是否不可迁移 | SDK 层 | promotion task pinning | + +**存储后端指标** + +| 指标名 | 类型 | 描述 | 采集来源 | 现有代码锚点 | +| ------------------- | --------- | ----------------- | ---------- | ---------------------------------------------------------------------------- | +| `ssd_read_latency` | histogram | SSD 读延迟分布 | 存储后端层 | `client_metric.h:SsdMetric` | +| `ssd_write_latency` | histogram | SSD 写延迟分布 | 存储后端层 | `client_metric.h:SsdMetric` | +| `ssd_gc_status` | enum | SSD GC 状态 | 存储后端层 | Nof TGT | +| `mem_used_ratio` | float | DRAM 内存水位比例 | SDK 层 | `MasterMetricManager::get_global_mem_used_ratio()` (Master 侧全局 DRAM 水位) | +| `ssd_used_bytes` | int64 | SSD 已用容量 | 存储后端层 | `allocation_strategy.h:SsdMetricsProvider` | + +#### 5.4.2.1.3 指标采集接口 + +``` +// IO Pattern Collector 接口 (新增, 位于 include/io_pattern_collector.h) +class IoPatternCollector { + public: + virtual ~IoPatternCollector() = default; + + // 推理框架层指标 (通过 CFM Client 上报) + virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; + + // SDK 层指标 (内部埋点) + virtual void RecordAccess(const std::string& key, + const AccessRecord& record) = 0; + + // 存储后端层指标 + virtual void RecordStorageMetric(const StorageMetric& metric) = 0; + + // 获取聚合后的指标快照 + virtual IoPatternSnapshot GetSnapshot() const = 0; +}; + +struct AccessRecord { + std::string key; + std::chrono::steady_clock::time_point access_time; + uint64_t block_size; + ReplicaType replica_type; + bool is_hit; + std::chrono::microseconds latency; +}; + +struct InferenceMetrics { + std::string session_id; + uint32_t prefix_depth; + uint32_t prefix_fanout; + uint32_t match_length; + uint32_t continuous_prefix_length; + uint32_t token_count; + float recompute_cost; + uint8_t request_priority; +}; +``` + +### 5.4.2.2 分析层 (IO Pattern Analyzer) + +分析层对采集的原始指标进行模式识别和特征提取,输出结构化的 IO Pattern 描述。 + +**分析能力:** + +| 分析类型 | 描述 | 输入指标 | 输出 | 对应需求 | +| ------------- | ------------------------------ | ------------------------------------------------------ | ------------------------------------ | -------------- | +| 热度分析 | 基于 LFU/滑动窗口的频率统计 | access_count_window, last_access_time | hot/cold 分类, 频率评分 | 淘汰/准入/预取 | +| 前缀分析 | Prefix tree 深度和 fanout 分析 | prefix_depth, prefix_fanout, match_length | 前缀共享度, 预取候选 | 预取/准入 | +| 时序预测 | 访问间隔和空闲时间分析 | idle_time, access pattern, access_count_window | 空闲评分, 预取优先级 | 淘汰/Tier down | +| 代价评估 | 重计算代价和迁移代价评估 | recompute_cost, token 数, block_size, transfer_eta | 代价评分, 迁移 ROI | 淘汰/Tier up | +| 访问模式识别 | 顺序/随机、大包/小包、读写比 | IO size distribution, access sequence | 模式分类 (SEQ/RANDOM/KV_LOOKUP等) | 缓存分区/分流 | +| 副本分析 | 多层副本分布分析 | replica_tiers, active/pinned | 副本冗余度, 迁移安全性 | 淘汰/Tier down | +| 写路径分析 | 写入模式分析 | write_batch_size, write_burst, overwrite_ratio | 写穿风险, GC 预警 | 准入/GC | +| Workload 识别 | 推理场景类型自动识别 | token_count, prefix_fanout, block_size, frequency 分布 | workload_type (Code Agent/推荐/对话) | 策略模板选择 | + +**Workload Type 感知策略模板** + +不同推理场景的 IO Pattern 差异巨大,单一通用评分公式无法覆盖所有场景。 + +**配置参数:** + +| 参数 | 默认值 | 说明 | +| -------------------------------------- | ------------- | ------------------------------------------------------------------ | +| `workload_detection_window_sec` | 60 | workload 识别滑动窗口大小 | +| `workload_detection_method` | `auto` | 识别方法:`auto`(阈值+聚类)、`threshold`(仅阈值)、`kmeans`(仅聚类) | +| `workload_template_transition_windows` | 3 | 模板切换过渡窗口数,控制平滑度 | +| `workload_mixed_load_mode` | `per_session` | 混合负载处理:`per_session`(按会话标记)、`global`(全局统一) | + +**运维观测:** + +| 指标 | 描述 | +| --------------------------------------- | ------------------------ | +| `workload_current_type` | 当前识别的 workload type | +| `workload_type_switch_count` | workload type 切换次数 | +| `workload_detection_latency_us` | 单次识别延迟 | +| `workload_template_transition_progress` | 模板过渡进度 (0.0-1.0) | + +**Workload 识别机制:** + +不同推理场景的 KVCache 访问模式差异巨大,单一通用评分公式无法覆盖所有场景。IO Pattern 分析层基于滑动窗口内的指标分布统计,自动识别 workload type 并切换对应策略模板。 + +**Workload Type 特征矩阵:** + +| Workload Type | 典型场景 | 访问特征 | 识别信号 | +| ------------- | ------------------------------ | ------------------------------------------------------------------------- | ----------------------------------------------------------------------- | +| Code Agent | Cursor/Copilot 长程代码生成 | 长上下文(>32K token)、高前缀复用、大 block_size(>512KB)、多轮访问为主 | 高 token_count + 低 prefix_fanout + 大 block_size + 低 frequency | +| 生成式推荐 | 京东/字节 GR 精排召回 | 高频小 block(<128KB)、高复用、密集访问、低重计算代价 | 低 token_count + 高 frequency + 小 block_size + 低 recompute_cost | +| 多轮对话 | ChatGPT 类对话、Agent 工具调用 | 中等 block、高前缀共享、渐进式增长、prefix cache 命中率高 | 中 token_count + 高 prefix_fanout + 高 match_length + 高 recompute_cost | + +**识别算法:** + +分析层维护一个滑动窗口(默认 60s),统计窗口内所有请求的 `token_count`/`prefix_fanout`/`block_size`/`frequency` 四维分布。采用两阶段识别: + +``` +阶段 1: 特征提取 + for each request in window: + feature_vector = (median(token_count), p90(prefix_fanout), + median(block_size), median(frequency)) + +阶段 2: 分类决策 + if feature_vector matches阈值规则: + -> 直接分类 (快速路径, 延迟 < 1ms) + else: + -> K-means 聚类 (慢速路径, 延迟 < 10ms, 用于混合负载场景) +``` + +**阈值规则(快速路径):** + +| 判定条件 | → Workload Type | +| ---------------------------------------------------------------------------------- | ----------------------- | +| `median(token_count) > 16KB && p90(prefix_fanout) > 16 && p90(match_length) > 256` | Code Agent | +| `median(block_size) < 128KB && median(frequency) > 20` | 生成式推荐 | +| `p90(prefix_fanout) > 16 && p90(match_length) > 256` | 多轮对话 | +| 不满足以上任一 | 混合负载 → K-means 聚类 | + +**策略模板对照:** + +每种 workload type 对应一组完整的策略参数模板,覆盖淘汰/准入/预取/Tier 四个维度: + +| 策略维度 | Code Agent | 生成式推荐 | 多轮对话 | +| ------------- | ----------------------------------------- | --------------------------------------- | -------------------------------------- | +| **预取** | 保守(仅 prefix > 512 预取,best_effort) | 激进(prefix > 64 预取,wait_complete) | 前缀优先(prefix > 256 预取,timeout) | +| **淘汰** | 激进(低 idle_thres,快速释放 L0) | 保守(高 idle_thres,保留热数据) | 前缀感知(prefix_fanout 高权重保留) | +| **准入** | 低阈值(access_count > 2 即准入) | 高阈值(access_count > 20 才准入) | 前缀准入(match_length > 128 即准入) | +| **Tier down** | 快速降级(L0→L2 跳级,跳过 L1) | 缓慢降级(L0→L1→L2 逐层) | 前缀亲和(共享前缀的 block 同层迁移) | +| **淘汰权重** | α=0.8, γ=0.2, δ=0.3, ε=0.1 | α=0.3, γ=0.8, δ=0.2, ε=0.1 | α=0.5, γ=0.4, δ=0.6, ε=0.8 | + +**模板切换机制:** + +``` +flowchart TD + WIN["滑动窗口指标统计
(60s)"] + FEAT["特征提取
4 维分布向量"] + RULE["阈值规则匹配"] + KMEANS["K-means 聚类
(混合负载)"] + CLASSIFY["Workload Type 判定"] + TEMPLATE["策略模板加载
(淘汰/准入/预取/Tier 参数)"] + APPLY["应用至 Policy Engine"] + + WIN --> FEAT --> RULE + RULE -->|"匹配成功"| CLASSIFY + RULE -->|"不匹配"| KMEANS --> CLASSIFY + CLASSIFY --> TEMPLATE --> APPLY +``` + +**切换平滑性:** workload type 变化时,策略参数不是瞬间切换,而是通过加权过渡(新旧模板权重在 3 个窗口周期内从 100:0 渐变到 0:100),避免策略突变导致缓存抖动。 + +**混合负载处理:** 当 K-means 识别出多种 workload type 共存时(如同一集群同时服务对话和推荐),采用 per-session workload 标记——在请求入口处根据 session 特征打标签,各 session 独立使用对应模板,而非全局统一。 + +### 5.4.2.3 策略层 (IO Pattern Policy Engine) + +策略层基于分析层的输出,通过可注册的 Ops 接口驱动各缓存机制: + +``` +flowchart TD + PE["Policy Engine"] + + subgraph EvictionOps ["EvictionOps (淘汰策略)"] + LRU["LRU"] + LFU["LFU"] + SBE["ScoreBased
(L0-L3 四层)"] + end + + subgraph PrefetchOps ["PrefetchOps (预取策略)"] + BE["BestEffort"] + TO["Timeout"] + WC["WaitComplete"] + TB["TraceBased"] + end + + subgraph AdmissionOps ["AdmissionOps (准入策略)"] + FREQ["Frequency"] + PM["PrefixMatch"] + WM["Watermark"] + CA["CostAware"] + end + + PE --> EvictionOps + PE --> PrefetchOps + PE --> AdmissionOps +``` + +## 5.4.3 缓存机制集成与关键流程 + +IO Pattern 模块不是重写现有机制,而是在现有机制之上增加统一的数据采集和分析层,通过 Ops 抽象接口驱动各机制。 + +### 5.4.3.2 淘汰 (Eviction) + +**现有机制**:`EvictionStrategy` 抽象类(`eviction_strategy.h`)提供 LRU 和 FIFO 两种实现;`storage_backend.h` 中基于 `last_access_ns_` 维护 LRU 索引。 + +**IO Pattern 增强**:引入基于评分的淘汰策略 (ScoreBasedEviction),按4层缓存层级分别使用不同评分公式。所有指标先经归一化处理(`normalize(x) = x / max_observed_x`,映射到 `[0, 1]`),消除量纲差异后再加权求和。归一化基准基于滑动窗口(默认 60s)内的最大观测值动态更新。 + +``` +flowchart TD + SNAP["IO Pattern Snapshot"] + NORM["归一化处理
norm(x) = x / max_observed"] + PE["Policy Engine
选择层级策略"] + SNAP --> NORM --> PE + + PE --> L0S["L0 HBM Evict"] + PE --> L1S["L1 Host DRAM/SSD Evict"] + PE --> L2S["L2 Segment DRAM Evict"] + PE --> L3S["L3 Nof SSD Evict"] + + L0S --> L0F["α\*norm(idle) - γ\*norm(freq)
- δ\*norm(recompute) - ε\*norm(fanout)"] + L1S --> L1F["α\*norm(idle) - γ\*norm(freq)
+ δ\*norm(lower_replica)
- ε\*norm(fanout) - ζ*norm(recompute)"] + L2S --> L2F["α\*norm(idle) - γ\*norm(freq)
+ δ\*norm(lower_replica)
- ε\*norm(fanout) - ζ*norm(recompute)"] + L3S --> L3F["α\*norm(idle)\*norm(block_size)
- γ\*norm(freq) - δ\*norm(recompute)
+ η*norm(other_replica)"] + + L0F --> L0OUT["-> L1"] + L1F --> L1OUT["-> L2"] + L2F --> L2OUT["-> L3"] + L3F --> L3OUT["-> 丢弃"] +``` + +**各层淘汰策略说明:** + +| 层级 | 评分侧重 | 淘汰去向 | 说明 | +| ---------------- | --------------------------------------------------------------------- | -------- | ------------------------- | +| L0 HBM | `idle_time` 主导,`recompute_cost`/`prefix_fanout` 高权重保留 | → L1 | HBM 最贵,冷数据快速降级 | +| L1 Host DRAM/SSD | 下层已有副本可安全淘汰,`prefix_fanout`/`recompute_cost` 高的数据保留 | → L2 | 本地 DRAM/SSD 到远端 DRAM | +| L2 Segment DRAM | `prefix_fanout`/`recompute_cost` 高的数据保留 | → L3 | 池化内存到远端 SSD | +| L3 Nof SSD | `block_size` 大 + 其他层已有副本优先淘汰 | → 丢弃 | 最底层,无下降空间 | + +**集成方式**:扩展现有 `EvictionStrategy` 接口,新增 `ScoreBasedEvictionStrategy`,由 Policy Engine 根据层级动态选择策略。 + +### 5.4.3.3 准入 (Admission) + +**现有机制**: + +- Client 侧:`CountMinSketch` + `admission_threshold_` 频率准入(`client_service.cpp`),仅频繁访问的 key 提升 hot cache +- Master 侧:Promotion-on-Hit 的 `promotion_admission_threshold_` 频率门控 + watermark 门控(`master_service.cpp`) + +**IO Pattern 增强**:扩展准入策略为多层逐级准入控制,每层提升需满足对应条件: + +| 准入路径 | 条件 | 说明 | +| -------- | -------------------------------------------------------------- | ----------------------------------------------------- | +| L3→L2 | `access_count_window >= threshold` | 频率达标才从 Nof SSD 提升至 Segment DRAM | +| L2→L1 | `access_count_window >= threshold && upper_space <= max_space` | 频率达标且上层有空间才提升 Segment DRAM→Host DRAM/SSD | +| L1→L0 | `max_length >= threshold (64)` | 前缀长度达标才从 Host DRAM/SSD 提升至 HBM | + +> +> SSD→HBM 跨层直达(跳过中间层)仅由推理框架 prefix cache 命中时触发,Mooncake 侧不自主执行跨层晋升到 HBM。 + +``` +flowchart TD + REQ["访问请求"] + ANA["IO Pattern Analyzer
计算准入条件"] + REQ --> ANA + + ANA --> P1["L3→L2
access_cnt >= thres"] + ANA --> P2["L2→L1
access_cnt >= thres
&& upper_space <= max"] + ANA --> P3["L1→L0
max_length >= 64"] + + P1 --> FA["频率准入
(CountMin Sketch)"] + P2 --> FA2["频率+空间准入
(Frequency + Watermark)"] + P3 --> PA["前缀准入
(PrefixMatch Admission)"] +``` + +**集成方式**:扩展现有 `CountMinSketch` 准入逻辑,新增 `PrefixMatchAdmission` 和 `CostAwareAdmission` 策略。 + +### 5.4.3.4 Tier Down / SSD Offload + +**现有机制**: + +- `enable_ssd_offload` + `ssd_offload_path` 配置 SSD offload 路径(`real_client.cpp`) +- `offload_on_evict` 模式:在淘汰时延迟 offload 到 LOCAL_DISK(`master_service.cpp`) +- `offload_force_evict`:超过 offload cap 时直接淘汰不 offload + +**IO Pattern 增强**:基于热度阈值的逐级 tier down,数据按 L0→L1→L2→L3 顺序逐层降级: + +``` +flowchart TD + START["L0 HBM 容量/水位检测"] + C1{"idle_time >= L0 cold_thres || frequency < L0 hot_thres ?"} + C2{"L2 seg_dram_avail ?"} + C3{"L3 nof SSD avail ?"} + C4{"L1 host DRAM<= thres ?"} + C5{"xds available ?"} + C6{"idle_time >= L1 cold_thres || frequency < L1 hot_thres ?"} + C7{"idle_time >= L2 cold_thres || frequency < L2 hot_thres ?"} + TD_L1A["L1 Host DRAM"] + TD_L1B["L1 SSD(xds)"] + TD_L2["L2 Segment DRAM"] + TD_L3["L3 Nof SSD"] + + START --> C1 + C1 -->|是| C4 + C4 -->|是| TD_L1A + C4 -->|否| C5 + C5 -->|是| TD_L1B + C5 -->|否| C2 + C2 -->|是| TD_L2 + C2 -->|否| C3 + C3 -->|否| WAIT["下层均不可用
等待重试 / 强制 evict"] + C3 -->|是| TD_L3 + TD_L1A --> C6 + C6 -->|是| C2 + TD_L2 --> C7 + C7 -->|是| C3 +``` + +> +> 当所有下层均不可用时,数据暂留当前层并等待下层恢复,或触发强制 evict 释放空间。冷数据不会保留在高速层——高速层是最昂贵的资源,冷数据必须逐级降级。 + +### 5.4.3.5 Tier Up / Promotion-on-Hit + +**现有机制**: + +- `promotion_on_hit` 模式(`master_service.cpp:379`):Get 观察到 LOCAL_DISK-only key 时队列异步拷贝回 MEMORY +- `CountMinSketch` 频率门控(`master_service.cpp:6944`) +- watermark 门控:DRAM 低于 `eviction_high_watermark_ratio_` 才允许 promotion +- `promotion_queue_limit` + `promotion_max_per_heartbeat` 控制 promotion 速率 +- `PromotionCandidate` 跟踪 + 重试 + TTL 过期 + +**IO Pattern 增强**:IO Pattern 分析层为 promotion 提供更丰富的决策输入: + +- 前缀匹配度:高前缀匹配的 key 优先 promotion +- 重计算代价:高 recompute_cost 的 key 优先 promotion +- 迁移 ETA:根据带宽和 block_size 估算 transfer_eta,避免迁移耗时过长 + +> +> Mooncake 侧 promotion 仅执行逐级提升(L3→L2→L1),不自主晋升到 L0 HBM。L0 HBM 层的数据加载由推理框架 prefix cache 命中时自主触发。 + +``` +flowchart TD + START["prefix cache 命中/预取触发/get"] + CALC["Tier Up Priority 计算:
priority = w0 * recompute_cost
+ w1 * continuous_prefix
+ w2 * request_priority
- w3 * transfer_eta"] + SORT["按优先级排序"] + EXEC["执行逐级 tier up
L3→L2→L1"] + + START --> CALC --> SORT --> EXEC +``` + +### 5.4.3.6 预取 (Prefetch) + +**现有机制**:当前无显式预取机制,Promotion-on-Hit 在 Get 命中 LOCAL_DISK 时异步提升到 MEMORY。 + +**IO Pattern 增强**:新增 `PrefetchOps` 抽象,SubMaster 根据 trace 和置信阈值生成预取器: + +**预取触发条件:** 低速层 prefix match length > 阈值 (256) 时触发预取至上一层(如 L3→L2、L2→L1) + +**预取策略(三种模式):** + +| 策略 | 描述 | 适用场景 | +| --------------- | ----------------------------------- | -------------------- | +| `best_effort` | check & match,无论是否完成立即返回 | 对 TTFT 时延敏感业务 | +| `timeout` | 预取完成或超时立即返回 | 兼顾时延和命中率 | +| `wait_complete` | 死等数据加载完成 | 追求极致命中率 | + +分析层基于 trace 历史命中率和置信阈值生成策略输入。置信度 = 滑动窗口内命中次数 / 总访问次数,低于阈值时不触发操作避免误判。例如预取器生成: + +- SubMaster 根据 trace 历史 + 置信阈值(如 `confidence > 0.6 && prefix match length > 256`)生成预取器 +- 置信度低于阈值时不触发操作,避免误判导致的缓存污染 +- 置信阈值精确定义和各策略默认值详见 + +**置信度计算:** 基于滑动窗口内的历史命中率,衡量当前预测的可信程度。 + +``` +confidence = hit_count_in_window / total_access_in_window +``` + +**置信阈值应用:** + +| 策略 | 置信阈值 | 含义 | 默认值 | +| -------- | ----------------------------------------------- | --------------------------------------- | ------------------------------------ | +| 预取触发 | `confidence > 0.6 && match_length > 256` | 历史命中率 > 60% 且前缀匹配足够长才预取 | prefix_threshold=256, confidence=0.6 | +| 准入提升 | `confidence > 0.5 && access_count >= threshold` | 历史命中率 > 50% 且频率达标才提升 | confidence=0.5 | +| 淘汰保守 | `confidence > 0.8` 时降低淘汰权重 | 高置信热数据更保守淘汰 | confidence=0.8 | + +> +> 置信度低于阈值时不触发操作,避免误判导致的缓存污染。置信度窗口默认 60s,可通过 `confidence_window_sec` 配置。 + +预取流程仅看 `match_length > 256` 触发 + +``` +flowchart TD + START["低速层 prefix match
(L1-L3)"] + C1{"match_length > 256 ?"} + NOP["不预取"] + SEL["选择预取策略"] + + START --> C1 + C1 -->|否| NOP + C1 -->|是| SEL + SEL --> BE["best_effort"] + SEL --> TO["timeout"] + SEL --> WC["wait_complete"] +``` + +- `max_prefetch_ratio`:预取占用带宽上限比例,默认 20%,可通过 `prefetch_max_bw_ratio` 配置 +- 带宽不足时延迟重试,而非直接丢弃预取请求 + +**集成方式**:在 SubMaster 中新增预取器,根据 IO Pattern 分析层的置信阈值异步预取 key 至上层。 + +## 5.4.5 上层推理框架对接 + +### 5.4.5.1 vLLM 集成 + +**现有对接**:`MooncakeConnector`(`mooncake_connector_v1.py`)实现 vLLM `KVConnectorBase_V1` 接口,支持 PD disaggregation(Prefill/Decode 分离)。 + +> +> **上层框架改动**:vLLM 侧无需改动。`MooncakeConnector` 作为 vLLM 的 out-of-tree connector(通过 `--kv_connector_module_path` 加载),在 connector 内部新增 CFM Client 调用即可上报指标和接收策略指令,不涉及 vLLM scheduler/engine 接口变更。vLLM v0.13.0+ 已内置 mooncake connector,后续可考虑将 CFM Client 合入上游。 + +**IO Pattern 对接增强**: + +``` +flowchart TD + VLLM["vLLM Engine"] + + subgraph MC ["MooncakeConnector (KVConnectorBase_V1)"] + GNMT["get_num_new_matched_tokens()
上报 match_length"] + USA["update_state_after_alloc()
上报 prefix_depth"] + RF["request_finished()
上报 token_count, recompute_cost"] + end + + subgraph CFMC ["CFM Client (新增)"] + RIM["IoPatternCollector
.ReportInferenceMetrics()"] + POP["PrefetchOps
接收预取指令"] + AOP["AdmissionOps
接收准入策略"] + end + + VLLM --> MC + VLLM --> CFMC + GNMT --> RIM + USA --> RIM + RF --> RIM +``` + +**采集对接**:在 `MooncakeConnector` 中增加 CFM Client 调用,将以下指标上报至 IO Pattern Collector: + +- `match_length`:prefix cache 命中长度(来自 `get_num_new_matched_tokens()`) +- `prefix_depth` / `prefix_fanout`:前缀树结构(来自 `update_state_after_alloc()` 及 prefix cache manager) +- `token_count`:请求 token 数(来自 `request_finished()`) +- `recompute_cost`:重计算代价估算(connector 侧基于 token_count 和模型 FLOPS 估算) +- `request_priority`:请求优先级(connector 层从 request metadata 提取) + +**策略对接**:CFM Client 接收 Policy Engine 的策略指令: + +- 预取指令:根据 prefix match length > 256 触发异步预取 +- 准入指令:根据频率/前缀匹配控制数据提升层级 +- 淘汰指令:根据淘汰评分驱动 L0-L3 层间淘汰 + +### 5.4.5.2 SGLang 集成 + +**现有对接**:SGLang HiCache 通过 `--hicache-storage-backend: mooncake` 将 Mooncake 作为存储后端,支持 layer_first / page_first 布局。 + +**IO Pattern 对接增强**: + +``` +flowchart TD + SGL["SGLang Engine"] + + subgraph HCC ["HiCache Connector"] + HR["hicache-ratio
容量配比"] + HML["hicache-mem-layout
layer_first / page_first"] + HIO["hicache-io-backend
direct / async"] + end + + subgraph CFMS ["CFM Client (新增)"] + RIM2["IoPatternCollector
.ReportInferenceMetrics()"] + DLA["数据布局适配
(layer_first / page_first)"] + PAS["预取/准入/淘汰策略"] + end + + SGL --> HCC + SGL --> CFMS + HR --> RIM2 + HML --> DLA + HIO --> RIM2 +``` + +**数据布局适配**:IO Pattern 需感知推理框架的 KV cache 布局模式,vLLM 和 SGLang 均需适配: + +| 框架 | 布局模式 | 描述 | IO Pattern 适配 | +| ------ | ------------------------------ | ------------------------------------------------------------------------- | ----------------------------------------------- | +| SGLang | `layer_first` | (2, layer, slot, num_head, head_dim) | 前缀分析按 layer 维度,预取按 layer 批量 | +| SGLang | `page_first` | (2, page_num, layer, num_head, head_dim) | 前缀分析按 page 维度,预取按 page 批量 | +| SGLang | `page_first_direct` | 混合模型 (Full Attention + SWA/Mamba) | 分区准入,full KV 固定分区 + SWA/mamba 灵活分配 | +| vLLM | `page-based` (block_size 粒度) | vLLM v1 默认 page-based 布局,connector 通过 `get_kv_cache_layout()` 检测 | 前缀分析按 block 维度,预取按 block 批量 | +| vLLM | `HMA multi-group` | 混合模型 (attention + Mamba2),`SupportsHMA` 多 group 布局 | 分组准入,各 group 独立淘汰/预取策略 | + +> +> vLLM connector 已在初始化时调用 `get_kv_cache_layout()` 检测布局(`mooncake_connector_v1.py:511`),并通过 `SupportsHMA` 支持 hybrid 模型多 group 布局。SGLang 通过 `--hicache-mem-layout` 参数显式配置布局。两者均需在 CFM Client 上报时附带布局信息,供 IO Pattern 分析层选择对应的预取/准入粒度。 + +### 5.4.5.3 CFM Client 设计 + +CFM Client 部署在推理节点侧,作为推理框架与 SuperCache 之间的策略桥梁: + +``` +flowchart TD + subgraph CFMClient ["CFM Client"] + MR["Metrics Reporter
(采集上报)"] + PR["Policy Receiver
(策略接收)"] + PE["Prefetch Executor
(预取执行)"] + RPC["CFM RPC Channel
(to SubMaster / PrefixCache Master)"] + MR --> RPC + PR --> RPC + PE --> RPC + end +``` + +**职责:** + +1. **Metrics Reporter**:定期(100ms)批量上报推理框架指标至 SubMaster +2. **Policy Receiver**:接收 Policy Engine 的淘汰/预取/准入策略指令 +3. **Prefetch Executor**:执行异步预取,支持 best_effort / timeout / wait_complete 三种模式 + +## 5.4.6 Ops 抽象接口设计 + +> **接口修订(2026-09)**:本节原始的 string/vector 简化签名仅用于 +> 查询示例,不能承载租户、字节预算、评分、Tier、超时和置信度等执行 +> 元数据。实际实现统一采用文末“Revised Ops contract”中的完整计划接口。 + +### 5.4.6.1 EvictionOps + +``` +// include/eviction_ops.h (扩展现有 eviction_strategy.h) +class EvictionOps { + public: + virtual ~EvictionOps() = default; + + // 基于 IO Pattern 评分选择淘汰 key + virtual std::vector SelectEvictionCandidates( + const IoPatternSnapshot& snapshot, + CacheTier tier, + size_t target_bytes) = 0; + + // 注册淘汰算法 + static void Register(const std::string& name, + std::function()> factory); +}; + +// 已有实现: LRU, FIFO (eviction_strategy.h) +// 新增实现: ScoreBasedEviction (L0-L3 四层不同评分公式) +``` + +### 5.4.6.2 PrefetchOps + +``` +// include/prefetch_ops.h (新增) +class PrefetchOps { + public: + virtual ~PrefetchOps() = default; + + // 基于 trace 和置信阈值生成预取候选 + virtual std::vector GeneratePrefetchPlan( + const IoPatternSnapshot& snapshot, + const TraceHistory& trace) = 0; + + // 执行预取 + virtual ErrorCode ExecutePrefetch( + const std::vector& candidates, + PrefetchStrategy strategy) = 0; + + static void Register(const std::string& name, + std::function()> factory); +}; + +enum class PrefetchStrategy { + kBestEffort, // 无论是否完成立即返回 + kTimeout, // 预取完成或超时立即返回 + kWaitComplete, // 死等数据加载完成 +}; +``` + +### 5.4.6.3 AdmissionOps + +``` +// include/admission_ops.h (新增, 扩展现有 CountMinSketch 准入) +class AdmissionOps { + public: + virtual ~AdmissionOps() = default; + + // 准入决策:是否允许数据进入目标层 + virtual AdmissionDecision CheckAdmission( + const std::string& key, + CacheTier target_tier, + const IoPatternSnapshot& snapshot) = 0; + + static void Register(const std::string& name, + std::function()> factory); +}; + +enum class AdmissionDecision { + kAdmit, // 允许进入 + kRejectFrequency, // 频率不足 + kRejectWatermark, // 水位过高 + kRejectPrefix, // 前缀匹配不足 + kDefer, // 延迟决策 (记录候选) +}; + +// 已有实现: FrequencyAdmission (CountMinSketch) +// 新增实现: PrefixMatchAdmission, CostAwareAdmission +``` + +### 5.4.6.4 Ops 注册机制 + +``` +classDiagram + class EvictionOps { + <> + +SelectEvictionCandidates(snapshot, tier, bytes) vector~string~ + +Register(name, factory) void + } + class PrefetchOps { + <> + +GeneratePrefetchPlan(snapshot, trace) vector~PrefetchCandidate~ + +ExecutePrefetch(candidates, strategy) ErrorCode + +Register(name, factory) void + } + class AdmissionOps { + <> + +CheckAdmission(key, tier, snapshot) AdmissionDecision + +Register(name, factory) void + } + class PolicyEngine { + -ops_registry_ : map + +SelectOps(type, name) Ops + +ExecutePolicy(snapshot) PolicyResult + } + + PolicyEngine --> EvictionOps : 查找/执行 + PolicyEngine --> PrefetchOps : 查找/执行 + PolicyEngine --> AdmissionOps : 查找/执行 +``` + +## 5.4.7 写路径 IO Pattern + +### 5.4.7.1 写路径采集指标 + +| 指标名 | 类型 | 描述 | 采集来源 | +| ------------------- | ------ | ------------------------------ | ------------------- | +| `write_batch_size` | uint32 | 批量写入 key 数 | SDK 层 (`BatchPut`) | +| `write_object_size` | uint64 | 单次写入数据大小 | SDK 层 | +| `write_burst` | bool | 是否突发写入(短时间大量 Put) | 分析层计算 | +| `write_frequency` | uint32 | key 写入频率 | SDK 层 | +| `overwrite_ratio` | float | 覆盖写比例 (同 key 重复 Put) | 分析层计算 | + +### 5.4.7.2 写路径 Pattern 对策略的影响 + +| Pattern | 影响策略 | 处理方式 | +| ---------- | ---------------------- | ------------------------------------------------------------------ | +| 突发写入 | 准入:避免写穿 SSD | 突发写入期间提高 `admission_threshold`,冷数据暂留 DRAM 不 offload | +| 高覆盖写 | 准入:跳过 SSD offload | 覆盖写比例高的 key 不 offload 到 SSD,避免无效写入 | +| 大批量写入 | GC:提前触发 | 预估写入量,提前通知 SSD 后端准备 GC 空间 | +| 低频写入 | 淘汰:降低保留优先级 | 低频写入的 key 在淘汰评分中 `frequency` 低,优先淘汰 | + +## 5.4.8 健壮性与可观测性 + +### 5.4.8.1 失败降级 + +IO Pattern 模块自身故障时,必须不影响数据路径,降级到现有基础机制: + +``` +flowchart TD + START["策略执行请求"] + C1{"IO Pattern 模块可用?"} + C2{"分析层响应
超时?"} + NORMAL["正常路径:
ScoreBasedEviction / PrefixMatchAdmission / TraceBasedPrefetch"] + DEGRADE["降级路径:
LRU / FIFO / FrequencyAdmission
(现有基础机制)"] + + START --> C1 + C1 -->|是| C2 + C1 -->|否| DEGRADE + C2 -->|否| NORMAL + C2 -->|是| DEGRADE +``` + +| 故障场景 | 降级行为 | 触发条件 | +| -------------- | --------------------------------- | ------------------- | +| SubMaster 崩溃 | 回退到 Client 本地 LRU/FIFO | RPC 连续失败 > 3 次 | +| 分析层超时 | 使用上一次成功快照 | 响应延迟 > 500ms | +| 分析层 OOM | 丢弃 per-key 指标,仅保留全局指标 | 内存占用 > 阈值 | +| RPC 网络抖动 | 延长上报间隔,本地缓存策略 | 丢包率 > 5% | + +### 5.4.8.2 反馈闭环 + +策略执行后需评估效果并自适应调优参数,形成闭环: + +``` +flowchart LR + EXEC["策略执行
(eviction/prefetch/admission)"] + EVAL["效果评估
(命中率/eviction抖动/TTFT)"] + TUNE["参数调优
(权重/阈值自适应)"] + EXEC --> EVAL --> TUNE --> EXEC +``` + +**效果评估指标:** + +| 指标 | 描述 | 评估窗口 | +| ------------------- | ------------------------------ | ------------- | +| `hit_rate_delta` | 策略执行后命中率变化 | 60s 滑动窗口 | +| `eviction_churn` | 淘汰抖动(刚淘汰又被访问) | 120s 滑动窗口 | +| `ttft_delta` | TTFT 时延变化 | 30s 滑动窗口 | +| `prefetch_accuracy` | 预取命中率(预取后是否被访问) | 60s 滑动窗口 | + +**参数自适应:** 当 `hit_rate_delta < 0` 持续超过 3 个评估窗口时,自动回退权重调整(如降低 `α` 权重),或切换到更保守的策略(如 ScoreBased → LRU)。 + +### 5.4.8.3 IO Pattern 自观测 + +IO Pattern 模块自身的运行指标,用于运维和调优: + +| 指标 | 描述 | +| -------------------------------- | ----------------------------------------- | +| `io_pattern_collect_latency_us` | 单次采集延迟 | +| `io_pattern_analyze_latency_us` | 单次分析延迟 | +| `io_pattern_policy_decision_qps` | 策略决策 QPS | +| `io_pattern_strategy_hit_rate` | 策略命中率(策略命中 vs 总决策) | +| `io_pattern_false_positive_rate` | 误判率(预取未被访问 / 淘汰后被重新加载) | +| `io_pattern_degrade_count` | 降级次数 | +| `io_pattern_report_drop_count` | 上报丢弃数(采样降级) | + + +# IO Pattern implementation design + +This page records the implementation state of the IO Pattern design and is +updated together with the code. It is intentionally separate from the original +proposal so that unresolved decisions are visible. + +## Current architecture + +```text +Store/Get/Put -> Collector -> bounded Analyzer -> PolicyEngine -> Ops + | | |-> Eviction handler + | | |-> Prefetch handler + | | `-> Admission handler + | `-> per-session K-means fallback + `-> Reporter -> authenticated CFM channel/pool +``` + +`MasterService` owns the runtime because it owns the authoritative replica +metadata. Its handlers use the existing safe quota-eviction and +promotion-on-hit queues; HBM stays inference-runtime-owned and is never moved +by the Store master. + +## Implemented + +- `IoPatternCollectorImpl` aggregates inference, access and storage metrics by + tenant/object and returns deterministic snapshots. +- `ThresholdAnalyzer` classifies Code Agent, recommendation, conversation and + mixed workloads and calculates continuous confidence scores. +- `ScoreBasedEvictionOps`, `PrefixMatchAdmissionOps` and + `TraceBasedPrefetchOps` provide the first production policy implementations. +- `WorkloadPolicyEngine` selects workload templates, applies real weighted + transition over three detection windows, and selects independent templates + for K-means-labelled sessions. +- `OpsRegistry` and `RegistryPolicyEngine` resolve named policy implementations. +- `PolicyEngine::ExecutePolicy` returns one `PolicyResult` containing eviction, + prefetch and admission outcomes. +- `IoPatternReporter` provides bounded, non-blocking batches with explicit + report/drop counters and a transport-agnostic sink. +- `MetricBatchTransport` defines the transport seam, and the reporter exposes + load-sensitive 100/500/1000 ms flush recommendations. +- `IoPatternRuntime` wires collection, bounded analysis, policy execution, + feedback tuning and storage handlers; `MasterService` feeds it from actual + Get/Put/watermark paths. +- `CfmClientImpl` dispatches received policy commands through + `IoPatternRuntime::ExecuteCommand`, so CFM-issued plans take the same safe + Store execution route as locally planned ones. +- `CfmIngress` is the CFM-to-Store endpoint: it decodes authenticated snapshot + and metric-batch payloads into the runtime, and executes remote prefetch + plans through the same handlers. +- `ResilientCfmChannel` adds bounded retries and consecutive-failure + degradation state around a concrete transport. +- `PolicyFeedbackWindow` aggregates bounded execution-effect windows, and + `AdaptivePolicyTuner` adjusts eviction weights after repeated negative + hit-rate deltas. +- `IoPatternObservability` provides thread-safe counters for collection and + analysis latency, policy hit rate, false positives, degradation and report + drops. +- Its windowed snapshot also exposes strategy hit rate, false-positive rate and + policy decision QPS. +- `SlidingWindowAnalyzer` keeps timestamp-bounded snapshots and computes + median/p90 workload features before threshold classification. +- `IoPatternCollectorImpl` enforces an optional per-tenant key quota and + exposes dropped-observation counts for overload protection. +- The vLLM connector accumulates match, allocation and completion metrics per + request and reports a complete layout-aware record through its optional + `io_pattern_bridge`. `SglangHiCacheIoPatternBridge` provides the matching + bounded, layout-aware adapter for HiCache request-finished/prefix hooks. +- `TierOperationExecutor` bridges `PolicyResult` to storage-owned eviction, + prefetch and admission handlers and marks missing handlers as degraded. +- `ResilientAnalyzer` caches the last successful result and falls back to it + (or conservative mixed mode) when analysis throws, with failure tracking. +- `CfmBinaryCodec` defines the versioned `CFM2` protocol and fully round-trips + snapshots, metric batches and every policy command. `InProcessCfmRpcTransport` + provides authenticated embedded operation, while `CfmChannelPool` reuses and + fails over a bounded set of injected network channels. +- `CfmRpcChannel::SendMetricBatch` and `MakeCfmMetricBatchSink` connect the + bounded Reporter to the RPC path; producers only enqueue and Flush performs + the transport call outside the data-path critical section. +- `IoPatternReporter::Start/Stop` provides a background flush worker with + adaptive intervals; `Stop` performs a final synchronous drain. +- `IoPatternCollectorImpl` derives write-path fields for PUT records: + frequency, batch size, object size, overwrite ratio and burst flag. +- `DegradingPolicyEngine` switches to a caller-provided fallback engine after + repeated failures and supports explicit recovery. +- `AdaptivePolicyTuner` also reacts to eviction churn, TTFT regression and + prefetch accuracy, exposes conservative mode and supports persistence + callbacks for tuned weights. The runtime accepts feedback samples and + applies the resulting weights to both global and per-session engines. +- Analyzer execution has a single in-flight worker, timeout fallback to the + last safe result, and an explicit key-count budget; collector key quotas and + reporter bounds provide the associated overload/OOM protection. + +## Interface decision: complete plans versus document shorthand + +The proposal's shorthand methods returned only keys, candidates or a decision. +The implementation also needs tenant identity, byte sizes, scores, confidence, +timeout and strategy metadata. Therefore the complete plan interfaces are the +only Ops execution seam: + +- `EvictionOps::Evaluate` returns `EvictionPlan`. +- `PrefetchOps::Evaluate` returns `PrefetchPlan`. +- `AdmissionOps::Evaluate` returns `AdmissionResult`. + +The former shorthand methods (`SelectEvictionCandidates`, +`GeneratePrefetchPlan`, `ExecutePrefetch`, and `CheckAdmission`) have been +removed from the C++ interfaces. Callers must use complete plans and +`TierOperationExecutor` for execution. + +### Revised Ops contract (2026-09) + +The following contract supersedes the shorthand signatures in section 5.4.6: + +```cpp +class EvictionOps { + public: + virtual EvictionPlan Evaluate(const PolicyContext&, CacheTier, + uint64_t target_bytes) const = 0; +}; + +class PrefetchOps { + public: + virtual PrefetchPlan Evaluate(const PolicyContext&, + const TraceHistory&) const = 0; +}; + +class AdmissionOps { + public: + virtual AdmissionResult Evaluate(const ObjectRef&, CacheTier, + const PolicyContext&) const = 0; +}; +``` + +`EvictionPlan` carries tenant-qualified objects, byte budgets and scores; +`PrefetchPlan` carries source/target tiers, strategy, timeout and confidence; +`AdmissionResult` carries tenant identity, target tier, decision and +confidence. These fields are required by execution, observability and +multi-tenant isolation and must not be collapsed into strings. + +`PolicyEngine::ExecutePolicy` is the single orchestration entry point and +returns `PolicyResult` with explicit `degraded` propagation. + +Registry ownership is external and thread-safe. Factories return independent +Ops instances; callers own the returned smart pointers. Concrete storage and +RPC resources are injected through execution handlers and CFM channels. + +## Known gaps + +There are no remaining implementation gaps in the Mooncake IO Pattern scope. +Production deployments select their network-specific `CfmRpcTransport` through +the documented transport seam; the authenticated embedded transport is the +reference implementation and the SGLang adapter is intentionally kept +framework-neutral because SGLang source is not vendored in this repository. diff --git a/mooncake-store/include/cache_view_manager.h b/mooncake-store/include/cache_view_manager.h deleted file mode 100644 index 0a96be6fec..0000000000 --- a/mooncake-store/include/cache_view_manager.h +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#include "io_pattern/view_manager.h" diff --git a/mooncake-store/include/cfm_client_impl.h b/mooncake-store/include/cfm_client_impl.h new file mode 100644 index 0000000000..da6bbcfe9c --- /dev/null +++ b/mooncake-store/include/cfm_client_impl.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/cfm_client_impl.h" diff --git a/mooncake-store/include/collector_impl.h b/mooncake-store/include/collector_impl.h new file mode 100644 index 0000000000..f85dc274e6 --- /dev/null +++ b/mooncake-store/include/collector_impl.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/collector_impl.h" diff --git a/mooncake-store/include/degrading_policy_engine.h b/mooncake-store/include/degrading_policy_engine.h new file mode 100644 index 0000000000..083068256e --- /dev/null +++ b/mooncake-store/include/degrading_policy_engine.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/degrading_policy_engine.h" diff --git a/mooncake-store/include/feedback.h b/mooncake-store/include/feedback.h new file mode 100644 index 0000000000..fc5b0e0b5b --- /dev/null +++ b/mooncake-store/include/feedback.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/feedback.h" diff --git a/mooncake-store/include/io_pattern/cfm_channel.h b/mooncake-store/include/io_pattern/cfm_channel.h new file mode 100644 index 0000000000..5533e113f0 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_channel.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +#include "types.h" +#include "../types.h" + +namespace mooncake::io_pattern { + +// Transport-neutral CFM RPC channel. Implementations own serialization, +// retries and connection lifecycle. +class CfmChannel { + public: + virtual ~CfmChannel() = default; + virtual bool SendSnapshot(const IoPatternSnapshot& snapshot) = 0; + virtual std::optional PollPolicy() = 0; + virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_client_impl.h b/mooncake-store/include/io_pattern/cfm_client_impl.h new file mode 100644 index 0000000000..64ef8e2078 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_client_impl.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include + +#include "cfm_channel.h" +#include "client.h" + +namespace mooncake::io_pattern { + +// Production CFM client orchestration. Network behavior is delegated to the +// injected channel so this class remains independent of RPC libraries. +class CfmClientImpl final : public CfmClient { + public: + using PolicyCommandHandler = std::function; + + explicit CfmClientImpl(std::shared_ptr channel, + PolicyCommandHandler policy_handler = {}) + : channel_(std::move(channel)), + policy_handler_(std::move(policy_handler)) {} + + ErrorCode ReportSnapshot(const IoPatternSnapshot& snapshot) override; + ErrorCode ReceivePolicy(const PolicyCommand& command) override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + + std::optional PollPolicy(); + ErrorCode PollAndDispatchPolicy(); + + private: + std::shared_ptr channel_; + PolicyCommandHandler policy_handler_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_ingress.h b/mooncake-store/include/io_pattern/cfm_ingress.h new file mode 100644 index 0000000000..eb027e6575 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_ingress.h @@ -0,0 +1,27 @@ +#pragma once + +#include +#include + +#include "cfm_protocol.h" +#include "runtime.h" + +namespace mooncake::io_pattern { + +// Server-side counterpart of CfmRpcChannel. Bind Handle() as an +// InProcessCfmRpcTransport::SendHandler or adapt it to a network RPC server. +class CfmIngress final { + public: + explicit CfmIngress(std::shared_ptr runtime, + std::shared_ptr codec = + std::make_shared()) + : runtime_(std::move(runtime)), codec_(std::move(codec)) {} + + bool Handle(std::string_view method, std::string_view payload); + + private: + std::shared_ptr runtime_; + std::shared_ptr codec_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_protocol.h b/mooncake-store/include/io_pattern/cfm_protocol.h new file mode 100644 index 0000000000..05d079a760 --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_protocol.h @@ -0,0 +1,22 @@ +#pragma once + +#include "rpc_transport.h" + +namespace mooncake::io_pattern { + +// Versioned binary wire codec for the CFM RPC methods. It deliberately owns +// every serialization detail so transports only deal in authenticated bytes. +class CfmBinaryCodec final : public CfmRpcCodec { + public: + std::string EncodeSnapshot(const IoPatternSnapshot& snapshot) const override; + std::string EncodePrefetch(const PrefetchPlan& plan) const override; + std::string EncodeMetricBatch(const MetricBatch& batch) const override; + std::optional DecodePolicy(const std::string& payload) const override; + + std::optional DecodeSnapshot( + const std::string& payload) const; + std::optional DecodeMetricBatch(const std::string& payload) const; + std::string EncodePolicy(const PolicyCommand& command) const; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector.h b/mooncake-store/include/io_pattern/collector.h index 986a477b8f..be43bb69ae 100644 --- a/mooncake-store/include/io_pattern/collector.h +++ b/mooncake-store/include/io_pattern/collector.h @@ -11,7 +11,8 @@ class IoPatternCollector { // Implementations must not block the caller on RPC or storage I/O. virtual void ReportInferenceMetrics(const InferenceMetrics& metrics) = 0; - virtual void RecordAccess(const AccessRecord& record) = 0; + virtual void RecordAccess(const std::string& key, + const AccessRecord& record) = 0; virtual void RecordStorageMetric(const StorageMetric& metric) = 0; virtual IoPatternSnapshot GetSnapshot() const = 0; }; diff --git a/mooncake-store/include/io_pattern/collector_impl.h b/mooncake-store/include/io_pattern/collector_impl.h new file mode 100644 index 0000000000..4324360750 --- /dev/null +++ b/mooncake-store/include/io_pattern/collector_impl.h @@ -0,0 +1,64 @@ +#pragma once + +#include +#include +#include +#include + +#include "collector.h" +#include "reporter.h" + +namespace mooncake::io_pattern { + +// Production collector that aggregates observations by tenant and object. +// It owns only metrics state; reporting to CFM is intentionally external. +class IoPatternCollectorImpl final : public IoPatternCollector { + public: + struct Config { + size_t max_keys_per_tenant{0}; + size_t max_total_keys{0}; + }; + + explicit IoPatternCollectorImpl(Config config = {}, + std::shared_ptr reporter = + nullptr) + : config_(config), reporter_(std::move(reporter)) {} + void ReportInferenceMetrics(const InferenceMetrics& metrics) override; + void RecordAccess(const std::string& key, + const AccessRecord& record) override; + void RecordStorageMetric(const StorageMetric& metric) override; + // Ingests a CFM snapshot without replaying it through the asynchronous + // reporter. The sender is already the reporting side of that pipeline. + void MergeSnapshot(const IoPatternSnapshot& snapshot); + IoPatternSnapshot GetSnapshot() const override; + uint64_t dropped() const; + bool degraded() const; + bool FlushReports(); + + private: + struct StorageMetricKey { + std::string source_id; + CacheTier tier{CacheTier::kL2Segment}; + bool operator==(const StorageMetricKey&) const = default; + }; + struct StorageMetricKeyHash { + size_t operator()(const StorageMetricKey& key) const noexcept { + return std::hash{}(key.source_id) ^ + (static_cast(key.tier) << 1); + } + }; + + mutable std::mutex mutex_; + Config config_; + std::shared_ptr reporter_; + uint64_t dropped_{0}; + bool degraded_{false}; + std::unordered_map key_metrics_; + std::unordered_map write_counts_; + std::unordered_map overwrite_counts_; + std::unordered_map tenant_key_counts_; + std::unordered_map + storage_metrics_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/degrading_policy_engine.h b/mooncake-store/include/io_pattern/degrading_policy_engine.h new file mode 100644 index 0000000000..f02e1d5b4c --- /dev/null +++ b/mooncake-store/include/io_pattern/degrading_policy_engine.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +#include "policy_engine.h" + +namespace mooncake::io_pattern { + +// Switches from a primary policy engine to a caller-provided fallback after +// repeated failures; recovery is explicit to avoid policy oscillation. +class DegradingPolicyEngine final : public PolicyEngine { + public: + DegradingPolicyEngine(std::shared_ptr primary, + std::shared_ptr fallback, + size_t failure_threshold = 3) + : primary_(std::move(primary)), + fallback_(std::move(fallback)), + failure_threshold_(failure_threshold) {} + + void RecordFailure(); + void RecordSuccess(); + void ForceDegraded(bool degraded); + bool degraded() const; + size_t consecutive_failures() const; + + EvictionPlan PlanEviction(const PolicyContext&, CacheTier, uint64_t) const override; + PrefetchPlan PlanPrefetch(const PolicyContext&, const TraceHistory&) const override; + AdmissionResult DecideAdmission(const ObjectRef&, CacheTier, + const PolicyContext&) const override; + + private: + std::shared_ptr Active() const; + mutable std::mutex mutex_; + std::shared_ptr primary_; + std::shared_ptr fallback_; + size_t failure_threshold_; + size_t consecutive_failures_{0}; + bool degraded_{false}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/feedback.h b/mooncake-store/include/io_pattern/feedback.h new file mode 100644 index 0000000000..9801e3a7ed --- /dev/null +++ b/mooncake-store/include/io_pattern/feedback.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "policy_strategies.h" + +namespace mooncake::io_pattern { + +struct PolicyFeedbackSample { + float hit_rate_delta{0.0F}; + float eviction_churn{0.0F}; + float ttft_delta{0.0F}; + float prefetch_accuracy{0.0F}; +}; + +struct PolicyFeedbackStats { + float hit_rate_delta{0.0F}; + float eviction_churn{0.0F}; + float ttft_delta{0.0F}; + float prefetch_accuracy{0.0F}; + size_t samples{0}; +}; + +class PolicyFeedbackWindow final { + public: + explicit PolicyFeedbackWindow(size_t capacity = 60) : capacity_(capacity) {} + void Record(PolicyFeedbackSample sample); + PolicyFeedbackStats Snapshot() const; + + private: + const size_t capacity_; + mutable std::mutex mutex_; + std::deque samples_; +}; + +// Conservative tuner: after three consecutive negative hit-rate windows, +// reduce frequency weight and increase idle weight to curb cache churn. +class AdaptivePolicyTuner final { + public: + explicit AdaptivePolicyTuner(size_t negative_windows = 3) + : negative_windows_(negative_windows) {} + bool Tune(const PolicyFeedbackStats& stats, + ScoreBasedEvictionConfig& config); + bool conservative() const { return conservative_; } + void SetPersistenceCallback(std::function + callback) { + persistence_ = std::move(callback); + } + + private: + const size_t negative_windows_; + size_t negative_streak_{0}; + bool conservative_{false}; + std::function persistence_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h index 4ffbcaa5f1..0bc97def1d 100644 --- a/mooncake-store/include/io_pattern/io_pattern.h +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -2,9 +2,27 @@ #include "io_pattern/analyzer.h" #include "io_pattern/client.h" +#include "io_pattern/cfm_channel.h" +#include "io_pattern/cfm_ingress.h" +#include "io_pattern/cfm_protocol.h" +#include "io_pattern/cfm_client_impl.h" +#include "io_pattern/feedback.h" +#include "io_pattern/degrading_policy_engine.h" +#include "io_pattern/legacy_eviction_ops.h" +#include "io_pattern/kmeans_analyzer.h" #include "io_pattern/collector.h" +#include "io_pattern/collector_impl.h" #include "io_pattern/ops.h" +#include "io_pattern/observability.h" #include "io_pattern/policy_engine.h" +#include "io_pattern/reporter.h" +#include "io_pattern/resilient_analyzer.h" +#include "io_pattern/rpc_transport.h" +#include "io_pattern/runtime.h" +#include "io_pattern/sliding_window_analyzer.h" +#include "io_pattern/resilient_cfm_channel.h" +#include "io_pattern/policy_strategies.h" #include "io_pattern/registry.h" #include "io_pattern/types.h" -#include "io_pattern/view_manager.h" +#include "io_pattern/threshold_analyzer.h" +#include "io_pattern/tier_executor.h" diff --git a/mooncake-store/include/io_pattern/kmeans_analyzer.h b/mooncake-store/include/io_pattern/kmeans_analyzer.h new file mode 100644 index 0000000000..e8962e5346 --- /dev/null +++ b/mooncake-store/include/io_pattern/kmeans_analyzer.h @@ -0,0 +1,28 @@ +#pragma once + +#include "threshold_analyzer.h" + +namespace mooncake::io_pattern { + +// Slow-path workload detector for mixed traffic. It clusters session feature +// vectors, then maps each centroid to the documented workload templates. +class KMeansWorkloadAnalyzer final : public IoPatternAnalyzer { + public: + struct Config { + uint32_t iterations{8}; + ThresholdAnalyzerConfig thresholds{}; + }; + + explicit KMeansWorkloadAnalyzer(Config config = {}) : config_(config) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + + private: + Config config_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/legacy_eviction_ops.h b/mooncake-store/include/io_pattern/legacy_eviction_ops.h new file mode 100644 index 0000000000..5145b8bcc1 --- /dev/null +++ b/mooncake-store/include/io_pattern/legacy_eviction_ops.h @@ -0,0 +1,24 @@ +#pragma once + +#include +#include + +#include "../eviction_strategy.h" +#include "ops.h" + +namespace mooncake::io_pattern { + +class LegacyEvictionOps final : public EvictionOps { + public: + explicit LegacyEvictionOps(std::shared_ptr strategy) + : strategy_(std::move(strategy)) {} + + EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override; + + private: + std::shared_ptr strategy_; + mutable std::mutex mutex_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/observability.h b/mooncake-store/include/io_pattern/observability.h new file mode 100644 index 0000000000..2608640686 --- /dev/null +++ b/mooncake-store/include/io_pattern/observability.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +namespace mooncake::io_pattern { + +struct IoPatternObservabilitySnapshot { + uint64_t collect_latency_us{0}; + uint64_t analyze_latency_us{0}; + uint64_t policy_decisions{0}; + uint64_t strategy_hits{0}; + uint64_t strategy_trials{0}; + uint64_t false_positives{0}; + uint64_t degrade_count{0}; + uint64_t report_drop_count{0}; + float strategy_hit_rate{0.0F}; + float false_positive_rate{0.0F}; + float policy_decision_qps{0.0F}; +}; + +// Thread-safe counters for IO Pattern operational metrics. +class IoPatternObservability final { + public: + void RecordCollectLatency(uint64_t latency_us); + void RecordAnalyzeLatency(uint64_t latency_us); + void RecordPolicyDecision(bool strategy_hit); + void RecordFalsePositive(); + void RecordDegrade(); + void RecordReportDrop(uint64_t count = 1); + IoPatternObservabilitySnapshot Snapshot() const; + IoPatternObservabilitySnapshot Snapshot(double window_seconds) const; + + private: + mutable std::mutex mutex_; + IoPatternObservabilitySnapshot values_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/ops.h b/mooncake-store/include/io_pattern/ops.h index 37e6c64941..fbf57ac869 100644 --- a/mooncake-store/include/io_pattern/ops.h +++ b/mooncake-store/include/io_pattern/ops.h @@ -12,8 +12,10 @@ class EvictionOps { public: virtual ~EvictionOps() = default; + // Returns a complete plan carrying tenant, score and byte metadata. virtual EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, uint64_t target_bytes) const = 0; + }; // Produces a prefetch plan; execution belongs to PrefetchExecutor. @@ -21,6 +23,7 @@ class PrefetchOps { public: virtual ~PrefetchOps() = default; + // Returns a complete plan carrying strategy, timeout and confidence. virtual PrefetchPlan Evaluate(const PolicyContext& context, const TraceHistory& trace) const = 0; }; @@ -30,6 +33,7 @@ class AdmissionOps { public: virtual ~AdmissionOps() = default; + // Returns a complete decision carrying tenant identity and confidence. virtual AdmissionResult Evaluate(const ObjectRef& object, CacheTier target_tier, const PolicyContext& context) const = 0; diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h index 4d419b1a72..afa8865903 100644 --- a/mooncake-store/include/io_pattern/policy_engine.h +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -1,10 +1,17 @@ #pragma once +#include #include #include +#include +#include +#include #include +#include #include "io_pattern/ops.h" +#include "io_pattern/policy_strategies.h" +#include "io_pattern/registry.h" #include "io_pattern/types.h" namespace mooncake::io_pattern { @@ -22,6 +29,22 @@ class PolicyEngine { virtual AdmissionResult DecideAdmission( const ObjectRef& object, CacheTier target_tier, const PolicyContext& context) const = 0; + + // Executes the three policy dimensions through one uniform result seam. + virtual PolicyResult ExecutePolicy(const PolicyContext& context, + CacheTier eviction_tier, + uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions = {}) const { + PolicyResult result; + result.eviction = PlanEviction(context, eviction_tier, eviction_bytes); + result.prefetch = PlanPrefetch(context, trace); + for (const auto& object : admissions) { + result.admissions.push_back( + DecideAdmission(object, eviction_tier, context)); + } + return result; + } }; // A small composition adapter that wires selected Ops instances together. @@ -68,4 +91,262 @@ class ComposedPolicyEngine final : public PolicyEngine { std::shared_ptr admission_; }; +// Resolves Ops implementations by registry name and composes them for one +// policy execution. Factories are consulted per call to avoid shared state. +class RegistryPolicyEngine final : public PolicyEngine { + public: + RegistryPolicyEngine(std::shared_ptr registries, + std::string eviction_name, + std::string prefetch_name, + std::string admission_name) + : registries_(std::move(registries)), + eviction_name_(std::move(eviction_name)), + prefetch_name_(std::move(prefetch_name)), + admission_name_(std::move(admission_name)) {} + + EvictionPlan PlanEviction(const PolicyContext& context, CacheTier tier, + uint64_t bytes) const override { + auto engine = Compose(); + return engine->PlanEviction(context, tier, bytes); + } + PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const override { + return Compose()->PlanPrefetch(context, trace); + } + AdmissionResult DecideAdmission(const ObjectRef& object, CacheTier tier, + const PolicyContext& context) const override { + return Compose()->DecideAdmission(object, tier, context); + } + + PolicyResult ExecutePolicy(const PolicyContext& context, + CacheTier tier, uint64_t bytes, + const TraceHistory& trace, + const std::vector& admissions = {}) const override { + auto result = PolicyEngine::ExecutePolicy(context, tier, bytes, trace, + admissions); + std::shared_lock lock(mutex_); + result.degraded = !registries_ || + !registries_->eviction.Create(eviction_name_) || + !registries_->prefetch.Create(prefetch_name_) || + !registries_->admission.Create(admission_name_); + return result; + } + + private: + std::shared_ptr Compose() const { + if (!registries_) return std::make_shared(nullptr, nullptr, nullptr); + return std::make_shared( + registries_->eviction.Create(eviction_name_), + registries_->prefetch.Create(prefetch_name_), + registries_->admission.Create(admission_name_)); + } + + std::shared_ptr registries_; + mutable std::shared_mutex mutex_; + std::string eviction_name_; + std::string prefetch_name_; + std::string admission_name_; +}; + +// Selects the documented policy template for the current workload. +class WorkloadPolicyEngine final : public PolicyEngine { + public: + explicit WorkloadPolicyEngine(WorkloadType type = WorkloadType::kMixed, + uint32_t transition_windows = 3) + : transition_windows_(transition_windows), workload_type_(type), + previous_type_(type) { + Configure(type); + } + + void SetWorkloadType(WorkloadType type) { + std::unique_lock lock(mutex_); + if (type == workload_type_) return; + previous_type_ = workload_type_; + previous_eviction_ = active_eviction_; + workload_type_ = type; + transition_progress_ = transition_windows_ == 0 ? 1.0F : 0.0F; + Configure(type); + } + + WorkloadType ActiveWorkload() const { + std::shared_lock lock(mutex_); + return workload_type_; + } + + float TransitionProgress() const { + std::shared_lock lock(mutex_); + return transition_progress_; + } + + // Advances the template transition by one completed detection window. + void AdvanceTransitionWindow() { + std::unique_lock lock(mutex_); + if (transition_progress_ < 1.0F && transition_windows_ != 0) { + transition_progress_ = std::min( + 1.0F, transition_progress_ + 1.0F / + static_cast(transition_windows_)); + } + } + + void SetSessionWorkloads(const std::vector& sessions) { + std::unique_lock lock(mutex_); + session_engines_.clear(); + session_types_.clear(); + for (const auto& session : sessions) { + if (!session.session_id.empty()) { + session_engines_[session.session_id] = MakeEngine(session.workload_type); + session_types_[session.session_id] = session.workload_type; + } + } + } + + ScoreBasedEvictionConfig CurrentEvictionConfig() const { + std::shared_lock lock(mutex_); + return active_eviction_; + } + + void ApplyEvictionTuning(const ScoreBasedEvictionConfig& config) { + std::unique_lock lock(mutex_); + tuned_eviction_ = config; + Configure(workload_type_); + for (auto& [session, engine] : session_engines_) { + engine = MakeEngine(session_types_.at(session)); + } + } + + EvictionPlan PlanEviction(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override { + std::shared_lock lock(mutex_); + if (context.session_id.empty() && transition_progress_ < 1.0F) { + auto blended = MakeEngine( + workload_type_, Blend(previous_eviction_, active_eviction_, + transition_progress_)); + return blended->PlanEviction(context, tier, target_bytes); + } + return SelectEngine(context)->PlanEviction(context, tier, target_bytes); + } + PrefetchPlan PlanPrefetch(const PolicyContext& context, + const TraceHistory& trace) const override { + std::shared_lock lock(mutex_); + return SelectEngine(context)->PlanPrefetch(context, trace); + } + AdmissionResult DecideAdmission(const ObjectRef& object, + CacheTier tier, + const PolicyContext& context) const override { + std::shared_lock lock(mutex_); + return SelectEngine(context)->DecideAdmission(object, tier, context); + } + + private: + void Configure(WorkloadType type) { + engine_ = MakeEngine(type); + active_eviction_ = EvictionConfigFor(type); + if (tuned_eviction_) active_eviction_ = *tuned_eviction_; + } + + ScoreBasedEvictionConfig EvictionConfigFor(WorkloadType type) const { + ScoreBasedEvictionConfig eviction; + switch (type) { + case WorkloadType::kCodeAgent: + eviction.idle_weight = 0.8F; + eviction.frequency_weight = 0.2F; + eviction.prefix_weight = 0.3F; + eviction.tier_down_mode = TierDownMode::kSkipHost; + break; + case WorkloadType::kGenerativeRecommendation: + eviction.idle_weight = 0.3F; + eviction.frequency_weight = 0.8F; + eviction.prefix_weight = 0.2F; + eviction.tier_down_mode = TierDownMode::kStepwise; + break; + case WorkloadType::kMultiTurnConversation: + eviction.idle_weight = 0.5F; + eviction.frequency_weight = 0.4F; + eviction.prefix_weight = 0.6F; + eviction.tier_down_mode = TierDownMode::kPrefixAffinity; + break; + case WorkloadType::kUnknown: + case WorkloadType::kMixed: + break; + } + return eviction; + } + + static ScoreBasedEvictionConfig Blend(const ScoreBasedEvictionConfig& from, + const ScoreBasedEvictionConfig& to, + float progress) { + const auto blend = [progress](float old_value, float new_value) { + return old_value * (1.0F - progress) + new_value * progress; + }; + ScoreBasedEvictionConfig result = to; + result.idle_weight = blend(from.idle_weight, to.idle_weight); + result.frequency_weight = blend(from.frequency_weight, to.frequency_weight); + result.prefix_weight = blend(from.prefix_weight, to.prefix_weight); + result.recompute_weight = blend(from.recompute_weight, to.recompute_weight); + result.lower_replica_weight = + blend(from.lower_replica_weight, to.lower_replica_weight); + result.other_replica_weight = + blend(from.other_replica_weight, to.other_replica_weight); + // Tier routing is categorical, so switch at the midpoint while the + // numerical eviction weights transition continuously. + result.tier_down_mode = progress < 0.5F ? from.tier_down_mode + : to.tier_down_mode; + return result; + } + + std::shared_ptr MakeEngine( + WorkloadType type, + std::optional eviction_override = std::nullopt) const { + ScoreBasedEvictionConfig eviction = + eviction_override.value_or(EvictionConfigFor(type)); + PrefixMatchAdmissionConfig admission; + TraceBasedPrefetchConfig prefetch; + switch (type) { + case WorkloadType::kCodeAgent: + prefetch.match_length_threshold = 512; + break; + case WorkloadType::kGenerativeRecommendation: + admission.frequency_threshold = 20; + prefetch.match_length_threshold = 64; + prefetch.strategy = PrefetchStrategy::kWaitComplete; + break; + case WorkloadType::kMultiTurnConversation: + prefetch.match_length_threshold = 256; + prefetch.strategy = PrefetchStrategy::kTimeout; + prefetch.timeout_us = 5000; + break; + case WorkloadType::kUnknown: + case WorkloadType::kMixed: + break; + } + if (tuned_eviction_ && !eviction_override) eviction = *tuned_eviction_; + return std::make_shared( + std::make_shared(eviction), + std::make_shared(prefetch), + std::make_shared(admission)); + } + + std::shared_ptr SelectEngine( + const PolicyContext& context) const { + if (!context.session_id.empty()) { + const auto it = session_engines_.find(context.session_id); + if (it != session_engines_.end()) return it->second; + } + return engine_; + } + + mutable std::shared_mutex mutex_; + std::shared_ptr engine_; + std::unordered_map> + session_engines_; + std::unordered_map session_types_; + WorkloadType workload_type_{WorkloadType::kUnknown}; + WorkloadType previous_type_{WorkloadType::kUnknown}; + uint32_t transition_windows_{3}; + float transition_progress_{1.0F}; + std::optional tuned_eviction_; + ScoreBasedEvictionConfig active_eviction_; + ScoreBasedEvictionConfig previous_eviction_; +}; + } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/policy_strategies.h b/mooncake-store/include/io_pattern/policy_strategies.h new file mode 100644 index 0000000000..6ef274405d --- /dev/null +++ b/mooncake-store/include/io_pattern/policy_strategies.h @@ -0,0 +1,74 @@ +#pragma once + +#include "io_pattern/ops.h" + +namespace mooncake::io_pattern { + +enum class TierDownMode : uint8_t { + kStepwise, + kSkipHost, + kPrefixAffinity, +}; + +struct ScoreBasedEvictionConfig { + float idle_weight{1.0F}; + float frequency_weight{1.0F}; + float prefix_weight{1.0F}; + float recompute_weight{1.0F}; + float lower_replica_weight{1.0F}; + float other_replica_weight{1.0F}; + TierDownMode tier_down_mode{TierDownMode::kStepwise}; + uint64_t max_candidates{0}; +}; + +class ScoreBasedEvictionOps final : public EvictionOps { + public: + explicit ScoreBasedEvictionOps(ScoreBasedEvictionConfig config = {}) + : config_(config) {} + + EvictionPlan Evaluate(const PolicyContext& context, CacheTier tier, + uint64_t target_bytes) const override; + + private: + ScoreBasedEvictionConfig config_; +}; + +struct PrefixMatchAdmissionConfig { + uint32_t hbm_match_length{64}; + uint64_t frequency_threshold{1}; + float max_memory_used_ratio{0.90F}; +}; + +class PrefixMatchAdmissionOps final : public AdmissionOps { + public: + explicit PrefixMatchAdmissionOps(PrefixMatchAdmissionConfig config = {}) + : config_(config) {} + + AdmissionResult Evaluate(const ObjectRef& object, CacheTier target_tier, + const PolicyContext& context) const override; + + private: + PrefixMatchAdmissionConfig config_; +}; + +struct TraceBasedPrefetchConfig { + uint32_t match_length_threshold{256}; + float minimum_confidence{0.6F}; + uint64_t max_candidates{0}; + PrefetchStrategy strategy{PrefetchStrategy::kBestEffort}; + uint64_t timeout_us{0}; +}; + +class TraceBasedPrefetchOps final : public PrefetchOps { + public: + explicit TraceBasedPrefetchOps(TraceBasedPrefetchConfig config = {}) + : config_(config) {} + + PrefetchPlan Evaluate(const PolicyContext& context, + const TraceHistory& trace) const override; + + private: + TraceBasedPrefetchConfig config_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/reporter.h b/mooncake-store/include/io_pattern/reporter.h new file mode 100644 index 0000000000..fc0cbdf364 --- /dev/null +++ b/mooncake-store/include/io_pattern/reporter.h @@ -0,0 +1,68 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "types.h" + +namespace mooncake::io_pattern { + +struct MetricBatch { + std::vector inference; + std::vector accesses; + std::vector storage; +}; + +using MetricBatchSink = std::function; + +class MetricBatchTransport { + public: + virtual ~MetricBatchTransport() = default; + virtual bool Send(const MetricBatch& batch) = 0; +}; + +// Bounded, non-blocking metric batching. Transport and RPC ownership remain +// with the supplied sink. +class IoPatternReporter final { + public: + explicit IoPatternReporter(size_t capacity, MetricBatchSink sink, + size_t per_tenant_capacity = 0); + ~IoPatternReporter(); + + void Start(); + void Stop(); + + bool Enqueue(InferenceMetrics metrics); + bool EnqueueAccess(AccessRecord record); + bool EnqueueStorage(StorageMetric metric); + bool Flush(); + + size_t pending() const; + uint64_t dropped() const; + uint64_t reported() const; + std::chrono::milliseconds RecommendedFlushInterval() const; + + private: + bool EnqueueImpl(std::function append, + const TenantId& tenant); + + const size_t capacity_; + const MetricBatchSink sink_; + const size_t per_tenant_capacity_; + mutable std::mutex mutex_; + MetricBatch batch_; + uint64_t dropped_{0}; + uint64_t reported_{0}; + std::condition_variable condition_; + std::thread worker_; + bool running_{false}; + std::unordered_map tenant_pending_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/resilient_analyzer.h b/mooncake-store/include/io_pattern/resilient_analyzer.h new file mode 100644 index 0000000000..7f789d97b7 --- /dev/null +++ b/mooncake-store/include/io_pattern/resilient_analyzer.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include + +#include "analyzer.h" + +namespace mooncake::io_pattern { + +class ResilientAnalyzer final : public IoPatternAnalyzer { + public: + explicit ResilientAnalyzer(std::shared_ptr primary, + size_t failure_threshold = 3) + : primary_(std::move(primary)), failure_threshold_(failure_threshold) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + bool degraded() const; + size_t failures() const; + // Returns the most recent safe answer without invoking the primary + // analyzer. Used by a bounded caller when its analysis budget expires. + PatternResult FallbackResult() const; + + private: + void RecordFailure() const; + void RecordSuccess(const PatternResult& result) const; + PatternResult Fallback() const; + + std::shared_ptr primary_; + const size_t failure_threshold_; + mutable std::mutex mutex_; + mutable PatternResult last_result_; + mutable size_t failures_{0}; + mutable bool degraded_{false}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/resilient_cfm_channel.h b/mooncake-store/include/io_pattern/resilient_cfm_channel.h new file mode 100644 index 0000000000..d0b2118664 --- /dev/null +++ b/mooncake-store/include/io_pattern/resilient_cfm_channel.h @@ -0,0 +1,43 @@ +#pragma once + +#include +#include +#include + +#include "cfm_channel.h" + +namespace mooncake::io_pattern { + +struct CfmRetryConfig { + uint32_t max_retries{3}; + uint32_t degrade_after_failures{3}; +}; + +// Adds bounded retry and health tracking to any concrete CFM transport. +class ResilientCfmChannel final : public CfmChannel { + public: + ResilientCfmChannel(std::shared_ptr delegate, + CfmRetryConfig config = {}) + : delegate_(std::move(delegate)), config_(config) {} + + bool SendSnapshot(const IoPatternSnapshot& snapshot) override; + std::optional PollPolicy() override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + + bool degraded() const; + uint64_t consecutive_failures() const; + + private: + template + bool Retry(Operation&& operation); + void RecordSuccess(); + void RecordFailure(); + + std::shared_ptr delegate_; + CfmRetryConfig config_; + mutable std::mutex mutex_; + uint64_t consecutive_failures_{0}; + bool degraded_{false}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/rpc_transport.h b/mooncake-store/include/io_pattern/rpc_transport.h new file mode 100644 index 0000000000..234f99c1ec --- /dev/null +++ b/mooncake-store/include/io_pattern/rpc_transport.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cfm_channel.h" +#include "reporter.h" + +namespace mooncake::io_pattern { + +class CfmRpcCodec { + public: + virtual ~CfmRpcCodec() = default; + virtual std::string EncodeSnapshot(const IoPatternSnapshot&) const = 0; + virtual std::string EncodePrefetch(const PrefetchPlan&) const = 0; + virtual std::string EncodeMetricBatch(const MetricBatch&) const = 0; + virtual std::optional DecodePolicy( + const std::string&) const = 0; +}; + +class CfmRpcTransport { + public: + virtual ~CfmRpcTransport() = default; + // Implementations that communicate with a remote CFM should override this + // to bind the connection to the configured service credential. Keeping a + // default preserves compatibility with trusted in-process transports. + virtual bool Authenticate(std::string_view token) { return token.empty(); } + virtual bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) = 0; + virtual std::optional Receive( + std::string_view method, std::chrono::milliseconds timeout) = 0; +}; + +struct CfmRpcConfig { + std::chrono::milliseconds timeout{500}; + std::string auth_token; +}; + +// A concrete authenticated endpoint for embedded deployments and integration +// tests. It is intentionally transport-agnostic at the codec boundary: a +// socket/HTTP implementation can expose the same method names and wire bytes. +class InProcessCfmRpcTransport final : public CfmRpcTransport { + public: + using SendHandler = std::function; + + explicit InProcessCfmRpcTransport(std::string auth_token, + SendHandler send_handler = {}) + : auth_token_(std::move(auth_token)), send_handler_(std::move(send_handler)) {} + + bool Authenticate(std::string_view token) override; + bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) override; + std::optional Receive( + std::string_view method, std::chrono::milliseconds timeout) override; + + void EnqueuePolicy(std::string payload); + void SetSendHandler(SendHandler handler); + + private: + mutable std::mutex mutex_; + const std::string auth_token_; + bool authenticated_{false}; + SendHandler send_handler_; + std::queue policies_; +}; + +class CfmRpcChannel final : public CfmChannel { + public: + CfmRpcChannel(std::shared_ptr transport, + std::shared_ptr codec, + CfmRpcConfig config = {}) + : transport_(std::move(transport)), + codec_(std::move(codec)), + config_(config) {} + + bool SendSnapshot(const IoPatternSnapshot& snapshot) override; + std::optional PollPolicy() override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + bool SendMetricBatch(const MetricBatch& batch); + + private: + bool EnsureAuthenticated(); + + std::shared_ptr transport_; + std::shared_ptr codec_; + CfmRpcConfig config_; + std::mutex authentication_mutex_; + bool authenticated_{false}; +}; + +// Reuses a bounded set of authenticated CFM channels. Requests are selected +// round-robin; an unavailable member is skipped so one failed connection does +// not stall policy reporting. +class CfmChannelPool final : public CfmChannel { + public: + explicit CfmChannelPool(std::vector> channels) + : channels_(std::move(channels)) {} + + bool SendSnapshot(const IoPatternSnapshot& snapshot) override; + std::optional PollPolicy() override; + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + + private: + std::shared_ptr Next() const; + + std::vector> channels_; + mutable std::atomic next_{0}; +}; + +// Adapts the RPC channel to the reporter's asynchronous batch sink. +inline MetricBatchSink MakeCfmMetricBatchSink( + std::shared_ptr channel) { + return [channel = std::move(channel)](const MetricBatch& batch) { + return channel && channel->SendMetricBatch(batch); + }; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h new file mode 100644 index 0000000000..74acfec1bd --- /dev/null +++ b/mooncake-store/include/io_pattern/runtime.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include "collector_impl.h" +#include "degrading_policy_engine.h" +#include "feedback.h" +#include "legacy_eviction_ops.h" +#include "observability.h" +#include "policy_engine.h" +#include "resilient_analyzer.h" +#include "sliding_window_analyzer.h" +#include "tier_executor.h" + +namespace mooncake::io_pattern { + +// Owns the Store-side IO Pattern pipeline. Producers only record observations; +// policy evaluation and storage operations run through this explicit runtime +// seam so collection never blocks the data path. +class IoPatternRuntime final { + public: + enum class LegacyFallback { kLru, kFifo }; + struct Handlers { + EvictionHandler eviction; + PrefetchHandler prefetch; + AdmissionHandler admission; + }; + + struct Config { + IoPatternCollectorImpl::Config collector; + uint64_t analysis_window_ns{60'000'000'000ULL}; + uint64_t analysis_timeout_us{500'000}; + size_t max_analysis_keys{100'000}; + size_t feedback_window{60}; + size_t report_capacity{4096}; + size_t report_per_tenant_capacity{0}; + size_t max_pending_prefetches{4096}; + MetricBatchSink report_sink; + LegacyFallback legacy_fallback{LegacyFallback::kLru}; + }; + + explicit IoPatternRuntime(Handlers handlers, Config config = {}); + ~IoPatternRuntime(); + + void ReportInferenceMetrics(const InferenceMetrics& metrics); + void RecordAccess(const std::string& key, const AccessRecord& record); + void RecordStorageMetric(const StorageMetric& metric); + void MergeSnapshot(const IoPatternSnapshot& snapshot); + + PolicyExecutionStatus Execute( + CacheTier eviction_tier, uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions = {}, + const std::string& session_id = {}); + // Applies a CFM-issued command through the same storage handlers as a + // locally planned policy. This is the CFM-to-Store execution endpoint. + ErrorCode ExecuteCommand(const PolicyCommand& command); + + void RecordFeedback(PolicyFeedbackSample sample); + IoPatternSnapshot Snapshot() const; + IoPatternObservabilitySnapshot ObservabilitySnapshot( + double window_seconds = 0.0) const; + bool degraded() const; + + private: + PatternResult AnalyzeWithinBudget(const IoPatternSnapshot& snapshot, + bool& degraded); + + Config config_; + std::shared_ptr reporter_; + std::shared_ptr collector_; + std::shared_ptr analyzer_; + std::shared_ptr workload_policy_; + std::shared_ptr policy_; + TierOperationExecutor executor_; + PolicyFeedbackWindow feedback_; + AdaptivePolicyTuner tuner_; + IoPatternObservability observability_; + mutable std::mutex feedback_state_mutex_; + std::unordered_set pending_prefetches_; + uint64_t feedback_accesses_{0}; + uint64_t feedback_hits_{0}; + float previous_hit_rate_{0.0F}; + // Shared with a timed-out detached analyzer so runtime teardown cannot + // leave a worker holding a pointer into a destroyed runtime instance. + std::shared_ptr> analysis_in_flight_{ + std::make_shared>(false)}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/sliding_window_analyzer.h b/mooncake-store/include/io_pattern/sliding_window_analyzer.h new file mode 100644 index 0000000000..583190d254 --- /dev/null +++ b/mooncake-store/include/io_pattern/sliding_window_analyzer.h @@ -0,0 +1,50 @@ +#pragma once + +#include +#include +#include + +#include "threshold_analyzer.h" +#include "kmeans_analyzer.h" + +namespace mooncake::io_pattern { + +struct WorkloadFeatureStats { + uint32_t token_median{0}; + uint32_t token_p90{0}; + uint32_t fanout_p90{0}; + uint64_t block_median{0}; + uint64_t block_p90{0}; + uint32_t match_p90{0}; + uint32_t frequency_median{0}; + size_t samples{0}; +}; + +// Maintains a timestamp-bounded history of snapshots for workload detection. +class SlidingWindowAnalyzer final : public IoPatternAnalyzer { + public: + explicit SlidingWindowAnalyzer(uint64_t window_ns = 60'000'000'000ULL, + ThresholdAnalyzerConfig config = {}) + : window_ns_(window_ns), + analyzer_(config), + kmeans_(KMeansWorkloadAnalyzer::Config{.thresholds = config}) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + WorkloadFeatureStats FeatureStats() const; + + private: + IoPatternSnapshot Aggregate(const IoPatternSnapshot& current) const; + void Append(const IoPatternSnapshot& snapshot) const; + + const uint64_t window_ns_; + mutable std::mutex mutex_; + mutable std::deque history_; + ThresholdAnalyzer analyzer_; + KMeansWorkloadAnalyzer kmeans_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/threshold_analyzer.h b/mooncake-store/include/io_pattern/threshold_analyzer.h new file mode 100644 index 0000000000..2fd3baa0c4 --- /dev/null +++ b/mooncake-store/include/io_pattern/threshold_analyzer.h @@ -0,0 +1,38 @@ +#pragma once + +#include "io_pattern/analyzer.h" + +namespace mooncake::io_pattern { + +struct ThresholdAnalyzerConfig { + uint32_t code_agent_token_count{16 * 1024}; + uint32_t code_agent_prefix_fanout{16}; + uint32_t code_agent_match_length{256}; + uint64_t recommendation_block_size{128 * 1024}; + uint32_t recommendation_frequency{20}; + uint32_t conversation_prefix_fanout{16}; + uint32_t conversation_match_length{256}; +}; + +// Deterministic, low-latency analyzer for the documented threshold path. +// Mixed workloads are intentionally returned when no rule matches. +class ThresholdAnalyzer final : public IoPatternAnalyzer { + public: + explicit ThresholdAnalyzer(ThresholdAnalyzerConfig config = {}) + : config_(config) {} + + PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; + WorkloadType DetectWorkloadType( + const IoPatternSnapshot& snapshot) const override; + float CalculateConfidence(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const override; + + private: + const KeyMetrics* FindKey(const ObjectRef& object, + const IoPatternSnapshot& snapshot) const; + float KeyConfidence(const KeyMetrics& key) const; + + ThresholdAnalyzerConfig config_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/tier_executor.h b/mooncake-store/include/io_pattern/tier_executor.h new file mode 100644 index 0000000000..3ac0265d7f --- /dev/null +++ b/mooncake-store/include/io_pattern/tier_executor.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +#include "types.h" + +namespace mooncake::io_pattern { + +using EvictionHandler = std::function; +using PrefetchHandler = std::function; +using AdmissionHandler = std::function; + +struct PolicyExecutionStatus { + ErrorCode eviction{ErrorCode::OK}; + ErrorCode prefetch{ErrorCode::OK}; + std::vector admissions; + bool degraded{false}; +}; + +// Bridges policy output to storage/tier mechanisms owned by other modules. +class TierOperationExecutor final { + public: + TierOperationExecutor(EvictionHandler eviction, + PrefetchHandler prefetch, + AdmissionHandler admission) + : eviction_(std::move(eviction)), + prefetch_(std::move(prefetch)), + admission_(std::move(admission)) {} + + PolicyExecutionStatus Execute(const PolicyResult& result) const; + + private: + EvictionHandler eviction_; + PrefetchHandler prefetch_; + AdmissionHandler admission_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h index 1fb936e56d..503766e0f3 100644 --- a/mooncake-store/include/io_pattern/types.h +++ b/mooncake-store/include/io_pattern/types.h @@ -60,6 +60,15 @@ struct ObjectRef { bool operator==(const ObjectRef&) const = default; }; +struct ObjectRefHash { + size_t operator()(const ObjectRef& object) const noexcept { + const size_t tenant_hash = TenantIdHash{}(object.tenant_id); + const size_t key_hash = std::hash{}(object.key); + return tenant_hash ^ (key_hash + 0x9e3779b9 + (tenant_hash << 6) + + (tenant_hash >> 2)); + } +}; + struct InferenceMetrics { ObjectRef object; std::string session_id; @@ -82,6 +91,8 @@ struct AccessRecord { CacheTier tier{CacheTier::kL2Segment}; IoOperation operation{IoOperation::kGet}; bool is_hit{false}; + uint32_t write_batch_size{0}; + bool overwrite{false}; }; struct StorageMetric { @@ -101,6 +112,7 @@ struct StorageMetric { struct KeyMetrics { ObjectRef object; + std::string session_id; uint64_t last_access_time_ns{0}; uint64_t access_count_window{0}; uint64_t idle_time_us{0}; @@ -144,15 +156,23 @@ struct KeyPattern { bool migration_safe{false}; }; +struct SessionPattern { + std::string session_id; + WorkloadType workload_type{WorkloadType::kUnknown}; + float confidence{0.0F}; +}; + struct PatternResult { WorkloadType workload_type{WorkloadType::kUnknown}; float workload_confidence{0.0F}; std::vector keys; + std::vector sessions; }; struct PolicyContext { IoPatternSnapshot snapshot; PatternResult analysis; + std::string session_id; }; struct TraceEvent { @@ -191,6 +211,7 @@ struct EvictionCandidate { ObjectRef object; uint64_t bytes{0}; float score{0.0F}; + CacheTier target_tier{CacheTier::kL3NofSsd}; }; struct EvictionPlan { @@ -214,6 +235,13 @@ struct AdmissionResult { float confidence{0.0F}; }; +struct PolicyResult { + EvictionPlan eviction; + PrefetchPlan prefetch; + std::vector admissions; + bool degraded{false}; +}; + struct CacheViewEntry { ObjectRef object; CacheTier tier{CacheTier::kL2Segment}; diff --git a/mooncake-store/include/io_pattern/view_manager.h b/mooncake-store/include/io_pattern/view_manager.h deleted file mode 100644 index 4ef636b71d..0000000000 --- a/mooncake-store/include/io_pattern/view_manager.h +++ /dev/null @@ -1,17 +0,0 @@ -#pragma once - -#include "io_pattern/types.h" - -namespace mooncake::io_pattern { - -// Owns the published cache view, not the storage operations that realize it. -class CacheViewManager { - public: - virtual ~CacheViewManager() = default; - - virtual CacheView ComputeView() const = 0; - virtual void PublishEvent(const CacheEvent& event) = 0; - virtual KVMappingTable GetGlobalMapping() const = 0; -}; - -} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern_analyzer.h b/mooncake-store/include/io_pattern_analyzer.h index 7579ac39c1..9ff4932701 100644 --- a/mooncake-store/include/io_pattern_analyzer.h +++ b/mooncake-store/include/io_pattern_analyzer.h @@ -1,3 +1,4 @@ #pragma once #include "io_pattern/analyzer.h" +#include "io_pattern/threshold_analyzer.h" diff --git a/mooncake-store/include/legacy_eviction_ops.h b/mooncake-store/include/legacy_eviction_ops.h new file mode 100644 index 0000000000..8fdcc44f50 --- /dev/null +++ b/mooncake-store/include/legacy_eviction_ops.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/legacy_eviction_ops.h" diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 2c17142084..1fba696f1a 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -52,6 +52,10 @@ namespace mooncake { +namespace io_pattern { +class IoPatternRuntime; +} + // Forward declaration for MasterSnapshotManager class MasterSnapshotManager; class MasterSnapshotRepository; @@ -2204,6 +2208,11 @@ class MasterService { // from any GetReplicaList caller without additional locking. std::unique_ptr promotion_sketch_; + // The IO-pattern pipeline is deliberately owned by MasterService: the + // master has the authoritative replica map and is the only component that + // can safely translate a policy plan into promotion/eviction operations. + std::unique_ptr io_pattern_runtime_; + const std::string ha_backend_type_; const std::string ha_backend_connstring_; diff --git a/mooncake-store/include/observability.h b/mooncake-store/include/observability.h new file mode 100644 index 0000000000..8e4b83cffc --- /dev/null +++ b/mooncake-store/include/observability.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/observability.h" diff --git a/mooncake-store/include/policy_strategies.h b/mooncake-store/include/policy_strategies.h new file mode 100644 index 0000000000..7a7c6001d4 --- /dev/null +++ b/mooncake-store/include/policy_strategies.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/policy_strategies.h" diff --git a/mooncake-store/include/reporter.h b/mooncake-store/include/reporter.h new file mode 100644 index 0000000000..8877123fab --- /dev/null +++ b/mooncake-store/include/reporter.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/reporter.h" diff --git a/mooncake-store/include/resilient_analyzer.h b/mooncake-store/include/resilient_analyzer.h new file mode 100644 index 0000000000..13e25d7901 --- /dev/null +++ b/mooncake-store/include/resilient_analyzer.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/resilient_analyzer.h" diff --git a/mooncake-store/include/resilient_cfm_channel.h b/mooncake-store/include/resilient_cfm_channel.h new file mode 100644 index 0000000000..654ac93cf9 --- /dev/null +++ b/mooncake-store/include/resilient_cfm_channel.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/resilient_cfm_channel.h" diff --git a/mooncake-store/include/rpc_transport.h b/mooncake-store/include/rpc_transport.h new file mode 100644 index 0000000000..b5a8f32cfc --- /dev/null +++ b/mooncake-store/include/rpc_transport.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/rpc_transport.h" diff --git a/mooncake-store/include/sliding_window_analyzer.h b/mooncake-store/include/sliding_window_analyzer.h new file mode 100644 index 0000000000..e9f03d51db --- /dev/null +++ b/mooncake-store/include/sliding_window_analyzer.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/sliding_window_analyzer.h" diff --git a/mooncake-store/include/threshold_analyzer.h b/mooncake-store/include/threshold_analyzer.h new file mode 100644 index 0000000000..a0b1ffad03 --- /dev/null +++ b/mooncake-store/include/threshold_analyzer.h @@ -0,0 +1,3 @@ +#pragma once + +#include "io_pattern/threshold_analyzer.h" diff --git a/mooncake-store/include/tier_executor.h b/mooncake-store/include/tier_executor.h new file mode 100644 index 0000000000..efac66ffdd --- /dev/null +++ b/mooncake-store/include/tier_executor.h @@ -0,0 +1,2 @@ +#pragma once +#include "io_pattern/tier_executor.h" diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 41d3392057..003454fda2 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -68,6 +68,24 @@ set(MOONCAKE_STORE_SOURCES utils/file_util.cpp task_manager.cpp local_hot_cache.cpp + io_pattern/collector_impl.cpp + io_pattern/reporter.cpp + io_pattern/cfm_client_impl.cpp + io_pattern/cfm_ingress.cpp + io_pattern/cfm_protocol.cpp + io_pattern/resilient_cfm_channel.cpp + io_pattern/feedback.cpp + io_pattern/degrading_policy_engine.cpp + io_pattern/legacy_eviction_ops.cpp + io_pattern/kmeans_analyzer.cpp + io_pattern/observability.cpp + io_pattern/sliding_window_analyzer.cpp + io_pattern/tier_executor.cpp + io_pattern/resilient_analyzer.cpp + io_pattern/rpc_transport.cpp + io_pattern/runtime.cpp + io_pattern/threshold_analyzer.cpp + io_pattern/policy_strategies.cpp ha/oplog/oplog_types.cpp ha/oplog/oplog_batch_codec.cpp ha/oplog/oplog_batch_storage.cpp diff --git a/mooncake-store/src/io_pattern/cfm_client_impl.cpp b/mooncake-store/src/io_pattern/cfm_client_impl.cpp new file mode 100644 index 0000000000..2bd502af2b --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_client_impl.cpp @@ -0,0 +1,31 @@ +#include "io_pattern/cfm_client_impl.h" + +namespace mooncake::io_pattern { + +ErrorCode CfmClientImpl::ReportSnapshot(const IoPatternSnapshot& snapshot) { + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + return channel_->SendSnapshot(snapshot) ? ErrorCode::OK + : ErrorCode::RPC_FAIL; +} + +ErrorCode CfmClientImpl::ReceivePolicy(const PolicyCommand& command) { + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + if (!policy_handler_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + return policy_handler_(command); +} + +ErrorCode CfmClientImpl::ExecutePrefetch(const PrefetchPlan& plan) { + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + return channel_->ExecutePrefetch(plan); +} + +std::optional CfmClientImpl::PollPolicy() { + return channel_ ? channel_->PollPolicy() : std::nullopt; +} + +ErrorCode CfmClientImpl::PollAndDispatchPolicy() { + const auto command = PollPolicy(); + return command ? ReceivePolicy(*command) : ErrorCode::RPC_TIMEOUT; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_ingress.cpp b/mooncake-store/src/io_pattern/cfm_ingress.cpp new file mode 100644 index 0000000000..effabd7b1a --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_ingress.cpp @@ -0,0 +1,36 @@ +#include "io_pattern/cfm_ingress.h" + +namespace mooncake::io_pattern { + +bool CfmIngress::Handle(std::string_view method, std::string_view payload) { + if (!runtime_ || !codec_) return false; + const std::string wire(payload); + if (method == "report_snapshot") { + const auto snapshot = codec_->DecodeSnapshot(wire); + if (!snapshot) return false; + runtime_->MergeSnapshot(*snapshot); + return true; + } + if (method == "report_metric_batch") { + const auto batch = codec_->DecodeMetricBatch(wire); + if (!batch) return false; + for (const auto& metric : batch->inference) { + runtime_->ReportInferenceMetrics(metric); + } + for (const auto& access : batch->accesses) { + runtime_->RecordAccess(access.object.key, access); + } + for (const auto& storage : batch->storage) { + runtime_->RecordStorageMetric(storage); + } + return true; + } + if (method == "execute_prefetch") { + const auto command = codec_->DecodePolicy(wire); + const auto* plan = command ? std::get_if(&*command) : nullptr; + return plan && runtime_->ExecuteCommand(*plan) == ErrorCode::OK; + } + return false; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_protocol.cpp b/mooncake-store/src/io_pattern/cfm_protocol.cpp new file mode 100644 index 0000000000..229025a017 --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_protocol.cpp @@ -0,0 +1,425 @@ +#include "io_pattern/cfm_protocol.h" + +#include +#include +#include + +namespace mooncake::io_pattern { +namespace { + +constexpr char kWireVersion[] = "CFM2"; +constexpr size_t kMaxWireStringBytes = 16 * 1024 * 1024; +constexpr size_t kMaxWirePayloadBytes = 64 * 1024 * 1024; +constexpr uint32_t kMaxWireRecords = 1'000'000; + +template +void Append(std::string& out, T value) { + static_assert(std::is_trivially_copyable_v); + const auto* bytes = reinterpret_cast(&value); + out.append(bytes, sizeof(value)); +} + +template +bool Read(const std::string& input, size_t& offset, T& value) { + static_assert(std::is_trivially_copyable_v); + if (input.size() - offset < sizeof(value)) return false; + std::memcpy(&value, input.data() + offset, sizeof(value)); + offset += sizeof(value); + return true; +} + +void AppendString(std::string& out, const std::string& value) { + const auto size = static_cast(value.size()); + Append(out, size); + out.append(value); +} + +bool ReadString(const std::string& input, size_t& offset, std::string& value) { + uint32_t size = 0; + if (!Read(input, offset, size) || size > kMaxWireStringBytes || + input.size() - offset < size) { + return false; + } + value.assign(input.data() + offset, size); + offset += size; + return true; +} + +void AppendObject(std::string& out, const ObjectRef& object) { + AppendString(out, object.tenant_id.value()); + AppendString(out, object.key); +} + +bool ReadObject(const std::string& input, size_t& offset, ObjectRef& object) { + std::string tenant; + if (!ReadString(input, offset, tenant) || !ReadString(input, offset, object.key)) { + return false; + } + object.tenant_id = TenantId(std::move(tenant)); + return true; +} + +template +void AppendEnum(std::string& out, Enum value) { + Append(out, static_cast(value)); +} + +template +bool ReadEnum(const std::string& input, size_t& offset, Enum& value) { + uint8_t raw = 0; + if (!Read(input, offset, raw)) return false; + value = static_cast(raw); + return true; +} + +void AppendPrefetchPlan(std::string& out, const PrefetchPlan& plan) { + AppendEnum(out, plan.strategy); + Append(out, plan.timeout_us); + Append(out, static_cast(plan.candidates.size())); + for (const auto& candidate : plan.candidates) { + AppendObject(out, candidate.object); + AppendEnum(out, candidate.source_tier); + AppendEnum(out, candidate.target_tier); + Append(out, candidate.bytes); + Append(out, candidate.priority); + Append(out, candidate.confidence); + } +} + +bool ReadPrefetchPlan(const std::string& input, size_t& offset, PrefetchPlan& plan) { + uint32_t count = 0; + if (!ReadEnum(input, offset, plan.strategy) || !Read(input, offset, plan.timeout_us) || + !Read(input, offset, count) || count > kMaxWireRecords) { + return false; + } + plan.candidates.clear(); + plan.candidates.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + PrefetchCandidate candidate; + if (!ReadObject(input, offset, candidate.object) || + !ReadEnum(input, offset, candidate.source_tier) || + !ReadEnum(input, offset, candidate.target_tier) || + !Read(input, offset, candidate.bytes) || + !Read(input, offset, candidate.priority) || + !Read(input, offset, candidate.confidence)) { + return false; + } + plan.candidates.push_back(std::move(candidate)); + } + return true; +} + +void AppendStorageMetric(std::string& out, const StorageMetric& storage) { + AppendString(out, storage.source_id); + Append(out, storage.observed_at_ns); + AppendEnum(out, storage.tier); + AppendEnum(out, storage.gc_state); + Append(out, storage.read_bandwidth_bytes_per_sec); + Append(out, storage.write_bandwidth_bytes_per_sec); + Append(out, storage.read_latency_us); + Append(out, storage.write_latency_us); + Append(out, storage.used_bytes); + Append(out, storage.capacity_bytes); + Append(out, storage.rpc_latency_us); + Append(out, storage.memory_used_ratio); +} + +bool ReadStorageMetric(const std::string& input, size_t& offset, + StorageMetric& storage) { + return ReadString(input, offset, storage.source_id) && + Read(input, offset, storage.observed_at_ns) && + ReadEnum(input, offset, storage.tier) && + ReadEnum(input, offset, storage.gc_state) && + Read(input, offset, storage.read_bandwidth_bytes_per_sec) && + Read(input, offset, storage.write_bandwidth_bytes_per_sec) && + Read(input, offset, storage.read_latency_us) && + Read(input, offset, storage.write_latency_us) && + Read(input, offset, storage.used_bytes) && + Read(input, offset, storage.capacity_bytes) && + Read(input, offset, storage.rpc_latency_us) && + Read(input, offset, storage.memory_used_ratio); +} + +void AppendHeader(std::string& out, char type) { + out.append(kWireVersion, sizeof(kWireVersion) - 1); + out.push_back(type); +} + +bool ReadHeader(const std::string& input, size_t& offset, char& type) { + if (input.size() < sizeof(kWireVersion) || input.size() > kMaxWirePayloadBytes || + input.compare(0, sizeof(kWireVersion) - 1, kWireVersion) != 0) { + return false; + } + offset = sizeof(kWireVersion) - 1; + return Read(input, offset, type); +} + +} // namespace + +std::string CfmBinaryCodec::EncodeSnapshot(const IoPatternSnapshot& snapshot) const { + std::string output; + AppendHeader(output, 'S'); + Append(output, snapshot.generated_at_ns); + Append(output, static_cast(snapshot.keys.size())); + for (const auto& key : snapshot.keys) { + AppendObject(output, key.object); + AppendString(output, key.session_id); + Append(output, key.last_access_time_ns); + Append(output, key.access_count_window); + Append(output, key.idle_time_us); + Append(output, key.block_size); + Append(output, key.transfer_eta_us); + Append(output, key.token_count); + Append(output, key.prefix_depth); + Append(output, key.prefix_fanout); + Append(output, key.match_length); + Append(output, key.continuous_prefix_length); + Append(output, key.other_replica_count); + Append(output, key.write_batch_size); + Append(output, key.write_frequency); + Append(output, key.write_object_size); + Append(output, key.recompute_cost); + Append(output, key.overwrite_ratio); + Append(output, key.replica_tiers); + AppendEnum(output, key.layout); + Append(output, key.layout_group); + Append(output, key.request_priority); + Append(output, key.active); + Append(output, key.pinned); + Append(output, key.ssd_replica_exists); + Append(output, key.write_burst); + } + Append(output, static_cast(snapshot.storage.size())); + for (const auto& storage : snapshot.storage) { + AppendStorageMetric(output, storage); + } + return output; +} + +std::string CfmBinaryCodec::EncodePrefetch(const PrefetchPlan& plan) const { + std::string output; + AppendHeader(output, 'P'); + AppendPrefetchPlan(output, plan); + return output; +} + +std::string CfmBinaryCodec::EncodeMetricBatch(const MetricBatch& batch) const { + std::string output; + AppendHeader(output, 'M'); + Append(output, static_cast(batch.inference.size())); + for (const auto& metric : batch.inference) { + AppendObject(output, metric.object); + AppendString(output, metric.session_id); + Append(output, metric.prefix_depth); + Append(output, metric.prefix_fanout); + Append(output, metric.match_length); + Append(output, metric.continuous_prefix_length); + Append(output, metric.token_count); + Append(output, metric.recompute_cost); + Append(output, metric.request_priority); + AppendEnum(output, metric.layout); + Append(output, metric.layout_group); + } + Append(output, static_cast(batch.accesses.size())); + for (const auto& access : batch.accesses) { + AppendObject(output, access.object); + Append(output, access.observed_at_ns); + Append(output, access.block_size); + Append(output, access.latency_us); + AppendEnum(output, access.tier); + AppendEnum(output, access.operation); + Append(output, access.is_hit); + Append(output, access.write_batch_size); + Append(output, access.overwrite); + } + Append(output, static_cast(batch.storage.size())); + for (const auto& storage : batch.storage) { + AppendStorageMetric(output, storage); + } + return output; +} + +std::string CfmBinaryCodec::EncodePolicy(const PolicyCommand& command) const { + std::string output; + if (const auto* eviction = std::get_if(&command)) { + AppendHeader(output, 'E'); + AppendEnum(output, eviction->source_tier); + Append(output, eviction->target_bytes); + Append(output, static_cast(eviction->candidates.size())); + for (const auto& candidate : eviction->candidates) { + AppendObject(output, candidate.object); + Append(output, candidate.bytes); + Append(output, candidate.score); + AppendEnum(output, candidate.target_tier); + } + } else if (const auto* prefetch = std::get_if(&command)) { + AppendHeader(output, 'P'); + AppendPrefetchPlan(output, *prefetch); + } else { + const auto& admission = std::get(command); + AppendHeader(output, 'A'); + AppendObject(output, admission.object); + AppendEnum(output, admission.target_tier); + AppendEnum(output, admission.decision); + Append(output, admission.confidence); + } + return output; +} + +std::optional CfmBinaryCodec::DecodePolicy( + const std::string& payload) const { + size_t offset = 0; + char type = 0; + if (!ReadHeader(payload, offset, type)) return std::nullopt; + if (type == 'P') { + PrefetchPlan plan; + return ReadPrefetchPlan(payload, offset, plan) && offset == payload.size() + ? std::optional(std::move(plan)) + : std::nullopt; + } + if (type == 'E') { + EvictionPlan plan; + uint32_t count = 0; + if (!ReadEnum(payload, offset, plan.source_tier) || + !Read(payload, offset, plan.target_bytes) || + !Read(payload, offset, count) || count > kMaxWireRecords) { + return std::nullopt; + } + plan.candidates.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + EvictionCandidate candidate; + if (!ReadObject(payload, offset, candidate.object) || + !Read(payload, offset, candidate.bytes) || + !Read(payload, offset, candidate.score) || + !ReadEnum(payload, offset, candidate.target_tier)) { + return std::nullopt; + } + plan.candidates.push_back(std::move(candidate)); + } + return offset == payload.size() + ? std::optional(std::move(plan)) + : std::nullopt; + } + if (type == 'A') { + AdmissionResult result; + if (!ReadObject(payload, offset, result.object) || + !ReadEnum(payload, offset, result.target_tier) || + !ReadEnum(payload, offset, result.decision) || + !Read(payload, offset, result.confidence) || offset != payload.size()) { + return std::nullopt; + } + return result; + } + return std::nullopt; +} + +std::optional CfmBinaryCodec::DecodeSnapshot( + const std::string& payload) const { + size_t offset = 0; + char type = 0; + IoPatternSnapshot snapshot; + uint32_t key_count = 0; + if (!ReadHeader(payload, offset, type) || type != 'S' || + !Read(payload, offset, snapshot.generated_at_ns) || + !Read(payload, offset, key_count) || key_count > kMaxWireRecords) { + return std::nullopt; + } + snapshot.keys.reserve(key_count); + for (uint32_t index = 0; index < key_count; ++index) { + KeyMetrics key; + if (!ReadObject(payload, offset, key.object) || + !ReadString(payload, offset, key.session_id) || + !Read(payload, offset, key.last_access_time_ns) || + !Read(payload, offset, key.access_count_window) || + !Read(payload, offset, key.idle_time_us) || + !Read(payload, offset, key.block_size) || + !Read(payload, offset, key.transfer_eta_us) || + !Read(payload, offset, key.token_count) || + !Read(payload, offset, key.prefix_depth) || + !Read(payload, offset, key.prefix_fanout) || + !Read(payload, offset, key.match_length) || + !Read(payload, offset, key.continuous_prefix_length) || + !Read(payload, offset, key.other_replica_count) || + !Read(payload, offset, key.write_batch_size) || + !Read(payload, offset, key.write_frequency) || + !Read(payload, offset, key.write_object_size) || + !Read(payload, offset, key.recompute_cost) || + !Read(payload, offset, key.overwrite_ratio) || + !Read(payload, offset, key.replica_tiers) || + !ReadEnum(payload, offset, key.layout) || + !Read(payload, offset, key.layout_group) || + !Read(payload, offset, key.request_priority) || + !Read(payload, offset, key.active) || !Read(payload, offset, key.pinned) || + !Read(payload, offset, key.ssd_replica_exists) || + !Read(payload, offset, key.write_burst)) { + return std::nullopt; + } + snapshot.keys.push_back(std::move(key)); + } + uint32_t storage_count = 0; + if (!Read(payload, offset, storage_count) || storage_count > kMaxWireRecords) { + return std::nullopt; + } + snapshot.storage.reserve(storage_count); + for (uint32_t index = 0; index < storage_count; ++index) { + StorageMetric storage; + if (!ReadStorageMetric(payload, offset, storage)) return std::nullopt; + snapshot.storage.push_back(std::move(storage)); + } + return offset == payload.size() ? std::optional(std::move(snapshot)) + : std::nullopt; +} + +std::optional CfmBinaryCodec::DecodeMetricBatch( + const std::string& payload) const { + size_t offset = 0; + char type = 0; + MetricBatch batch; + uint32_t count = 0; + if (!ReadHeader(payload, offset, type) || type != 'M' || + !Read(payload, offset, count) || count > kMaxWireRecords) { + return std::nullopt; + } + batch.inference.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + InferenceMetrics metric; + if (!ReadObject(payload, offset, metric.object) || + !ReadString(payload, offset, metric.session_id) || + !Read(payload, offset, metric.prefix_depth) || + !Read(payload, offset, metric.prefix_fanout) || + !Read(payload, offset, metric.match_length) || + !Read(payload, offset, metric.continuous_prefix_length) || + !Read(payload, offset, metric.token_count) || + !Read(payload, offset, metric.recompute_cost) || + !Read(payload, offset, metric.request_priority) || + !ReadEnum(payload, offset, metric.layout) || + !Read(payload, offset, metric.layout_group)) return std::nullopt; + batch.inference.push_back(std::move(metric)); + } + if (!Read(payload, offset, count) || count > kMaxWireRecords) return std::nullopt; + batch.accesses.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + AccessRecord access; + if (!ReadObject(payload, offset, access.object) || + !Read(payload, offset, access.observed_at_ns) || + !Read(payload, offset, access.block_size) || + !Read(payload, offset, access.latency_us) || + !ReadEnum(payload, offset, access.tier) || + !ReadEnum(payload, offset, access.operation) || + !Read(payload, offset, access.is_hit) || + !Read(payload, offset, access.write_batch_size) || + !Read(payload, offset, access.overwrite)) return std::nullopt; + batch.accesses.push_back(std::move(access)); + } + if (!Read(payload, offset, count) || count > kMaxWireRecords) return std::nullopt; + batch.storage.reserve(count); + for (uint32_t index = 0; index < count; ++index) { + StorageMetric storage; + if (!ReadStorageMetric(payload, offset, storage)) return std::nullopt; + batch.storage.push_back(std::move(storage)); + } + return offset == payload.size() ? std::optional(std::move(batch)) + : std::nullopt; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/collector_impl.cpp b/mooncake-store/src/io_pattern/collector_impl.cpp new file mode 100644 index 0000000000..e31b06c223 --- /dev/null +++ b/mooncake-store/src/io_pattern/collector_impl.cpp @@ -0,0 +1,183 @@ +#include "io_pattern/collector_impl.h" + +#include +#include +#include + +namespace mooncake::io_pattern { +namespace { +uint64_t NowNs() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} +} + +void IoPatternCollectorImpl::ReportInferenceMetrics( + const InferenceMetrics& metrics) { + std::lock_guard lock(mutex_); + if (reporter_ && !reporter_->Enqueue(metrics)) ++dropped_; + if (!key_metrics_.contains(metrics.object) && config_.max_total_keys != 0 && + key_metrics_.size() >= config_.max_total_keys) { + degraded_ = true; + ++dropped_; + return; + } + if (!key_metrics_.contains(metrics.object) && + config_.max_keys_per_tenant != 0 && + tenant_key_counts_[metrics.object.tenant_id] >= + config_.max_keys_per_tenant) { + ++dropped_; + return; + } + if (!key_metrics_.contains(metrics.object)) + ++tenant_key_counts_[metrics.object.tenant_id]; + auto& value = key_metrics_[metrics.object]; + value.object = metrics.object; + value.session_id = metrics.session_id; + value.layout = metrics.layout; + value.layout_group = metrics.layout_group; + value.prefix_depth = metrics.prefix_depth; + value.prefix_fanout = metrics.prefix_fanout; + value.match_length = metrics.match_length; + value.continuous_prefix_length = metrics.continuous_prefix_length; + value.token_count = metrics.token_count; + value.recompute_cost = metrics.recompute_cost; + value.request_priority = metrics.request_priority; +} + +void IoPatternCollectorImpl::RecordAccess(const std::string& key, + const AccessRecord& record) { + std::lock_guard lock(mutex_); + if (reporter_ && !reporter_->EnqueueAccess(record)) ++dropped_; + ObjectRef object = record.object; + if (!key.empty()) object.key = key; + if (!key_metrics_.contains(object) && config_.max_total_keys != 0 && + key_metrics_.size() >= config_.max_total_keys) { + degraded_ = true; + ++dropped_; + return; + } + if (!key_metrics_.contains(object) && config_.max_keys_per_tenant != 0 && + tenant_key_counts_[object.tenant_id] >= config_.max_keys_per_tenant) { + ++dropped_; + return; + } + if (!key_metrics_.contains(object)) ++tenant_key_counts_[object.tenant_id]; + auto& value = key_metrics_[object]; + value.object = object; + ++value.access_count_window; + value.last_access_time_ns = + std::max(value.last_access_time_ns, record.observed_at_ns); + value.block_size = std::max(value.block_size, record.block_size); + value.replica_tiers |= CacheTierBit(record.tier); + value.active = value.active || record.is_hit; + if (record.operation == IoOperation::kPut) { + ++write_counts_[object]; + if (record.overwrite) ++overwrite_counts_[object]; + value.write_frequency = + static_cast(std::min(write_counts_[object], + UINT32_MAX)); + value.write_batch_size = + std::max(value.write_batch_size, record.write_batch_size); + value.write_object_size = std::max(value.write_object_size, + record.block_size); + value.overwrite_ratio = + static_cast(overwrite_counts_[object]) / + static_cast(write_counts_[object]); + value.write_burst = record.write_batch_size >= 16; + } +} + +void IoPatternCollectorImpl::RecordStorageMetric(const StorageMetric& metric) { + std::lock_guard lock(mutex_); + if (reporter_ && !reporter_->EnqueueStorage(metric)) ++dropped_; + StorageMetricKey key{metric.source_id, metric.tier}; + auto it = storage_metrics_.find(key); + if (it == storage_metrics_.end() || + metric.observed_at_ns >= it->second.observed_at_ns) { + storage_metrics_[std::move(key)] = metric; + } +} + +void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { + std::lock_guard lock(mutex_); + for (const auto& metrics : snapshot.keys) { + if (!key_metrics_.contains(metrics.object) && + config_.max_total_keys != 0 && + key_metrics_.size() >= config_.max_total_keys) { + degraded_ = true; + ++dropped_; + continue; + } + if (!key_metrics_.contains(metrics.object) && + config_.max_keys_per_tenant != 0 && + tenant_key_counts_[metrics.object.tenant_id] >= + config_.max_keys_per_tenant) { + ++dropped_; + continue; + } + if (!key_metrics_.contains(metrics.object)) { + ++tenant_key_counts_[metrics.object.tenant_id]; + } + key_metrics_[metrics.object] = metrics; + } + for (const auto& metric : snapshot.storage) { + StorageMetricKey key{metric.source_id, metric.tier}; + auto it = storage_metrics_.find(key); + if (it == storage_metrics_.end() || + metric.observed_at_ns >= it->second.observed_at_ns) { + storage_metrics_[std::move(key)] = metric; + } + } +} + +IoPatternSnapshot IoPatternCollectorImpl::GetSnapshot() const { + std::lock_guard lock(mutex_); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = NowNs(); + snapshot.keys.reserve(key_metrics_.size()); + for (const auto& [object, metrics] : key_metrics_) { + (void)object; + auto copy = metrics; + if (copy.last_access_time_ns != 0 && + snapshot.generated_at_ns > copy.last_access_time_ns) { + copy.idle_time_us = + (snapshot.generated_at_ns - copy.last_access_time_ns) / 1000; + } + snapshot.keys.push_back(std::move(copy)); + } + snapshot.storage.reserve(storage_metrics_.size()); + for (const auto& [key, metric] : storage_metrics_) { + (void)key; + snapshot.storage.push_back(metric); + } + std::sort(snapshot.keys.begin(), snapshot.keys.end(), + [](const KeyMetrics& a, const KeyMetrics& b) { + if (a.object.tenant_id != b.object.tenant_id) + return a.object.tenant_id < b.object.tenant_id; + return a.object.key < b.object.key; + }); + return snapshot; +} + +uint64_t IoPatternCollectorImpl::dropped() const { + std::lock_guard lock(mutex_); + return dropped_; +} + +bool IoPatternCollectorImpl::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +bool IoPatternCollectorImpl::FlushReports() { + std::shared_ptr reporter; + { + std::lock_guard lock(mutex_); + reporter = reporter_; + } + return !reporter || reporter->Flush(); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/degrading_policy_engine.cpp b/mooncake-store/src/io_pattern/degrading_policy_engine.cpp new file mode 100644 index 0000000000..bdcbe6d06f --- /dev/null +++ b/mooncake-store/src/io_pattern/degrading_policy_engine.cpp @@ -0,0 +1,60 @@ +#include "io_pattern/degrading_policy_engine.h" + +namespace mooncake::io_pattern { + +void DegradingPolicyEngine::RecordFailure() { + std::lock_guard lock(mutex_); + ++consecutive_failures_; + if (failure_threshold_ != 0 && consecutive_failures_ >= failure_threshold_) + degraded_ = true; +} + +void DegradingPolicyEngine::RecordSuccess() { + std::lock_guard lock(mutex_); + consecutive_failures_ = 0; +} + +void DegradingPolicyEngine::ForceDegraded(bool value) { + std::lock_guard lock(mutex_); + degraded_ = value; + if (!value) consecutive_failures_ = 0; +} + +bool DegradingPolicyEngine::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +size_t DegradingPolicyEngine::consecutive_failures() const { + std::lock_guard lock(mutex_); + return consecutive_failures_; +} + +std::shared_ptr DegradingPolicyEngine::Active() const { + std::lock_guard lock(mutex_); + return degraded_ ? fallback_ : primary_; +} + +EvictionPlan DegradingPolicyEngine::PlanEviction(const PolicyContext& context, + CacheTier tier, + uint64_t bytes) const { + auto engine = Active(); + return engine ? engine->PlanEviction(context, tier, bytes) + : EvictionPlan{.source_tier = tier, .target_bytes = bytes}; +} + +PrefetchPlan DegradingPolicyEngine::PlanPrefetch( + const PolicyContext& context, const TraceHistory& trace) const { + auto engine = Active(); + return engine ? engine->PlanPrefetch(context, trace) : PrefetchPlan{}; +} + +AdmissionResult DegradingPolicyEngine::DecideAdmission( + const ObjectRef& object, CacheTier tier, + const PolicyContext& context) const { + auto engine = Active(); + return engine ? engine->DecideAdmission(object, tier, context) + : AdmissionResult{.object = object, .target_tier = tier}; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/feedback.cpp b/mooncake-store/src/io_pattern/feedback.cpp new file mode 100644 index 0000000000..7598dbc99b --- /dev/null +++ b/mooncake-store/src/io_pattern/feedback.cpp @@ -0,0 +1,58 @@ +#include "io_pattern/feedback.h" + +namespace mooncake::io_pattern { + +void PolicyFeedbackWindow::Record(PolicyFeedbackSample sample) { + std::lock_guard lock(mutex_); + if (capacity_ == 0) return; + if (samples_.size() == capacity_) samples_.pop_front(); + samples_.push_back(sample); +} + +PolicyFeedbackStats PolicyFeedbackWindow::Snapshot() const { + std::lock_guard lock(mutex_); + PolicyFeedbackStats stats; + stats.samples = samples_.size(); + for (const auto& sample : samples_) { + stats.hit_rate_delta += sample.hit_rate_delta; + stats.eviction_churn += sample.eviction_churn; + stats.ttft_delta += sample.ttft_delta; + stats.prefetch_accuracy += sample.prefetch_accuracy; + } + if (stats.samples != 0) { + const float divisor = static_cast(stats.samples); + stats.hit_rate_delta /= divisor; + stats.eviction_churn /= divisor; + stats.ttft_delta /= divisor; + stats.prefetch_accuracy /= divisor; + } + return stats; +} + +bool AdaptivePolicyTuner::Tune(const PolicyFeedbackStats& stats, + ScoreBasedEvictionConfig& config) { + if (stats.hit_rate_delta < 0.0F) { + ++negative_streak_; + } else { + negative_streak_ = 0; + } + if (negative_windows_ == 0 || negative_streak_ < negative_windows_) { + if (stats.eviction_churn > 0.5F || stats.prefetch_accuracy < 0.2F || + stats.ttft_delta > 0.1F) { + conservative_ = true; + config.prefix_weight *= 0.9F; + config.recompute_weight *= 0.9F; + if (persistence_) persistence_(config); + return true; + } + return false; + } + config.frequency_weight *= 0.8F; + config.idle_weight *= 1.1F; + conservative_ = true; + negative_streak_ = 0; + if (persistence_) persistence_(config); + return true; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/kmeans_analyzer.cpp b/mooncake-store/src/io_pattern/kmeans_analyzer.cpp new file mode 100644 index 0000000000..971e1a92a3 --- /dev/null +++ b/mooncake-store/src/io_pattern/kmeans_analyzer.cpp @@ -0,0 +1,161 @@ +#include "io_pattern/kmeans_analyzer.h" + +#include +#include +#include +#include + +namespace mooncake::io_pattern { +namespace { + +// Token length, fanout, block size, access frequency and prefix-match length +// are the five documented dimensions. They are normalized against the active +// sliding window before distance calculation. +using Feature = std::array; + +struct SessionFeatures { + Feature values{}; + size_t samples{0}; +}; + +Feature ToFeature(const KeyMetrics& key) { + return {static_cast(key.token_count), + static_cast(key.prefix_fanout), + static_cast(key.block_size), + static_cast(key.access_count_window), + static_cast(key.match_length)}; +} + +float Distance(const Feature& lhs, const Feature& rhs, const Feature& scale) { + float distance = 0.0F; + for (size_t index = 0; index < lhs.size(); ++index) { + const float normalized = (lhs[index] - rhs[index]) / + std::max(1.0F, scale[index]); + distance += normalized * normalized; + } + return distance; +} + +WorkloadType Classify(const Feature& feature, + const ThresholdAnalyzerConfig& config) { + if (feature[0] > config.code_agent_token_count && + feature[1] > config.code_agent_prefix_fanout && + feature[4] > config.code_agent_match_length) { + return WorkloadType::kCodeAgent; + } + if (feature[2] < config.recommendation_block_size && + feature[3] > config.recommendation_frequency) { + return WorkloadType::kGenerativeRecommendation; + } + if (feature[1] > config.conversation_prefix_fanout && + feature[4] > config.conversation_match_length) { + return WorkloadType::kMultiTurnConversation; + } + return WorkloadType::kMixed; +} + +} // namespace + +PatternResult KMeansWorkloadAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + PatternResult result; + if (snapshot.keys.empty()) { + result.workload_type = WorkloadType::kMixed; + return result; + } + + std::unordered_map by_session; + Feature scale{1.0F, 1.0F, 1.0F, 1.0F}; + for (const auto& key : snapshot.keys) { + const std::string session = key.session_id.empty() + ? key.object.tenant_id.value() + ":" + key.object.key + : key.session_id; + auto& aggregate = by_session[session]; + const auto feature = ToFeature(key); + for (size_t index = 0; index < feature.size(); ++index) { + aggregate.values[index] += feature[index]; + scale[index] = std::max(scale[index], feature[index]); + } + ++aggregate.samples; + } + + std::vector session_ids; + std::vector samples; + session_ids.reserve(by_session.size()); + samples.reserve(by_session.size()); + for (auto& [session, aggregate] : by_session) { + for (auto& value : aggregate.values) { + value /= static_cast(aggregate.samples); + } + session_ids.push_back(session); + samples.push_back(aggregate.values); + } + + const size_t cluster_count = std::min(3, samples.size()); + std::vector centroids(samples.begin(), samples.begin() + cluster_count); + std::vector assignments(samples.size(), 0); + for (uint32_t iteration = 0; iteration < config_.iterations; ++iteration) { + std::vector sums(cluster_count); + std::vector counts(cluster_count, 0); + for (size_t sample_index = 0; sample_index < samples.size(); ++sample_index) { + size_t best = 0; + float best_distance = Distance(samples[sample_index], centroids[0], scale); + for (size_t cluster = 1; cluster < cluster_count; ++cluster) { + const float distance = Distance(samples[sample_index], centroids[cluster], scale); + if (distance < best_distance) { + best = cluster; + best_distance = distance; + } + } + assignments[sample_index] = best; + ++counts[best]; + for (size_t field = 0; field < samples[sample_index].size(); ++field) { + sums[best][field] += samples[sample_index][field]; + } + } + for (size_t cluster = 0; cluster < cluster_count; ++cluster) { + if (counts[cluster] == 0) continue; + for (size_t field = 0; field < centroids[cluster].size(); ++field) { + centroids[cluster][field] = sums[cluster][field] / + static_cast(counts[cluster]); + } + } + } + + std::vector labels; + labels.reserve(cluster_count); + for (const auto& centroid : centroids) labels.push_back(Classify(centroid, config_.thresholds)); + WorkloadType global = labels[assignments.front()]; + bool mixed = false; + for (size_t index = 0; index < samples.size(); ++index) { + const auto type = labels[assignments[index]]; + result.sessions.push_back( + SessionPattern{.session_id = session_ids[index], .workload_type = type, + .confidence = 1.0F / (1.0F + Distance( + samples[index], centroids[assignments[index]], scale))}); + if (type != global) mixed = true; + } + result.workload_type = mixed ? WorkloadType::kMixed : global; + result.workload_confidence = mixed ? 0.5F : result.sessions.front().confidence; + + ThresholdAnalyzer key_analyzer(config_.thresholds); + result.keys = key_analyzer.Analyze(snapshot).keys; + return result; +} + +WorkloadType KMeansWorkloadAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + return Analyze(snapshot).workload_type; +} + +float KMeansWorkloadAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto result = Analyze(snapshot); + const auto it = std::find_if(result.keys.begin(), result.keys.end(), + [&object](const KeyPattern& key) { + return key.object == object; + }); + return it == result.keys.end() ? 0.0F : it->confidence; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/legacy_eviction_ops.cpp b/mooncake-store/src/io_pattern/legacy_eviction_ops.cpp new file mode 100644 index 0000000000..ea35682864 --- /dev/null +++ b/mooncake-store/src/io_pattern/legacy_eviction_ops.cpp @@ -0,0 +1,35 @@ +#include "io_pattern/legacy_eviction_ops.h" + +#include + +namespace mooncake::io_pattern { + +EvictionPlan LegacyEvictionOps::Evaluate(const PolicyContext& context, + CacheTier tier, + uint64_t target_bytes) const { + EvictionPlan plan{.source_tier = tier, .target_bytes = target_bytes}; + if (!strategy_ || target_bytes == 0) return plan; + std::lock_guard lock(mutex_); + for (const auto& key : context.snapshot.keys) { + if (key.replica_tiers & CacheTierBit(tier)) { + strategy_->AddKey(key.object.tenant_id.MakeScopedKey(key.object.key)); + } + } + uint64_t bytes = 0; + while (bytes < target_bytes) { + const auto scoped = strategy_->EvictKey(); + if (scoped.empty()) break; + auto [tenant, key] = TenantId::ParseScopedKey(scoped); + auto it = std::find_if(context.snapshot.keys.begin(), context.snapshot.keys.end(), + [&](const KeyMetrics& value) { + return value.object.tenant_id == tenant && + value.object.key == key; + }); + if (it == context.snapshot.keys.end()) continue; + plan.candidates.push_back({it->object, it->block_size, 0.0F}); + bytes += it->block_size; + } + return plan; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/observability.cpp b/mooncake-store/src/io_pattern/observability.cpp new file mode 100644 index 0000000000..bb920fb786 --- /dev/null +++ b/mooncake-store/src/io_pattern/observability.cpp @@ -0,0 +1,63 @@ +#include "io_pattern/observability.h" + +#include + +namespace mooncake::io_pattern { + +void IoPatternObservability::RecordCollectLatency(uint64_t value) { + std::lock_guard lock(mutex_); + values_.collect_latency_us = std::max(values_.collect_latency_us, value); +} + +void IoPatternObservability::RecordAnalyzeLatency(uint64_t value) { + std::lock_guard lock(mutex_); + values_.analyze_latency_us = std::max(values_.analyze_latency_us, value); +} + +void IoPatternObservability::RecordPolicyDecision(bool strategy_hit) { + std::lock_guard lock(mutex_); + ++values_.policy_decisions; + ++values_.strategy_trials; + if (strategy_hit) ++values_.strategy_hits; +} + +void IoPatternObservability::RecordFalsePositive() { + std::lock_guard lock(mutex_); + ++values_.false_positives; +} + +void IoPatternObservability::RecordDegrade() { + std::lock_guard lock(mutex_); + ++values_.degrade_count; +} + +void IoPatternObservability::RecordReportDrop(uint64_t count) { + std::lock_guard lock(mutex_); + values_.report_drop_count += count; +} + +IoPatternObservabilitySnapshot IoPatternObservability::Snapshot() const { + std::lock_guard lock(mutex_); + auto result = values_; + result.strategy_hit_rate = result.strategy_trials == 0 + ? 0.0F + : static_cast(result.strategy_hits) / + static_cast(result.strategy_trials); + result.false_positive_rate = result.strategy_trials == 0 + ? 0.0F + : static_cast(result.false_positives) / + static_cast(result.strategy_trials); + return result; +} + +IoPatternObservabilitySnapshot IoPatternObservability::Snapshot( + double window_seconds) const { + auto result = Snapshot(); + if (window_seconds > 0.0) { + result.policy_decision_qps = + static_cast(result.policy_decisions / window_seconds); + } + return result; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp new file mode 100644 index 0000000000..98e00f308c --- /dev/null +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -0,0 +1,213 @@ +#include "io_pattern/policy_strategies.h" + +#include +#include + +namespace mooncake::io_pattern { +namespace { + +const KeyPattern* FindPattern(const ObjectRef& object, + const PatternResult& result) { + const auto it = std::find_if( + result.keys.begin(), result.keys.end(), + [&object](const KeyPattern& pattern) { return pattern.object == object; }); + return it == result.keys.end() ? nullptr : &*it; +} + +const KeyMetrics* FindMetrics(const ObjectRef& object, + const IoPatternSnapshot& snapshot) { + const auto it = std::find_if( + snapshot.keys.begin(), snapshot.keys.end(), + [&object](const KeyMetrics& key) { return key.object == object; }); + return it == snapshot.keys.end() ? nullptr : &*it; +} + +bool HasLowerTierReplica(const KeyMetrics& key, CacheTier tier) { + const auto tier_index = static_cast(tier); + for (uint8_t index = tier_index + 1; + index <= static_cast(CacheTier::kL3NofSsd); ++index) { + if (key.replica_tiers & static_cast(1U << index)) { + return true; + } + } + return false; +} + +CacheTier TierDownTarget(CacheTier source, TierDownMode mode) { + if (source == CacheTier::kL3NofSsd) return CacheTier::kL3NofSsd; + if (mode == TierDownMode::kSkipHost && source == CacheTier::kL0Hbm) { + return CacheTier::kL2Segment; + } + // Prefix-affinity keeps the immediate next tier as the placement target; + // callers may co-locate grouped prefixes within that tier. + return static_cast(static_cast(source) + 1); +} + +} // namespace + +EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, + CacheTier tier, + uint64_t target_bytes) const { + EvictionPlan plan{.source_tier = tier, .target_bytes = target_bytes}; + uint64_t max_block_size = 0; + uint32_t max_other_replicas = 0; + for (const auto& key : context.snapshot.keys) { + if ((key.replica_tiers & CacheTierBit(tier)) == 0 || key.pinned) { + continue; + } + max_block_size = std::max(max_block_size, key.block_size); + max_other_replicas = std::max(max_other_replicas, + key.other_replica_count); + } + for (const auto& key : context.snapshot.keys) { + if ((key.replica_tiers & CacheTierBit(tier)) == 0 || key.pinned) { + continue; + } + const auto* pattern = FindPattern(key.object, context.analysis); + if (pattern == nullptr) { + continue; + } + const float normalized_block = max_block_size == 0 + ? 0.0F + : static_cast(key.block_size) / + static_cast(max_block_size); + const float normalized_other_replicas = + max_other_replicas == 0 + ? 0.0F + : static_cast(key.other_replica_count) / + static_cast(max_other_replicas); + float score = 0.0F; + switch (tier) { + case CacheTier::kL0Hbm: + score = config_.idle_weight * pattern->idle_score - + config_.frequency_weight * pattern->frequency_score - + config_.prefix_weight * pattern->prefix_score - + config_.recompute_weight * pattern->recompute_score; + break; + case CacheTier::kL1Host: + case CacheTier::kL2Segment: + score = config_.idle_weight * pattern->idle_score - + config_.frequency_weight * pattern->frequency_score + + config_.lower_replica_weight * + (HasLowerTierReplica(key, tier) ? 1.0F : 0.0F) - + config_.prefix_weight * pattern->prefix_score - + config_.recompute_weight * pattern->recompute_score; + break; + case CacheTier::kL3NofSsd: + score = config_.idle_weight * pattern->idle_score * + normalized_block - + config_.frequency_weight * pattern->frequency_score - + config_.recompute_weight * pattern->recompute_score + + config_.other_replica_weight * normalized_other_replicas; + break; + } + plan.candidates.push_back( + EvictionCandidate{.object = key.object, + .bytes = key.block_size, + .score = score, + .target_tier = TierDownTarget( + tier, config_.tier_down_mode)}); + } + std::sort(plan.candidates.begin(), plan.candidates.end(), + [](const EvictionCandidate& lhs, const EvictionCandidate& rhs) { + return lhs.score > rhs.score; + }); + if (target_bytes == 0) { + plan.candidates.clear(); + return plan; + } + if (config_.max_candidates != 0 && + plan.candidates.size() > config_.max_candidates) { + plan.candidates.resize(config_.max_candidates); + } + uint64_t selected_bytes = 0; + auto end = plan.candidates.begin(); + while (end != plan.candidates.end()) { + if (end->bytes > target_bytes - selected_bytes) { + break; + } + selected_bytes += end->bytes; + ++end; + } + plan.candidates.erase(end, plan.candidates.end()); + return plan; +} + +AdmissionResult PrefixMatchAdmissionOps::Evaluate( + const ObjectRef& object, CacheTier target_tier, + const PolicyContext& context) const { + AdmissionResult result{.object = object, .target_tier = target_tier}; + const auto* key = FindMetrics(object, context.snapshot); + if (key == nullptr) { + return result; + } + if (target_tier == CacheTier::kL0Hbm) { + result.decision = key->match_length >= config_.hbm_match_length + ? AdmissionDecision::kAdmit + : AdmissionDecision::kRejectPrefix; + const auto threshold = std::max(1U, config_.hbm_match_length); + result.confidence = key->match_length == 0 + ? 0.0F + : std::min(1.0F, static_cast( + key->match_length) / + threshold); + return result; + } + result.decision = key->access_count_window >= config_.frequency_threshold + ? AdmissionDecision::kAdmit + : AdmissionDecision::kRejectFrequency; + if (result.decision == AdmissionDecision::kAdmit && + !context.snapshot.storage.empty() && + context.snapshot.storage.front().memory_used_ratio >= + config_.max_memory_used_ratio) { + result.decision = AdmissionDecision::kRejectWatermark; + } + result.confidence = + result.decision == AdmissionDecision::kAdmit ? 1.0F : 0.0F; + return result; +} + +PrefetchPlan TraceBasedPrefetchOps::Evaluate( + const PolicyContext& context, const TraceHistory& trace) const { + PrefetchPlan plan{.strategy = config_.strategy, + .timeout_us = config_.timeout_us}; + std::unordered_map seen; + for (const auto& event : trace.events) { + if (!event.is_hit || + event.match_length <= config_.match_length_threshold || + seen.contains(event.object)) { + continue; + } + const auto* key = FindMetrics(event.object, context.snapshot); + if (key == nullptr) { + continue; + } + PrefetchCandidate candidate; + candidate.object = event.object; + candidate.bytes = key->block_size; + const auto* pattern = FindPattern(event.object, context.analysis); + candidate.confidence = pattern == nullptr ? 0.0F : pattern->confidence; + if (candidate.confidence < config_.minimum_confidence) { + continue; + } + if (key->replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) { + candidate.source_tier = CacheTier::kL3NofSsd; + candidate.target_tier = CacheTier::kL2Segment; + } else if (key->replica_tiers & CacheTierBit(CacheTier::kL2Segment)) { + candidate.source_tier = CacheTier::kL2Segment; + candidate.target_tier = CacheTier::kL1Host; + } else { + continue; + } + candidate.priority = static_cast(event.match_length); + plan.candidates.push_back(candidate); + seen.emplace(event.object, true); + if (config_.max_candidates != 0 && + plan.candidates.size() >= config_.max_candidates) { + break; + } + } + return plan; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/reporter.cpp b/mooncake-store/src/io_pattern/reporter.cpp new file mode 100644 index 0000000000..c8bf7f4fe8 --- /dev/null +++ b/mooncake-store/src/io_pattern/reporter.cpp @@ -0,0 +1,143 @@ +#include "io_pattern/reporter.h" + +#include + +namespace mooncake::io_pattern { + +IoPatternReporter::IoPatternReporter(size_t capacity, MetricBatchSink sink, + size_t per_tenant_capacity) + : capacity_(capacity), + sink_(std::move(sink)), + per_tenant_capacity_(per_tenant_capacity) {} + +IoPatternReporter::~IoPatternReporter() { Stop(); } + +void IoPatternReporter::Start() { + std::lock_guard lock(mutex_); + if (running_) return; + running_ = true; + worker_ = std::thread([this] { + std::unique_lock lock(mutex_); + while (running_) { + const size_t size = batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); + const auto interval = + (capacity_ == 0 || size * 2 >= capacity_) + ? std::chrono::milliseconds(100) + : (size == 0 ? std::chrono::milliseconds(1000) + : std::chrono::milliseconds(500)); + condition_.wait_for(lock, interval, [this] { return !running_; }); + if (!running_) break; + lock.unlock(); + Flush(); + lock.lock(); + } + }); +} + +void IoPatternReporter::Stop() { + { + std::lock_guard lock(mutex_); + if (!running_) return; + running_ = false; + } + condition_.notify_all(); + if (worker_.joinable()) worker_.join(); + Flush(); +} + +bool IoPatternReporter::Enqueue(InferenceMetrics metrics) { + const auto tenant = metrics.object.tenant_id; + return EnqueueImpl( + [value = std::move(metrics)](MetricBatch& batch) { + batch.inference.push_back(value); + }, + tenant); +} + +bool IoPatternReporter::EnqueueAccess(AccessRecord record) { + const auto tenant = record.object.tenant_id; + return EnqueueImpl( + [value = std::move(record)](MetricBatch& batch) { + batch.accesses.push_back(value); + }, + tenant); +} + +bool IoPatternReporter::EnqueueStorage(StorageMetric metric) { + return EnqueueImpl([value = std::move(metric)](MetricBatch& batch) { + batch.storage.push_back(value); + }, TenantId::Default()); +} + +bool IoPatternReporter::EnqueueImpl(std::function append, + const TenantId& tenant) { + std::lock_guard lock(mutex_); + const size_t size = batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); + if (!sink_ || size >= capacity_) { + ++dropped_; + return false; + } + if (per_tenant_capacity_ != 0 && + tenant_pending_[tenant] >= per_tenant_capacity_) { + ++dropped_; + return false; + } + append(batch_); + ++tenant_pending_[tenant]; + condition_.notify_one(); + return true; +} + +bool IoPatternReporter::Flush() { + MetricBatch outgoing; + { + std::lock_guard lock(mutex_); + if (batch_.inference.empty() && batch_.accesses.empty() && + batch_.storage.empty()) { + return true; + } + outgoing = std::move(batch_); + batch_ = {}; + tenant_pending_.clear(); + } + if (!sink_(outgoing)) { + std::lock_guard lock(mutex_); + ++dropped_; + return false; + } + std::lock_guard lock(mutex_); + reported_ += outgoing.inference.size() + outgoing.accesses.size() + + outgoing.storage.size(); + return true; +} + +size_t IoPatternReporter::pending() const { + std::lock_guard lock(mutex_); + return batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); +} + +uint64_t IoPatternReporter::dropped() const { + std::lock_guard lock(mutex_); + return dropped_; +} + +uint64_t IoPatternReporter::reported() const { + std::lock_guard lock(mutex_); + return reported_; +} + +std::chrono::milliseconds IoPatternReporter::RecommendedFlushInterval() const { + std::lock_guard lock(mutex_); + const size_t size = batch_.inference.size() + batch_.accesses.size() + + batch_.storage.size(); + if (capacity_ == 0 || size * 2 >= capacity_) { + return std::chrono::milliseconds(100); + } + if (size == 0) return std::chrono::milliseconds(1000); + return std::chrono::milliseconds(500); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/resilient_analyzer.cpp b/mooncake-store/src/io_pattern/resilient_analyzer.cpp new file mode 100644 index 0000000000..b850a37398 --- /dev/null +++ b/mooncake-store/src/io_pattern/resilient_analyzer.cpp @@ -0,0 +1,67 @@ +#include "io_pattern/resilient_analyzer.h" + +namespace mooncake::io_pattern { + +PatternResult ResilientAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + if (!primary_) return Fallback(); + try { + auto result = primary_->Analyze(snapshot); + RecordSuccess(result); + return result; + } catch (...) { + RecordFailure(); + return Fallback(); + } +} + +WorkloadType ResilientAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + return Analyze(snapshot).workload_type; +} + +float ResilientAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto result = Analyze(snapshot); + for (const auto& key : result.keys) { + if (key.object == object) return key.confidence; + } + return 0.0F; +} + +void ResilientAnalyzer::RecordFailure() const { + std::lock_guard lock(mutex_); + ++failures_; + if (failure_threshold_ != 0 && failures_ >= failure_threshold_) + degraded_ = true; +} + +void ResilientAnalyzer::RecordSuccess(const PatternResult& result) const { + std::lock_guard lock(mutex_); + last_result_ = result; + failures_ = 0; + degraded_ = false; +} + +PatternResult ResilientAnalyzer::Fallback() const { + std::lock_guard lock(mutex_); + if (!last_result_.keys.empty() || last_result_.workload_type != WorkloadType::kUnknown) + return last_result_; + PatternResult result; + result.workload_type = WorkloadType::kMixed; + return result; +} + +bool ResilientAnalyzer::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +size_t ResilientAnalyzer::failures() const { + std::lock_guard lock(mutex_); + return failures_; +} + +PatternResult ResilientAnalyzer::FallbackResult() const { return Fallback(); } + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp new file mode 100644 index 0000000000..4786cdad53 --- /dev/null +++ b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp @@ -0,0 +1,80 @@ +#include "io_pattern/resilient_cfm_channel.h" + +namespace mooncake::io_pattern { + +template +bool ResilientCfmChannel::Retry(Operation&& operation) { + if (!delegate_) { + RecordFailure(); + return false; + } + for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { + if (operation()) { + RecordSuccess(); + return true; + } + } + RecordFailure(); + return false; +} + +bool ResilientCfmChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { + return Retry([&] { return delegate_->SendSnapshot(snapshot); }); +} + +std::optional ResilientCfmChannel::PollPolicy() { + if (!delegate_) { + RecordFailure(); + return std::nullopt; + } + for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { + auto result = delegate_->PollPolicy(); + if (result.has_value()) { + RecordSuccess(); + return result; + } + } + RecordFailure(); + return std::nullopt; +} + +ErrorCode ResilientCfmChannel::ExecutePrefetch(const PrefetchPlan& plan) { + ErrorCode result = ErrorCode::RPC_FAIL; + if (!delegate_) { + RecordFailure(); + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { + result = delegate_->ExecutePrefetch(plan); + if (result == ErrorCode::OK) { + RecordSuccess(); + return result; + } + } + RecordFailure(); + return result; +} + +void ResilientCfmChannel::RecordSuccess() { + std::lock_guard lock(mutex_); + consecutive_failures_ = 0; + degraded_ = false; +} + +void ResilientCfmChannel::RecordFailure() { + std::lock_guard lock(mutex_); + ++consecutive_failures_; + if (consecutive_failures_ >= config_.degrade_after_failures) degraded_ = true; +} + +bool ResilientCfmChannel::degraded() const { + std::lock_guard lock(mutex_); + return degraded_; +} + +uint64_t ResilientCfmChannel::consecutive_failures() const { + std::lock_guard lock(mutex_); + return consecutive_failures_; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/rpc_transport.cpp b/mooncake-store/src/io_pattern/rpc_transport.cpp new file mode 100644 index 0000000000..429d894bae --- /dev/null +++ b/mooncake-store/src/io_pattern/rpc_transport.cpp @@ -0,0 +1,111 @@ +#include "io_pattern/rpc_transport.h" + +namespace mooncake::io_pattern { + +bool InProcessCfmRpcTransport::Authenticate(std::string_view token) { + std::lock_guard lock(mutex_); + authenticated_ = token == auth_token_; + return authenticated_; +} + +bool InProcessCfmRpcTransport::Send(std::string_view method, + std::string_view payload, + std::chrono::milliseconds) { + std::lock_guard lock(mutex_); + if (!authenticated_) return false; + return !send_handler_ || send_handler_(method, payload); +} + +std::optional InProcessCfmRpcTransport::Receive( + std::string_view method, std::chrono::milliseconds) { + std::lock_guard lock(mutex_); + if (!authenticated_ || method != "poll_policy" || policies_.empty()) { + return std::nullopt; + } + auto payload = std::move(policies_.front()); + policies_.pop(); + return payload; +} + +void InProcessCfmRpcTransport::EnqueuePolicy(std::string payload) { + std::lock_guard lock(mutex_); + policies_.push(std::move(payload)); +} + +void InProcessCfmRpcTransport::SetSendHandler(SendHandler handler) { + std::lock_guard lock(mutex_); + send_handler_ = std::move(handler); +} + +bool CfmRpcChannel::EnsureAuthenticated() { + std::lock_guard lock(authentication_mutex_); + if (authenticated_) return true; + authenticated_ = transport_ && transport_->Authenticate(config_.auth_token); + return authenticated_; +} + +bool CfmRpcChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { + if (!transport_ || !codec_ || !EnsureAuthenticated()) return false; + return transport_->Send("report_snapshot", codec_->EncodeSnapshot(snapshot), + config_.timeout); +} + +std::optional CfmRpcChannel::PollPolicy() { + if (!transport_ || !codec_ || !EnsureAuthenticated()) return std::nullopt; + const auto payload = transport_->Receive("poll_policy", config_.timeout); + return payload ? codec_->DecodePolicy(*payload) : std::nullopt; +} + +ErrorCode CfmRpcChannel::ExecutePrefetch(const PrefetchPlan& plan) { + if (!transport_ || !codec_ || !EnsureAuthenticated()) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + return transport_->Send("execute_prefetch", codec_->EncodePrefetch(plan), + config_.timeout) + ? ErrorCode::OK + : ErrorCode::RPC_TIMEOUT; +} + +bool CfmRpcChannel::SendMetricBatch(const MetricBatch& batch) { + if (!transport_ || !codec_ || !EnsureAuthenticated()) return false; + return transport_->Send("report_metric_batch", codec_->EncodeMetricBatch(batch), + config_.timeout); +} + +std::shared_ptr CfmChannelPool::Next() const { + if (channels_.empty()) return nullptr; + const auto index = next_.fetch_add(1, std::memory_order_relaxed) % + channels_.size(); + return channels_[index]; +} + +bool CfmChannelPool::SendSnapshot(const IoPatternSnapshot& snapshot) { + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (channel && channel->SendSnapshot(snapshot)) return true; + } + return false; +} + +std::optional CfmChannelPool::PollPolicy() { + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (!channel) continue; + auto command = channel->PollPolicy(); + if (command) return command; + } + return std::nullopt; +} + +ErrorCode CfmChannelPool::ExecutePrefetch(const PrefetchPlan& plan) { + ErrorCode last_error = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (!channel) continue; + last_error = channel->ExecutePrefetch(plan); + if (last_error == ErrorCode::OK) return last_error; + } + return last_error; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp new file mode 100644 index 0000000000..f2a41f2cae --- /dev/null +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -0,0 +1,261 @@ +#include "io_pattern/runtime.h" + +#include +#include +#include + +namespace mooncake::io_pattern { + +IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) + : config_(config), + executor_(std::move(handlers.eviction), std::move(handlers.prefetch), + std::move(handlers.admission)), + feedback_(config.feedback_window) { + // A runtime always has an OOM guard even when a caller omits collector + // limits. The same bound is used by the bounded analyzer below. + if (config_.collector.max_total_keys == 0) { + config_.collector.max_total_keys = config_.max_analysis_keys; + } + if (config_.report_sink) { + reporter_ = std::make_shared( + config_.report_capacity, config_.report_sink, + config_.report_per_tenant_capacity); + reporter_->Start(); + } + collector_ = std::make_shared(config_.collector, + reporter_); + auto sliding = std::make_shared( + config.analysis_window_ns); + analyzer_ = std::make_shared(std::move(sliding)); + workload_policy_ = std::make_shared(); + std::shared_ptr legacy_strategy; + if (config_.legacy_fallback == LegacyFallback::kFifo) { + legacy_strategy = std::make_shared(); + } else { + legacy_strategy = std::make_shared(); + } + auto fallback = std::make_shared( + std::make_shared(std::move(legacy_strategy)), + nullptr, std::make_shared()); + policy_ = std::make_shared(workload_policy_, fallback); +} + +IoPatternRuntime::~IoPatternRuntime() { + if (reporter_) reporter_->Stop(); +} + +void IoPatternRuntime::ReportInferenceMetrics(const InferenceMetrics& metrics) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->ReportInferenceMetrics(metrics); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) + observability_.RecordReportDrop(dropped_after - dropped_before); +} + +void IoPatternRuntime::RecordAccess(const std::string& key, + const AccessRecord& record) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->RecordAccess(key, record); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) + observability_.RecordReportDrop(dropped_after - dropped_before); + + PolicyFeedbackSample feedback; + bool has_feedback = false; + { + std::lock_guard lock(feedback_state_mutex_); + ++feedback_accesses_; + feedback_hits_ += record.is_hit; + ObjectRef object = record.object; + if (!key.empty()) object.key = key; + if (pending_prefetches_.erase(object) != 0) { + feedback.prefetch_accuracy = record.is_hit ? 1.0F : 0.0F; + has_feedback = true; + if (!record.is_hit) observability_.RecordFalsePositive(); + } + // A completed 64-access window is a stable, bounded source of actual + // hit-rate deltas. TTFT remains supplied by the inference bridge via + // the public RecordFeedback API. + if (feedback_accesses_ >= 64) { + const auto hit_rate = static_cast(feedback_hits_) / + static_cast(feedback_accesses_); + feedback.hit_rate_delta = hit_rate - previous_hit_rate_; + previous_hit_rate_ = hit_rate; + feedback_accesses_ = 0; + feedback_hits_ = 0; + has_feedback = true; + } + } + if (has_feedback) RecordFeedback(feedback); +} + +void IoPatternRuntime::RecordStorageMetric(const StorageMetric& metric) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->RecordStorageMetric(metric); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) + observability_.RecordReportDrop(dropped_after - dropped_before); +} + +void IoPatternRuntime::MergeSnapshot(const IoPatternSnapshot& snapshot) { + const auto start = std::chrono::steady_clock::now(); + const auto dropped_before = collector_->dropped(); + collector_->MergeSnapshot(snapshot); + observability_.RecordCollectLatency( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + const auto dropped_after = collector_->dropped(); + if (dropped_after > dropped_before) { + observability_.RecordReportDrop(dropped_after - dropped_before); + } +} + +PatternResult IoPatternRuntime::AnalyzeWithinBudget( + const IoPatternSnapshot& snapshot, bool& degraded) { + degraded = config_.max_analysis_keys != 0 && + snapshot.keys.size() > config_.max_analysis_keys; + if (degraded || analysis_in_flight_->exchange(true, std::memory_order_acq_rel)) { + degraded = true; + return analyzer_->FallbackResult(); + } + + std::promise promise; + auto result = promise.get_future(); + auto analyzer = analyzer_; + auto in_flight = analysis_in_flight_; + std::thread([analyzer = std::move(analyzer), snapshot, + promise = std::move(promise), in_flight]() mutable { + try { + promise.set_value(analyzer->Analyze(snapshot)); + } catch (...) { + promise.set_value(analyzer->FallbackResult()); + } + in_flight->store(false, std::memory_order_release); + }).detach(); + + if (result.wait_for(std::chrono::microseconds(config_.analysis_timeout_us)) == + std::future_status::ready) { + return result.get(); + } + degraded = true; + return analyzer_->FallbackResult(); +} + +PolicyExecutionStatus IoPatternRuntime::Execute( + CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, + const std::vector& admissions, const std::string& session_id) { + const auto snapshot = collector_->GetSnapshot(); + const auto start = std::chrono::steady_clock::now(); + bool analysis_degraded = false; + const auto analysis = AnalyzeWithinBudget(snapshot, analysis_degraded); + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count(); + observability_.RecordAnalyzeLatency(elapsed); + + workload_policy_->SetWorkloadType(analysis.workload_type); + workload_policy_->SetSessionWorkloads(analysis.sessions); + workload_policy_->AdvanceTransitionWindow(); + const PolicyResult result = policy_->ExecutePolicy( + PolicyContext{.snapshot = snapshot, .analysis = analysis, + .session_id = session_id}, eviction_tier, + eviction_bytes, trace, admissions); + auto status = executor_.Execute(result); + status.degraded = status.degraded || result.degraded || collector_->degraded() || + analysis_degraded || + elapsed > static_cast(config_.analysis_timeout_us); + observability_.RecordPolicyDecision(!result.eviction.candidates.empty() || + !result.prefetch.candidates.empty()); + const bool failed = status.eviction != ErrorCode::OK || + status.prefetch != ErrorCode::OK || status.degraded; + if (failed) policy_->RecordFailure(); + else policy_->RecordSuccess(); + if (status.degraded || policy_->degraded()) observability_.RecordDegrade(); + status.degraded = status.degraded || policy_->degraded(); + + PolicyFeedbackSample feedback; + bool has_feedback = false; + { + std::lock_guard lock(feedback_state_mutex_); + for (const auto& candidate : result.prefetch.candidates) { + if (config_.max_pending_prefetches == 0 || + pending_prefetches_.size() < config_.max_pending_prefetches) { + pending_prefetches_.insert(candidate.object); + } + } + if (!result.prefetch.candidates.empty() && status.prefetch != ErrorCode::OK) { + feedback.prefetch_accuracy = 0.0F; + has_feedback = true; + } + if (!snapshot.keys.empty() && !result.eviction.candidates.empty()) { + feedback.eviction_churn = static_cast( + result.eviction.candidates.size()) / + static_cast(snapshot.keys.size()); + has_feedback = true; + } + } + if (has_feedback) RecordFeedback(feedback); + return status; +} + +ErrorCode IoPatternRuntime::ExecuteCommand(const PolicyCommand& command) { + PolicyResult result; + if (const auto* eviction = std::get_if(&command)) { + result.eviction = *eviction; + } else if (const auto* prefetch = std::get_if(&command)) { + result.prefetch = *prefetch; + } else { + result.admissions.push_back(std::get(command)); + } + const auto status = executor_.Execute(result); + if (status.degraded) { + observability_.RecordDegrade(); + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + if (const auto* eviction = std::get_if(&command)) { + return status.eviction; + } + if (const auto* prefetch = std::get_if(&command)) { + return status.prefetch; + } + return status.admissions.empty() ? ErrorCode::OK : status.admissions.front(); +} + +void IoPatternRuntime::RecordFeedback(PolicyFeedbackSample sample) { + feedback_.Record(sample); + auto config = workload_policy_->CurrentEvictionConfig(); + if (tuner_.Tune(feedback_.Snapshot(), config)) { + workload_policy_->ApplyEvictionTuning(config); + } +} + +IoPatternSnapshot IoPatternRuntime::Snapshot() const { + return collector_->GetSnapshot(); +} + +IoPatternObservabilitySnapshot IoPatternRuntime::ObservabilitySnapshot( + double window_seconds) const { + return observability_.Snapshot(window_seconds); +} + +bool IoPatternRuntime::degraded() const { + return collector_->degraded() || analyzer_->degraded() || policy_->degraded(); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp new file mode 100644 index 0000000000..ed992fc45e --- /dev/null +++ b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp @@ -0,0 +1,92 @@ +#include "io_pattern/sliding_window_analyzer.h" + +#include +#include + +namespace mooncake::io_pattern { +namespace { +template +T Percentile(std::vector values, size_t rank) { + if (values.empty()) return 0; + std::sort(values.begin(), values.end()); + return values[std::min(rank, values.size() - 1)]; +} +} + +void SlidingWindowAnalyzer::Append(const IoPatternSnapshot& snapshot) const { + std::lock_guard lock(mutex_); + if (history_.empty() || + history_.back().generated_at_ns != snapshot.generated_at_ns) { + history_.push_back(snapshot); + } + const uint64_t cutoff = snapshot.generated_at_ns > window_ns_ + ? snapshot.generated_at_ns - window_ns_ + : 0; + while (!history_.empty() && history_.front().generated_at_ns < cutoff) + history_.pop_front(); +} + +IoPatternSnapshot SlidingWindowAnalyzer::Aggregate( + const IoPatternSnapshot& current) const { + Append(current); + std::lock_guard lock(mutex_); + IoPatternSnapshot aggregate = current; + aggregate.keys.clear(); + for (const auto& snapshot : history_) { + aggregate.keys.insert(aggregate.keys.end(), snapshot.keys.begin(), + snapshot.keys.end()); + } + return aggregate; +} + +PatternResult SlidingWindowAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + const auto aggregate = Aggregate(snapshot); + auto result = analyzer_.Analyze(aggregate); + return result.workload_type == WorkloadType::kMixed + ? kmeans_.Analyze(aggregate) + : result; +} + +WorkloadType SlidingWindowAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + return Analyze(snapshot).workload_type; +} + +float SlidingWindowAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto result = Analyze(snapshot); + const auto it = std::find_if(result.keys.begin(), result.keys.end(), + [&object](const KeyPattern& key) { + return key.object == object; + }); + return it == result.keys.end() ? 0.0F : it->confidence; +} + +WorkloadFeatureStats SlidingWindowAnalyzer::FeatureStats() const { + std::lock_guard lock(mutex_); + std::vector tokens, fanouts, matches, frequencies; + std::vector blocks; + for (const auto& snapshot : history_) { + for (const auto& key : snapshot.keys) { + tokens.push_back(key.token_count); + fanouts.push_back(key.prefix_fanout); + matches.push_back(key.match_length); + frequencies.push_back(static_cast(key.access_count_window)); + blocks.push_back(key.block_size); + } + } + const auto p90 = [](size_t size) { return size == 0 ? 0 : (size * 9) / 10; }; + WorkloadFeatureStats stats; + stats.samples = tokens.size(); + stats.token_median = Percentile(tokens, tokens.size() / 2); + stats.token_p90 = Percentile(tokens, p90(tokens.size())); + stats.fanout_p90 = Percentile(fanouts, p90(fanouts.size())); + stats.block_median = Percentile(blocks, blocks.size() / 2); + stats.block_p90 = Percentile(blocks, p90(blocks.size())); + stats.match_p90 = Percentile(matches, p90(matches.size())); + stats.frequency_median = Percentile(frequencies, frequencies.size() / 2); + return stats; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/threshold_analyzer.cpp b/mooncake-store/src/io_pattern/threshold_analyzer.cpp new file mode 100644 index 0000000000..f84d8a2c1b --- /dev/null +++ b/mooncake-store/src/io_pattern/threshold_analyzer.cpp @@ -0,0 +1,134 @@ +#include "io_pattern/threshold_analyzer.h" + +#include + +namespace mooncake::io_pattern { +namespace { + +bool IsCodeAgent(const KeyMetrics& key, const ThresholdAnalyzerConfig& config) { + return key.token_count > config.code_agent_token_count && + key.prefix_fanout > config.code_agent_prefix_fanout && + key.match_length > config.code_agent_match_length; +} + +bool IsRecommendation(const KeyMetrics& key, + const ThresholdAnalyzerConfig& config) { + return key.block_size < config.recommendation_block_size && + key.access_count_window > config.recommendation_frequency; +} + +bool IsConversation(const KeyMetrics& key, + const ThresholdAnalyzerConfig& config) { + return key.prefix_fanout > config.conversation_prefix_fanout && + key.match_length > config.conversation_match_length; +} + +float RuleConfidence(const KeyMetrics& key, + const ThresholdAnalyzerConfig& config) { + float score = 0.0F; + if (IsCodeAgent(key, config)) score = std::max(score, 1.0F); + if (IsRecommendation(key, config)) score = std::max(score, 1.0F); + if (IsConversation(key, config)) score = std::max(score, 1.0F); + // A partial match is useful to policies, but must not look like a + // definitive workload classification. + if (score == 0.0F) { + const float code = std::min( + {static_cast(key.token_count) / + std::max(1.0F, static_cast(config.code_agent_token_count)), + static_cast(key.prefix_fanout) / + std::max(1.0F, static_cast(config.code_agent_prefix_fanout)), + static_cast(key.match_length) / + std::max(1.0F, static_cast(config.code_agent_match_length))}); + const float recommendation = std::min( + static_cast(config.recommendation_block_size) / + std::max(1.0F, static_cast(key.block_size)), + static_cast(key.access_count_window) / + std::max(1.0F, static_cast(config.recommendation_frequency))); + const float conversation = std::min( + static_cast(key.prefix_fanout) / + std::max(1.0F, static_cast(config.conversation_prefix_fanout)), + static_cast(key.match_length) / + std::max(1.0F, static_cast(config.conversation_match_length))); + score = std::clamp(std::max({code, recommendation, conversation}), + 0.0F, 1.0F); + } + return score; +} + +} // namespace + +PatternResult ThresholdAnalyzer::Analyze( + const IoPatternSnapshot& snapshot) const { + PatternResult result; + result.workload_type = DetectWorkloadType(snapshot); + if (!snapshot.keys.empty()) { + float total = 0.0F; + for (const auto& key : snapshot.keys) total += KeyConfidence(key); + result.workload_confidence = + std::clamp(total / static_cast(snapshot.keys.size()), 0.0F, + 1.0F); + } + result.keys.reserve(snapshot.keys.size()); + for (const auto& key : snapshot.keys) { + KeyPattern pattern; + pattern.object = key.object; + pattern.confidence = KeyConfidence(key); + pattern.frequency_score = std::min( + 1.0F, static_cast(key.access_count_window) / 20.0F); + pattern.idle_score = std::min( + 1.0F, static_cast(key.idle_time_us) / 1'000'000.0F); + pattern.prefix_score = std::min( + 1.0F, static_cast(key.match_length) / 256.0F); + pattern.recompute_score = std::min(1.0F, key.recompute_cost); + pattern.transfer_roi = key.transfer_eta_us == 0 + ? 0.0F + : key.recompute_cost / + static_cast(key.transfer_eta_us); + pattern.migration_safe = !key.pinned; + result.keys.push_back(pattern); + } + return result; +} + +WorkloadType ThresholdAnalyzer::DetectWorkloadType( + const IoPatternSnapshot& snapshot) const { + if (snapshot.keys.empty()) { + return WorkloadType::kMixed; + } + uint32_t code_agents = 0; + uint32_t recommendations = 0; + uint32_t conversations = 0; + for (const auto& key : snapshot.keys) { + code_agents += IsCodeAgent(key, config_); + recommendations += IsRecommendation(key, config_); + conversations += IsConversation(key, config_); + } + const uint32_t matched = static_cast(code_agents != 0) + + static_cast(recommendations != 0) + + static_cast(conversations != 0); + if (matched > 1) return WorkloadType::kMixed; + if (code_agents != 0) return WorkloadType::kCodeAgent; + if (recommendations != 0) return WorkloadType::kGenerativeRecommendation; + if (conversations != 0) return WorkloadType::kMultiTurnConversation; + return WorkloadType::kMixed; +} + +float ThresholdAnalyzer::CalculateConfidence( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto* key = FindKey(object, snapshot); + return key == nullptr ? 0.0F : KeyConfidence(*key); +} + +const KeyMetrics* ThresholdAnalyzer::FindKey( + const ObjectRef& object, const IoPatternSnapshot& snapshot) const { + const auto it = std::find_if( + snapshot.keys.begin(), snapshot.keys.end(), + [&object](const KeyMetrics& key) { return key.object == object; }); + return it == snapshot.keys.end() ? nullptr : &*it; +} + +float ThresholdAnalyzer::KeyConfidence(const KeyMetrics& key) const { + return RuleConfidence(key, config_); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/tier_executor.cpp b/mooncake-store/src/io_pattern/tier_executor.cpp new file mode 100644 index 0000000000..02ba4371de --- /dev/null +++ b/mooncake-store/src/io_pattern/tier_executor.cpp @@ -0,0 +1,35 @@ +#include "io_pattern/tier_executor.h" + +namespace mooncake::io_pattern { + +PolicyExecutionStatus TierOperationExecutor::Execute( + const PolicyResult& result) const { + PolicyExecutionStatus status; + if (eviction_ && (!result.eviction.candidates.empty() || + result.eviction.target_bytes != 0)) { + status.eviction = eviction_(result.eviction); + } else if (!result.eviction.candidates.empty() || + result.eviction.target_bytes != 0) { + status.eviction = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + status.degraded = true; + } + if (prefetch_ && !result.prefetch.candidates.empty()) { + status.prefetch = prefetch_(result.prefetch); + } else if (!result.prefetch.candidates.empty()) { + status.prefetch = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + status.degraded = true; + } + for (const auto& admission : result.admissions) { + if (admission_ && admission.decision == AdmissionDecision::kAdmit) { + status.admissions.push_back(admission_(admission)); + } else if (admission.decision == AdmissionDecision::kAdmit) { + status.admissions.push_back(ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); + status.degraded = true; + } else { + status.admissions.push_back(ErrorCode::OK); + } + } + return status; +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 03d890f099..e16e698570 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -59,6 +59,7 @@ #include "master_snapshot_repository.h" #include "ha_metric_manager.h" #include "metadata_store.h" +#include "io_pattern/runtime.h" namespace mooncake { @@ -436,6 +437,51 @@ MasterService::MasterService(const MasterServiceConfig& config) << ")"; } + io_pattern_runtime_ = std::make_unique( + io_pattern::IoPatternRuntime::Handlers{ + .eviction = [this](const io_pattern::EvictionPlan& plan) { + bool evicted = plan.candidates.empty(); + std::unordered_map targets; + for (const auto& candidate : plan.candidates) { + targets[candidate.object.tenant_id] += candidate.bytes; + } + for (const auto& [tenant, bytes] : targets) { + const auto result = EvictTenantMemoryForQuota(tenant, bytes); + evicted = evicted || result.freed_bytes != 0; + } + return evicted ? ErrorCode::OK : ErrorCode::OBJECT_NOT_FOUND; + }, + .prefetch = [this](const io_pattern::PrefetchPlan& plan) { + for (const auto& candidate : plan.candidates) { + // Store's safe promotion primitive is LOCAL_DISK -> MEMORY; + // HBM remains inference-runtime-owned and is never promoted + // from the master control plane. + if (candidate.target_tier == io_pattern::CacheTier::kL0Hbm) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + const ObjectIdentity object_id{candidate.object.tenant_id, + candidate.object.key}; + if (TryPushPromotionQueue(object_id, + /*record_candidate=*/false) != + PromotionQueueResult::kQueued) { + return ErrorCode::OBJECT_NOT_FOUND; + } + } + return ErrorCode::OK; + }, + .admission = [this](const io_pattern::AdmissionResult& result) { + if (result.target_tier == io_pattern::CacheTier::kL0Hbm) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + const ObjectIdentity object_id{result.object.tenant_id, + result.object.key}; + return TryPushPromotionQueue(object_id, + /*record_candidate=*/false) == + PromotionQueueResult::kQueued + ? ErrorCode::OK + : ErrorCode::OBJECT_NOT_FOUND; + }}); + kv_event_publisher_ = std::make_unique(BuildKvEventConfig(config)); @@ -4307,6 +4353,8 @@ auto MasterService::GetReplicaListLocal(const ObjectIdentity& object_id) GetReplicaListResponse resp({}, default_kv_lease_ttl_); bool promotion_eligible = false; + io_pattern::AccessRecord io_access; + bool record_io_access = false; { MetadataAccessorRO accessor(this, object_id); @@ -4384,11 +4432,26 @@ auto MasterService::GetReplicaListLocal(const ObjectIdentity& object_id) resp = GetReplicaListResponse(std::move(replica_list), default_kv_lease_ttl_, metadata.object_checksum); + io_access.object = {object_id.tenant_id, key}; + io_access.observed_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + io_access.block_size = metadata.size; + io_access.tier = resp.replicas[0].is_memory_replica() + ? io_pattern::CacheTier::kL1Host + : io_pattern::CacheTier::kL3NofSsd; + io_access.operation = io_pattern::IoOperation::kGet; + io_access.is_hit = true; + record_io_access = true; } // RO accessor released. Safe to take a fresh RW accessor now. if (promotion_eligible) { TryPushPromotionQueue(object_id); } + if (record_io_access && io_pattern_runtime_) { + io_pattern_runtime_->RecordAccess(key, io_access); + } return resp; } @@ -4539,6 +4602,7 @@ MasterService::BatchGetReplicaListLocal(const std::vector& keys, } std::vector promotion_candidates; + std::vector io_accesses; std::shared_lock shared_lock(snapshot_mutex_); { MetadataShardAccessorRO shard(this, shard_idx); @@ -4619,12 +4683,29 @@ MasterService::BatchGetReplicaListLocal(const std::vector& keys, results[original_idx] = GetReplicaListResponse( std::move(replica_list), default_kv_lease_ttl_, metadata.object_checksum); + io_accesses.push_back( + {.object = {normalized_tenant, key}, + .observed_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()), + .block_size = metadata.size, + .tier = results[original_idx]->replicas[0].is_memory_replica() + ? io_pattern::CacheTier::kL1Host + : io_pattern::CacheTier::kL3NofSsd, + .operation = io_pattern::IoOperation::kGet, + .is_hit = true}); } } for (const auto& object_id : promotion_candidates) { TryPushPromotionQueue(object_id); } + if (io_pattern_runtime_) { + for (const auto& access : io_accesses) { + io_pattern_runtime_->RecordAccess(access.object.key, access); + } + } } return results; @@ -5324,6 +5405,22 @@ auto MasterService::PutEnd(const UUID& client_id, const ObjectMeta& object_meta, metadata.GrantLease(0, default_kv_soft_pin_ttl_); PublishKvStored(key, replica_type, metadata, object_id.tenant_id); + if (io_pattern_runtime_) { + io_pattern_runtime_->RecordAccess( + key, {.object = {object_id.tenant_id, key}, + .observed_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()), + .block_size = metadata.size, + .tier = replica_type == ReplicaType::MEMORY + ? io_pattern::CacheTier::kL1Host + : io_pattern::CacheTier::kL3NofSsd, + .operation = io_pattern::IoOperation::kPut, + .is_hit = true, + .write_batch_size = 1}); + } + if (enable_oplog_ && ordered_oplog_writer_) { std::string payload = SerializeMetadataForOpLog(metadata); auto result = AppendOpLogVisibleBeforeDurable( @@ -8813,6 +8910,16 @@ void MasterService::EvictionThreadFunc() { double evict_ratio_lowerbound = std::max(evict_ratio_target * 0.5, used_ratio - eviction_high_watermark_ratio_); + if (io_pattern_runtime_) { + io_pattern_runtime_->RecordStorageMetric( + {.source_id = "master-memory", .tier = io_pattern::CacheTier::kL1Host, + .memory_used_ratio = static_cast(used_ratio)}); + const auto capacity = std::max( + 0, MasterMetricManager::instance().get_total_mem_capacity()); + io_pattern_runtime_->Execute( + io_pattern::CacheTier::kL1Host, + static_cast(evict_ratio_target * capacity), {}); + } BatchEvict(evict_ratio_target, evict_ratio_lowerbound); LOG(INFO) << "[EVICT-DONE] BatchEvict execution completed."; last_discard_time = now; @@ -9415,7 +9522,7 @@ MasterService::TenantQuotaEvictionResult MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, uint64_t target_bytes) { TenantQuotaEvictionResult total; - if (!enable_multi_tenants_ || target_bytes == 0) { + if (target_bytes == 0) { return total; } diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index e35e2c6dd7..b5836d306f 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1,9 +1,12 @@ #include "io_pattern/io_pattern.h" +#include "io_pattern/threshold_analyzer.h" +#include "io_pattern/policy_strategies.h" #include #include #include #include +#include #include @@ -25,7 +28,7 @@ class TestCollector final : public IoPatternCollector { void ReportInferenceMetrics(const InferenceMetrics& metrics) override { inference_metrics = metrics; } - void RecordAccess(const AccessRecord& record) override { + void RecordAccess(const std::string&, const AccessRecord& record) override { access_record = record; } void RecordStorageMetric(const StorageMetric& metric) override { @@ -57,6 +60,19 @@ class TestAnalyzer final : public IoPatternAnalyzer { .workload_confidence = 0.75F}; }; +class ThrowingAnalyzer final : public IoPatternAnalyzer { + public: + PatternResult Analyze(const IoPatternSnapshot&) const override { + throw std::runtime_error("analysis failure"); + } + WorkloadType DetectWorkloadType(const IoPatternSnapshot&) const override { + throw std::runtime_error("analysis failure"); + } + float CalculateConfidence(const ObjectRef&, const IoPatternSnapshot&) const override { + throw std::runtime_error("analysis failure"); + } +}; + class TestPrefetchOps final : public PrefetchOps { public: PrefetchPlan Evaluate(const PolicyContext&, @@ -87,11 +103,75 @@ class TestPrefetchExecutor final : public PrefetchExecutor { PrefetchPlan plan; }; +class TestCfmChannel final : public CfmChannel { + public: + bool SendSnapshot(const IoPatternSnapshot& value) override { + snapshot = value; + return send_ok; + } + std::optional PollPolicy() override { return policy; } + ErrorCode ExecutePrefetch(const PrefetchPlan& value) override { + plan = value; + return execute_code; + } + bool send_ok{true}; + ErrorCode execute_code{ErrorCode::OK}; + IoPatternSnapshot snapshot; + std::optional policy; + PrefetchPlan plan; +}; + +class FlakyCfmChannel final : public CfmChannel { + public: + bool SendSnapshot(const IoPatternSnapshot&) override { + return send_failures-- <= 0; + } + std::optional PollPolicy() override { return PrefetchPlan{}; } + ErrorCode ExecutePrefetch(const PrefetchPlan&) override { + return ErrorCode::RPC_FAIL; + } + int send_failures{0}; +}; + +class TestRpcTransport final : public CfmRpcTransport { + public: + bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) override { + last_method = std::string(method); + last_payload = std::string(payload); + last_timeout = timeout; + return send_ok; + } + std::optional Receive(std::string_view method, + std::chrono::milliseconds timeout) override { + last_method = std::string(method); + last_timeout = timeout; + return response; + } + bool send_ok{true}; + std::optional response; + std::string last_method; + std::string last_payload; + std::chrono::milliseconds last_timeout{0}; +}; + +class TestRpcCodec final : public CfmRpcCodec { + public: + std::string EncodeSnapshot(const IoPatternSnapshot&) const override { return "snapshot"; } + std::string EncodePrefetch(const PrefetchPlan&) const override { return "prefetch"; } + std::string EncodeMetricBatch(const MetricBatch&) const override { + return "batch"; + } + std::optional DecodePolicy(const std::string& value) const override { + return value == "policy" ? std::optional(PrefetchPlan{}) + : std::nullopt; + } +}; + TEST(IoPatternFrameworkTest, PublicSeamsRemainAbstract) { static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); - static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); static_assert(std::is_abstract_v); @@ -144,7 +224,7 @@ TEST(IoPatternFrameworkTest, CollectorAndAnalyzerExposeValueFlow) { access_record.object = collector.inference_metrics.object; access_record.observed_at_ns = 43; access_record.is_hit = true; - collector.RecordAccess(access_record); + collector.RecordAccess("key", access_record); collector.RecordStorageMetric(StorageMetric{.source_id = "segment-1"}); const auto snapshot = collector.GetSnapshot(); @@ -160,6 +240,268 @@ TEST(IoPatternFrameworkTest, CollectorAndAnalyzerExposeValueFlow) { EXPECT_EQ(analyzer.DetectWorkloadType(snapshot), WorkloadType::kMixed); } +TEST(IoPatternFrameworkTest, CollectorImplAggregatesAndIsolatesTenants) { + IoPatternCollectorImpl collector; + AccessRecord access; + access.object = {TenantId("tenant-a"), "ignored"}; + access.observed_at_ns = 200; + access.block_size = 4096; + access.tier = CacheTier::kL1Host; + access.is_hit = true; + collector.RecordAccess("shared-key", access); + access.object.tenant_id = TenantId("tenant-b"); + access.observed_at_ns = 100; + collector.RecordAccess("shared-key", access); + + const auto snapshot = collector.GetSnapshot(); + ASSERT_EQ(snapshot.keys.size(), 2); + EXPECT_EQ(snapshot.keys[0].object.tenant_id.value(), "tenant-a"); + EXPECT_EQ(snapshot.keys[1].object.tenant_id.value(), "tenant-b"); + EXPECT_EQ(snapshot.keys[0].object.key, "shared-key"); +} + +TEST(IoPatternFrameworkTest, CollectorImplKeepsLatestStorageObservation) { + IoPatternCollectorImpl collector; + collector.RecordStorageMetric(StorageMetric{.source_id = "segment", + .observed_at_ns = 20, + .used_bytes = 200}); + collector.RecordStorageMetric(StorageMetric{.source_id = "segment", + .observed_at_ns = 10, + .used_bytes = 100}); + const auto snapshot = collector.GetSnapshot(); + ASSERT_EQ(snapshot.storage.size(), 1); + EXPECT_EQ(snapshot.storage.front().used_bytes, 200); +} + +TEST(IoPatternFrameworkTest, CollectorImplEnforcesPerTenantKeyQuota) { + IoPatternCollectorImpl collector( + IoPatternCollectorImpl::Config{.max_keys_per_tenant = 1}); + InferenceMetrics first; + first.object = {TenantId("tenant-a"), "first"}; + collector.ReportInferenceMetrics(first); + InferenceMetrics second; + second.object = {TenantId("tenant-a"), "second"}; + collector.ReportInferenceMetrics(second); + InferenceMetrics other_tenant; + other_tenant.object = {TenantId("tenant-b"), "second"}; + collector.ReportInferenceMetrics(other_tenant); + EXPECT_EQ(collector.GetSnapshot().keys.size(), 2); + EXPECT_EQ(collector.dropped(), 1); +} + +TEST(IoPatternFrameworkTest, CollectorImplDegradesAtGlobalKeyLimit) { + IoPatternCollectorImpl collector( + IoPatternCollectorImpl::Config{.max_total_keys = 1}); + InferenceMetrics first; + first.object = {TenantId("tenant-a"), "first"}; + collector.ReportInferenceMetrics(first); + InferenceMetrics second; + second.object = {TenantId("tenant-b"), "second"}; + collector.ReportInferenceMetrics(second); + EXPECT_TRUE(collector.degraded()); + EXPECT_EQ(collector.dropped(), 1); + EXPECT_EQ(collector.GetSnapshot().keys.size(), 1); + collector.RecordStorageMetric(StorageMetric{.source_id = "segment"}); + EXPECT_EQ(collector.GetSnapshot().storage.size(), 1); +} + +TEST(IoPatternFrameworkTest, CollectorImplFeedsReporterWithoutInlineTransport) { + MetricBatch received; + auto reporter = std::make_shared(4, [&](const MetricBatch& batch) { + received = batch; + return true; + }); + IoPatternCollectorImpl collector({}, reporter); + collector.ReportInferenceMetrics(InferenceMetrics{}); + collector.RecordStorageMetric(StorageMetric{}); + EXPECT_EQ(reporter->pending(), 2); + EXPECT_TRUE(collector.FlushReports()); + EXPECT_EQ(received.inference.size(), 1); + EXPECT_EQ(received.storage.size(), 1); +} + +TEST(IoPatternFrameworkTest, CollectorImplDerivesWritePathMetrics) { + IoPatternCollectorImpl collector; + AccessRecord write; + write.object = {TenantId("tenant-a"), "write-key"}; + write.operation = IoOperation::kPut; + write.block_size = 4096; + write.write_batch_size = 32; + collector.RecordAccess("write-key", write); + write.overwrite = true; + collector.RecordAccess("write-key", write); + const auto snapshot = collector.GetSnapshot(); + const auto& key = snapshot.keys.front(); + EXPECT_EQ(key.write_frequency, 2); + EXPECT_EQ(key.write_batch_size, 32); + EXPECT_EQ(key.write_object_size, 4096); + EXPECT_FLOAT_EQ(key.overwrite_ratio, 0.5F); + EXPECT_TRUE(key.write_burst); +} + +TEST(IoPatternFrameworkTest, ThresholdAnalyzerClassifiesDocumentedWorkloads) { + ThresholdAnalyzer analyzer; + IoPatternSnapshot code_agent; + KeyMetrics code_key; + code_key.object = {TenantId("tenant-a"), "code"}; + code_key.token_count = 20 * 1024; + code_key.prefix_fanout = 20; + code_key.match_length = 512; + code_agent.keys.push_back(code_key); + EXPECT_EQ(analyzer.DetectWorkloadType(code_agent), + WorkloadType::kCodeAgent); + + IoPatternSnapshot recommendation; + KeyMetrics recommendation_key; + recommendation_key.object = {TenantId("tenant-a"), "recommendation"}; + recommendation_key.block_size = 64 * 1024; + recommendation_key.access_count_window = 30; + recommendation.keys.push_back(recommendation_key); + EXPECT_EQ(analyzer.DetectWorkloadType(recommendation), + WorkloadType::kGenerativeRecommendation); +} + +TEST(IoPatternFrameworkTest, ThresholdAnalyzerFallsBackToMixed) { + ThresholdAnalyzer analyzer; + IoPatternSnapshot snapshot; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "unknown"}; + snapshot.keys.push_back(key); + + const auto result = analyzer.Analyze(snapshot); + EXPECT_EQ(result.workload_type, WorkloadType::kMixed); + EXPECT_FLOAT_EQ(result.workload_confidence, 0.0F); + ASSERT_EQ(result.keys.size(), 1); + EXPECT_EQ(result.keys.front().object.key, "unknown"); + EXPECT_FLOAT_EQ(analyzer.CalculateConfidence(key.object, snapshot), 0.0F); +} + +TEST(IoPatternFrameworkTest, ThresholdAnalyzerReportsMixedAndPartialConfidence) { + ThresholdAnalyzer analyzer; + IoPatternSnapshot snapshot; + KeyMetrics code; + code.object = {TenantId("tenant-a"), "code"}; + code.token_count = 20 * 1024; + code.prefix_fanout = 20; + code.match_length = 512; + snapshot.keys.push_back(code); + KeyMetrics recommendation; + recommendation.object = {TenantId("tenant-b"), "recommendation"}; + recommendation.block_size = 64 * 1024; + recommendation.access_count_window = 30; + snapshot.keys.push_back(recommendation); + EXPECT_EQ(analyzer.DetectWorkloadType(snapshot), WorkloadType::kMixed); + EXPECT_FLOAT_EQ(analyzer.CalculateConfidence( + {TenantId("tenant-a"), "code"}, snapshot), + 1.0F); + KeyMetrics partial; + partial.object = {TenantId("tenant-c"), "partial"}; + partial.token_count = 8 * 1024; + snapshot.keys.push_back(partial); + EXPECT_GT(analyzer.CalculateConfidence(partial.object, snapshot), 0.0F); +} + +TEST(IoPatternFrameworkTest, ScoreEvictionSelectsColdObjectsWithinBudget) { + ScoreBasedEvictionOps eviction; + PolicyContext context; + KeyMetrics cold; + cold.object = {TenantId("tenant-a"), "cold"}; + cold.block_size = 100; + cold.replica_tiers = CacheTierBit(CacheTier::kL1Host); + KeyMetrics hot = cold; + hot.object.key = "hot"; + hot.block_size = 100; + context.snapshot.keys = {cold, hot}; + context.analysis.keys = { + {.object = cold.object, .frequency_score = 0.1F, .idle_score = 0.9F}, + {.object = hot.object, .frequency_score = 0.9F, .idle_score = 0.1F}, + }; + + const auto plan = eviction.Evaluate(context, CacheTier::kL1Host, 100); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().object.key, "cold"); + EXPECT_EQ(plan.candidates.front().bytes, 100); +} + +TEST(IoPatternFrameworkTest, ScoreEvictionSkipsPinnedAndZeroBudget) { + ScoreBasedEvictionOps eviction; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "pinned"}; + key.block_size = 1; + key.pinned = true; + key.replica_tiers = CacheTierBit(CacheTier::kL1Host); + context.snapshot.keys.push_back(key); + context.analysis.keys.push_back( + KeyPattern{.object = key.object, .frequency_score = 0.0F}); + + EXPECT_TRUE(eviction.Evaluate(context, CacheTier::kL1Host, 1024) + .candidates.empty()); + key.pinned = false; + context.snapshot.keys.front() = key; + EXPECT_TRUE(eviction.Evaluate(context, CacheTier::kL1Host, 0) + .candidates.empty()); +} + +TEST(IoPatternFrameworkTest, PrefixAdmissionUsesTierSpecificSignals) { + PrefixMatchAdmissionOps admission; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "prefix"}; + key.access_count_window = 10; + key.match_length = 64; + context.snapshot.keys.push_back(key); + + const auto hbm = admission.Evaluate(key.object, CacheTier::kL0Hbm, context); + EXPECT_EQ(hbm.decision, AdmissionDecision::kAdmit); + + key.match_length = 1; + context.snapshot.keys.front() = key; + const auto rejected = + admission.Evaluate(key.object, CacheTier::kL0Hbm, context); + EXPECT_EQ(rejected.decision, AdmissionDecision::kRejectPrefix); +} + +TEST(IoPatternFrameworkTest, TracePrefetchPlansOnlyLongPrefixMatches) { + TraceBasedPrefetchOps prefetch; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "block"}; + key.block_size = 4096; + key.replica_tiers = CacheTierBit(CacheTier::kL3NofSsd); + context.snapshot.keys.push_back(key); + + TraceHistory trace; + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 512, .is_hit = true}); + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 8, .is_hit = true}); + + const auto plan = prefetch.Evaluate(context, trace); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().source_tier, CacheTier::kL3NofSsd); + EXPECT_EQ(plan.candidates.front().target_tier, CacheTier::kL2Segment); + EXPECT_EQ(plan.candidates.front().bytes, 4096); +} + +TEST(IoPatternFrameworkTest, TracePrefetchDeduplicatesObjects) { + TraceBasedPrefetchOps prefetch; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "block"}; + key.block_size = 128; + key.replica_tiers = CacheTierBit(CacheTier::kL2Segment); + context.snapshot.keys.push_back(key); + + TraceHistory trace; + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 300, .is_hit = true}); + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 400, .is_hit = true}); + + EXPECT_EQ(prefetch.Evaluate(context, trace).candidates.size(), 1); +} + TEST(IoPatternFrameworkTest, PolicyContextCarriesRawAndDerivedViews) { PolicyContext context; context.snapshot.generated_at_ns = 123; @@ -242,5 +584,614 @@ TEST(IoPatternFrameworkTest, ComposedEngineDelegatesAndDegradesSafely) { EXPECT_EQ(executor.plan.strategy, PrefetchStrategy::kWaitComplete); } +TEST(IoPatternFrameworkTest, WorkloadPolicyEngineSelectsAndTransitionsTemplates) { + WorkloadPolicyEngine engine(WorkloadType::kCodeAgent, 3); + EXPECT_EQ(engine.ActiveWorkload(), WorkloadType::kCodeAgent); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 1.0F); + + engine.SetWorkloadType(WorkloadType::kGenerativeRecommendation); + EXPECT_EQ(engine.ActiveWorkload(), + WorkloadType::kGenerativeRecommendation); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 0.0F); + engine.AdvanceTransitionWindow(); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 1.0F / 3.0F); + engine.AdvanceTransitionWindow(); + engine.AdvanceTransitionWindow(); + EXPECT_FLOAT_EQ(engine.TransitionProgress(), 1.0F); + + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "item"}; + key.block_size = 64 * 1024; + key.access_count_window = 30; + context.snapshot.keys.push_back(key); + TraceHistory trace; + trace.events.push_back( + TraceEvent{.object = key.object, .match_length = 64, .is_hit = true}); + EXPECT_EQ(engine.PlanPrefetch(context, trace).strategy, + PrefetchStrategy::kWaitComplete); +} + +TEST(IoPatternFrameworkTest, UnifiedPolicyResultSeamDelegates) { + IoPatternSnapshot snapshot; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "key"}; + snapshot.keys.push_back(key); + ComposedPolicyEngine engine(std::make_shared(), + std::make_shared(), + std::make_shared()); + PolicyContext context; + context.snapshot = snapshot; + const auto result = engine.ExecutePolicy( + context, CacheTier::kL1Host, 1024, {}, {key.object}); + EXPECT_EQ(result.admissions.size(), 1); + EXPECT_EQ(result.admissions.front().object, key.object); +} + +TEST(IoPatternFrameworkTest, RegistryPolicyEngineResolvesNamedOps) { + auto registries = std::make_shared(); + ASSERT_TRUE(registries->eviction.Register( + "score", [] { return std::make_shared(); })); + ASSERT_TRUE(registries->prefetch.Register( + "trace", [] { return std::make_shared(); })); + ASSERT_TRUE(registries->admission.Register( + "prefix", [] { return std::make_shared(); })); + RegistryPolicyEngine engine(registries, "score", "trace", "prefix"); + const ObjectRef object{TenantId("tenant-a"), "key"}; + const auto result = engine.ExecutePolicy({}, CacheTier::kL1Host, 1024, {}, + {object}); + ASSERT_EQ(result.admissions.size(), 1); + EXPECT_EQ(result.admissions.front().object, object); + + RegistryPolicyEngine missing(registries, "missing", "trace", "prefix"); + EXPECT_TRUE(missing.ExecutePolicy({}, CacheTier::kL1Host, 0, {}).degraded); +} + +TEST(IoPatternFrameworkTest, ReporterBatchesBoundsAndCountsDrops) { + MetricBatch received; + IoPatternReporter reporter(2, [&](const MetricBatch& batch) { + received = batch; + return true; + }); + EXPECT_TRUE(reporter.Enqueue(InferenceMetrics{})); + EXPECT_TRUE(reporter.EnqueueStorage(StorageMetric{})); + EXPECT_FALSE(reporter.EnqueueAccess(AccessRecord{})); + EXPECT_EQ(reporter.dropped(), 1); + EXPECT_EQ(reporter.pending(), 2); + EXPECT_TRUE(reporter.Flush()); + EXPECT_EQ(reporter.pending(), 0); + EXPECT_EQ(reporter.reported(), 2); + EXPECT_EQ(received.inference.size(), 1); + EXPECT_EQ(received.storage.size(), 1); +} + +TEST(IoPatternFrameworkTest, ReporterAdaptsFlushIntervalToLoad) { + IoPatternReporter reporter(4, [](const MetricBatch&) { return true; }); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(1000)); + reporter.Enqueue(InferenceMetrics{}); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(500)); + reporter.Enqueue(InferenceMetrics{}); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(100)); +} + +TEST(IoPatternFrameworkTest, ReporterEnforcesPerTenantFairness) { + IoPatternReporter reporter(4, [](const MetricBatch&) { return true; }, 1); + InferenceMetrics first; + first.object = {TenantId("tenant-a"), "a"}; + InferenceMetrics second = first; + second.object.key = "b"; + InferenceMetrics other = first; + other.object.tenant_id = TenantId("tenant-b"); + EXPECT_TRUE(reporter.Enqueue(first)); + EXPECT_FALSE(reporter.Enqueue(second)); + EXPECT_TRUE(reporter.Enqueue(other)); + EXPECT_EQ(reporter.dropped(), 1); +} + +TEST(IoPatternFrameworkTest, CfmClientDelegatesToTransportChannel) { + auto channel = std::make_shared(); + CfmClientImpl client(channel); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = 42; + EXPECT_EQ(client.ReportSnapshot(snapshot), ErrorCode::OK); + EXPECT_EQ(channel->snapshot.generated_at_ns, 42); + channel->policy = PrefetchPlan{}; + EXPECT_TRUE(client.PollPolicy().has_value()); + EXPECT_EQ(client.ExecutePrefetch(PrefetchPlan{}), ErrorCode::OK); + channel->send_ok = false; + EXPECT_EQ(client.ReportSnapshot(snapshot), ErrorCode::RPC_FAIL); + CfmClientImpl unavailable(nullptr); + EXPECT_EQ(unavailable.ReportSnapshot(snapshot), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); +} + +TEST(IoPatternFrameworkTest, CfmClientDispatchesReceivedPolicyCommands) { + auto channel = std::make_shared(); + int dispatched = 0; + CfmClientImpl client(channel, [&](const PolicyCommand& command) { + EXPECT_TRUE(std::holds_alternative(command)); + ++dispatched; + return ErrorCode::OK; + }); + + EXPECT_EQ(client.ReceivePolicy(PolicyCommand{PrefetchPlan{}}), ErrorCode::OK); + EXPECT_EQ(dispatched, 1); + channel->policy = PolicyCommand{AdmissionResult{}}; + EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); + EXPECT_EQ(dispatched, 2); +} + +TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { + auto flaky = std::make_shared(); + flaky->send_failures = 2; + ResilientCfmChannel channel(flaky, CfmRetryConfig{.max_retries = 2, + .degrade_after_failures = 2}); + EXPECT_TRUE(channel.SendSnapshot({})); + EXPECT_FALSE(channel.degraded()); + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::RPC_FAIL); + EXPECT_FALSE(channel.degraded()); + EXPECT_EQ(channel.consecutive_failures(), 1); + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::RPC_FAIL); + EXPECT_EQ(channel.consecutive_failures(), 2); + EXPECT_TRUE(channel.degraded()); +} + +TEST(IoPatternFrameworkTest, ResilientAnalyzerFallsBackAfterFailure) { + ResilientAnalyzer analyzer(std::make_shared(), 2); + EXPECT_EQ(analyzer.DetectWorkloadType({}), WorkloadType::kMixed); + EXPECT_FALSE(analyzer.degraded()); + EXPECT_EQ(analyzer.DetectWorkloadType({}), WorkloadType::kMixed); + EXPECT_TRUE(analyzer.degraded()); + EXPECT_EQ(analyzer.failures(), 2); +} + +TEST(IoPatternFrameworkTest, RpcChannelUsesCodecTransportAndTimeout) { + auto transport = std::make_shared(); + auto codec = std::make_shared(); + CfmRpcChannel channel(transport, codec, CfmRpcConfig{.timeout = std::chrono::milliseconds(25)}); + EXPECT_TRUE(channel.SendSnapshot({})); + EXPECT_EQ(transport->last_method, "report_snapshot"); + EXPECT_EQ(transport->last_payload, "snapshot"); + EXPECT_EQ(transport->last_timeout, std::chrono::milliseconds(25)); + transport->response = "policy"; + EXPECT_TRUE(channel.PollPolicy().has_value()); + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::OK); + auto rpc_channel = std::make_shared(transport, codec); + IoPatternReporter reporter(2, MakeCfmMetricBatchSink(rpc_channel)); + reporter.Enqueue(InferenceMetrics{}); + EXPECT_TRUE(reporter.Flush()); + EXPECT_EQ(transport->last_method, "report_metric_batch"); + EXPECT_EQ(transport->last_payload, "batch"); + transport->send_ok = false; + EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::RPC_TIMEOUT); +} + +TEST(IoPatternFrameworkTest, BinaryCfmCodecRoundTripsAllPolicyCommands) { + CfmBinaryCodec codec; + PrefetchPlan prefetch{.strategy = PrefetchStrategy::kTimeout, + .timeout_us = 42, + .candidates = {PrefetchCandidate{ + .object = {TenantId("tenant-a"), "key"}, + .source_tier = CacheTier::kL3NofSsd, + .target_tier = CacheTier::kL2Segment, + .bytes = 512, + .priority = 0.8F, + .confidence = 0.9F}}}; + const auto decoded_prefetch = codec.DecodePolicy(codec.EncodePolicy(prefetch)); + ASSERT_TRUE(decoded_prefetch.has_value()); + const auto& decoded_plan = std::get(*decoded_prefetch); + ASSERT_EQ(decoded_plan.candidates.size(), 1); + EXPECT_EQ(decoded_plan.candidates.front().object.key, "key"); + EXPECT_EQ(decoded_plan.timeout_us, 42); + + AdmissionResult admission{.object = {TenantId("tenant-b"), "admit"}, + .target_tier = CacheTier::kL1Host, + .decision = AdmissionDecision::kAdmit, + .confidence = 0.75F}; + const auto decoded_admission = codec.DecodePolicy(codec.EncodePolicy(admission)); + ASSERT_TRUE(decoded_admission.has_value()); + EXPECT_EQ(std::get(*decoded_admission).object.key, "admit"); + + EvictionPlan eviction{.source_tier = CacheTier::kL1Host, + .target_bytes = 128, + .candidates = {EvictionCandidate{ + .object = {TenantId("tenant-c"), "evict"}, + .bytes = 128, + .score = 0.4F}}}; + const auto decoded_eviction = codec.DecodePolicy(codec.EncodePolicy(eviction)); + ASSERT_TRUE(decoded_eviction.has_value()); + EXPECT_EQ(std::get(*decoded_eviction).candidates.front().object.key, + "evict"); + + IoPatternSnapshot snapshot{.generated_at_ns = 9, + .keys = {KeyMetrics{.object = {TenantId("tenant-d"), "full"}, + .session_id = "session", + .token_count = 16, + .active = true}}, + .storage = {StorageMetric{.source_id = "master", + .used_bytes = 42}}}; + const auto decoded_snapshot = codec.DecodeSnapshot(codec.EncodeSnapshot(snapshot)); + ASSERT_TRUE(decoded_snapshot.has_value()); + EXPECT_EQ(decoded_snapshot->keys.front().session_id, "session"); + EXPECT_EQ(decoded_snapshot->storage.front().used_bytes, 42); + + MetricBatch batch{.inference = {InferenceMetrics{.object = {TenantId("tenant"), "metric"}, + .session_id = "s"}}, + .accesses = {AccessRecord{.object = {TenantId("tenant"), "metric"}, + .is_hit = true}}}; + const auto decoded_batch = codec.DecodeMetricBatch(codec.EncodeMetricBatch(batch)); + ASSERT_TRUE(decoded_batch.has_value()); + EXPECT_EQ(decoded_batch->inference.front().session_id, "s"); + EXPECT_TRUE(decoded_batch->accesses.front().is_hit); +} + +TEST(IoPatternFrameworkTest, InProcessCfmTransportAuthenticatesAndDispatches) { + CfmBinaryCodec codec; + bool received_snapshot = false; + auto transport = std::make_shared( + "shared-secret", [&received_snapshot](std::string_view method, + std::string_view) { + received_snapshot = method == "report_snapshot"; + return received_snapshot; + }); + CfmRpcChannel authorized(transport, std::make_shared(), + {.auth_token = "shared-secret"}); + EXPECT_TRUE(authorized.SendSnapshot({})); + EXPECT_TRUE(received_snapshot); + + transport->EnqueuePolicy(codec.EncodePolicy( + AdmissionResult{.object = {TenantId("tenant"), "key"}, + .decision = AdmissionDecision::kAdmit})); + ASSERT_TRUE(authorized.PollPolicy().has_value()); + + auto rejected = std::make_shared("secret"); + CfmRpcChannel unauthorized(rejected, std::make_shared(), + {.auth_token = "wrong"}); + EXPECT_FALSE(unauthorized.SendSnapshot({})); +} + +TEST(IoPatternFrameworkTest, CfmIngressFeedsRuntimeFromMetricBatches) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{.eviction = [](const EvictionPlan&) { + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { + return ErrorCode::OK; + }, + .admission = [](const AdmissionResult&) { + return ErrorCode::OK; + }}); + auto codec = std::make_shared(); + CfmIngress ingress(runtime, codec); + MetricBatch batch{.inference = {InferenceMetrics{ + .object = {TenantId("tenant"), "metric-key"}, + .session_id = "session", + .token_count = 32}}, + .accesses = {AccessRecord{ + .object = {TenantId("tenant"), "metric-key"}, + .block_size = 64, + .is_hit = true}}}; + EXPECT_TRUE(ingress.Handle("report_metric_batch", codec->EncodeMetricBatch(batch))); + const auto snapshot = runtime->Snapshot(); + ASSERT_EQ(snapshot.keys.size(), 1); + EXPECT_EQ(snapshot.keys.front().session_id, "session"); + EXPECT_EQ(snapshot.keys.front().access_count_window, 1); +} + +TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { + size_t batches = 0; + IoPatternReporter reporter(4, [&](const MetricBatch&) { + ++batches; + return true; + }); + reporter.Enqueue(InferenceMetrics{}); + reporter.Start(); + reporter.Stop(); + EXPECT_EQ(batches, 1); +} + +TEST(IoPatternFrameworkTest, DegradingPolicyEngineSwitchesToFallback) { + auto primary = std::make_shared( + std::make_shared(), nullptr, nullptr); + auto fallback = std::make_shared(nullptr, nullptr, + nullptr); + DegradingPolicyEngine engine(primary, fallback, 2); + EXPECT_FALSE(engine.degraded()); + EXPECT_EQ(engine.PlanEviction({}, CacheTier::kL1Host, 10).target_bytes, 10); + engine.RecordFailure(); + engine.RecordFailure(); + EXPECT_TRUE(engine.degraded()); + EXPECT_TRUE(engine.PlanEviction({}, CacheTier::kL1Host, 10) + .candidates.empty()); + engine.ForceDegraded(false); + EXPECT_FALSE(engine.degraded()); + EXPECT_EQ(engine.consecutive_failures(), 0); +} + +TEST(IoPatternFrameworkTest, FeedbackWindowAggregatesBoundedSamples) { + PolicyFeedbackWindow window(2); + window.Record({.hit_rate_delta = -0.2F, .prefetch_accuracy = 0.5F}); + window.Record({.hit_rate_delta = 0.1F, .prefetch_accuracy = 0.9F}); + window.Record({.hit_rate_delta = -0.4F, .prefetch_accuracy = 0.3F}); + const auto stats = window.Snapshot(); + EXPECT_EQ(stats.samples, 2); + EXPECT_FLOAT_EQ(stats.hit_rate_delta, (-0.2F - 0.4F) / 2.0F); + EXPECT_FLOAT_EQ(stats.prefetch_accuracy, (0.9F + 0.3F) / 2.0F); +} + +TEST(IoPatternFrameworkTest, AdaptiveTunerChangesWeightsAfterNegativeStreak) { + AdaptivePolicyTuner tuner(3); + ScoreBasedEvictionConfig config; + EXPECT_FALSE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); + EXPECT_FALSE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); + EXPECT_TRUE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); + EXPECT_FLOAT_EQ(config.frequency_weight, 0.8F); + EXPECT_FLOAT_EQ(config.idle_weight, 1.1F); + EXPECT_FALSE(tuner.Tune({.hit_rate_delta = 0.0F}, config)); +} + +TEST(IoPatternFrameworkTest, AdaptiveTunerHandlesChurnAndPersistsChanges) { + AdaptivePolicyTuner tuner(3); + ScoreBasedEvictionConfig config; + bool persisted = false; + tuner.SetPersistenceCallback( + [&](const ScoreBasedEvictionConfig&) { persisted = true; }); + EXPECT_TRUE(tuner.Tune({.eviction_churn = 0.8F}, config)); + EXPECT_TRUE(tuner.conservative()); + EXPECT_TRUE(persisted); +} + +TEST(IoPatternFrameworkTest, ObservabilityTracksPolicyAndDegradeCounters) { + IoPatternObservability metrics; + metrics.RecordCollectLatency(10); + metrics.RecordCollectLatency(3); + metrics.RecordAnalyzeLatency(20); + metrics.RecordPolicyDecision(true); + metrics.RecordPolicyDecision(false); + metrics.RecordFalsePositive(); + metrics.RecordDegrade(); + metrics.RecordReportDrop(2); + const auto snapshot = metrics.Snapshot(); + EXPECT_EQ(snapshot.collect_latency_us, 10); + EXPECT_EQ(snapshot.analyze_latency_us, 20); + EXPECT_EQ(snapshot.policy_decisions, 2); + EXPECT_EQ(snapshot.strategy_hits, 1); + EXPECT_EQ(snapshot.strategy_trials, 2); + EXPECT_EQ(snapshot.false_positives, 1); + EXPECT_EQ(snapshot.degrade_count, 1); + EXPECT_EQ(snapshot.report_drop_count, 2); + EXPECT_FLOAT_EQ(snapshot.strategy_hit_rate, 0.5F); + EXPECT_FLOAT_EQ(snapshot.false_positive_rate, 0.5F); + EXPECT_FLOAT_EQ(metrics.Snapshot(2.0).policy_decision_qps, 1.0F); +} + +TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { + SlidingWindowAnalyzer analyzer(100); + IoPatternSnapshot first; + first.generated_at_ns = 10; + first.keys.push_back(KeyMetrics{.token_count = 20 * 1024, + .prefix_fanout = 20, + .match_length = 512, + .block_size = 100, + .access_count_window = 1}); + IoPatternSnapshot second; + second.generated_at_ns = 50; + second.keys.push_back(KeyMetrics{.token_count = 30, + .prefix_fanout = 20, + .match_length = 300, + .block_size = 300, + .access_count_window = 5}); + EXPECT_EQ(analyzer.DetectWorkloadType(second), WorkloadType::kMixed); + const auto stats = analyzer.FeatureStats(); + EXPECT_EQ(stats.samples, 2); + EXPECT_EQ(stats.token_median, 30); + EXPECT_EQ(stats.fanout_p90, 20); + EXPECT_EQ(stats.block_p90, 300); +} + +TEST(IoPatternFrameworkTest, KMeansFallbackLabelsIndependentSessions) { + SlidingWindowAnalyzer analyzer; + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = 1; + snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "long"}, + .session_id = "code-session", + .token_count = 20 * 1024, + .prefix_fanout = 20, + .match_length = 512}, + KeyMetrics{.object = {TenantId("tenant-b"), "small"}, + .session_id = "recommendation-session", + .block_size = 64 * 1024, + .access_count_window = 30}, + }; + + const auto result = analyzer.Analyze(snapshot); + EXPECT_EQ(result.workload_type, WorkloadType::kMixed); + ASSERT_EQ(result.sessions.size(), 2); + EXPECT_NE(result.sessions[0].workload_type, result.sessions[1].workload_type); +} + +TEST(IoPatternFrameworkTest, TierExecutorBridgesPolicyResults) { + int evictions = 0; + int prefetches = 0; + int admissions = 0; + TierOperationExecutor executor( + [&](const EvictionPlan&) { ++evictions; return ErrorCode::OK; }, + [&](const PrefetchPlan&) { ++prefetches; return ErrorCode::OK; }, + [&](const AdmissionResult&) { ++admissions; return ErrorCode::OK; }); + PolicyResult result; + result.eviction.target_bytes = 1024; + result.prefetch.candidates.push_back(PrefetchCandidate{}); + result.admissions.push_back( + AdmissionResult{.decision = AdmissionDecision::kAdmit}); + const auto status = executor.Execute(result); + EXPECT_EQ(status.eviction, ErrorCode::OK); + EXPECT_EQ(status.prefetch, ErrorCode::OK); + ASSERT_EQ(status.admissions.size(), 1); + EXPECT_EQ(status.admissions.front(), ErrorCode::OK); + EXPECT_EQ(evictions, 1); + EXPECT_EQ(prefetches, 1); + EXPECT_EQ(admissions, 1); + + TierOperationExecutor degraded({}, {}, {}); + const auto degraded_status = degraded.Execute(result); + EXPECT_TRUE(degraded_status.degraded); + EXPECT_EQ(degraded_status.prefetch, + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); +} + +TEST(IoPatternFrameworkTest, LegacyEvictionAdapterUsesLruFallback) { + auto lru = std::make_shared(); + LegacyEvictionOps fallback(lru); + PolicyContext context; + KeyMetrics first{.object = {TenantId("tenant-a"), "first"}, + .block_size = 10, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}; + KeyMetrics second{.object = {TenantId("tenant-a"), "second"}, + .block_size = 20, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}; + context.snapshot.keys = {first, second}; + const auto plan = fallback.Evaluate(context, CacheTier::kL1Host, 10); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().object.key, "first"); +} + +TEST(IoPatternFrameworkTest, ScoreBasedEvictionUsesTierSpecificSignals) { + PolicyContext context; + context.snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "small-single-copy"}, + .block_size = 10, + .other_replica_count = 0, + .replica_tiers = CacheTierBit(CacheTier::kL3NofSsd)}, + KeyMetrics{.object = {TenantId("tenant-a"), "large-redundant"}, + .block_size = 100, + .other_replica_count = 1, + .replica_tiers = CacheTierBit(CacheTier::kL3NofSsd)}, + }; + context.analysis.keys = { + KeyPattern{.object = context.snapshot.keys[0].object, .idle_score = 1.0F}, + KeyPattern{.object = context.snapshot.keys[1].object, .idle_score = 1.0F}, + }; + + ScoreBasedEvictionOps eviction; + const auto plan = eviction.Evaluate(context, CacheTier::kL3NofSsd, 110); + + ASSERT_EQ(plan.candidates.size(), 2); + EXPECT_EQ(plan.candidates.front().object.key, "large-redundant"); + EXPECT_EQ(plan.candidates.front().target_tier, CacheTier::kL3NofSsd); +} + +TEST(IoPatternFrameworkTest, TierDownTemplatesChooseDocumentedTargets) { + PolicyContext context; + context.snapshot.keys = {KeyMetrics{ + .object = {TenantId("tenant"), "key"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL0Hbm)}}; + context.analysis.keys = {KeyPattern{.object = context.snapshot.keys.front().object}}; + + ScoreBasedEvictionOps code({.tier_down_mode = TierDownMode::kSkipHost}); + EXPECT_EQ(code.Evaluate(context, CacheTier::kL0Hbm, 64) + .candidates.front().target_tier, + CacheTier::kL2Segment); + ScoreBasedEvictionOps recommendation( + {.tier_down_mode = TierDownMode::kStepwise}); + EXPECT_EQ(recommendation.Evaluate(context, CacheTier::kL0Hbm, 64) + .candidates.front().target_tier, + CacheTier::kL1Host); +} + +TEST(IoPatternFrameworkTest, PrefetchRequiresConfidenceAndNeverPromotesToHbm) { + PolicyContext context; + context.snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "low-confidence"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL3NofSsd)}, + KeyMetrics{.object = {TenantId("tenant-a"), "host-only"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}, + }; + context.analysis.keys = { + KeyPattern{.object = context.snapshot.keys[0].object, .confidence = 0.5F}, + KeyPattern{.object = context.snapshot.keys[1].object, .confidence = 0.9F}, + }; + TraceHistory trace{.events = { + TraceEvent{.object = context.snapshot.keys[0].object, + .match_length = 512, + .is_hit = true}, + TraceEvent{.object = context.snapshot.keys[1].object, + .match_length = 512, + .is_hit = true}, + }}; + + TraceBasedPrefetchOps prefetch; + const auto plan = prefetch.Evaluate(context, trace); + + EXPECT_TRUE(plan.candidates.empty()); +} + +TEST(IoPatternFrameworkTest, RuntimeConnectsCollectionAnalysisPolicyAndHandlers) { + int evictions = 0; + int prefetches = 0; + int admissions = 0; + IoPatternRuntime runtime( + IoPatternRuntime::Handlers{ + .eviction = [&](const EvictionPlan&) { + ++evictions; + return ErrorCode::OK; + }, + .prefetch = [&](const PrefetchPlan&) { + ++prefetches; + return ErrorCode::OK; + }, + .admission = [&](const AdmissionResult&) { + ++admissions; + return ErrorCode::OK; + }, + }); + + AccessRecord access{.object = {TenantId("tenant-a"), "runtime-key"}, + .observed_at_ns = 1, + .block_size = 64, + .tier = CacheTier::kL2Segment, + .is_hit = true}; + runtime.RecordAccess(access.object.key, access); + runtime.ReportInferenceMetrics( + InferenceMetrics{.object = access.object, .match_length = 512}); + + const auto status = runtime.Execute(CacheTier::kL2Segment, 64, + TraceHistory{}, {access.object}); + + EXPECT_EQ(status.eviction, ErrorCode::OK); + ASSERT_EQ(status.admissions.size(), 1); + EXPECT_EQ(status.admissions.front(), ErrorCode::OK); + EXPECT_EQ(evictions, 1); + EXPECT_EQ(admissions, 1); + EXPECT_GE(runtime.Snapshot().keys.size(), 1); +} + +TEST(IoPatternFrameworkTest, RuntimeExecutesCfmCommandsThroughStorageHandlers) { + int admissions = 0; + IoPatternRuntime runtime( + {.eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [&admissions](const AdmissionResult&) { + ++admissions; + return ErrorCode::OK; + }}); + CfmClientImpl client( + std::make_shared(), + [&runtime](const PolicyCommand& command) { + return runtime.ExecuteCommand(command); + }); + EXPECT_EQ(client.ReceivePolicy( + AdmissionResult{.object = {TenantId("tenant"), "key"}, + .decision = AdmissionDecision::kAdmit}), + ErrorCode::OK); + EXPECT_EQ(admissions, 1); +} + } // namespace } // namespace mooncake::io_pattern diff --git a/mooncake-wheel/mooncake/io_pattern_bridge.py b/mooncake-wheel/mooncake/io_pattern_bridge.py new file mode 100644 index 0000000000..cd0c46d729 --- /dev/null +++ b/mooncake-wheel/mooncake/io_pattern_bridge.py @@ -0,0 +1,81 @@ +"""Framework-neutral, non-blocking CFM metric bridges. + +The vLLM connector accepts these objects through ``vllm_config``. SGLang's +HiCache integration can instantiate :class:`SglangHiCacheIoPatternBridge` at +its request-finished and prefix-match hooks without depending on vLLM. +""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from queue import Empty, Full, Queue +from threading import Event, Thread +from typing import Any + + +MetricSink = Callable[[Mapping[str, Any]], None] + + +class BatchedIoPatternBridge: + """Bounded asynchronous bridge to a CFM metric reporter. + + ``report`` receives complete records (for example, a CFM RPC client + method). Back pressure drops metrics instead of delaying inference. + """ + + def __init__(self, report: MetricSink, capacity: int = 4096) -> None: + self._report = report + self._queue: Queue[dict[str, Any]] = Queue(maxsize=capacity) + self._stopping = Event() + self._worker = Thread(target=self._run, name="io-pattern-cfm", + daemon=True) + self._worker.start() + self.dropped = 0 + + def report_inference_metrics(self, **metrics: Any) -> None: + try: + self._queue.put_nowait(dict(metrics)) + except Full: + self.dropped += 1 + + def close(self) -> None: + self._stopping.set() + self._worker.join(timeout=1.0) + + def _run(self) -> None: + while not self._stopping.is_set() or not self._queue.empty(): + try: + metrics = self._queue.get(timeout=0.1) + except Empty: + continue + try: + self._report(metrics) + except Exception: + # A failed CFM report must remain isolated from inference. + self.dropped += 1 + + +class SglangHiCacheIoPatternBridge(BatchedIoPatternBridge): + """Adapter for SGLang HiCache request and prefix-match hooks. + + ``layout`` must be the active ``--hicache-mem-layout`` value, normally + ``layer_first``, ``page_first`` or ``page_first_direct``. + """ + + def request_finished(self, *, session_id: str, token_count: int, + prefix_depth: int, prefix_fanout: int, + match_length: int, continuous_prefix_length: int, + recompute_cost: float, request_priority: int = 0, + layout: str = "layer_first", layout_group: int = 0) -> None: + self.report_inference_metrics( + session_id=session_id, + token_count=token_count, + prefix_depth=prefix_depth, + prefix_fanout=prefix_fanout, + match_length=match_length, + continuous_prefix_length=continuous_prefix_length, + recompute_cost=recompute_cost, + request_priority=request_priority, + layout=layout, + layout_group=layout_group, + ) diff --git a/mooncake-wheel/mooncake/mooncake_connector_v1.py b/mooncake-wheel/mooncake/mooncake_connector_v1.py index 18873b38fc..79a9b721ae 100644 --- a/mooncake-wheel/mooncake/mooncake_connector_v1.py +++ b/mooncake-wheel/mooncake/mooncake_connector_v1.py @@ -17,7 +17,7 @@ from dataclasses import dataclass from queue import Queue from os import getenv -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any, Callable, Optional import msgspec import numpy as np @@ -71,6 +71,31 @@ logger = init_logger(__name__) +class IoPatternBridge: + """Optional connector-side bridge for IO Pattern metric reporting. + + Deployments may attach an object implementing ``report_inference_metrics`` + to ``vllm_config``. The connector remains usable when it is absent. + """ + + def report_inference_metrics(self, **metrics: Any) -> None: + raise NotImplementedError + + +class CallbackIoPatternBridge(IoPatternBridge): + """Concrete bridge that forwards complete metric records to a CFM adapter. + + The callback is deliberately injected by the deployment so this connector + stays independent of a particular Python/C++ RPC binding. + """ + + def __init__(self, report: Callable[..., None]) -> None: + self._report = report + + def report_inference_metrics(self, **metrics: Any) -> None: + self._report(**metrics) + + class MooncakeAgentMetadata( msgspec.Struct, omit_defaults=True, # type: ignore[call-arg] @@ -131,6 +156,8 @@ def __init__(self, vllm_config: VllmConfig, role: KVConnectorRole): assert vllm_config.kv_transfer_config.engine_id is not None super().__init__(vllm_config, role) self.engine_id: EngineId = vllm_config.kv_transfer_config.engine_id + self.io_pattern_bridge: Optional[IoPatternBridge] = getattr( + vllm_config, "io_pattern_bridge", None) if role == KVConnectorRole.SCHEDULER: self.connector_scheduler: Optional[MooncakeConnectorScheduler] = \ @@ -235,6 +262,12 @@ class MooncakeConnectorScheduler: def __init__(self, vllm_config: VllmConfig, engine_id: str): self.vllm_config = vllm_config self.engine_id: EngineId = engine_id + self.io_pattern_bridge: Optional[IoPatternBridge] = getattr( + vllm_config, "io_pattern_bridge", None) + # Kept on the scheduler, which owns all three hook points below. + # Each request contributes partial observations until completion. + self._io_pattern_metrics: dict[ReqId, dict[str, Any]] = {} + self.io_pattern_layout = get_kv_cache_layout() self.side_channel_host = get_ip() self.side_channel_port = get_mooncake_side_channel_port(vllm_config) @@ -277,6 +310,11 @@ def get_num_new_matched_tokens( # Remote prefill: get all prompt blocks from remote. count = len(request.prompt_token_ids) - num_computed_tokens if count > 0: + self._io_pattern_metrics.setdefault(request.request_id, {}).update( + match_length=count, + continuous_prefix_length=count, + token_count=len(request.prompt_token_ids), + ) return count, True # No remote prefill for this request. @@ -295,6 +333,12 @@ def update_state_after_alloc(self, request: "Request", if not params: return + self._io_pattern_metrics.setdefault(request.request_id, {}).update( + prefix_depth=params.get("prefix_depth", len(blocks.get_unhashed_block_ids())), + prefix_fanout=params.get("prefix_fanout", 0), + continuous_prefix_length=num_external_tokens, + ) + if params.get("do_remote_prefill"): assert self.kv_role != "kv_producer" if all(p in params for p in ("remote_host", "remote_port")): @@ -357,7 +401,29 @@ def request_finished( "MooncakeConnector request_finished, request_status=%s, " "kv_transfer_params=%s", request.status, params) if not params: + # A request may finish before allocation; discard any partial + # observation so aborted requests cannot accumulate indefinitely. + self._io_pattern_metrics.pop(request.request_id, None) return False, None + if self.io_pattern_bridge is not None: + try: + metrics = self._io_pattern_metrics.pop(request.request_id, {}) + token_count = metrics.get("token_count", len(block_ids)) + self.io_pattern_bridge.report_inference_metrics( + session_id=request.request_id, + token_count=token_count, + prefix_depth=metrics.get("prefix_depth", params.get("prefix_depth", 0)), + prefix_fanout=metrics.get("prefix_fanout", params.get("prefix_fanout", 0)), + match_length=metrics.get("match_length", params.get("match_length", 0)), + continuous_prefix_length=metrics.get( + "continuous_prefix_length", params.get("continuous_prefix_length", 0)), + recompute_cost=params.get("recompute_cost", float(token_count)), + request_priority=getattr(request, "priority", 0), + layout=self.io_pattern_layout, + layout_group=params.get("layout_group", 0), + ) + except Exception: # metrics must never affect the data path + logger.debug("IO Pattern metric report failed", exc_info=True) if params.get("do_remote_prefill"): # If do_remote_prefill is still True when the request is finished, From efa0ddb5cfc2d5482758a8cf49e5e4ecd329c58b Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 2 Sep 2026 15:38:53 +0800 Subject: [PATCH 04/47] =?UTF-8?q?io=20pattern=E4=BB=A3=E7=A0=81=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E6=AD=A5=E5=AE=8C=E6=88=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/io_pattern_design.md | 71 +- .../include/io_pattern/cfm_channel.h | 30 +- .../include/io_pattern/cfm_ingress.h | 3 +- .../include/io_pattern/cfm_service.h | 91 +++ .../include/io_pattern/collector_impl.h | 22 +- .../include/io_pattern/io_pattern.h | 1 + mooncake-store/include/io_pattern/reporter.h | 4 + .../io_pattern/resilient_cfm_channel.h | 3 +- .../include/io_pattern/rpc_transport.h | 58 +- mooncake-store/include/io_pattern/runtime.h | 41 +- .../io_pattern/sliding_window_analyzer.h | 6 +- mooncake-store/include/master_config.h | 56 +- mooncake-store/include/master_service.h | 24 +- mooncake-store/include/rpc_service.h | 4 + mooncake-store/src/CMakeLists.txt | 1 + .../src/io_pattern/cfm_client_impl.cpp | 14 +- mooncake-store/src/io_pattern/cfm_ingress.cpp | 27 +- mooncake-store/src/io_pattern/cfm_service.cpp | 284 ++++++++ .../src/io_pattern/collector_impl.cpp | 118 +++- .../src/io_pattern/policy_strategies.cpp | 21 +- mooncake-store/src/io_pattern/reporter.cpp | 32 +- .../src/io_pattern/resilient_cfm_channel.cpp | 21 +- .../src/io_pattern/rpc_transport.cpp | 213 +++++- mooncake-store/src/io_pattern/runtime.cpp | 146 ++++- .../io_pattern/sliding_window_analyzer.cpp | 68 +- mooncake-store/src/master.cpp | 99 +++ mooncake-store/src/master_service.cpp | 609 +++++++++++------- mooncake-store/src/rpc_service.cpp | 9 +- .../tests/io_pattern_framework_test.cpp | 399 +++++++++++- .../tests/master_service_config_test.cpp | 22 + 30 files changed, 2102 insertions(+), 395 deletions(-) create mode 100644 mooncake-store/include/io_pattern/cfm_service.h create mode 100644 mooncake-store/src/io_pattern/cfm_service.cpp diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index 1c9ab5818c..59a7a656bf 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -982,7 +982,7 @@ by the Store master. - `IoPatternReporter` provides bounded, non-blocking batches with explicit report/drop counters and a transport-agnostic sink. - `MetricBatchTransport` defines the transport seam, and the reporter exposes - load-sensitive 100/500/1000 ms flush recommendations. + load-sensitive 100/200/500/1000 ms flush recommendations. - `IoPatternRuntime` wires collection, bounded analysis, policy execution, feedback tuning and storage handlers; `MasterService` feeds it from actual Get/Put/watermark paths. @@ -1034,6 +1034,24 @@ by the Store master. - Analyzer execution has a single in-flight worker, timeout fallback to the last safe result, and an explicit key-count budget; collector key quotas and reporter bounds provide the associated overload/OOM protection. +- Access and write frequencies use timestamped buckets pruned against a true + rolling 60-second cutoff. Sliding analysis deduplicates objects across + snapshots and enforces a hard total retained-key budget (including a single + oversized snapshot), so repeated high-watermark evaluations cannot multiply + complete snapshots without bound. CFM ingress rebases process-local monotonic + timestamps to receiver time, and each object also has a hard bucket-count cap. +- Store-side eviction executes only the tenant-qualified objects selected by + the policy. The legacy `BatchEvict` path runs only when policy execution + fails, avoiding a second unplanned eviction pass. +- Non-memory PUT completions enqueue bounded, asynchronous L1 retention/ + promotion evaluation. This is a post-write cache-admission hook, not initial + replica placement: the existing `PutStart` contract selects and allocates + replicas before write metrics such as batch and overwrite are known. +- CFM polling distinguishes a command, a healthy empty queue and a transport + error. Only transport errors contribute to consecutive-failure degradation. +- Reporter intervals follow the documented memory/RPC load thresholds + (100/200/500/1000 ms), and in-process transport callbacks execute outside the + transport mutex. ## Interface decision: complete plans versus document shorthand @@ -1088,10 +1106,47 @@ Registry ownership is external and thread-safe. Factories return independent Ops instances; callers own the returned smart pointers. Concrete storage and RPC resources are injected through execution handlers and CFM channels. -## Known gaps - -There are no remaining implementation gaps in the Mooncake IO Pattern scope. -Production deployments select their network-specific `CfmRpcTransport` through -the documented transport seam; the authenticated embedded transport is the -reference implementation and the SGLang adapter is intentionally kept -framework-neutral because SGLang source is not vendored in this repository. +## Production CFM wiring + +Master registers authenticated CFM handlers on its existing `coro_rpc` port. +`CoroRpcCfmTransport` is the production client: metric batches are delivered to +`CfmIngress`, while policy commands use a bounded per-node queue and are polled +by stable `node_id`. Received commands execute through +`IoPatternRuntime::ExecuteCommand`, preserving the same storage-safe handlers as +local policy decisions. Every report RPC also carries that `node_id`; ingress +uses it as the authoritative storage-metric source so central aggregation does +not merge watermarks from different Masters. + +Configure a central CFM receiver with `io_pattern_cfm_auth_token`. Configure each +reporting/policy-consuming Master with: + +- `io_pattern_cfm_endpoint=host:port` +- `io_pattern_cfm_node_id=` (defaults to `cluster_id`) +- the same `io_pattern_cfm_auth_token` +- on the central receiver only, a distinct + `io_pattern_cfm_producer_auth_token` for policy producers +- optional `io_pattern_cfm_timeout_ms` and + `io_pattern_cfm_policy_queue_capacity` + +An outbound Master authenticates during construction and fails startup if the +configured CFM endpoint cannot be reached or rejects the token. At runtime the +Reporter sends metric batches over the channel and a resilient poll loop +dispatches queued policies. On the receiver, each accepted metric batch is put +onto a bounded policy-production queue; the central runtime runs +Collector -> Analyzer -> PolicyEngine asynchronously and automatically queues +high-watermark eviction and trace-derived prefetch commands for the reporting +`node_id`. An external policy producer may also call the registered +`CfmRpcService::EnqueuePolicy` RPC with a target node id and an encoded +`PolicyCommand`. + +Node credentials cannot use that explicit enqueue RPC; an external producer +must authenticate with the separately configured producer credential. The +server validates commands before enqueueing them, assigns a delivery id, and +retains each command until the target node acknowledges successful execution. +Its poll response distinguishes an authenticated empty queue from a rejected +or invalid request, so authorization failures enter the normal degradation +path. The configured capacity is enforced for both pending production work and +policy delivery, with policy delivery bounded both per node and globally. + +The SGLang adapter remains framework-neutral because SGLang source is not +vendored in this repository. diff --git a/mooncake-store/include/io_pattern/cfm_channel.h b/mooncake-store/include/io_pattern/cfm_channel.h index 5533e113f0..7d0b84e125 100644 --- a/mooncake-store/include/io_pattern/cfm_channel.h +++ b/mooncake-store/include/io_pattern/cfm_channel.h @@ -1,19 +1,47 @@ #pragma once #include +#include #include "types.h" #include "../types.h" namespace mooncake::io_pattern { +struct CfmPollResult { + enum class Status { kCommand, kEmpty, kError }; + + static CfmPollResult Command(PolicyCommand command, + uint64_t delivery_id = 0) { + return {.status = Status::kCommand, + .command = std::move(command), + .delivery_id = delivery_id}; + } + static CfmPollResult Empty() { return {.status = Status::kEmpty}; } + static CfmPollResult Error() { return {.status = Status::kError}; } + + Status status{Status::kEmpty}; + std::optional command; + uint64_t delivery_id{0}; +}; + // Transport-neutral CFM RPC channel. Implementations own serialization, // retries and connection lifecycle. class CfmChannel { public: virtual ~CfmChannel() = default; virtual bool SendSnapshot(const IoPatternSnapshot& snapshot) = 0; - virtual std::optional PollPolicy() = 0; + virtual CfmPollResult PollPolicyResult() = 0; + std::optional PollPolicy() { + auto result = PollPolicyResult(); + if (result.status != CfmPollResult::Status::kCommand || + !result.command) { + return std::nullopt; + } + if (!AcknowledgePolicy(result.delivery_id, true)) return std::nullopt; + return std::move(result.command); + } + virtual bool AcknowledgePolicy(uint64_t, bool) { return true; } virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; }; diff --git a/mooncake-store/include/io_pattern/cfm_ingress.h b/mooncake-store/include/io_pattern/cfm_ingress.h index eb027e6575..f11a33079b 100644 --- a/mooncake-store/include/io_pattern/cfm_ingress.h +++ b/mooncake-store/include/io_pattern/cfm_ingress.h @@ -17,7 +17,8 @@ class CfmIngress final { std::make_shared()) : runtime_(std::move(runtime)), codec_(std::move(codec)) {} - bool Handle(std::string_view method, std::string_view payload); + bool Handle(std::string_view method, std::string_view payload, + std::string_view source_id = {}); private: std::shared_ptr runtime_; diff --git a/mooncake-store/include/io_pattern/cfm_service.h b/mooncake-store/include/io_pattern/cfm_service.h new file mode 100644 index 0000000000..40a53c2fab --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_service.h @@ -0,0 +1,91 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cfm_ingress.h" + +namespace mooncake::io_pattern { + +// Authenticated server-side CFM endpoint. The RPC layer delegates to this +// class, keeping authentication, bounded policy queues and runtime dispatch +// independent of the concrete network transport. +class CfmService final { + public: + CfmService(std::shared_ptr runtime, + std::string auth_token, size_t policy_queue_capacity = 4096, + std::string producer_auth_token = {}); + ~CfmService(); + + bool Authenticate(std::string_view token) const; + bool AuthenticateNode(std::string_view token) const; + bool AuthenticateProducer(std::string_view token) const; + bool Send(std::string_view node_id, std::string_view method, + std::string_view payload, std::string_view token); + std::optional> PollPolicy( + std::string_view node_id, std::string_view token); + bool AcknowledgePolicy(std::string_view node_id, uint64_t delivery_id, + bool success, std::string_view token); + bool EnqueuePolicy(std::string node_id, std::string payload, + std::string_view token); + + private: + bool EnqueueValidated(std::string node_id, std::string payload); + void SchedulePolicyProduction(std::string node_id, MetricBatch batch); + void PolicyProducerWorker(); + void ProducePolicies(std::string_view node_id, const MetricBatch& batch); + + std::shared_ptr runtime_; + std::shared_ptr codec_; + CfmIngress ingress_; + const std::string auth_token_; + const std::string producer_auth_token_; + const size_t policy_queue_capacity_; + std::mutex mutex_; + std::unordered_map>> + policy_queues_; + uint64_t next_delivery_id_{1}; + size_t total_queued_policies_{0}; + std::mutex producer_mutex_; + std::condition_variable producer_cv_; + std::deque> pending_metric_batches_; + bool producer_stopping_{false}; + std::thread producer_worker_; +}; + +// coro_rpc-facing adapter. Keeping RPC signatures here lets both the Master +// server and integration tests register the exact production endpoints. +class CfmRpcService final { + public: + explicit CfmRpcService(std::shared_ptr service) + : service_(std::move(service)) {} + + bool Authenticate(const std::string& auth_token); + bool Send(const std::string& node_id, const std::string& method, + const std::string& payload, const std::string& auth_token); + // The boolean explicitly distinguishes a rejected request from an + // authenticated queue that currently has no policy. + std::pair>> Receive( + const std::string& method, const std::string& node_id, + const std::string& auth_token); + bool Acknowledge(const std::string& node_id, uint64_t delivery_id, + bool success, const std::string& auth_token); + bool EnqueuePolicy(const std::string& node_id, const std::string& payload, + const std::string& auth_token); + + private: + std::shared_ptr service_; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/collector_impl.h b/mooncake-store/include/io_pattern/collector_impl.h index 4324360750..fb2a43844d 100644 --- a/mooncake-store/include/io_pattern/collector_impl.h +++ b/mooncake-store/include/io_pattern/collector_impl.h @@ -1,8 +1,10 @@ #pragma once +#include +#include +#include #include #include -#include #include #include "collector.h" @@ -17,6 +19,10 @@ class IoPatternCollectorImpl final : public IoPatternCollector { struct Config { size_t max_keys_per_tenant{0}; size_t max_total_keys{0}; + uint64_t access_window_ns{60'000'000'000ULL}; + uint64_t access_bucket_ns{1'000'000'000ULL}; + size_t max_access_buckets_per_key{64}; + std::function now_ns; }; explicit IoPatternCollectorImpl(Config config = {}, @@ -47,6 +53,16 @@ class IoPatternCollectorImpl final : public IoPatternCollector { (static_cast(key.tier) << 1); } }; + struct AccessWindowBucket { + uint64_t observed_at_ns{0}; + uint64_t access_count{0}; + uint64_t write_count{0}; + uint64_t overwrite_count{0}; + uint32_t max_write_batch_size{0}; + }; + + void ApplyAccessWindow(const ObjectRef& object, uint64_t now_ns, + KeyMetrics& metrics) const; mutable std::mutex mutex_; Config config_; @@ -54,8 +70,8 @@ class IoPatternCollectorImpl final : public IoPatternCollector { uint64_t dropped_{0}; bool degraded_{false}; std::unordered_map key_metrics_; - std::unordered_map write_counts_; - std::unordered_map overwrite_counts_; + std::unordered_map, ObjectRefHash> + access_windows_; std::unordered_map tenant_key_counts_; std::unordered_map storage_metrics_; diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h index 0bc97def1d..78f3dbef1d 100644 --- a/mooncake-store/include/io_pattern/io_pattern.h +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -4,6 +4,7 @@ #include "io_pattern/client.h" #include "io_pattern/cfm_channel.h" #include "io_pattern/cfm_ingress.h" +#include "io_pattern/cfm_service.h" #include "io_pattern/cfm_protocol.h" #include "io_pattern/cfm_client_impl.h" #include "io_pattern/feedback.h" diff --git a/mooncake-store/include/io_pattern/reporter.h b/mooncake-store/include/io_pattern/reporter.h index fc0cbdf364..8926550fa5 100644 --- a/mooncake-store/include/io_pattern/reporter.h +++ b/mooncake-store/include/io_pattern/reporter.h @@ -46,11 +46,13 @@ class IoPatternReporter final { size_t pending() const; uint64_t dropped() const; uint64_t reported() const; + void UpdateLoad(float memory_used_ratio, uint64_t rpc_latency_us); std::chrono::milliseconds RecommendedFlushInterval() const; private: bool EnqueueImpl(std::function append, const TenantId& tenant); + std::chrono::milliseconds RecommendedFlushIntervalLocked() const; const size_t capacity_; const MetricBatchSink sink_; @@ -62,6 +64,8 @@ class IoPatternReporter final { std::condition_variable condition_; std::thread worker_; bool running_{false}; + float memory_used_ratio_{0.0F}; + uint64_t rpc_latency_us_{0}; std::unordered_map tenant_pending_; }; diff --git a/mooncake-store/include/io_pattern/resilient_cfm_channel.h b/mooncake-store/include/io_pattern/resilient_cfm_channel.h index d0b2118664..e022de2157 100644 --- a/mooncake-store/include/io_pattern/resilient_cfm_channel.h +++ b/mooncake-store/include/io_pattern/resilient_cfm_channel.h @@ -21,7 +21,8 @@ class ResilientCfmChannel final : public CfmChannel { : delegate_(std::move(delegate)), config_(config) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - std::optional PollPolicy() override; + CfmPollResult PollPolicyResult() override; + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; bool degraded() const; diff --git a/mooncake-store/include/io_pattern/rpc_transport.h b/mooncake-store/include/io_pattern/rpc_transport.h index 234f99c1ec..98ff83d830 100644 --- a/mooncake-store/include/io_pattern/rpc_transport.h +++ b/mooncake-store/include/io_pattern/rpc_transport.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include "cfm_channel.h" @@ -16,6 +17,23 @@ namespace mooncake::io_pattern { +struct CfmReceiveResult { + enum class Status { kPayload, kEmpty, kError }; + + static CfmReceiveResult Payload(std::string payload, + uint64_t delivery_id = 0) { + return {.status = Status::kPayload, + .payload = std::move(payload), + .delivery_id = delivery_id}; + } + static CfmReceiveResult Empty() { return {.status = Status::kEmpty}; } + static CfmReceiveResult Error() { return {.status = Status::kError}; } + + Status status{Status::kEmpty}; + std::string payload; + uint64_t delivery_id{0}; +}; + class CfmRpcCodec { public: virtual ~CfmRpcCodec() = default; @@ -35,8 +53,34 @@ class CfmRpcTransport { virtual bool Authenticate(std::string_view token) { return token.empty(); } virtual bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds timeout) = 0; - virtual std::optional Receive( + virtual CfmReceiveResult Receive( std::string_view method, std::chrono::milliseconds timeout) = 0; + virtual bool Acknowledge(uint64_t delivery_id, bool success, + std::chrono::milliseconds timeout) = 0; +}; + +// Production CFM transport over Mooncake's existing coro_rpc connection pool. +// It targets the CFM handlers registered on the Master RPC service. +class CoroRpcCfmTransport final : public CfmRpcTransport { + public: + CoroRpcCfmTransport(std::string endpoint, std::string node_id, + std::chrono::milliseconds default_timeout = + std::chrono::milliseconds(500)); + ~CoroRpcCfmTransport() override; + + bool Authenticate(std::string_view token) override; + bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds timeout) override; + CfmReceiveResult Receive( + std::string_view method, std::chrono::milliseconds timeout) override; + bool Acknowledge(uint64_t delivery_id, bool success, + std::chrono::milliseconds timeout) override; + bool EnqueuePolicy(std::string_view node_id, std::string_view payload, + std::chrono::milliseconds timeout); + + private: + class Impl; + std::unique_ptr impl_; }; struct CfmRpcConfig { @@ -58,8 +102,12 @@ class InProcessCfmRpcTransport final : public CfmRpcTransport { bool Authenticate(std::string_view token) override; bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds timeout) override; - std::optional Receive( + CfmReceiveResult Receive( std::string_view method, std::chrono::milliseconds timeout) override; + bool Acknowledge(uint64_t, bool, + std::chrono::milliseconds) override { + return true; + } void EnqueuePolicy(std::string payload); void SetSendHandler(SendHandler handler); @@ -82,7 +130,8 @@ class CfmRpcChannel final : public CfmChannel { config_(config) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - std::optional PollPolicy() override; + CfmPollResult PollPolicyResult() override; + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; bool SendMetricBatch(const MetricBatch& batch); @@ -105,7 +154,8 @@ class CfmChannelPool final : public CfmChannel { : channels_(std::move(channels)) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - std::optional PollPolicy() override; + CfmPollResult PollPolicyResult() override; + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; private: diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 74acfec1bd..7ec9fffc27 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -1,8 +1,12 @@ #pragma once #include +#include +#include #include #include +#include +#include #include #include @@ -39,6 +43,7 @@ class IoPatternRuntime final { size_t report_capacity{4096}; size_t report_per_tenant_capacity{0}; size_t max_pending_prefetches{4096}; + size_t max_pending_admissions{4096}; MetricBatchSink report_sink; LegacyFallback legacy_fallback{LegacyFallback::kLru}; }; @@ -56,9 +61,18 @@ class IoPatternRuntime final { const TraceHistory& trace, const std::vector& admissions = {}, const std::string& session_id = {}); + // Runs Collector -> Analyzer -> PolicyEngine without invoking local + // storage handlers. Central CFM uses this to produce commands for a + // target node; Store data paths continue to use Execute(). + PolicyResult Plan(CacheTier eviction_tier, uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions = {}, + const std::string& session_id = {}); // Applies a CFM-issued command through the same storage handlers as a // locally planned policy. This is the CFM-to-Store execution endpoint. ErrorCode ExecuteCommand(const PolicyCommand& command); + bool ScheduleAdmission(ObjectRef object, CacheTier target_tier, + std::string session_id = {}); void RecordFeedback(PolicyFeedbackSample sample); IoPatternSnapshot Snapshot() const; @@ -68,7 +82,27 @@ class IoPatternRuntime final { private: PatternResult AnalyzeWithinBudget(const IoPatternSnapshot& snapshot, - bool& degraded); + bool& degraded); + struct PlannedPolicy { + IoPatternSnapshot snapshot; + PolicyResult result; + bool analysis_degraded{false}; + uint64_t analysis_elapsed_us{0}; + }; + PlannedPolicy BuildPolicy(CacheTier eviction_tier, + uint64_t eviction_bytes, + const TraceHistory& trace, + const std::vector& admissions, + const std::string& session_id); + void AdmissionWorker(); + ErrorCode ExecuteAdmission(const ObjectRef& object, CacheTier target_tier, + const std::string& session_id); + + struct PendingAdmission { + ObjectRef object; + CacheTier target_tier{CacheTier::kL1Host}; + std::string session_id; + }; Config config_; std::shared_ptr reporter_; @@ -89,6 +123,11 @@ class IoPatternRuntime final { // leave a worker holding a pointer into a destroyed runtime instance. std::shared_ptr> analysis_in_flight_{ std::make_shared>(false)}; + std::mutex admission_mutex_; + std::condition_variable admission_condition_; + std::deque pending_admissions_; + std::thread admission_worker_; + bool admission_stopping_{false}; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/sliding_window_analyzer.h b/mooncake-store/include/io_pattern/sliding_window_analyzer.h index 583190d254..dfbd701c6d 100644 --- a/mooncake-store/include/io_pattern/sliding_window_analyzer.h +++ b/mooncake-store/include/io_pattern/sliding_window_analyzer.h @@ -24,8 +24,10 @@ struct WorkloadFeatureStats { class SlidingWindowAnalyzer final : public IoPatternAnalyzer { public: explicit SlidingWindowAnalyzer(uint64_t window_ns = 60'000'000'000ULL, - ThresholdAnalyzerConfig config = {}) + ThresholdAnalyzerConfig config = {}, + size_t max_history_keys = 200'000) : window_ns_(window_ns), + max_history_keys_(max_history_keys), analyzer_(config), kmeans_(KMeansWorkloadAnalyzer::Config{.thresholds = config}) {} @@ -41,8 +43,10 @@ class SlidingWindowAnalyzer final : public IoPatternAnalyzer { void Append(const IoPatternSnapshot& snapshot) const; const uint64_t window_ns_; + const size_t max_history_keys_; mutable std::mutex mutex_; mutable std::deque history_; + mutable size_t history_key_count_{0}; ThresholdAnalyzer analyzer_; KMeansWorkloadAnalyzer kmeans_; }; diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index 43ab2e3a24..1bb0b49c99 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -17,6 +18,17 @@ namespace mooncake { // Forwarded to the HA serve phase via MasterServiceSupervisorConfig. class HttpMetadataServer; +struct IoPatternCfmConfig { + // Empty endpoint means this Master only serves the CFM RPC endpoints. + // Set host:port to report to and poll policies from a central CFM Master. + std::string endpoint; + std::string node_id; + std::string auth_token; + std::string producer_auth_token; + uint32_t timeout_ms{500}; + uint32_t policy_queue_capacity{4096}; +}; + inline std::string ResolveConfiguredHABackendConnstring( std::string_view ha_backend_type, std::string_view ha_backend_connstring, std::string_view etcd_endpoints) { @@ -39,6 +51,7 @@ struct MasterConfig { std::string rpc_interface; int32_t rpc_conn_timeout_seconds; bool rpc_enable_tcp_no_delay; + IoPatternCfmConfig io_pattern_cfm; uint64_t default_kv_lease_ttl; uint64_t default_kv_soft_pin_ttl; @@ -61,8 +74,7 @@ struct MasterConfig { // Master view lease TTL in seconds (HA leadership lease). // When the lease expires without successful renewal, the Master // is considered dead and a standby can take over. - int64_t master_view_lease_ttl_sec = - DEFAULT_MASTER_VIEW_LEASE_TTL_SEC; + int64_t master_view_lease_ttl_sec = DEFAULT_MASTER_VIEW_LEASE_TTL_SEC; // OpLog store configuration bool enable_oplog = false; @@ -78,7 +90,8 @@ struct MasterConfig { std::string cluster_id; // 集群中允许同时 serving 的 submaster 上限(CVM 名额协调,先到先得)。 - // 默认 1 保持单主行为;>1 时多 submaster 均分 slot,超出 k 名自动降级为 standby。 + // 默认 1 保持单主行为;>1 时多 submaster 均分 slot,超出 k 名自动降级为 + // standby。 uint32_t submaster_count = 1; // CVM external HTTP API (CvmHttpServer) bind config. Port 0 keeps the // HTTP server disabled. @@ -214,6 +227,7 @@ class MasterServiceSupervisorConfig { std::chrono::steady_clock::duration rpc_conn_timeout = std::chrono::seconds( 0); // Client connection timeout. 0 = no timeout (infinite) bool rpc_enable_tcp_no_delay = true; + IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; std::string etcd_endpoints = "0.0.0.0:2379"; @@ -359,6 +373,7 @@ class MasterServiceSupervisorConfig { rpc_conn_timeout = std::chrono::seconds(config.rpc_conn_timeout_seconds); rpc_enable_tcp_no_delay = config.rpc_enable_tcp_no_delay; + io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; etcd_endpoints = config.etcd_endpoints; ha_backend_connstring = ResolveConfiguredHABackendConnstring( @@ -440,8 +455,8 @@ class MasterServiceSupervisorConfig { enable_cxl = config.enable_cxl; vchunk_config = config.vchunk_config; vchunk_etcd_endpoints = config.vchunk_etcd_endpoints.empty() - ? config.etcd_endpoints - : config.vchunk_etcd_endpoints; + ? config.etcd_endpoints + : config.vchunk_etcd_endpoints; pod_name = config.pod_name; pod_namespace = config.pod_namespace; @@ -553,6 +568,7 @@ class WrappedMasterServiceConfig { bool kv_events_emit_legacy_compat = true; bool kv_events_emit_object_key = true; uint32_t kv_events_queue_capacity = 65536; + IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; // OpLog store configuration @@ -570,7 +586,8 @@ class WrappedMasterServiceConfig { uint16_t cvm_http_port = 0; std::string cvm_http_host = "0.0.0.0"; // 集群中允许同时 serving 的 submaster 上限(CVM 名额协调,先到先得)。 - // 默认 1 保持单主行为;>1 时多 submaster 均分 slot,超出 k 名自动降级为 standby。 + // 默认 1 保持单主行为;>1 时多 submaster 均分 slot,超出 k 名自动降级为 + // standby。 uint32_t submaster_count = 1; std::string root_fs_dir = DEFAULT_ROOT_FS_DIR; int64_t global_file_segment_size = DEFAULT_GLOBAL_FILE_SEGMENT_SIZE; @@ -657,6 +674,7 @@ class WrappedMasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; + io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = ResolveConfiguredHABackendConnstring( ha_backend_type, config.ha_backend_connstring, @@ -729,8 +747,8 @@ class WrappedMasterServiceConfig { enable_cxl = config.enable_cxl; vchunk_config = config.vchunk_config; vchunk_etcd_endpoints = config.vchunk_etcd_endpoints.empty() - ? config.etcd_endpoints - : config.vchunk_etcd_endpoints; + ? config.etcd_endpoints + : config.vchunk_etcd_endpoints; } // From MasterServiceSupervisorConfig, enable_ha is set to true @@ -781,6 +799,7 @@ class WrappedMasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; + io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = ResolveConfiguredHABackendConnstring( ha_backend_type, config.ha_backend_connstring, @@ -895,8 +914,6 @@ class MasterServiceConfigBuilder { std::string cxl_path_ = DEFAULT_CXL_PATH; size_t cxl_size_ = DEFAULT_CXL_SIZE; bool enable_cxl_ = false; - VChunkConfig vchunk_config_{}; - std::shared_ptr vchunk_metadata_store_; public: MasterServiceConfigBuilder() = default; @@ -1197,6 +1214,11 @@ class MasterServiceConfigBuilder { return *this; } + MasterServiceConfigBuilder& set_io_pattern_cfm(IoPatternCfmConfig config) { + io_pattern_cfm_ = std::move(config); + return *this; + } + MasterServiceConfig build() const; }; @@ -1269,7 +1291,8 @@ class MasterServiceConfig { uint16_t cvm_http_port = 0; std::string cvm_http_host = "0.0.0.0"; // 集群中允许同时 serving 的 submaster 上限(CVM 名额协调,先到先得)。 - // 默认 1 保持单主行为;>1 时多 submaster 均分 slot,超出 k 名自动降级为 standby。 + // 默认 1 保持单主行为;>1 时多 submaster 均分 slot,超出 k 名自动降级为 + // standby。 uint32_t submaster_count = 1; std::string root_fs_dir = DEFAULT_ROOT_FS_DIR; int64_t global_file_segment_size = DEFAULT_GLOBAL_FILE_SEGMENT_SIZE; @@ -1306,9 +1329,6 @@ class MasterServiceConfig { std::string cxl_path = DEFAULT_CXL_PATH; size_t cxl_size = DEFAULT_CXL_SIZE; bool enable_cxl = false; - VChunkConfig vchunk_config{}; - std::string vchunk_etcd_endpoints; - std::shared_ptr vchunk_metadata_store; MasterServiceConfig() = default; // From WrappedMasterServiceConfig @@ -1401,12 +1421,6 @@ class MasterServiceConfig { cxl_path = config.cxl_path; cxl_size = config.cxl_size; enable_cxl = config.enable_cxl; - vchunk_config = config.vchunk_config; - vchunk_etcd_endpoints = config.vchunk_etcd_endpoints; - if (vchunk_config.enabled && !vchunk_etcd_endpoints.empty()) { - vchunk_metadata_store = std::make_shared( - vchunk_etcd_endpoints, vchunk_config, cluster_id); - } } // Static factory method to create a builder @@ -1473,8 +1487,6 @@ inline MasterServiceConfig MasterServiceConfigBuilder::build() const { config.cxl_path = cxl_path_; config.cxl_size = cxl_size_; config.enable_cxl = enable_cxl_; - config.vchunk_config = vchunk_config_; - config.vchunk_metadata_store = vchunk_metadata_store_; return config; } diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 1fba696f1a..9f2c720ecd 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -53,6 +53,9 @@ namespace mooncake { namespace io_pattern { +class CfmChannel; +class CfmClientImpl; +class CfmService; class IoPatternRuntime; } @@ -437,6 +440,10 @@ class MasterService { bool KvEventsEnabled() const; KvEventPublisher::Stats GetKvEventStats() const; + std::shared_ptr GetCfmService() const { + return io_pattern_cfm_service_; + } + /** * @brief Batch clear KV cache replicas for specified object keys. * @param object_keys Vector of object key strings to clear. @@ -1016,7 +1023,8 @@ class MasterService { uint64_t evicted_objects{0}; }; TenantQuotaEvictionResult EvictTenantMemoryForQuota( - const TenantId& tenant_id, uint64_t target_bytes); + const TenantId& tenant_id, uint64_t target_bytes, + const std::unordered_set* candidate_keys = nullptr); // Helper to get a snapshot of alive clients (under client_mutex_ shared // lock) @@ -1698,6 +1706,11 @@ class MasterService { const std::chrono::system_clock::time_point& now) -> tl::expected, ErrorCode>; + auto PutEndInternal(const UUID& client_id, const ObjectMeta& object_meta, + const TenantId& tenant_id, ReplicaType replica_type, + uint32_t write_batch_size, bool overwrite) + -> tl::expected; + /** * @brief Helper to discard expired processing keys. */ @@ -2211,7 +2224,14 @@ class MasterService { // The IO-pattern pipeline is deliberately owned by MasterService: the // master has the authoritative replica map and is the only component that // can safely translate a policy plan into promotion/eviction operations. - std::unique_ptr io_pattern_runtime_; + std::shared_ptr io_pattern_runtime_; + std::shared_ptr io_pattern_cfm_service_; + std::shared_ptr io_pattern_cfm_channel_; + std::unique_ptr io_pattern_cfm_client_; + std::atomic io_pattern_cfm_polling_{false}; + std::mutex io_pattern_cfm_poll_mutex_; + std::condition_variable io_pattern_cfm_poll_cv_; + std::thread io_pattern_cfm_poll_thread_; const std::string ha_backend_type_; diff --git a/mooncake-store/include/rpc_service.h b/mooncake-store/include/rpc_service.h index 52f062c4df..25f7f56ac8 100644 --- a/mooncake-store/include/rpc_service.h +++ b/mooncake-store/include/rpc_service.h @@ -13,6 +13,7 @@ #include "rpc_types.h" #include "master_config.h" #include "kv_event/kv_event_publisher.h" +#include "io_pattern/cfm_service.h" #include "segment.h" namespace mooncake { @@ -397,8 +398,11 @@ class WrappedMasterService { bool KvEventsEnabled() const; KvEventPublisher::Stats GetKvEventStats() const; + io_pattern::CfmRpcService& CfmRpcEndpoint() { return cfm_rpc_service_; } + private: MasterService master_service_; + io_pattern::CfmRpcService cfm_rpc_service_; }; void RegisterRpcService(coro_rpc::coro_rpc_server& server, diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index 003454fda2..f1f153aef9 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -72,6 +72,7 @@ set(MOONCAKE_STORE_SOURCES io_pattern/reporter.cpp io_pattern/cfm_client_impl.cpp io_pattern/cfm_ingress.cpp + io_pattern/cfm_service.cpp io_pattern/cfm_protocol.cpp io_pattern/resilient_cfm_channel.cpp io_pattern/feedback.cpp diff --git a/mooncake-store/src/io_pattern/cfm_client_impl.cpp b/mooncake-store/src/io_pattern/cfm_client_impl.cpp index 2bd502af2b..88425020fd 100644 --- a/mooncake-store/src/io_pattern/cfm_client_impl.cpp +++ b/mooncake-store/src/io_pattern/cfm_client_impl.cpp @@ -24,8 +24,18 @@ std::optional CfmClientImpl::PollPolicy() { } ErrorCode CfmClientImpl::PollAndDispatchPolicy() { - const auto command = PollPolicy(); - return command ? ReceivePolicy(*command) : ErrorCode::RPC_TIMEOUT; + if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + auto result = channel_->PollPolicyResult(); + if (result.status == CfmPollResult::Status::kEmpty) return ErrorCode::OK; + if (result.status == CfmPollResult::Status::kError || !result.command) { + return ErrorCode::RPC_TIMEOUT; + } + const auto execution = ReceivePolicy(*result.command); + if (!channel_->AcknowledgePolicy(result.delivery_id, + execution == ErrorCode::OK)) { + return ErrorCode::RPC_FAIL; + } + return execution; } } // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_ingress.cpp b/mooncake-store/src/io_pattern/cfm_ingress.cpp index effabd7b1a..3ce568c6c3 100644 --- a/mooncake-store/src/io_pattern/cfm_ingress.cpp +++ b/mooncake-store/src/io_pattern/cfm_ingress.cpp @@ -1,8 +1,11 @@ #include "io_pattern/cfm_ingress.h" +#include + namespace mooncake::io_pattern { -bool CfmIngress::Handle(std::string_view method, std::string_view payload) { +bool CfmIngress::Handle(std::string_view method, std::string_view payload, + std::string_view source_id) { if (!runtime_ || !codec_) return false; const std::string wire(payload); if (method == "report_snapshot") { @@ -14,14 +17,28 @@ bool CfmIngress::Handle(std::string_view method, std::string_view payload) { if (method == "report_metric_batch") { const auto batch = codec_->DecodeMetricBatch(wire); if (!batch) return false; + const auto received_at_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); for (const auto& metric : batch->inference) { runtime_->ReportInferenceMetrics(metric); } for (const auto& access : batch->accesses) { - runtime_->RecordAccess(access.object.key, access); + auto normalized = access; + // steady_clock epochs are process-local. CFM observations must be + // rebased to the receiving Store's clock before windowing. + normalized.observed_at_ns = received_at_ns; + runtime_->RecordAccess(normalized.object.key, normalized); } for (const auto& storage : batch->storage) { - runtime_->RecordStorageMetric(storage); + auto normalized = storage; + // The transport identity is authoritative for remote metrics. It + // keeps per-node watermarks distinct even when a producer omitted + // or accidentally reused StorageMetric::source_id. + if (!source_id.empty()) normalized.source_id = source_id; + normalized.observed_at_ns = received_at_ns; + runtime_->RecordStorageMetric(normalized); } return true; } @@ -30,6 +47,10 @@ bool CfmIngress::Handle(std::string_view method, std::string_view payload) { const auto* plan = command ? std::get_if(&*command) : nullptr; return plan && runtime_->ExecuteCommand(*plan) == ErrorCode::OK; } + if (method == "execute_policy") { + const auto command = codec_->DecodePolicy(wire); + return command && runtime_->ExecuteCommand(*command) == ErrorCode::OK; + } return false; } diff --git a/mooncake-store/src/io_pattern/cfm_service.cpp b/mooncake-store/src/io_pattern/cfm_service.cpp new file mode 100644 index 0000000000..f0efe6d048 --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_service.cpp @@ -0,0 +1,284 @@ +#include "io_pattern/cfm_service.h" + +#include +#include +#include + +namespace mooncake::io_pattern { + +CfmService::CfmService(std::shared_ptr runtime, + std::string auth_token, + size_t policy_queue_capacity, + std::string producer_auth_token) + : runtime_(std::move(runtime)), + codec_(std::make_shared()), + ingress_(runtime_, codec_), + auth_token_(std::move(auth_token)), + producer_auth_token_(std::move(producer_auth_token)), + policy_queue_capacity_(policy_queue_capacity), + producer_worker_(&CfmService::PolicyProducerWorker, this) {} + +CfmService::~CfmService() { + { + std::lock_guard lock(producer_mutex_); + producer_stopping_ = true; + pending_metric_batches_.clear(); + } + producer_cv_.notify_all(); + if (producer_worker_.joinable()) producer_worker_.join(); +} + +bool CfmService::Authenticate(std::string_view token) const { + return AuthenticateNode(token) || AuthenticateProducer(token); +} + +bool CfmService::AuthenticateNode(std::string_view token) const { + return !auth_token_.empty() && token == auth_token_; +} + +bool CfmService::AuthenticateProducer(std::string_view token) const { + return !producer_auth_token_.empty() && producer_auth_token_ != auth_token_ && + token == producer_auth_token_; +} + +bool CfmService::Send(std::string_view node_id, std::string_view method, + std::string_view payload, std::string_view token) { + const bool executes_policy = + method == "execute_policy" || method == "execute_prefetch"; + if (node_id.empty() || + (executes_policy ? !AuthenticateProducer(token) + : !AuthenticateNode(token))) { + return false; + } + std::optional metric_batch; + if (method == "report_metric_batch") { + metric_batch = codec_->DecodeMetricBatch(std::string(payload)); + if (!metric_batch) return false; + } + if (!ingress_.Handle(method, payload, node_id)) return false; + if (metric_batch) { + SchedulePolicyProduction(std::string(node_id), + std::move(*metric_batch)); + } + return true; +} + +std::optional> CfmService::PollPolicy( + std::string_view node_id, std::string_view token) { + if (!AuthenticateNode(token) || node_id.empty()) return std::nullopt; + std::lock_guard lock(mutex_); + auto it = policy_queues_.find(std::string(node_id)); + if (it == policy_queues_.end() || it->second.empty()) return std::nullopt; + return it->second.front(); +} + +bool CfmService::AcknowledgePolicy(std::string_view node_id, + uint64_t delivery_id, bool success, + std::string_view token) { + if (!AuthenticateNode(token) || node_id.empty() || delivery_id == 0) { + return false; + } + std::lock_guard lock(mutex_); + auto it = policy_queues_.find(std::string(node_id)); + if (it == policy_queues_.end() || it->second.empty() || + it->second.front().first != delivery_id) { + return false; + } + if (!success) { + if (it->second.size() > 1) { + auto failed = std::move(it->second.front()); + it->second.pop_front(); + it->second.push_back(std::move(failed)); + } + return true; + } + it->second.pop_front(); + --total_queued_policies_; + if (it->second.empty()) policy_queues_.erase(it); + return true; +} + +bool CfmService::EnqueuePolicy(std::string node_id, std::string payload, + std::string_view token) { + if (!AuthenticateProducer(token) || node_id.empty() || payload.empty() || + !codec_->DecodePolicy(payload)) { + return false; + } + return EnqueueValidated(std::move(node_id), std::move(payload)); +} + +bool CfmService::EnqueueValidated(std::string node_id, std::string payload) { + std::lock_guard lock(mutex_); + auto& queue = policy_queues_[node_id]; + if (std::any_of(queue.begin(), queue.end(), [&](const auto& queued) { + return queued.second == payload; + })) { + return true; + } + if (policy_queue_capacity_ == 0 || + total_queued_policies_ >= policy_queue_capacity_ || + queue.size() >= policy_queue_capacity_) { + if (queue.empty()) policy_queues_.erase(node_id); + return false; + } + if (next_delivery_id_ == 0) next_delivery_id_ = 1; + const uint64_t delivery_id = next_delivery_id_++; + queue.emplace_back(delivery_id, std::move(payload)); + ++total_queued_policies_; + return true; +} + +void CfmService::SchedulePolicyProduction(std::string node_id, + MetricBatch batch) { + { + std::lock_guard lock(producer_mutex_); + if (producer_stopping_ || policy_queue_capacity_ == 0 || + pending_metric_batches_.size() >= policy_queue_capacity_) { + return; + } + pending_metric_batches_.emplace_back(std::move(node_id), + std::move(batch)); + } + producer_cv_.notify_one(); +} + +void CfmService::PolicyProducerWorker() { + while (true) { + std::pair pending; + { + std::unique_lock lock(producer_mutex_); + producer_cv_.wait(lock, [this] { + return producer_stopping_ || !pending_metric_batches_.empty(); + }); + if (producer_stopping_) return; + pending = std::move(pending_metric_batches_.front()); + pending_metric_batches_.pop_front(); + } + try { + ProducePolicies(pending.first, pending.second); + } catch (...) { + // Policy production is best effort and must never terminate the + // RPC service. The next metric batch will trigger a fresh plan. + } + } +} + +void CfmService::ProducePolicies(std::string_view node_id, + const MetricBatch& batch) { + if (!runtime_ || node_id.empty()) return; + + std::unordered_map match_lengths; + std::string session_id; + for (const auto& metric : batch.inference) { + match_lengths[metric.object] = metric.match_length; + if (session_id.empty()) session_id = metric.session_id; + } + TraceHistory trace; + trace.events.reserve(batch.accesses.size()); + for (const auto& access : batch.accesses) { + const auto match = match_lengths.find(access.object); + trace.events.push_back( + {.object = access.object, + .observed_at_ns = access.observed_at_ns, + .match_length = match == match_lengths.end() ? 0U + : match->second, + .is_hit = access.is_hit}); + } + + bool produced_prefetch = false; + for (const auto& storage : batch.storage) { + if (static_cast(storage.tier) > + static_cast(CacheTier::kL3NofSsd)) { + continue; + } + if (!std::isfinite(storage.memory_used_ratio) || + storage.memory_used_ratio < 0.90F) { + continue; + } + const float used_ratio = + std::clamp(storage.memory_used_ratio, 0.0F, 1.0F); + uint64_t target_bytes = 0; + if (storage.capacity_bytes != 0) { + const uint64_t low_watermark = + storage.capacity_bytes - storage.capacity_bytes / 5; + if (storage.used_bytes > low_watermark) { + target_bytes = storage.used_bytes - low_watermark; + } + } + if (target_bytes == 0) { + uint64_t tier_bytes = 0; + for (const auto& key : runtime_->Snapshot().keys) { + if ((key.replica_tiers & CacheTierBit(storage.tier)) == 0) { + continue; + } + tier_bytes = + key.block_size > + std::numeric_limits::max() - tier_bytes + ? std::numeric_limits::max() + : tier_bytes + key.block_size; + } + const auto excess_ratio = std::max( + 0.0F, used_ratio - 0.80F); + target_bytes = static_cast( + static_cast(tier_bytes) * excess_ratio / + std::max(0.01F, used_ratio)); + } + auto result = runtime_->Plan(storage.tier, target_bytes, trace, {}, + session_id); + if (result.degraded) continue; + if (!result.eviction.candidates.empty()) { + EnqueueValidated(std::string(node_id), + codec_->EncodePolicy(result.eviction)); + } + if (!produced_prefetch && !result.prefetch.candidates.empty()) { + EnqueueValidated(std::string(node_id), + codec_->EncodePolicy(result.prefetch)); + produced_prefetch = true; + } + } + + if (!produced_prefetch && !trace.events.empty()) { + auto result = runtime_->Plan(CacheTier::kL1Host, 0, trace, {}, + session_id); + if (!result.degraded && !result.prefetch.candidates.empty()) { + EnqueueValidated(std::string(node_id), + codec_->EncodePolicy(result.prefetch)); + } + } +} + +bool CfmRpcService::Authenticate(const std::string& auth_token) { + return service_ && service_->Authenticate(auth_token); +} + +bool CfmRpcService::Send(const std::string& node_id, const std::string& method, + const std::string& payload, + const std::string& auth_token) { + return service_ && service_->Send(node_id, method, payload, auth_token); +} + +std::pair>> +CfmRpcService::Receive( + const std::string& method, const std::string& node_id, + const std::string& auth_token) { + if (!service_ || method != "poll_policy" || + !service_->AuthenticateNode(auth_token) || node_id.empty()) { + return {false, std::nullopt}; + } + return {true, service_->PollPolicy(node_id, auth_token)}; +} + +bool CfmRpcService::Acknowledge(const std::string& node_id, + uint64_t delivery_id, bool success, + const std::string& auth_token) { + return service_ && service_->AcknowledgePolicy( + node_id, delivery_id, success, auth_token); +} + +bool CfmRpcService::EnqueuePolicy(const std::string& node_id, + const std::string& payload, + const std::string& auth_token) { + return service_ && service_->EnqueuePolicy(node_id, payload, auth_token); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/collector_impl.cpp b/mooncake-store/src/io_pattern/collector_impl.cpp index e31b06c223..ff03f54ade 100644 --- a/mooncake-store/src/io_pattern/collector_impl.cpp +++ b/mooncake-store/src/io_pattern/collector_impl.cpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace mooncake::io_pattern { @@ -13,6 +14,43 @@ uint64_t NowNs() { } } +void IoPatternCollectorImpl::ApplyAccessWindow(const ObjectRef& object, + uint64_t now_ns, + KeyMetrics& metrics) const { + metrics.access_count_window = 0; + metrics.write_frequency = 0; + metrics.overwrite_ratio = 0.0F; + metrics.write_batch_size = 0; + metrics.write_burst = false; + const auto it = access_windows_.find(object); + if (it == access_windows_.end()) return; + + const uint64_t cutoff = config_.access_window_ns != 0 && + now_ns > config_.access_window_ns + ? now_ns - config_.access_window_ns + : 0; + uint64_t writes = 0; + uint64_t overwrites = 0; + for (const auto& bucket : it->second) { + if (config_.access_window_ns != 0 && bucket.observed_at_ns < cutoff) { + continue; + } + metrics.access_count_window += bucket.access_count; + writes += bucket.write_count; + overwrites += bucket.overwrite_count; + metrics.write_batch_size = + std::max(metrics.write_batch_size, bucket.max_write_batch_size); + } + metrics.write_frequency = + static_cast(std::min( + writes, std::numeric_limits::max())); + if (writes != 0) { + metrics.overwrite_ratio = static_cast(overwrites) / + static_cast(writes); + } + metrics.write_burst = metrics.write_batch_size >= 16; +} + void IoPatternCollectorImpl::ReportInferenceMetrics( const InferenceMetrics& metrics) { std::lock_guard lock(mutex_); @@ -66,32 +104,70 @@ void IoPatternCollectorImpl::RecordAccess(const std::string& key, if (!key_metrics_.contains(object)) ++tenant_key_counts_[object.tenant_id]; auto& value = key_metrics_[object]; value.object = object; - ++value.access_count_window; + const uint64_t observed_at_ns = + record.observed_at_ns == 0 + ? (config_.now_ns ? config_.now_ns() : NowNs()) + : record.observed_at_ns; + auto& window = access_windows_[object]; + const uint64_t bucket_ns = std::max(1, config_.access_bucket_ns); + const uint64_t bucket_timestamp = + observed_at_ns - (observed_at_ns % bucket_ns); + const uint64_t reference_ns = + window.empty() ? observed_at_ns + : std::max(observed_at_ns, + window.back().observed_at_ns); + AccessWindowBucket bucket{.observed_at_ns = bucket_timestamp, + .access_count = 1}; + if (record.operation == IoOperation::kPut) { + bucket.write_count = 1; + bucket.overwrite_count = record.overwrite ? 1 : 0; + bucket.max_write_batch_size = record.write_batch_size; + } + const auto position = std::lower_bound( + window.begin(), window.end(), bucket_timestamp, + [](const AccessWindowBucket& existing, uint64_t timestamp) { + return existing.observed_at_ns < timestamp; + }); + if (position != window.end() && + position->observed_at_ns == bucket_timestamp) { + position->access_count += bucket.access_count; + position->write_count += bucket.write_count; + position->overwrite_count += bucket.overwrite_count; + position->max_write_batch_size = std::max( + position->max_write_batch_size, bucket.max_write_batch_size); + } else { + window.insert(position, bucket); + } + const uint64_t cutoff = config_.access_window_ns != 0 && + reference_ns > config_.access_window_ns + ? reference_ns - config_.access_window_ns + : 0; + while (!window.empty() && config_.access_window_ns != 0 && + window.front().observed_at_ns < cutoff) { + window.pop_front(); + } + while (config_.max_access_buckets_per_key != 0 && + window.size() > config_.max_access_buckets_per_key) { + window.pop_front(); + } value.last_access_time_ns = - std::max(value.last_access_time_ns, record.observed_at_ns); + std::max(value.last_access_time_ns, observed_at_ns); value.block_size = std::max(value.block_size, record.block_size); value.replica_tiers |= CacheTierBit(record.tier); value.active = value.active || record.is_hit; if (record.operation == IoOperation::kPut) { - ++write_counts_[object]; - if (record.overwrite) ++overwrite_counts_[object]; - value.write_frequency = - static_cast(std::min(write_counts_[object], - UINT32_MAX)); - value.write_batch_size = - std::max(value.write_batch_size, record.write_batch_size); value.write_object_size = std::max(value.write_object_size, record.block_size); - value.overwrite_ratio = - static_cast(overwrite_counts_[object]) / - static_cast(write_counts_[object]); - value.write_burst = record.write_batch_size >= 16; } + ApplyAccessWindow(object, observed_at_ns, value); } void IoPatternCollectorImpl::RecordStorageMetric(const StorageMetric& metric) { std::lock_guard lock(mutex_); - if (reporter_ && !reporter_->EnqueueStorage(metric)) ++dropped_; + if (reporter_) { + reporter_->UpdateLoad(metric.memory_used_ratio, metric.rpc_latency_us); + if (!reporter_->EnqueueStorage(metric)) ++dropped_; + } StorageMetricKey key{metric.source_id, metric.tier}; auto it = storage_metrics_.find(key); if (it == storage_metrics_.end() || @@ -102,6 +178,7 @@ void IoPatternCollectorImpl::RecordStorageMetric(const StorageMetric& metric) { void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { std::lock_guard lock(mutex_); + const uint64_t received_at_ns = config_.now_ns ? config_.now_ns() : NowNs(); for (const auto& metrics : snapshot.keys) { if (!key_metrics_.contains(metrics.object) && config_.max_total_keys != 0 && @@ -121,6 +198,15 @@ void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { ++tenant_key_counts_[metrics.object.tenant_id]; } key_metrics_[metrics.object] = metrics; + auto& window = access_windows_[metrics.object]; + window.clear(); + window.push_back({.observed_at_ns = received_at_ns, + .access_count = metrics.access_count_window, + .write_count = metrics.write_frequency, + .overwrite_count = static_cast( + metrics.overwrite_ratio * + static_cast(metrics.write_frequency)), + .max_write_batch_size = metrics.write_batch_size}); } for (const auto& metric : snapshot.storage) { StorageMetricKey key{metric.source_id, metric.tier}; @@ -135,11 +221,11 @@ void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { IoPatternSnapshot IoPatternCollectorImpl::GetSnapshot() const { std::lock_guard lock(mutex_); IoPatternSnapshot snapshot; - snapshot.generated_at_ns = NowNs(); + snapshot.generated_at_ns = config_.now_ns ? config_.now_ns() : NowNs(); snapshot.keys.reserve(key_metrics_.size()); for (const auto& [object, metrics] : key_metrics_) { - (void)object; auto copy = metrics; + ApplyAccessWindow(object, snapshot.generated_at_ns, copy); if (copy.last_access_time_ns != 0 && snapshot.generated_at_ns > copy.last_access_time_ns) { copy.idle_time_us = diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp index 98e00f308c..d11a542bfc 100644 --- a/mooncake-store/src/io_pattern/policy_strategies.cpp +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -1,6 +1,7 @@ #include "io_pattern/policy_strategies.h" #include +#include #include namespace mooncake::io_pattern { @@ -122,11 +123,11 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, } uint64_t selected_bytes = 0; auto end = plan.candidates.begin(); - while (end != plan.candidates.end()) { - if (end->bytes > target_bytes - selected_bytes) { - break; - } - selected_bytes += end->bytes; + while (end != plan.candidates.end() && selected_bytes < target_bytes) { + selected_bytes = + end->bytes > std::numeric_limits::max() - selected_bytes + ? std::numeric_limits::max() + : selected_bytes + end->bytes; ++end; } plan.candidates.erase(end, plan.candidates.end()); @@ -156,10 +157,14 @@ AdmissionResult PrefixMatchAdmissionOps::Evaluate( result.decision = key->access_count_window >= config_.frequency_threshold ? AdmissionDecision::kAdmit : AdmissionDecision::kRejectFrequency; + const bool target_over_watermark = std::any_of( + context.snapshot.storage.begin(), context.snapshot.storage.end(), + [&](const StorageMetric& metric) { + return metric.tier == target_tier && + metric.memory_used_ratio >= config_.max_memory_used_ratio; + }); if (result.decision == AdmissionDecision::kAdmit && - !context.snapshot.storage.empty() && - context.snapshot.storage.front().memory_used_ratio >= - config_.max_memory_used_ratio) { + target_over_watermark) { result.decision = AdmissionDecision::kRejectWatermark; } result.confidence = diff --git a/mooncake-store/src/io_pattern/reporter.cpp b/mooncake-store/src/io_pattern/reporter.cpp index c8bf7f4fe8..a9428a45a8 100644 --- a/mooncake-store/src/io_pattern/reporter.cpp +++ b/mooncake-store/src/io_pattern/reporter.cpp @@ -19,13 +19,7 @@ void IoPatternReporter::Start() { worker_ = std::thread([this] { std::unique_lock lock(mutex_); while (running_) { - const size_t size = batch_.inference.size() + batch_.accesses.size() + - batch_.storage.size(); - const auto interval = - (capacity_ == 0 || size * 2 >= capacity_) - ? std::chrono::milliseconds(100) - : (size == 0 ? std::chrono::milliseconds(1000) - : std::chrono::milliseconds(500)); + const auto interval = RecommendedFlushIntervalLocked(); condition_.wait_for(lock, interval, [this] { return !running_; }); if (!running_) break; lock.unlock(); @@ -131,13 +125,25 @@ uint64_t IoPatternReporter::reported() const { std::chrono::milliseconds IoPatternReporter::RecommendedFlushInterval() const { std::lock_guard lock(mutex_); - const size_t size = batch_.inference.size() + batch_.accesses.size() + - batch_.storage.size(); - if (capacity_ == 0 || size * 2 >= capacity_) { - return std::chrono::milliseconds(100); + return RecommendedFlushIntervalLocked(); +} + +void IoPatternReporter::UpdateLoad(float memory_used_ratio, + uint64_t rpc_latency_us) { + std::lock_guard lock(mutex_); + memory_used_ratio_ = memory_used_ratio; + rpc_latency_us_ = rpc_latency_us; + condition_.notify_one(); +} + +std::chrono::milliseconds +IoPatternReporter::RecommendedFlushIntervalLocked() const { + if (memory_used_ratio_ >= 0.95F || rpc_latency_us_ > 100'000) { + return std::chrono::milliseconds(1000); } - if (size == 0) return std::chrono::milliseconds(1000); - return std::chrono::milliseconds(500); + if (memory_used_ratio_ >= 0.80F) return std::chrono::milliseconds(500); + if (memory_used_ratio_ >= 0.50F) return std::chrono::milliseconds(200); + return std::chrono::milliseconds(100); } } // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp index 4786cdad53..013f7e725d 100644 --- a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp +++ b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp @@ -22,20 +22,31 @@ bool ResilientCfmChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { return Retry([&] { return delegate_->SendSnapshot(snapshot); }); } -std::optional ResilientCfmChannel::PollPolicy() { +CfmPollResult ResilientCfmChannel::PollPolicyResult() { if (!delegate_) { RecordFailure(); - return std::nullopt; + return CfmPollResult::Error(); } for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { - auto result = delegate_->PollPolicy(); - if (result.has_value()) { + auto result = delegate_->PollPolicyResult(); + if (result.status == CfmPollResult::Status::kCommand) { + RecordSuccess(); + return result; + } + if (result.status == CfmPollResult::Status::kEmpty) { RecordSuccess(); return result; } } RecordFailure(); - return std::nullopt; + return CfmPollResult::Error(); +} + +bool ResilientCfmChannel::AcknowledgePolicy(uint64_t delivery_id, + bool success) { + return Retry([&] { + return delegate_->AcknowledgePolicy(delivery_id, success); + }); } ErrorCode ResilientCfmChannel::ExecutePrefetch(const PrefetchPlan& plan) { diff --git a/mooncake-store/src/io_pattern/rpc_transport.cpp b/mooncake-store/src/io_pattern/rpc_transport.cpp index 429d894bae..8f9e992890 100644 --- a/mooncake-store/src/io_pattern/rpc_transport.cpp +++ b/mooncake-store/src/io_pattern/rpc_transport.cpp @@ -1,7 +1,155 @@ #include "io_pattern/rpc_transport.h" +#include +#include +#include + +#include + +#include "io_pattern/cfm_service.h" +#include "store_rpc_client_io_context.h" + namespace mooncake::io_pattern { +class CoroRpcCfmTransport::Impl { + public: + Impl(std::string endpoint, std::string node_id, + std::chrono::milliseconds default_timeout) + : endpoint_(std::move(endpoint)), + node_id_(std::move(node_id)), + default_timeout_(default_timeout) {} + + template + std::optional Invoke(std::chrono::milliseconds timeout, + Args&&... args) { + auto pool = GetPool(timeout.count() > 0 ? timeout : default_timeout_); + return async_simple::coro::syncAwait( + [&]() -> async_simple::coro::Lazy> { + auto request = co_await pool->send_request( + [&](coro_io::client_reuse_hint, + coro_rpc::coro_rpc_client& client) { + return client.send_request( + std::forward(args)...); + }); + if (!request) co_return std::nullopt; + auto response = co_await std::move(request.value()); + if (!response) co_return std::nullopt; + co_return response->result(); + }()); + } + + std::shared_ptr> GetPool( + std::chrono::milliseconds timeout) { + std::lock_guard lock(mutex_); + const auto key = timeout.count(); + const auto existing = pools_.find(key); + if (existing != pools_.end()) return existing->second; + coro_io::client_pool::pool_config config; + config.client_config.request_timeout_duration = timeout; + config.host_alive_detect_duration = std::chrono::seconds(0); + auto pool = coro_io::client_pool::create( + endpoint_, config, GetStoreRpcClientIoContextPool()); + pools_.emplace(key, pool); + return pool; + } + + std::string endpoint_; + std::string node_id_; + std::chrono::milliseconds default_timeout_; + std::mutex mutex_; + std::string auth_token_; + std::unordered_map< + int64_t, + std::shared_ptr>> + pools_; +}; + +CoroRpcCfmTransport::CoroRpcCfmTransport( + std::string endpoint, std::string node_id, + std::chrono::milliseconds default_timeout) + : impl_(std::make_unique(std::move(endpoint), std::move(node_id), + default_timeout)) {} + +CoroRpcCfmTransport::~CoroRpcCfmTransport() = default; + +bool CoroRpcCfmTransport::Authenticate(std::string_view token) { + if (!impl_ || token.empty()) return false; + const std::string wire_token(token); + const auto result = impl_->Invoke<&CfmRpcService::Authenticate, bool>( + impl_->default_timeout_, wire_token); + if (!result || !*result) return false; + std::lock_guard lock(impl_->mutex_); + impl_->auth_token_ = wire_token; + return true; +} + +bool CoroRpcCfmTransport::Send(std::string_view method, + std::string_view payload, + std::chrono::milliseconds timeout) { + if (!impl_) return false; + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return false; + const auto result = impl_->Invoke<&CfmRpcService::Send, bool>( + timeout, impl_->node_id_, std::string(method), std::string(payload), + auth_token); + return result && *result; +} + +CfmReceiveResult CoroRpcCfmTransport::Receive( + std::string_view method, std::chrono::milliseconds timeout) { + if (!impl_) return CfmReceiveResult::Error(); + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return CfmReceiveResult::Error(); + const auto result = + impl_->Invoke<&CfmRpcService::Receive, + std::pair< + bool, + std::optional>>>( + timeout, std::string(method), impl_->node_id_, auth_token); + if (!result || !result->first) return CfmReceiveResult::Error(); + if (!result->second) return CfmReceiveResult::Empty(); + return CfmReceiveResult::Payload(std::move(result->second->second), + result->second->first); +} + +bool CoroRpcCfmTransport::Acknowledge( + uint64_t delivery_id, bool success, std::chrono::milliseconds timeout) { + if (!impl_ || delivery_id == 0) return false; + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return false; + const auto result = impl_->Invoke<&CfmRpcService::Acknowledge, bool>( + timeout, impl_->node_id_, delivery_id, success, auth_token); + return result && *result; +} + +bool CoroRpcCfmTransport::EnqueuePolicy( + std::string_view node_id, std::string_view payload, + std::chrono::milliseconds timeout) { + if (!impl_) return false; + std::string auth_token; + { + std::lock_guard lock(impl_->mutex_); + auth_token = impl_->auth_token_; + } + if (auth_token.empty()) return false; + const auto result = + impl_->Invoke<&CfmRpcService::EnqueuePolicy, bool>( + timeout, std::string(node_id), std::string(payload), auth_token); + return result && *result; +} + bool InProcessCfmRpcTransport::Authenticate(std::string_view token) { std::lock_guard lock(mutex_); authenticated_ = token == auth_token_; @@ -11,20 +159,23 @@ bool InProcessCfmRpcTransport::Authenticate(std::string_view token) { bool InProcessCfmRpcTransport::Send(std::string_view method, std::string_view payload, std::chrono::milliseconds) { - std::lock_guard lock(mutex_); - if (!authenticated_) return false; - return !send_handler_ || send_handler_(method, payload); + SendHandler handler; + { + std::lock_guard lock(mutex_); + if (!authenticated_) return false; + handler = send_handler_; + } + return !handler || handler(method, payload); } -std::optional InProcessCfmRpcTransport::Receive( +CfmReceiveResult InProcessCfmRpcTransport::Receive( std::string_view method, std::chrono::milliseconds) { std::lock_guard lock(mutex_); - if (!authenticated_ || method != "poll_policy" || policies_.empty()) { - return std::nullopt; - } + if (!authenticated_ || method != "poll_policy") return CfmReceiveResult::Error(); + if (policies_.empty()) return CfmReceiveResult::Empty(); auto payload = std::move(policies_.front()); policies_.pop(); - return payload; + return CfmReceiveResult::Payload(std::move(payload)); } void InProcessCfmRpcTransport::EnqueuePolicy(std::string payload) { @@ -50,10 +201,28 @@ bool CfmRpcChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { config_.timeout); } -std::optional CfmRpcChannel::PollPolicy() { - if (!transport_ || !codec_ || !EnsureAuthenticated()) return std::nullopt; - const auto payload = transport_->Receive("poll_policy", config_.timeout); - return payload ? codec_->DecodePolicy(*payload) : std::nullopt; +CfmPollResult CfmRpcChannel::PollPolicyResult() { + if (!transport_ || !codec_ || !EnsureAuthenticated()) { + return CfmPollResult::Error(); + } + auto received = transport_->Receive("poll_policy", config_.timeout); + if (received.status == CfmReceiveResult::Status::kEmpty) { + return CfmPollResult::Empty(); + } + if (received.status == CfmReceiveResult::Status::kError) { + return CfmPollResult::Error(); + } + auto command = codec_->DecodePolicy(received.payload); + if (!command) { + transport_->Acknowledge(received.delivery_id, false, config_.timeout); + return CfmPollResult::Error(); + } + return CfmPollResult::Command(std::move(*command), received.delivery_id); +} + +bool CfmRpcChannel::AcknowledgePolicy(uint64_t delivery_id, bool success) { + return transport_ && delivery_id != 0 && EnsureAuthenticated() && + transport_->Acknowledge(delivery_id, success, config_.timeout); } ErrorCode CfmRpcChannel::ExecutePrefetch(const PrefetchPlan& plan) { @@ -87,14 +256,26 @@ bool CfmChannelPool::SendSnapshot(const IoPatternSnapshot& snapshot) { return false; } -std::optional CfmChannelPool::PollPolicy() { +CfmPollResult CfmChannelPool::PollPolicyResult() { + bool saw_empty = false; for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { auto channel = Next(); if (!channel) continue; - auto command = channel->PollPolicy(); - if (command) return command; + auto result = channel->PollPolicyResult(); + if (result.status == CfmPollResult::Status::kCommand) return result; + saw_empty = saw_empty || result.status == CfmPollResult::Status::kEmpty; + } + return saw_empty ? CfmPollResult::Empty() : CfmPollResult::Error(); +} + +bool CfmChannelPool::AcknowledgePolicy(uint64_t delivery_id, bool success) { + for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { + auto channel = Next(); + if (channel && channel->AcknowledgePolicy(delivery_id, success)) { + return true; + } } - return std::nullopt; + return false; } ErrorCode CfmChannelPool::ExecutePrefetch(const PrefetchPlan& plan) { diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index f2a41f2cae..1e4bc31d02 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -38,9 +38,17 @@ IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) std::make_shared(std::move(legacy_strategy)), nullptr, std::make_shared()); policy_ = std::make_shared(workload_policy_, fallback); + admission_worker_ = std::thread(&IoPatternRuntime::AdmissionWorker, this); } IoPatternRuntime::~IoPatternRuntime() { + { + std::lock_guard lock(admission_mutex_); + admission_stopping_ = true; + pending_admissions_.clear(); + } + admission_condition_.notify_all(); + if (admission_worker_.joinable()) admission_worker_.join(); if (reporter_) reporter_->Stop(); } @@ -160,32 +168,18 @@ PatternResult IoPatternRuntime::AnalyzeWithinBudget( PolicyExecutionStatus IoPatternRuntime::Execute( CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, const std::vector& admissions, const std::string& session_id) { - const auto snapshot = collector_->GetSnapshot(); - const auto start = std::chrono::steady_clock::now(); - bool analysis_degraded = false; - const auto analysis = AnalyzeWithinBudget(snapshot, analysis_degraded); - const auto elapsed = std::chrono::duration_cast( - std::chrono::steady_clock::now() - start) - .count(); - observability_.RecordAnalyzeLatency(elapsed); - - workload_policy_->SetWorkloadType(analysis.workload_type); - workload_policy_->SetSessionWorkloads(analysis.sessions); - workload_policy_->AdvanceTransitionWindow(); - const PolicyResult result = policy_->ExecutePolicy( - PolicyContext{.snapshot = snapshot, .analysis = analysis, - .session_id = session_id}, eviction_tier, - eviction_bytes, trace, admissions); + auto planned = BuildPolicy(eviction_tier, eviction_bytes, trace, admissions, + session_id); + const auto& snapshot = planned.snapshot; + const auto& result = planned.result; auto status = executor_.Execute(result); - status.degraded = status.degraded || result.degraded || collector_->degraded() || - analysis_degraded || - elapsed > static_cast(config_.analysis_timeout_us); - observability_.RecordPolicyDecision(!result.eviction.candidates.empty() || - !result.prefetch.candidates.empty()); + status.degraded = status.degraded || result.degraded; const bool failed = status.eviction != ErrorCode::OK || status.prefetch != ErrorCode::OK || status.degraded; - if (failed) policy_->RecordFailure(); - else policy_->RecordSuccess(); + if (failed) + policy_->RecordFailure(); + else + policy_->RecordSuccess(); if (status.degraded || policy_->degraded()) observability_.RecordDegrade(); status.degraded = status.degraded || policy_->degraded(); @@ -199,7 +193,8 @@ PolicyExecutionStatus IoPatternRuntime::Execute( pending_prefetches_.insert(candidate.object); } } - if (!result.prefetch.candidates.empty() && status.prefetch != ErrorCode::OK) { + if (!result.prefetch.candidates.empty() && + status.prefetch != ErrorCode::OK) { feedback.prefetch_accuracy = 0.0F; has_feedback = true; } @@ -214,6 +209,45 @@ PolicyExecutionStatus IoPatternRuntime::Execute( return status; } +PolicyResult IoPatternRuntime::Plan( + CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, + const std::vector& admissions, const std::string& session_id) { + return BuildPolicy(eviction_tier, eviction_bytes, trace, admissions, + session_id) + .result; +} + +IoPatternRuntime::PlannedPolicy IoPatternRuntime::BuildPolicy( + CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, + const std::vector& admissions, const std::string& session_id) { + PlannedPolicy planned; + planned.snapshot = collector_->GetSnapshot(); + const auto start = std::chrono::steady_clock::now(); + const auto analysis = + AnalyzeWithinBudget(planned.snapshot, planned.analysis_degraded); + planned.analysis_elapsed_us = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now() - start) + .count()); + observability_.RecordAnalyzeLatency(planned.analysis_elapsed_us); + + workload_policy_->SetWorkloadType(analysis.workload_type); + workload_policy_->SetSessionWorkloads(analysis.sessions); + workload_policy_->AdvanceTransitionWindow(); + planned.result = policy_->ExecutePolicy( + PolicyContext{.snapshot = planned.snapshot, .analysis = analysis, + .session_id = session_id}, eviction_tier, + eviction_bytes, trace, admissions); + planned.result.degraded = + planned.result.degraded || collector_->degraded() || + planned.analysis_degraded || + planned.analysis_elapsed_us > config_.analysis_timeout_us; + observability_.RecordPolicyDecision( + !planned.result.eviction.candidates.empty() || + !planned.result.prefetch.candidates.empty()); + return planned; +} + ErrorCode IoPatternRuntime::ExecuteCommand(const PolicyCommand& command) { PolicyResult result; if (const auto* eviction = std::get_if(&command)) { @@ -237,6 +271,70 @@ ErrorCode IoPatternRuntime::ExecuteCommand(const PolicyCommand& command) { return status.admissions.empty() ? ErrorCode::OK : status.admissions.front(); } +bool IoPatternRuntime::ScheduleAdmission(ObjectRef object, CacheTier target_tier, + std::string session_id) { + { + std::lock_guard lock(admission_mutex_); + if (admission_stopping_ || + (config_.max_pending_admissions != 0 && + pending_admissions_.size() >= config_.max_pending_admissions)) { + return false; + } + pending_admissions_.push_back( + {.object = std::move(object), + .target_tier = target_tier, + .session_id = std::move(session_id)}); + } + admission_condition_.notify_one(); + return true; +} + +void IoPatternRuntime::AdmissionWorker() { + while (true) { + PendingAdmission pending; + { + std::unique_lock lock(admission_mutex_); + admission_condition_.wait(lock, [this] { + return admission_stopping_ || !pending_admissions_.empty(); + }); + if (admission_stopping_) return; + pending = std::move(pending_admissions_.front()); + pending_admissions_.pop_front(); + } + try { + ExecuteAdmission(pending.object, pending.target_tier, + pending.session_id); + } catch (...) { + policy_->RecordFailure(); + observability_.RecordDegrade(); + } + } +} + +ErrorCode IoPatternRuntime::ExecuteAdmission(const ObjectRef& object, + CacheTier target_tier, + const std::string& session_id) { + const auto snapshot = collector_->GetSnapshot(); + bool analysis_degraded = false; + const auto analysis = AnalyzeWithinBudget(snapshot, analysis_degraded); + workload_policy_->SetWorkloadType(analysis.workload_type); + workload_policy_->SetSessionWorkloads(analysis.sessions); + const auto admission = policy_->DecideAdmission( + object, target_tier, + PolicyContext{.snapshot = snapshot, + .analysis = analysis, + .session_id = session_id}); + PolicyResult result; + result.admissions.push_back(admission); + auto status = executor_.Execute(result); + if (analysis_degraded) status.degraded = true; + const auto code = status.admissions.empty() ? ErrorCode::OK + : status.admissions.front(); + if (code != ErrorCode::OK || status.degraded) + observability_.RecordDegrade(); + return code; +} + void IoPatternRuntime::RecordFeedback(PolicyFeedbackSample sample) { feedback_.Record(sample); auto config = workload_policy_->CurrentEvictionConfig(); diff --git a/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp index ed992fc45e..2020f8d021 100644 --- a/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp +++ b/mooncake-store/src/io_pattern/sliding_window_analyzer.cpp @@ -1,6 +1,7 @@ #include "io_pattern/sliding_window_analyzer.h" #include +#include #include namespace mooncake::io_pattern { @@ -11,19 +12,60 @@ T Percentile(std::vector values, size_t rank) { std::sort(values.begin(), values.end()); return values[std::min(rank, values.size() - 1)]; } + +std::vector LatestKeys( + const std::deque& history) { + std::unordered_map latest; + for (const auto& snapshot : history) { + for (const auto& key : snapshot.keys) latest[key.object] = key; + } + std::vector keys; + keys.reserve(latest.size()); + for (auto& [object, key] : latest) { + (void)object; + keys.push_back(std::move(key)); + } + std::sort(keys.begin(), keys.end(), [](const KeyMetrics& left, + const KeyMetrics& right) { + if (left.object.tenant_id != right.object.tenant_id) { + return left.object.tenant_id < right.object.tenant_id; + } + return left.object.key < right.object.key; + }); + return keys; +} } void SlidingWindowAnalyzer::Append(const IoPatternSnapshot& snapshot) const { std::lock_guard lock(mutex_); + IoPatternSnapshot bounded = snapshot; + if (max_history_keys_ != 0 && + bounded.keys.size() > max_history_keys_) { + std::sort(bounded.keys.begin(), bounded.keys.end(), + [](const KeyMetrics& left, const KeyMetrics& right) { + if (left.object.tenant_id != right.object.tenant_id) { + return left.object.tenant_id < right.object.tenant_id; + } + return left.object.key < right.object.key; + }); + bounded.keys.resize(max_history_keys_); + } if (history_.empty() || - history_.back().generated_at_ns != snapshot.generated_at_ns) { - history_.push_back(snapshot); + history_.back().generated_at_ns != bounded.generated_at_ns) { + history_key_count_ += bounded.keys.size(); + history_.push_back(std::move(bounded)); } const uint64_t cutoff = snapshot.generated_at_ns > window_ns_ ? snapshot.generated_at_ns - window_ns_ : 0; - while (!history_.empty() && history_.front().generated_at_ns < cutoff) + while (!history_.empty() && history_.front().generated_at_ns < cutoff) { + history_key_count_ -= history_.front().keys.size(); history_.pop_front(); + } + while (max_history_keys_ != 0 && history_key_count_ > max_history_keys_) { + history_key_count_ -= history_.front().keys.size(); + history_.pop_front(); + } } IoPatternSnapshot SlidingWindowAnalyzer::Aggregate( @@ -31,11 +73,7 @@ IoPatternSnapshot SlidingWindowAnalyzer::Aggregate( Append(current); std::lock_guard lock(mutex_); IoPatternSnapshot aggregate = current; - aggregate.keys.clear(); - for (const auto& snapshot : history_) { - aggregate.keys.insert(aggregate.keys.end(), snapshot.keys.begin(), - snapshot.keys.end()); - } + aggregate.keys = LatestKeys(history_); return aggregate; } @@ -67,14 +105,12 @@ WorkloadFeatureStats SlidingWindowAnalyzer::FeatureStats() const { std::lock_guard lock(mutex_); std::vector tokens, fanouts, matches, frequencies; std::vector blocks; - for (const auto& snapshot : history_) { - for (const auto& key : snapshot.keys) { - tokens.push_back(key.token_count); - fanouts.push_back(key.prefix_fanout); - matches.push_back(key.match_length); - frequencies.push_back(static_cast(key.access_count_window)); - blocks.push_back(key.block_size); - } + for (const auto& key : LatestKeys(history_)) { + tokens.push_back(key.token_count); + fanouts.push_back(key.prefix_fanout); + matches.push_back(key.match_length); + frequencies.push_back(static_cast(key.access_count_window)); + blocks.push_back(key.block_size); } const auto p90 = [](size_t size) { return size == 0 ? 0 : (size * 9) / 10; }; WorkloadFeatureStats stats; diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index b6176b11e7..c23e65261f 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -160,6 +160,19 @@ DEFINE_int32(rpc_conn_timeout_seconds, 0, "Connection timeout in seconds (0 = no timeout)"); DEFINE_bool(rpc_enable_tcp_no_delay, true, "Enable TCP_NODELAY for RPC connections"); +DEFINE_string(io_pattern_cfm_endpoint, "", + "Central CFM Master RPC endpoint (host:port); empty serves CFM " + "requests without outbound reporting"); +DEFINE_string(io_pattern_cfm_node_id, "", + "Stable node id used for CFM policy polling; defaults to cluster_id"); +DEFINE_string(io_pattern_cfm_auth_token, "", + "Authentication token for CFM node report/poll RPCs"); +DEFINE_string(io_pattern_cfm_producer_auth_token, "", + "Separate token authorized to enqueue CFM policies"); +DEFINE_uint32(io_pattern_cfm_timeout_ms, 500, + "CFM RPC request timeout in milliseconds"); +DEFINE_uint32(io_pattern_cfm_policy_queue_capacity, 4096, + "Maximum queued CFM policies per node"); DEFINE_validator(eviction_ratio, [](const char* flagname, double value) { if (value < 0.0 || value > 1.0) { LOG(FATAL) << "Mem eviction ratio must be between 0.0 and 1.0"; @@ -513,6 +526,26 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, default_config.GetBool("rpc_enable_tcp_no_delay", &master_config.rpc_enable_tcp_no_delay, FLAGS_rpc_enable_tcp_no_delay); + default_config.GetString("io_pattern_cfm_endpoint", + &master_config.io_pattern_cfm.endpoint, + FLAGS_io_pattern_cfm_endpoint); + default_config.GetString("io_pattern_cfm_node_id", + &master_config.io_pattern_cfm.node_id, + FLAGS_io_pattern_cfm_node_id); + default_config.GetString("io_pattern_cfm_auth_token", + &master_config.io_pattern_cfm.auth_token, + FLAGS_io_pattern_cfm_auth_token); + default_config.GetString( + "io_pattern_cfm_producer_auth_token", + &master_config.io_pattern_cfm.producer_auth_token, + FLAGS_io_pattern_cfm_producer_auth_token); + default_config.GetUInt32("io_pattern_cfm_timeout_ms", + &master_config.io_pattern_cfm.timeout_ms, + FLAGS_io_pattern_cfm_timeout_ms); + default_config.GetUInt32( + "io_pattern_cfm_policy_queue_capacity", + &master_config.io_pattern_cfm.policy_queue_capacity, + FLAGS_io_pattern_cfm_policy_queue_capacity); default_config.GetDurationMs("default_kv_lease_ttl", &master_config.default_kv_lease_ttl, mooncake::DEFAULT_DEFAULT_KV_LEASE_TTL); @@ -809,6 +842,41 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, } google::CommandLineFlagInfo info; + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_endpoint", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.endpoint = FLAGS_io_pattern_cfm_endpoint; + } + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_node_id", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.node_id = FLAGS_io_pattern_cfm_node_id; + } + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_auth_token", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.auth_token = + FLAGS_io_pattern_cfm_auth_token; + } + if ((google::GetCommandLineFlagInfo( + "io_pattern_cfm_producer_auth_token", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.producer_auth_token = + FLAGS_io_pattern_cfm_producer_auth_token; + } + if ((google::GetCommandLineFlagInfo("io_pattern_cfm_timeout_ms", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.timeout_ms = FLAGS_io_pattern_cfm_timeout_ms; + } + if ((google::GetCommandLineFlagInfo( + "io_pattern_cfm_policy_queue_capacity", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cfm.policy_queue_capacity = + FLAGS_io_pattern_cfm_policy_queue_capacity; + } if ((google::GetCommandLineFlagInfo("enable_cxl", &info) && !info.is_default) || !conf_set) { @@ -1522,6 +1590,29 @@ int main(int argc, char* argv[]) { << ", must be 'cachelib' or 'offset'"; return 1; } + if (!master_config.io_pattern_cfm.endpoint.empty() && + master_config.io_pattern_cfm.auth_token.empty()) { + LOG(FATAL) << "io_pattern_cfm_auth_token is required when " + "io_pattern_cfm_endpoint is configured"; + return 1; + } + if (!master_config.io_pattern_cfm.producer_auth_token.empty() && + master_config.io_pattern_cfm.producer_auth_token == + master_config.io_pattern_cfm.auth_token) { + LOG(FATAL) << "io_pattern_cfm_producer_auth_token must differ from " + "io_pattern_cfm_auth_token"; + return 1; + } + if (master_config.io_pattern_cfm.timeout_ms == 0 || + master_config.io_pattern_cfm.timeout_ms > 10'000 || + master_config.io_pattern_cfm.policy_queue_capacity == 0) { + LOG(FATAL) << "io_pattern_cfm_timeout_ms must be in [1, 10000] and " + "io_pattern_cfm_policy_queue_capacity must be non-zero"; + return 1; + } + if (master_config.io_pattern_cfm.node_id.empty()) { + master_config.io_pattern_cfm.node_id = master_config.cluster_id; + } const char* value = std::getenv("MC_RPC_PROTOCOL"); std::string protocol = "tcp"; @@ -1591,6 +1682,14 @@ int main(int argc, char* argv[]) { << ", client_ttl=" << master_config.client_live_ttl_sec << ", rpc_thread_num=" << master_config.rpc_thread_num << ", rpc_port=" << master_config.rpc_port + << ", io_pattern_cfm_endpoint=" + << (master_config.io_pattern_cfm.endpoint.empty() + ? "" + : master_config.io_pattern_cfm.endpoint) + << ", io_pattern_cfm_node_id=" + << master_config.io_pattern_cfm.node_id + << ", io_pattern_cfm_server_enabled=" + << !master_config.io_pattern_cfm.auth_token.empty() << ", rpc_address=" << master_config.rpc_address << ", rpc_interface=" << master_config.rpc_interface << ", rpc_conn_timeout_seconds=" diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index e16e698570..00ee88b62d 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -60,6 +60,11 @@ #include "ha_metric_manager.h" #include "metadata_store.h" #include "io_pattern/runtime.h" +#include "io_pattern/cfm_client_impl.h" +#include "io_pattern/cfm_protocol.h" +#include "io_pattern/cfm_service.h" +#include "io_pattern/resilient_cfm_channel.h" +#include "io_pattern/rpc_transport.h" namespace mooncake { @@ -247,8 +252,7 @@ MasterService::MasterService(const MasterServiceConfig& config) if (partitioned_vchunk) { vchunk_recovery_pending_ = true; } else { - const auto error = - vchunk_manager_.Recover(getCurrentTimeInMilli()); + const auto error = vchunk_manager_.Recover(getCurrentTimeInMilli()); if (error != ErrorCode::OK) { throw std::runtime_error("failed to recover vchunk metadata"); } @@ -437,50 +441,115 @@ MasterService::MasterService(const MasterServiceConfig& config) << ")"; } - io_pattern_runtime_ = std::make_unique( + io_pattern::IoPatternRuntime::Config io_pattern_config; + std::shared_ptr cfm_rpc_channel; + if (!config.io_pattern_cfm.endpoint.empty()) { + if (config.io_pattern_cfm.timeout_ms == 0 || + config.io_pattern_cfm.timeout_ms > 10'000) { + throw std::invalid_argument( + "io_pattern_cfm_timeout_ms must be in [1, 10000]"); + } + if (config.io_pattern_cfm.auth_token.empty()) { + throw std::invalid_argument( + "io_pattern_cfm_auth_token is required when " + "io_pattern_cfm_endpoint is configured"); + } + const std::string node_id = config.io_pattern_cfm.node_id.empty() + ? config.cluster_id + : config.io_pattern_cfm.node_id; + auto transport = std::make_shared( + config.io_pattern_cfm.endpoint, node_id, + std::chrono::milliseconds(config.io_pattern_cfm.timeout_ms)); + if (!transport->Authenticate(config.io_pattern_cfm.auth_token)) { + throw std::runtime_error( + "failed to authenticate with configured IO-pattern CFM " + "endpoint " + + config.io_pattern_cfm.endpoint); + } + cfm_rpc_channel = std::make_shared( + std::move(transport), + std::make_shared(), + io_pattern::CfmRpcConfig{ + .timeout = + std::chrono::milliseconds(config.io_pattern_cfm.timeout_ms), + .auth_token = config.io_pattern_cfm.auth_token}); + io_pattern_config.report_sink = + io_pattern::MakeCfmMetricBatchSink(cfm_rpc_channel); + io_pattern_cfm_channel_ = + std::make_shared(cfm_rpc_channel); + } + + io_pattern_runtime_ = std::make_shared( io_pattern::IoPatternRuntime::Handlers{ - .eviction = [this](const io_pattern::EvictionPlan& plan) { - bool evicted = plan.candidates.empty(); - std::unordered_map targets; - for (const auto& candidate : plan.candidates) { - targets[candidate.object.tenant_id] += candidate.bytes; - } - for (const auto& [tenant, bytes] : targets) { - const auto result = EvictTenantMemoryForQuota(tenant, bytes); - evicted = evicted || result.freed_bytes != 0; - } - return evicted ? ErrorCode::OK : ErrorCode::OBJECT_NOT_FOUND; - }, - .prefetch = [this](const io_pattern::PrefetchPlan& plan) { - for (const auto& candidate : plan.candidates) { - // Store's safe promotion primitive is LOCAL_DISK -> MEMORY; - // HBM remains inference-runtime-owned and is never promoted - // from the master control plane. - if (candidate.target_tier == io_pattern::CacheTier::kL0Hbm) { - return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; - } - const ObjectIdentity object_id{candidate.object.tenant_id, - candidate.object.key}; - if (TryPushPromotionQueue(object_id, - /*record_candidate=*/false) != - PromotionQueueResult::kQueued) { + .eviction = + [this](const io_pattern::EvictionPlan& plan) { + struct TenantCandidates { + uint64_t bytes{0}; + std::unordered_set keys; + }; + if (plan.target_bytes == 0) return ErrorCode::OK; + if (plan.candidates.empty()) return ErrorCode::OBJECT_NOT_FOUND; + uint64_t total_freed = 0; + std::unordered_map + targets; + for (const auto& candidate : plan.candidates) { + auto& target = targets[candidate.object.tenant_id]; + target.bytes += candidate.bytes; + target.keys.insert(candidate.object.key); } - } - return ErrorCode::OK; - }, - .admission = [this](const io_pattern::AdmissionResult& result) { - if (result.target_tier == io_pattern::CacheTier::kL0Hbm) { - return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; - } - const ObjectIdentity object_id{result.object.tenant_id, - result.object.key}; - return TryPushPromotionQueue(object_id, - /*record_candidate=*/false) == - PromotionQueueResult::kQueued - ? ErrorCode::OK - : ErrorCode::OBJECT_NOT_FOUND; - }}); + for (const auto& [tenant, target] : targets) { + const auto result = EvictTenantMemoryForQuota( + tenant, target.bytes, &target.keys); + total_freed = + result.freed_bytes > + std::numeric_limits::max() - + total_freed + ? std::numeric_limits::max() + : total_freed + result.freed_bytes; + } + return total_freed >= plan.target_bytes + ? ErrorCode::OK + : ErrorCode::OBJECT_NOT_FOUND; + }, + .prefetch = + [this](const io_pattern::PrefetchPlan& plan) { + for (const auto& candidate : plan.candidates) { + // Store's safe promotion primitive is LOCAL_DISK -> + // MEMORY; HBM remains inference-runtime-owned and is + // never promoted from the master control plane. + if (candidate.target_tier == + io_pattern::CacheTier::kL0Hbm) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + const ObjectIdentity object_id{ + candidate.object.tenant_id, candidate.object.key}; + if (TryPushPromotionQueue(object_id, + /*record_candidate=*/false) != + PromotionQueueResult::kQueued) { + return ErrorCode::OBJECT_NOT_FOUND; + } + } + return ErrorCode::OK; + }, + .admission = + [this](const io_pattern::AdmissionResult& result) { + if (result.target_tier == io_pattern::CacheTier::kL0Hbm) { + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + } + const ObjectIdentity object_id{result.object.tenant_id, + result.object.key}; + return TryPushPromotionQueue(object_id, + /*record_candidate=*/false) == + PromotionQueueResult::kQueued + ? ErrorCode::OK + : ErrorCode::OBJECT_NOT_FOUND; + }}, + std::move(io_pattern_config)); + io_pattern_cfm_service_ = std::make_shared( + io_pattern_runtime_, config.io_pattern_cfm.auth_token, + config.io_pattern_cfm.policy_queue_capacity, + config.io_pattern_cfm.producer_auth_token); kv_event_publisher_ = std::make_unique(BuildKvEventConfig(config)); @@ -606,9 +675,38 @@ MasterService::MasterService(const MasterServiceConfig& config) segment_manager_.initializeCxlAllocator(cxl_path_, cxl_size_); VLOG(1) << "action=start_cxl_global_allocator"; } + if (vchunk_enabled_ && !vchunk_recovery_pending_) { StartVChunkReaper(); } + + // Start the CFM consumer last. If any preceding initialization throws, + // constructor unwinding must not encounter a joinable std::thread. + if (io_pattern_cfm_channel_) { + io_pattern_cfm_client_ = std::make_unique( + io_pattern_cfm_channel_, + [this](const io_pattern::PolicyCommand& command) { + return io_pattern_runtime_ + ? io_pattern_runtime_->ExecuteCommand(command) + : ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + }); + io_pattern_cfm_polling_ = true; + io_pattern_cfm_poll_thread_ = std::thread([this] { + while (io_pattern_cfm_polling_.load(std::memory_order_acquire)) { + const auto result = + io_pattern_cfm_client_->PollAndDispatchPolicy(); + std::unique_lock lock(io_pattern_cfm_poll_mutex_); + io_pattern_cfm_poll_cv_.wait_for( + lock, + result == ErrorCode::OK ? std::chrono::milliseconds(100) + : std::chrono::seconds(1), + [this] { + return !io_pattern_cfm_polling_.load( + std::memory_order_acquire); + }); + } + }); + } } tl::expected MasterService::VChunkPutStart( @@ -623,9 +721,8 @@ tl::expected MasterService::VChunkPutStart( } auto allocator_access = segment_manager_.getAllocatorAccess(); return vchunk_manager_.PutStart(allocator_access.getAllocatorManager(), - tenant_id, key, total_size, - is_ssd_segment, now_ms, - excluded_segments); + tenant_id, key, total_size, is_ssd_segment, + now_ms, excluded_segments); } ErrorCode MasterService::VChunkPutEnd(const TenantId& tenant_id, @@ -677,14 +774,11 @@ MasterService::AcquireVChunkRead(const TenantId& tenant_id, return vchunk_manager_.AcquireRead(tenant_id, key); } -tl::expected -MasterService::AcquireVChunkReadLease(const TenantId& tenant_id, - const std::string& key, - int64_t now_ms) { +tl::expected MasterService::AcquireVChunkReadLease( + const TenantId& tenant_id, const std::string& key, int64_t now_ms) { constexpr int64_t kRemoteReadLeaseTtlMs = 5 * 60 * 1000; if (now_ms < 0 || - now_ms > std::numeric_limits::max() - - kRemoteReadLeaseTtlMs) { + now_ms > std::numeric_limits::max() - kRemoteReadLeaseTtlMs) { return tl::make_unexpected(ErrorCode::INVALID_PARAMS); } auto handle = AcquireVChunkRead(tenant_id, key); @@ -702,8 +796,7 @@ MasterService::AcquireVChunkReadLease(const TenantId& tenant_id, return lease; } -ErrorCode MasterService::ReleaseVChunkReadLease( - const std::string& lease_id) { +ErrorCode MasterService::ReleaseVChunkReadLease(const std::string& lease_id) { if (lease_id.empty()) { return ErrorCode::INVALID_PARAMS; } @@ -713,8 +806,7 @@ ErrorCode MasterService::ReleaseVChunkReadLease( } ErrorCode MasterService::RemoveVChunk(const TenantId& tenant_id, - const std::string& key, - int64_t now_ms) { + const std::string& key, int64_t now_ms) { if (!vchunk_enabled_) { return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; } @@ -829,10 +921,9 @@ void MasterService::StopSlotOwnerHeartbeat() { #ifdef STORE_USE_ETCD ErrorCode MasterService::StartSlotOwnerHeartbeat() { - const bool kv_partition_enabled = enable_ha_ && - ha_backend_type_ == "etcd" && - !master_id_.empty() && - !cluster_id_.empty(); + const bool kv_partition_enabled = + enable_ha_ && ha_backend_type_ == "etcd" && !master_id_.empty() && + !cluster_id_.empty(); if (!kv_partition_enabled) { return ErrorCode::OK; } @@ -856,7 +947,8 @@ ErrorCode MasterService::StartSlotOwnerHeartbeat() { UpdateOwnedSlots(initial_slots); if (vchunk_recovery_pending_) { const auto error = vchunk_manager_.Recover( - getCurrentTimeInMilli(), [this](const VChunkMetadataRecord& record) { + getCurrentTimeInMilli(), + [this](const VChunkMetadataRecord& record) { return OwnsVChunkSlot( cvm::KeySlot(TenantId(record.tenant_id), record.key)); }); @@ -930,8 +1022,8 @@ ErrorCode MasterService::StartInterMasterRpc() { inter_master_rpc_ = std::make_unique(); ErrorCode rc = inter_master_rpc_->Start(cluster_id_, master_id_); if (rc != ErrorCode::OK) { - LOG(WARNING) << "StartInterMasterRpc: refresh loop not started: " - << rc << " (manual member updates still work)"; + LOG(WARNING) << "StartInterMasterRpc: refresh loop not started: " << rc + << " (manual member updates still work)"; // Keep the client object for manual member updates; only the // etcd-driven refresh thread is unavailable. } @@ -962,9 +1054,8 @@ uint32_t MasterService::GetOwnedSlotCount() const { tl::expected, ErrorCode> MasterService::InterMasterAllocateReplicas( - const std::string& tenant_id, const std::string& key, - uint64_t slice_length, uint64_t replica_num, - const std::vector& preferred_segments) { + const std::string& tenant_id, const std::string& key, uint64_t slice_length, + uint64_t replica_num, const std::vector& preferred_segments) { if (key.empty() || slice_length == 0 || replica_num == 0) { return tl::make_unexpected(ErrorCode::INVALID_PARAMS); } @@ -1023,10 +1114,10 @@ MasterService::InterMasterAllocateReplicas( inter_master_keepalive_.emplace(scoped_key, std::move(allocation.value())); } - LOG(INFO) << "InterMasterAllocateReplicas: allocated " - << descriptors.size() << " replica(s) for scoped_key=" - << scoped_key << ", slice_length=" << slice_length - << ", preferred_segments=" << preferred_segments.size(); + LOG(INFO) << "InterMasterAllocateReplicas: allocated " << descriptors.size() + << " replica(s) for scoped_key=" << scoped_key + << ", slice_length=" << slice_length + << ", preferred_segments=" << preferred_segments.size(); return descriptors; } @@ -1067,9 +1158,11 @@ MasterService::InterMasterBatchGetReplicaList( } tl::expected, ErrorCode> -MasterService::InterMasterPutStart( - const UUID& client_id, const std::string& key, const std::string& tenant_id, - uint64_t slice_length, const ReplicateConfig& config) { +MasterService::InterMasterPutStart(const UUID& client_id, + const std::string& key, + const std::string& tenant_id, + uint64_t slice_length, + const ReplicateConfig& config) { const TenantId tenant(tenant_id); // peer 互信:调用方已按 slot 归属解析本机为 owner,直接执行完整本地 // PutStart(分配 + 写元数据 + keepalive)。本机 OwnsSlot==true,不会再 @@ -1078,9 +1171,11 @@ MasterService::InterMasterPutStart( } tl::expected, ErrorCode> -MasterService::InterMasterUpsertStart( - const UUID& client_id, const std::string& key, const std::string& tenant_id, - uint64_t slice_length, const ReplicateConfig& config) { +MasterService::InterMasterUpsertStart(const UUID& client_id, + const std::string& key, + const std::string& tenant_id, + uint64_t slice_length, + const ReplicateConfig& config) { const TenantId tenant(tenant_id); // Upsert 转发:本机为 slot owner,执行完整本地 UpsertStart 以保留 // "已存在则覆盖(preemption)"语义;由 PutStart 转发走 PutStart 会丢失 @@ -1135,14 +1230,13 @@ MasterService::TryAllocateReplicasRemotely( auto& slot = remote_replica_allocator_keepalive_[endpoint]; if (!slot) { slot = std::make_shared(endpoint, - endpoint); + endpoint); } alloc = slot; } - replicas.emplace_back( - std::make_unique( - alloc, mem_desc.buffer_descriptor), - desc.status); + replicas.emplace_back(std::make_unique( + alloc, mem_desc.buffer_descriptor), + desc.status); } if (replicas.size() != replica_num) { // Unexpected descriptor types: undo at the peer and fail. @@ -1355,10 +1449,9 @@ ErrorCode MasterService::ImportSlotMetadata(uint16_t slot) { } else if (desc.is_local_disk_replica()) { const auto& local_disk_desc = desc.get_local_disk_descriptor(); - replicas.emplace_back(local_disk_desc.client_id, - local_disk_desc.object_size, - local_disk_desc.transport_endpoint, - desc.status); + replicas.emplace_back( + local_disk_desc.client_id, local_disk_desc.object_size, + local_disk_desc.transport_endpoint, desc.status); } } @@ -1371,8 +1464,9 @@ ErrorCode MasterService::ImportSlotMetadata(uint16_t slot) { standby_meta.group_id, tenant_id, user_key)); if (!inserted) { // 新获得 slot 时理论上不应碰撞;若碰撞则跳过以避免重复记账。 - LOG(WARNING) << "ImportSlotMetadata: duplicate key slot=" << slot - << ", key=" << entry.key << ", skipped"; + LOG(WARNING) + << "ImportSlotMetadata: duplicate key slot=" << slot + << ", key=" << entry.key << ", skipped"; continue; } auto& metadata = metadata_it->second; @@ -1451,8 +1545,8 @@ void MasterService::PublishSegmentOwnerForCvm(const Segment& segment) { // when no lease has been injected yet. ErrorCode err; if (cvm_lease_id_ != 0) { - err = cvm::EtcdViewStore::SaveSegmentOwnerWithLease( - cluster_id_, owner, cvm_lease_id_); + err = cvm::EtcdViewStore::SaveSegmentOwnerWithLease(cluster_id_, owner, + cvm_lease_id_); } else { err = cvm::EtcdViewStore::SaveSegmentOwner(cluster_id_, owner); } @@ -1487,8 +1581,8 @@ std::vector MasterService::ResolveOwnedSlotsForCvm() { cvm::EtcdViewStore::LoadAllMasters(cluster_id_, masters, version); if (err != ErrorCode::OK) { std::lock_guard lock(cvm_resolver_mutex_); - LOG(WARNING) << "ResolveOwnedSlotsForCvm: LoadAllMasters failed: " << err - << ", keeping previous owned set (sticky), count=" + LOG(WARNING) << "ResolveOwnedSlotsForCvm: LoadAllMasters failed: " + << err << ", keeping previous owned set (sticky), count=" << cvm_last_resolved_owned_slots_.size(); return cvm_last_resolved_owned_slots_; } @@ -1603,14 +1697,22 @@ bool MasterService::OwnsSlot(uint16_t slot) const { bool MasterService::OwnsVChunkSlot(uint16_t slot) const { std::shared_lock lock(owned_slots_mutex_); if (!owned_slots_ready_) { - const bool partitioned = enable_ha_ && ha_backend_type_ == "etcd" && - submaster_count_ > 1; + const bool partitioned = + enable_ha_ && ha_backend_type_ == "etcd" && submaster_count_ > 1; return !partitioned; } return slot < owned_slot_lookup_.size() && owned_slot_lookup_[slot]; } MasterService::~MasterService() { + io_pattern_cfm_polling_.store(false, std::memory_order_release); + io_pattern_cfm_poll_cv_.notify_all(); + if (io_pattern_cfm_poll_thread_.joinable()) { + io_pattern_cfm_poll_thread_.join(); + } + io_pattern_cfm_client_.reset(); + io_pattern_cfm_channel_.reset(); + if (ordered_oplog_writer_) { ordered_oplog_writer_->Stop(); } @@ -1666,6 +1768,11 @@ MasterService::~MasterService() { vchunk_reaper_thread_.join(); } + // Its admission worker executes handlers that capture this service. Stop + // and join it while all handler dependencies are still alive. + io_pattern_cfm_service_.reset(); + io_pattern_runtime_.reset(); + // Reset snapshot manager after all other threads have joined // This triggers the destructor which joins the snapshot thread if (snapshot_manager_) { @@ -3053,8 +3160,8 @@ auto RetryOplogPersist(F&& persist_fn) -> decltype(std::declval()()) { // KV backend. Recovery depends on the backend; bail out // earlier to avoid spinning on a persistent outage. if (attempt >= kOplogRetryMaxAttemptsUnavailable) { - LOG(WARNING) << "Oplog writer not accepting after " - << attempt << " retries, falling back to local"; + LOG(WARNING) << "Oplog writer not accepting after " << attempt + << " retries, falling back to local"; return result; } std::this_thread::sleep_for(std::chrono::milliseconds( @@ -3283,13 +3390,11 @@ void MasterService::ClearInvalidHandles( if (!cleanup_plan.removed_ids.empty()) { if (enable_ha_) { if (enable_oplog_) { - auto persist_result = - RetryOplogPersist([&]() { - return PersistStaleHandleCleanupForHA( - "ClearInvalidHandles", - tenant_it->first, it->first, - it->second, cleanup_plan); - }); + auto persist_result = RetryOplogPersist([&]() { + return PersistStaleHandleCleanupForHA( + "ClearInvalidHandles", tenant_it->first, + it->first, it->second, cleanup_plan); + }); if (persist_result) { ++it; continue; @@ -3306,19 +3411,16 @@ void MasterService::ClearInvalidHandles( } else if (!it->second.IsValid()) { if (enable_ha_) { if (enable_oplog_) { - auto persist_result = - RetryOplogPersist([&]() { - return AppendOpLogWithDurableFinalize( - OpType::REMOVE, - tenant_it->first.value(), - it->first, {}, - [this]( - const OpLogEntry& durable_entry) { - FinalizeMetadataEraseAfterDurable( - durable_entry, - QuotaEraseMode::kFull); - }); - }); + auto persist_result = RetryOplogPersist([&]() { + return AppendOpLogWithDurableFinalize( + OpType::REMOVE, tenant_it->first.value(), + it->first, {}, + [this](const OpLogEntry& durable_entry) { + FinalizeMetadataEraseAfterDurable( + durable_entry, + QuotaEraseMode::kFull); + }); + }); if (persist_result) { // OPLog path succeeded – skip local erase. ++it; @@ -3426,10 +3528,9 @@ auto MasterService::UnmountSegment(const UUID& segment_id, return {}; } -auto MasterService::GracefulUnmountSegment(const UUID& segment_id, - const UUID& client_id, - uint64_t grace_period_ms) - -> tl::expected { +auto MasterService::GracefulUnmountSegment( + const UUID& segment_id, const UUID& client_id, + uint64_t grace_period_ms) -> tl::expected { std::unique_lock lock(snapshot_mutex_); ScopedSegmentAccess segment_access = segment_manager_.getSegmentAccess(); @@ -3776,11 +3877,10 @@ void MasterService::RestoreFromStandbySnapshot( std::vector owned_slot_lookup; { const bool kv_partition_enabled = - enable_ha_ && ha_backend_type_ == "etcd" && - !master_id_.empty() && !cluster_id_.empty(); + enable_ha_ && ha_backend_type_ == "etcd" && !master_id_.empty() && + !cluster_id_.empty(); if (kv_partition_enabled) { - const std::vector owned_slots = - ResolveOwnedSlotsForCvm(); + const std::vector owned_slots = ResolveOwnedSlotsForCvm(); if (!owned_slots.empty()) { owned_slot_lookup.assign(cvm::kSlotCount, false); for (uint16_t slot : owned_slots) { @@ -4292,8 +4392,7 @@ auto MasterService::GetOffloadEndpoints() [&unique_endpoints](const Replica& replica) { const auto desc = replica.get_descriptor(); const auto& endpoint = - desc.get_local_disk_descriptor() - .transport_endpoint; + desc.get_local_disk_descriptor().transport_endpoint; if (!endpoint.empty()) { unique_endpoints.emplace(endpoint); } @@ -4549,8 +4648,7 @@ MasterService::BatchGetReplicaList(const std::vector& keys, } auto group_result = inter_master_rpc_->BatchGetReplicaList( owner, group_keys, normalized_tenant.value()); - for (size_t j = 0; j < indices.size() && j < group_result.size(); - ++j) { + for (size_t j = 0; j < indices.size() && j < group_result.size(); ++j) { results[indices[j]] = std::move(group_result[j]); } } @@ -4687,12 +4785,14 @@ MasterService::BatchGetReplicaListLocal(const std::vector& keys, {.object = {normalized_tenant, key}, .observed_at_ns = static_cast( std::chrono::duration_cast( - std::chrono::steady_clock::now().time_since_epoch()) + std::chrono::steady_clock::now() + .time_since_epoch()) .count()), .block_size = metadata.size, - .tier = results[original_idx]->replicas[0].is_memory_replica() - ? io_pattern::CacheTier::kL1Host - : io_pattern::CacheTier::kL3NofSsd, + .tier = + results[original_idx]->replicas[0].is_memory_replica() + ? io_pattern::CacheTier::kL1Host + : io_pattern::CacheTier::kL3NofSsd, .operation = io_pattern::IoOperation::kGet, .is_hit = true}); } @@ -4866,16 +4966,15 @@ auto MasterService::AllocateAndInsertMetadata( } } - auto append_preferred_segment = [&preferred_segments]( - const std::string& - segment_name) { - if (!segment_name.empty() && - std::find(preferred_segments.begin(), - preferred_segments.end(), - segment_name) == preferred_segments.end()) { - preferred_segments.push_back(segment_name); - } - }; + auto append_preferred_segment = + [&preferred_segments](const std::string& segment_name) { + if (!segment_name.empty() && + std::find(preferred_segments.begin(), + preferred_segments.end(), + segment_name) == preferred_segments.end()) { + preferred_segments.push_back(segment_name); + } + }; if (!config.preferred_segment.empty()) { append_preferred_segment(config.preferred_segment); } else { @@ -4943,8 +5042,7 @@ auto MasterService::AllocateAndInsertMetadata( need_mem_eviction_ = true; } abort_reserved_quota(); - return tl::make_unexpected( - ErrorCode::NO_AVAILABLE_HANDLE); + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); } } } else { @@ -5300,9 +5398,11 @@ auto MasterService::PutStart(const UUID& client_id, const std::string& key, return tl::make_unexpected(ErrorCode::TENANT_QUOTA_EXCEEDED); } -auto MasterService::PutEnd(const UUID& client_id, const ObjectMeta& object_meta, - const TenantId& tenant_id, ReplicaType replica_type) - -> tl::expected { +auto MasterService::PutEndInternal( + const UUID& client_id, const ObjectMeta& object_meta, + const TenantId& tenant_id, ReplicaType replica_type, + uint32_t write_batch_size, + bool overwrite) -> tl::expected { const auto& key = object_meta.key; std::shared_lock shared_lock(snapshot_mutex_); const auto object_id = MakeObjectIdentityForRequest(key, tenant_id); @@ -5418,7 +5518,12 @@ auto MasterService::PutEnd(const UUID& client_id, const ObjectMeta& object_meta, : io_pattern::CacheTier::kL3NofSsd, .operation = io_pattern::IoOperation::kPut, .is_hit = true, - .write_batch_size = 1}); + .write_batch_size = write_batch_size, + .overwrite = overwrite}); + if (replica_type != ReplicaType::MEMORY) { + io_pattern_runtime_->ScheduleAdmission( + {object_id.tenant_id, key}, io_pattern::CacheTier::kL1Host); + } } if (enable_oplog_ && ordered_oplog_writer_) { @@ -5525,10 +5630,10 @@ auto MasterService::AddReplica(const UUID& client_id, const std::string& key, OpType::PUT_END, object_id.tenant_id.value(), key, oplog_payload); if (!persist_result) { - LOG(WARNING) << "AddReplica: OpLog skipped for local_disk" - << " (metadata already updated), key=" << key - << ", err=" - << static_cast(persist_result.error()); + LOG(WARNING) + << "AddReplica: OpLog skipped for local_disk" + << " (metadata already updated), key=" << key + << ", err=" << static_cast(persist_result.error()); } } return true; @@ -5566,16 +5671,16 @@ auto MasterService::AddReplica(const UUID& client_id, const std::string& key, if (!persist_result) { LOG(WARNING) << "AddReplica: OpLog skipped for local_disk" << " (metadata already updated), key=" << key - << ", err=" << static_cast(persist_result.error()); + << ", err=" + << static_cast(persist_result.error()); } } return false; } -auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, - const TenantId& tenant_id, - ReplicaType replica_type) - -> tl::expected { +auto MasterService::PutRevoke( + const UUID& client_id, const std::string& key, const TenantId& tenant_id, + ReplicaType replica_type) -> tl::expected { std::shared_lock shared_lock(snapshot_mutex_); const auto object_id = MakeObjectIdentityForRequest(key, tenant_id); MetadataAccessorRW accessor(this, object_id); @@ -5678,6 +5783,13 @@ auto MasterService::PutRevoke(const UUID& client_id, const std::string& key, return {}; } +auto MasterService::PutEnd(const UUID& client_id, const ObjectMeta& object_meta, + const TenantId& tenant_id, ReplicaType replica_type) + -> tl::expected { + return PutEndInternal(client_id, object_meta, tenant_id, replica_type, + /*write_batch_size=*/1, /*overwrite=*/false); +} + auto MasterService::PutEnd(const UUID& client_id, const std::string& key, const TenantId& tenant_id, ReplicaType replica_type) -> tl::expected { @@ -5692,8 +5804,11 @@ std::vector> MasterService::BatchPutEnd( std::vector> results; results.reserve(object_metas.size()); for (const auto& object_meta : object_metas) { - results.emplace_back( - PutEnd(client_id, object_meta, tenant_id, replica_type)); + results.emplace_back(PutEndInternal( + client_id, object_meta, tenant_id, replica_type, + static_cast(std::min( + object_metas.size(), std::numeric_limits::max())), + /*overwrite=*/false)); } return results; } @@ -6091,26 +6206,24 @@ auto MasterService::UpsertStart(const UUID& client_id, const std::string& key, return tl::make_unexpected(ErrorCode::TENANT_QUOTA_EXCEEDED); } -auto MasterService::UpsertEnd(const UUID& client_id, - const ObjectMeta& object_meta, - const TenantId& tenant_id, - ReplicaType replica_type) - -> tl::expected { - return PutEnd(client_id, object_meta, tenant_id, replica_type); +auto MasterService::UpsertEnd( + const UUID& client_id, const ObjectMeta& object_meta, + const TenantId& tenant_id, + ReplicaType replica_type) -> tl::expected { + return PutEndInternal(client_id, object_meta, tenant_id, replica_type, + /*write_batch_size=*/1, /*overwrite=*/true); } -auto MasterService::UpsertEnd(const UUID& client_id, const std::string& key, - const TenantId& tenant_id, - ReplicaType replica_type) - -> tl::expected { +auto MasterService::UpsertEnd( + const UUID& client_id, const std::string& key, const TenantId& tenant_id, + ReplicaType replica_type) -> tl::expected { return UpsertEnd(client_id, ObjectMeta{key, std::nullopt}, tenant_id, replica_type); } -auto MasterService::UpsertRevoke(const UUID& client_id, const std::string& key, - const TenantId& tenant_id, - ReplicaType replica_type) - -> tl::expected { +auto MasterService::UpsertRevoke( + const UUID& client_id, const std::string& key, const TenantId& tenant_id, + ReplicaType replica_type) -> tl::expected { return PutRevoke(client_id, key, tenant_id, replica_type); } @@ -6151,7 +6264,17 @@ MasterService::BatchUpsertStart(const UUID& client_id, std::vector> MasterService::BatchUpsertEnd( const UUID& client_id, const std::vector& object_metas, const TenantId& tenant_id) { - return BatchPutEnd(client_id, object_metas, tenant_id, ReplicaType::ALL); + assert(tenant_id.IsValid()); + std::vector> results; + results.reserve(object_metas.size()); + const auto batch_size = static_cast(std::min( + object_metas.size(), std::numeric_limits::max())); + for (const auto& object_meta : object_metas) { + results.emplace_back(PutEndInternal(client_id, object_meta, tenant_id, + ReplicaType::ALL, batch_size, + /*overwrite=*/true)); + } + return results; } std::vector> MasterService::BatchUpsertRevoke( @@ -6160,11 +6283,9 @@ std::vector> MasterService::BatchUpsertRevoke( return BatchPutRevoke(client_id, keys, tenant_id); } -auto MasterService::EvictDiskReplica(const UUID& client_id, - const std::string& key, - const TenantId& tenant_id, - ReplicaType replica_type) - -> tl::expected { +auto MasterService::EvictDiskReplica( + const UUID& client_id, const std::string& key, const TenantId& tenant_id, + ReplicaType replica_type) -> tl::expected { const auto object_id = MakeObjectIdentityForRequest(key, tenant_id); MetadataAccessorRW accessor(this, object_id); if (!accessor.Exists()) { @@ -6971,9 +7092,7 @@ auto MasterService::Remove(const std::string& key, const TenantId& tenant_id, auto& metadata = accessor.Get(); std::vector local_disk_holders; metadata.VisitReplicas( - [](const Replica& replica) { - return replica.is_local_disk_replica(); - }, + [](const Replica& replica) { return replica.is_local_disk_replica(); }, [&local_disk_holders](Replica& replica) { auto client_id = replica.get_local_disk_client_id(); if (client_id.has_value()) { @@ -7017,15 +7136,13 @@ auto MasterService::Remove(const std::string& key, const TenantId& tenant_id, auto persist_result = AppendReservedOpLogWithDurableFinalize( std::move(reservation.value()), OpType::REMOVE, object_id.tenant_id.value(), key, {}, - [this, removed_ids = std::move(removed_ids), - local_disk_holders, - tenant_id_for_task = object_id.tenant_id.value(), key]( - const OpLogEntry& durable_entry) { + [this, removed_ids = std::move(removed_ids), local_disk_holders, + tenant_id_for_task = object_id.tenant_id.value(), + key](const OpLogEntry& durable_entry) { FinalizeRemovedReplicasAfterDurable( durable_entry, removed_ids, QuotaEraseMode::kFull); - EnqueueRemoveTasks( - local_disk_holders, - RemoveTaskItem{tenant_id_for_task, key}); + EnqueueRemoveTasks(local_disk_holders, + RemoveTaskItem{tenant_id_for_task, key}); }); if (!persist_result) { return tl::make_unexpected(persist_result.error()); @@ -7040,14 +7157,15 @@ auto MasterService::Remove(const std::string& key, const TenantId& tenant_id, accessor.Erase(); // Push removed key to each LOCAL_DISK holder's removed_keys queue. - EnqueueRemoveTasks(local_disk_holders, RemoveTaskItem{tenant_id.value(), key}); + EnqueueRemoveTasks(local_disk_holders, + RemoveTaskItem{tenant_id.value(), key}); return {}; } auto MasterService::RemoveByRegex(const std::string& regex_pattern, - const TenantId& tenant_id, bool force) - -> tl::expected { + const TenantId& tenant_id, + bool force) -> tl::expected { assert(tenant_id.IsValid()); long removed_count = 0; std::regex pattern; @@ -7494,8 +7612,8 @@ auto MasterService::BatchRemove(const std::vector& keys, normalized_tenant.value(), key, {}, [this, removed_ids = std::move(removed_ids), batch_local_disk_holders, - tenant_id = normalized_tenant.value(), key]( - const OpLogEntry& durable_entry) { + tenant_id = normalized_tenant.value(), + key](const OpLogEntry& durable_entry) { FinalizeRemovedReplicasAfterDurable( durable_entry, removed_ids, QuotaEraseMode::kFull); @@ -7522,9 +7640,8 @@ auto MasterService::BatchRemove(const std::vector& keys, } // Push removed key to each LOCAL_DISK holder's removed_keys queue. - EnqueueRemoveTasks( - batch_local_disk_holders, - RemoveTaskItem{normalized_tenant.value(), key}); + EnqueueRemoveTasks(batch_local_disk_holders, + RemoveTaskItem{normalized_tenant.value(), key}); results[original_idx] = {}; // Success } @@ -7767,8 +7884,8 @@ auto MasterService::RemoveObjectHeartbeat(const UUID& client_id) } } -void MasterService::EnqueueRemoveTasks( - const std::vector& holder_ids, const RemoveTaskItem& task) { +void MasterService::EnqueueRemoveTasks(const std::vector& holder_ids, + const RemoveTaskItem& task) { if (holder_ids.empty()) return; ScopedLocalDiskSegmentAccess access = segment_manager_.getLocalDiskSegmentAccess(); @@ -7778,16 +7895,16 @@ void MasterService::EnqueueRemoveTasks( if (it == segments.end()) continue; MutexLocker locker(&it->second->offloading_mutex_); if (std::find(it->second->removed_keys.begin(), - it->second->removed_keys.end(), task) == - it->second->removed_keys.end()) { + it->second->removed_keys.end(), + task) == it->second->removed_keys.end()) { it->second->removed_keys.push_back(task); } } } auto MasterService::AckRemoveObjectHeartbeat( - const UUID& client_id, const std::vector& tasks) - -> tl::expected { + const UUID& client_id, + const std::vector& tasks) -> tl::expected { std::shared_lock shared_lock(snapshot_mutex_); ScopedLocalDiskSegmentAccess access = segment_manager_.getLocalDiskSegmentAccess(); @@ -7801,8 +7918,8 @@ auto MasterService::AckRemoveObjectHeartbeat( pending.erase(std::remove_if(pending.begin(), pending.end(), [&tasks](const RemoveTaskItem& task) { return std::find(tasks.begin(), - tasks.end(), task) != - tasks.end(); + tasks.end(), + task) != tasks.end(); }), pending.end()); return {}; @@ -7887,8 +8004,8 @@ auto MasterService::NotifyOffloadSuccess( return t.source_client_id == client_id; }); if (offload_it != tasks.end()) { - auto source = - accessor.Get().GetReplicaByID(offload_it->source_id); + auto source = accessor.Get().GetReplicaByID( + offload_it->source_id); if (source != nullptr) { source->dec_refcnt(); } @@ -7958,8 +8075,7 @@ auto MasterService::NotifyOffloadSuccess( size_t updated = obj_metadata.VisitReplicas( [client_id](const Replica& rep) { return rep.type() == ReplicaType::LOCAL_DISK && - rep.get_local_disk_client_id() == - client_id; + rep.get_local_disk_client_id() == client_id; }, [&metadata](Replica& rep) { rep.update_local_disk_location( @@ -7996,11 +8112,11 @@ auto MasterService::NotifyOffloadSuccess( if (res.error() == ErrorCode::OBJECT_NOT_FOUND) { continue; } - LOG(WARNING) << "Failed to add replica, skipping object: " - << "error=" << res.error() - << ", client_id=" << client_id - << ", tenant_id=" << object_id.tenant_id.value() - << ", key=" << object_id.user_key; + LOG(WARNING) + << "Failed to add replica, skipping object: " + << "error=" << res.error() << ", client_id=" << client_id + << ", tenant_id=" << object_id.tenant_id.value() + << ", key=" << object_id.user_key; continue; } added_new_local_disk_replica = res.value(); @@ -8688,10 +8804,9 @@ auto MasterService::PromotionAllocStart( return PromotionAllocStartResponse{std::move(desc)}; } -auto MasterService::NotifyPromotionSuccess(const UUID& client_id, - const std::string& key, - const TenantId& tenant_id) - -> tl::expected { +auto MasterService::NotifyPromotionSuccess( + const UUID& client_id, const std::string& key, + const TenantId& tenant_id) -> tl::expected { std::shared_lock shared_lock(snapshot_mutex_); const auto object_id = MakeObjectIdentityForRequest(key, tenant_id); MetadataAccessorRW accessor(this, object_id); @@ -8823,10 +8938,9 @@ auto MasterService::NotifyPromotionSuccess(const UUID& client_id, return {}; } -auto MasterService::NotifyPromotionFailure(const UUID& client_id, - const std::string& key, - const TenantId& tenant_id) - -> tl::expected { +auto MasterService::NotifyPromotionFailure( + const UUID& client_id, const std::string& key, + const TenantId& tenant_id) -> tl::expected { std::shared_lock shared_lock(snapshot_mutex_); const auto object_id = MakeObjectIdentityForRequest(key, tenant_id); MetadataAccessorRW accessor(this, object_id); @@ -8912,16 +9026,22 @@ void MasterService::EvictionThreadFunc() { used_ratio - eviction_high_watermark_ratio_); if (io_pattern_runtime_) { io_pattern_runtime_->RecordStorageMetric( - {.source_id = "master-memory", .tier = io_pattern::CacheTier::kL1Host, + {.source_id = "master-memory", + .tier = io_pattern::CacheTier::kL1Host, .memory_used_ratio = static_cast(used_ratio)}); const auto capacity = std::max( - 0, MasterMetricManager::instance().get_total_mem_capacity()); - io_pattern_runtime_->Execute( + 0, + MasterMetricManager::instance().get_total_mem_capacity()); + const auto status = io_pattern_runtime_->Execute( io_pattern::CacheTier::kL1Host, static_cast(evict_ratio_target * capacity), {}); + if (status.eviction != ErrorCode::OK) { + BatchEvict(evict_ratio_target, evict_ratio_lowerbound); + } + } else { + BatchEvict(evict_ratio_target, evict_ratio_lowerbound); } - BatchEvict(evict_ratio_target, evict_ratio_lowerbound); - LOG(INFO) << "[EVICT-DONE] BatchEvict execution completed."; + LOG(INFO) << "[EVICT-DONE] eviction execution completed."; last_discard_time = now; } else if (now - last_discard_time > put_start_release_timeout_sec_) { // Try discarding expired processing keys and ongoing replication @@ -9185,21 +9305,21 @@ void MasterService::DiscardExpiredProcessingReplicas( auto& tasks = task_it->second; auto metadata_it = tenant_state.metadata.find(task_it->first); for (auto t = tasks.begin(); t != tasks.end();) { - const auto ttl = - t->start_time + put_start_release_timeout_sec_; + const auto ttl = t->start_time + put_start_release_timeout_sec_; if (ttl > now) { t++; continue; } if (metadata_it != tenant_state.metadata.end()) { - auto source = metadata_it->second.GetReplicaByID( - t->source_id); + auto source = + metadata_it->second.GetReplicaByID(t->source_id); if (source != nullptr) { source->dec_refcnt(); } } - LOG(WARNING) << "Offloading task expired for key: " - << task_it->first << " tenant=" << tenant_it->first; + LOG(WARNING) + << "Offloading task expired for key: " << task_it->first + << " tenant=" << tenant_it->first; t = tasks.erase(t); } if (tasks.empty()) { @@ -9519,8 +9639,9 @@ tl::expected MasterService::ApplySnapshotState( } MasterService::TenantQuotaEvictionResult -MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, - uint64_t target_bytes) { +MasterService::EvictTenantMemoryForQuota( + const TenantId& tenant_id, uint64_t target_bytes, + const std::unordered_set* candidate_keys) { TenantQuotaEvictionResult total; if (target_bytes == 0) { return total; @@ -9637,6 +9758,17 @@ MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, .evicted_objects = freed > 0 ? 1U : 0U}; } + // Group eviction is atomic. A policy plan naming only part of a group + // must not silently expand into unplanned objects; let the caller take + // the legacy fallback path instead. + if (candidate_keys && + std::any_of(group_it->second.begin(), group_it->second.end(), + [candidate_keys](const std::string& member_key) { + return !candidate_keys->contains(member_key); + })) { + return {}; + } + for (const auto& member_key : group_it->second) { auto member_it = tenant_state.metadata.find(member_key); if (member_it != tenant_state.metadata.end() && @@ -9691,6 +9823,11 @@ MasterService::EvictTenantMemoryForQuota(const TenantId& tenant_id, for (auto it = tenant_state.metadata.begin(); it != tenant_state.metadata.end() && total.freed_bytes < target_bytes;) { + if (candidate_keys && + !candidate_keys->contains(it->first)) { + ++it; + continue; + } auto& metadata = it->second; if (metadata.IsHardPinned() || !metadata.IsLeaseExpired(now) || @@ -12694,8 +12831,8 @@ ErrorCode MasterService::InitializeBatchOpLogWriter( return ErrorCode::INVALID_PARAMS; } - auto storage = std::make_unique(cluster_id_, *backend, - master_id_); + auto storage = + std::make_unique(cluster_id_, *backend, master_id_); DurablePrefix durable_prefix; ErrorCode err = storage->InitDurablePrefix(durable_prefix); if (err != ErrorCode::OK) { diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index 11fe6f60c4..d8c1666976 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -101,7 +101,8 @@ WrappedMasterService::WrappedMasterService( const WrappedMasterServiceConfig& config, HttpMetadataServer* http_metadata_server, const std::string& http_metadata_remote_url) - : master_service_(MasterServiceConfig(config)) { + : master_service_(MasterServiceConfig(config)), + cfm_rpc_service_(master_service_.GetCfmService()) { // Configure metadata cleanup on client timeout. Prefer the co-located // in-process server; otherwise fall back to a separately-deployed HTTP // metadata server derived from the cluster configuration. @@ -2068,6 +2069,12 @@ void RegisterRpcService( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::PutStart>( &wrapped_master_service); + auto& cfm = wrapped_master_service.CfmRpcEndpoint(); + server.register_handler<&io_pattern::CfmRpcService::Authenticate>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::Send>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::Receive>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::Acknowledge>(&cfm); + server.register_handler<&io_pattern::CfmRpcService::EnqueuePolicy>(&cfm); server.register_handler<&mooncake::WrappedMasterService::PutEnd>( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::PutRevoke>( diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index b5836d306f..51f5329d09 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -3,12 +3,16 @@ #include "io_pattern/policy_strategies.h" #include +#include +#include +#include #include #include #include #include #include +#include namespace mooncake::io_pattern { namespace { @@ -109,12 +113,23 @@ class TestCfmChannel final : public CfmChannel { snapshot = value; return send_ok; } - std::optional PollPolicy() override { return policy; } + CfmPollResult PollPolicyResult() override { + return policy ? CfmPollResult::Command(*policy, 42) + : CfmPollResult::Empty(); + } + bool AcknowledgePolicy(uint64_t delivery_id, bool success) override { + acknowledged_delivery_id = delivery_id; + acknowledged_success = success; + return acknowledge_ok; + } ErrorCode ExecutePrefetch(const PrefetchPlan& value) override { plan = value; return execute_code; } bool send_ok{true}; + bool acknowledge_ok{true}; + bool acknowledged_success{false}; + uint64_t acknowledged_delivery_id{0}; ErrorCode execute_code{ErrorCode::OK}; IoPatternSnapshot snapshot; std::optional policy; @@ -126,7 +141,9 @@ class FlakyCfmChannel final : public CfmChannel { bool SendSnapshot(const IoPatternSnapshot&) override { return send_failures-- <= 0; } - std::optional PollPolicy() override { return PrefetchPlan{}; } + CfmPollResult PollPolicyResult() override { + return CfmPollResult::Command(PrefetchPlan{}); + } ErrorCode ExecutePrefetch(const PrefetchPlan&) override { return ErrorCode::RPC_FAIL; } @@ -142,13 +159,24 @@ class TestRpcTransport final : public CfmRpcTransport { last_timeout = timeout; return send_ok; } - std::optional Receive(std::string_view method, - std::chrono::milliseconds timeout) override { + CfmReceiveResult Receive(std::string_view method, + std::chrono::milliseconds timeout) override { last_method = std::string(method); last_timeout = timeout; - return response; + return response ? CfmReceiveResult::Payload(*response) + : CfmReceiveResult::Empty(); + } + bool Acknowledge(uint64_t delivery_id, bool success, + std::chrono::milliseconds timeout) override { + acknowledged_delivery_id = delivery_id; + acknowledged_success = success; + last_timeout = timeout; + return acknowledge_ok; } bool send_ok{true}; + bool acknowledge_ok{true}; + bool acknowledged_success{false}; + uint64_t acknowledged_delivery_id{0}; std::optional response; std::string last_method; std::string last_payload; @@ -339,6 +367,65 @@ TEST(IoPatternFrameworkTest, CollectorImplDerivesWritePathMetrics) { EXPECT_TRUE(key.write_burst); } +TEST(IoPatternFrameworkTest, CollectorImplBoundsAccessCountByTimeWindow) { + uint64_t now_ns = 10; + IoPatternCollectorImpl collector(IoPatternCollectorImpl::Config{ + .access_window_ns = 100, + .access_bucket_ns = 1, + .now_ns = [&] { return now_ns; }}); + AccessRecord access{.object = {TenantId("tenant-a"), "key"}, + .observed_at_ns = 10}; + collector.RecordAccess(access.object.key, access); + access.observed_at_ns = 50; + collector.RecordAccess(access.object.key, access); + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 2); + + access.observed_at_ns = 111; + now_ns = 111; + collector.RecordAccess(access.object.key, access); + // A true sliding window retains the event at 50 even though the first + // event's fixed 10..110 bucket has ended. + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 2); + + now_ns = 212; + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 0); +} + +TEST(IoPatternFrameworkTest, CollectorExpiresMergedAccessWindowsLocally) { + uint64_t now_ns = 10; + IoPatternCollectorImpl collector(IoPatternCollectorImpl::Config{ + .access_window_ns = 100, + .access_bucket_ns = 1, + .now_ns = [&] { return now_ns; }}); + IoPatternSnapshot remote; + remote.keys.push_back( + KeyMetrics{.object = {TenantId("tenant-a"), "remote"}, + .access_count_window = 7, + .write_frequency = 3}); + + collector.MergeSnapshot(remote); + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 7); + now_ns = 111; + const auto expired = collector.GetSnapshot().keys.front(); + EXPECT_EQ(expired.access_count_window, 0); + EXPECT_EQ(expired.write_frequency, 0); +} + +TEST(IoPatternFrameworkTest, CollectorHardCapsBucketsForOutOfOrderInput) { + uint64_t now_ns = 3; + IoPatternCollectorImpl collector(IoPatternCollectorImpl::Config{ + .access_window_ns = 1'000, + .access_bucket_ns = 1, + .max_access_buckets_per_key = 2, + .now_ns = [&] { return now_ns; }}); + AccessRecord access{.object = {TenantId("tenant-a"), "key"}}; + for (uint64_t timestamp : {1ULL, 2ULL, 3ULL}) { + access.observed_at_ns = timestamp; + collector.RecordAccess(access.object.key, access); + } + EXPECT_EQ(collector.GetSnapshot().keys.front().access_count_window, 2); +} + TEST(IoPatternFrameworkTest, ThresholdAnalyzerClassifiesDocumentedWorkloads) { ThresholdAnalyzer analyzer; IoPatternSnapshot code_agent; @@ -460,6 +547,17 @@ TEST(IoPatternFrameworkTest, PrefixAdmissionUsesTierSpecificSignals) { const auto rejected = admission.Evaluate(key.object, CacheTier::kL0Hbm, context); EXPECT_EQ(rejected.decision, AdmissionDecision::kRejectPrefix); + + context.snapshot.storage = { + StorageMetric{.source_id = "ssd", + .tier = CacheTier::kL3NofSsd, + .memory_used_ratio = 0.1F}, + StorageMetric{.source_id = "host", + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.95F}}; + EXPECT_EQ(admission.Evaluate(key.object, CacheTier::kL1Host, context) + .decision, + AdmissionDecision::kRejectWatermark); } TEST(IoPatternFrameworkTest, TracePrefetchPlansOnlyLongPrefixMatches) { @@ -667,14 +765,21 @@ TEST(IoPatternFrameworkTest, ReporterBatchesBoundsAndCountsDrops) { TEST(IoPatternFrameworkTest, ReporterAdaptsFlushIntervalToLoad) { IoPatternReporter reporter(4, [](const MetricBatch&) { return true; }); + reporter.UpdateLoad(0.25F, 0); EXPECT_EQ(reporter.RecommendedFlushInterval(), - std::chrono::milliseconds(1000)); - reporter.Enqueue(InferenceMetrics{}); + std::chrono::milliseconds(100)); + reporter.UpdateLoad(0.50F, 0); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(200)); + reporter.UpdateLoad(0.80F, 0); EXPECT_EQ(reporter.RecommendedFlushInterval(), std::chrono::milliseconds(500)); - reporter.Enqueue(InferenceMetrics{}); + reporter.UpdateLoad(0.95F, 0); EXPECT_EQ(reporter.RecommendedFlushInterval(), - std::chrono::milliseconds(100)); + std::chrono::milliseconds(1000)); + reporter.UpdateLoad(0.25F, 101'000); + EXPECT_EQ(reporter.RecommendedFlushInterval(), + std::chrono::milliseconds(1000)); } TEST(IoPatternFrameworkTest, ReporterEnforcesPerTenantFairness) { @@ -722,6 +827,11 @@ TEST(IoPatternFrameworkTest, CfmClientDispatchesReceivedPolicyCommands) { channel->policy = PolicyCommand{AdmissionResult{}}; EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); EXPECT_EQ(dispatched, 2); + EXPECT_EQ(channel->acknowledged_delivery_id, 42); + EXPECT_TRUE(channel->acknowledged_success); + channel->policy.reset(); + EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); + EXPECT_EQ(dispatched, 2); } TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { @@ -739,6 +849,17 @@ TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { EXPECT_TRUE(channel.degraded()); } +TEST(IoPatternFrameworkTest, EmptyPolicyPollKeepsChannelHealthy) { + auto idle = std::make_shared(); + ResilientCfmChannel channel( + idle, CfmRetryConfig{.max_retries = 2, .degrade_after_failures = 2}); + + EXPECT_FALSE(channel.PollPolicy().has_value()); + EXPECT_FALSE(channel.PollPolicy().has_value()); + EXPECT_EQ(channel.consecutive_failures(), 0); + EXPECT_FALSE(channel.degraded()); +} + TEST(IoPatternFrameworkTest, ResilientAnalyzerFallsBackAfterFailure) { ResilientAnalyzer analyzer(std::make_shared(), 2); EXPECT_EQ(analyzer.DetectWorkloadType({}), WorkloadType::kMixed); @@ -853,6 +974,39 @@ TEST(IoPatternFrameworkTest, InProcessCfmTransportAuthenticatesAndDispatches) { EXPECT_FALSE(unauthorized.SendSnapshot({})); } +TEST(IoPatternFrameworkTest, InProcessTransportDoesNotHoldLockAcrossHandler) { + std::promise handler_entered; + std::promise release_handler; + auto release = release_handler.get_future().share(); + auto transport = std::make_shared( + "shared-secret", [&](std::string_view, std::string_view) { + handler_entered.set_value(); + release.wait(); + return true; + }); + ASSERT_TRUE(transport->Authenticate("shared-secret")); + + std::thread sender([&] { + EXPECT_TRUE(transport->Send("report_snapshot", {}, + std::chrono::milliseconds(10))); + }); + if (handler_entered.get_future().wait_for(std::chrono::seconds(1)) != + std::future_status::ready) { + release_handler.set_value(); + sender.join(); + FAIL() << "send handler did not start"; + return; + } + auto enqueue = std::async(std::launch::async, [&] { + transport->EnqueuePolicy("policy"); + return true; + }); + EXPECT_EQ(enqueue.wait_for(std::chrono::milliseconds(100)), + std::future_status::ready); + release_handler.set_value(); + sender.join(); +} + TEST(IoPatternFrameworkTest, CfmIngressFeedsRuntimeFromMetricBatches) { auto runtime = std::make_shared( IoPatternRuntime::Handlers{.eviction = [](const EvictionPlan&) { @@ -881,6 +1035,158 @@ TEST(IoPatternFrameworkTest, CfmIngressFeedsRuntimeFromMetricBatches) { EXPECT_EQ(snapshot.keys.front().access_count_window, 1); } +TEST(IoPatternFrameworkTest, CfmServiceAuthenticatesAndBoundsPolicyQueues) { + int admissions = 0; + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [&admissions](const AdmissionResult&) { + ++admissions; + return ErrorCode::OK; + }}); + CfmService service(runtime, "secret", 1, "producer"); + CfmBinaryCodec codec; + + EXPECT_FALSE(service.Authenticate("wrong")); + EXPECT_TRUE(service.Authenticate("secret")); + EXPECT_FALSE(service.EnqueuePolicy("node-a", "first", "secret")); + const auto admission = codec.EncodePolicy(AdmissionResult{ + .object = {TenantId("tenant"), "key"}, + .target_tier = CacheTier::kL1Host, + .decision = AdmissionDecision::kAdmit}); + const auto second = codec.EncodePolicy(PrefetchPlan{}); + EXPECT_FALSE(service.EnqueuePolicy("node-a", "malformed", "producer")); + EXPECT_TRUE(service.EnqueuePolicy("node-a", admission, "producer")); + EXPECT_FALSE(service.EnqueuePolicy("node-a", second, "producer")); + const auto delivery = service.PollPolicy("node-a", "secret"); + ASSERT_TRUE(delivery.has_value()); + EXPECT_EQ(delivery->second, admission); + EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, false, + "secret")); + EXPECT_TRUE(service.PollPolicy("node-a", "secret").has_value()); + EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, true, + "secret")); + EXPECT_FALSE(service.PollPolicy("node-a", "secret").has_value()); + + EXPECT_FALSE(service.Send("", "execute_policy", admission, "secret")); + EXPECT_FALSE( + service.Send("node-a", "execute_policy", admission, "secret")); + EXPECT_TRUE( + service.Send("node-a", "execute_policy", admission, "producer")); + EXPECT_EQ(admissions, 1); +} + +TEST(IoPatternFrameworkTest, CoroRpcCfmTransportRunsTheProductionWirePath) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + auto service = + std::make_shared(runtime, "secret", 2, "producer"); + CfmRpcService endpoint(service); + coro_rpc::coro_rpc_server server(1, 0, "127.0.0.1"); + server.register_handler<&CfmRpcService::Authenticate>(&endpoint); + server.register_handler<&CfmRpcService::Send>(&endpoint); + server.register_handler<&CfmRpcService::Receive>(&endpoint); + server.register_handler<&CfmRpcService::Acknowledge>(&endpoint); + server.register_handler<&CfmRpcService::EnqueuePolicy>(&endpoint); + ASSERT_FALSE(server.async_start().hasResult()); + + const auto rejected_poll = + endpoint.Receive("poll_policy", "node-a", "wrong"); + EXPECT_FALSE(rejected_poll.first); + const auto empty_poll = endpoint.Receive("poll_policy", "node-a", "secret"); + EXPECT_TRUE(empty_poll.first); + EXPECT_FALSE(empty_poll.second.has_value()); + + CoroRpcCfmTransport transport( + "127.0.0.1:" + std::to_string(server.port()), "node-a", + std::chrono::milliseconds(500)); + EXPECT_FALSE(transport.Authenticate("wrong")); + ASSERT_TRUE(transport.Authenticate("secret")); + + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "remote-key"}, + .is_hit = true}); + batch.storage.push_back(StorageMetric{.source_id = "spoofed", + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.75F}); + EXPECT_TRUE(transport.Send("report_metric_batch", + codec.EncodeMetricBatch(batch), + std::chrono::milliseconds(500))); + const auto snapshot = runtime->Snapshot(); + ASSERT_EQ(snapshot.keys.size(), 1); + ASSERT_EQ(snapshot.storage.size(), 1); + EXPECT_EQ(snapshot.keys.front().object.key, "remote-key"); + EXPECT_EQ(snapshot.storage.front().source_id, "node-a"); + + const auto policy = codec.EncodePolicy(PrefetchPlan{}); + CoroRpcCfmTransport producer( + "127.0.0.1:" + std::to_string(server.port()), "producer", + std::chrono::milliseconds(500)); + ASSERT_TRUE(producer.Authenticate("producer")); + EXPECT_TRUE(producer.EnqueuePolicy("node-a", policy, + std::chrono::milliseconds(500))); + const auto received = + transport.Receive("poll_policy", std::chrono::milliseconds(500)); + EXPECT_EQ(received.status, CfmReceiveResult::Status::kPayload); + EXPECT_EQ(received.payload, policy); + EXPECT_NE(received.delivery_id, 0); + EXPECT_TRUE(transport.Acknowledge(received.delivery_id, false, + std::chrono::milliseconds(500))); + const auto redelivered = + transport.Receive("poll_policy", std::chrono::milliseconds(500)); + EXPECT_EQ(redelivered.delivery_id, received.delivery_id); + EXPECT_EQ(redelivered.payload, policy); + EXPECT_TRUE(transport.Acknowledge(redelivered.delivery_id, true, + std::chrono::milliseconds(500))); + EXPECT_EQ(transport.Receive("poll_policy", std::chrono::milliseconds(500)) + .status, + CfmReceiveResult::Status::kEmpty); + server.stop(); +} + +TEST(IoPatternFrameworkTest, CfmProducesNodePolicyFromHighWatermarkReport) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + CfmService service(runtime, "node-secret", 8, "producer-secret"); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "cold-key"}, + .block_size = 1024, + .tier = CacheTier::kL1Host, + .is_hit = false}); + batch.storage.push_back(StorageMetric{.tier = CacheTier::kL1Host, + .used_bytes = 950, + .capacity_bytes = 1000, + .memory_used_ratio = 0.95F}); + ASSERT_TRUE(service.Send("node-a", "report_metric_batch", + codec.EncodeMetricBatch(batch), "node-secret")); + + std::optional> delivery; + for (size_t attempt = 0; attempt < 100 && !delivery; ++attempt) { + delivery = service.PollPolicy("node-a", "node-secret"); + if (!delivery) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + ASSERT_TRUE(delivery.has_value()); + const auto command = codec.DecodePolicy(delivery->second); + ASSERT_TRUE(command.has_value()); + const auto* eviction = std::get_if(&*command); + ASSERT_NE(eviction, nullptr); + ASSERT_EQ(eviction->candidates.size(), 1); + EXPECT_EQ(eviction->candidates.front().object.key, "cold-key"); + EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, true, + "node-secret")); +} + TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { size_t batches = 0; IoPatternReporter reporter(4, [&](const MetricBatch&) { @@ -972,14 +1278,16 @@ TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { SlidingWindowAnalyzer analyzer(100); IoPatternSnapshot first; first.generated_at_ns = 10; - first.keys.push_back(KeyMetrics{.token_count = 20 * 1024, + first.keys.push_back(KeyMetrics{.object = {TenantId("tenant"), "first"}, + .token_count = 20 * 1024, .prefix_fanout = 20, .match_length = 512, .block_size = 100, .access_count_window = 1}); IoPatternSnapshot second; second.generated_at_ns = 50; - second.keys.push_back(KeyMetrics{.token_count = 30, + second.keys.push_back(KeyMetrics{.object = {TenantId("tenant"), "second"}, + .token_count = 30, .prefix_fanout = 20, .match_length = 300, .block_size = 300, @@ -992,6 +1300,39 @@ TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { EXPECT_EQ(stats.block_p90, 300); } +TEST(IoPatternFrameworkTest, SlidingWindowDeduplicatesObjectsAndBoundsHistory) { + SlidingWindowAnalyzer analyzer(1'000, {}, 2); + const ObjectRef object{TenantId("tenant-a"), "same-key"}; + for (uint64_t timestamp = 1; timestamp <= 3; ++timestamp) { + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = timestamp; + snapshot.keys.push_back(KeyMetrics{.object = object, + .token_count = + static_cast(timestamp), + .access_count_window = timestamp}); + analyzer.Analyze(snapshot); + } + + const auto stats = analyzer.FeatureStats(); + EXPECT_EQ(stats.samples, 1); + EXPECT_EQ(stats.token_median, 3); + EXPECT_EQ(stats.frequency_median, 3); +} + +TEST(IoPatternFrameworkTest, SlidingWindowCapsASingleOversizedSnapshot) { + SlidingWindowAnalyzer analyzer(1'000, {}, 2); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = 1; + snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant"), "c"}}, + KeyMetrics{.object = {TenantId("tenant"), "a"}}, + KeyMetrics{.object = {TenantId("tenant"), "b"}}, + }; + + analyzer.Analyze(snapshot); + EXPECT_EQ(analyzer.FeatureStats().samples, 2); +} + TEST(IoPatternFrameworkTest, KMeansFallbackLabelsIndependentSessions) { SlidingWindowAnalyzer analyzer; IoPatternSnapshot snapshot; @@ -1059,6 +1400,21 @@ TEST(IoPatternFrameworkTest, LegacyEvictionAdapterUsesLruFallback) { EXPECT_EQ(plan.candidates.front().object.key, "first"); } +TEST(IoPatternFrameworkTest, ScoreBasedEvictionMayCrossTheByteTarget) { + PolicyContext context; + context.snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant-a"), "large"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}}; + context.analysis.keys = { + KeyPattern{.object = {TenantId("tenant-a"), "large"}}}; + ScoreBasedEvictionOps eviction; + + const auto plan = eviction.Evaluate(context, CacheTier::kL1Host, 32); + ASSERT_EQ(plan.candidates.size(), 1); + EXPECT_EQ(plan.candidates.front().bytes, 64); +} + TEST(IoPatternFrameworkTest, ScoreBasedEvictionUsesTierSpecificSignals) { PolicyContext context; context.snapshot.keys = { @@ -1193,5 +1549,26 @@ TEST(IoPatternFrameworkTest, RuntimeExecutesCfmCommandsThroughStorageHandlers) { EXPECT_EQ(admissions, 1); } +TEST(IoPatternFrameworkTest, RuntimeSchedulesAdmissionOffTheProducerPath) { + std::promise handled; + IoPatternRuntime runtime( + {.eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [&handled](const AdmissionResult& result) { + handled.set_value(result); + return ErrorCode::OK; + }}); + AccessRecord access{.object = {TenantId("tenant"), "disk-key"}, + .tier = CacheTier::kL3NofSsd, + .operation = IoOperation::kPut}; + runtime.RecordAccess(access.object.key, access); + + EXPECT_TRUE(runtime.ScheduleAdmission(access.object, CacheTier::kL1Host)); + auto result = handled.get_future(); + ASSERT_EQ(result.wait_for(std::chrono::seconds(1)), + std::future_status::ready); + EXPECT_EQ(result.get().object, access.object); +} + } // namespace } // namespace mooncake::io_pattern diff --git a/mooncake-store/tests/master_service_config_test.cpp b/mooncake-store/tests/master_service_config_test.cpp index 4b69bcf5f0..ed50c77325 100644 --- a/mooncake-store/tests/master_service_config_test.cpp +++ b/mooncake-store/tests/master_service_config_test.cpp @@ -46,4 +46,26 @@ TEST(MasterServiceConfigTest, OplogBatchMaxEntriesBuilderOverrideRespected) { EXPECT_EQ(17u, config.oplog_batch_max_entries); } +TEST(MasterServiceConfigTest, IoPatternCfmPropagatesToServingConfig) { + MasterConfig master_config{}; + master_config.io_pattern_cfm = {.endpoint = "cfm.example:50051", + .node_id = "master-a", + .auth_token = "secret", + .producer_auth_token = "producer-secret", + .timeout_ms = 750, + .policy_queue_capacity = 32}; + + MasterServiceSupervisorConfig supervisor_config(master_config); + WrappedMasterServiceConfig wrapped_config(supervisor_config, 1); + MasterServiceConfig service_config(wrapped_config); + + EXPECT_EQ(service_config.io_pattern_cfm.endpoint, "cfm.example:50051"); + EXPECT_EQ(service_config.io_pattern_cfm.node_id, "master-a"); + EXPECT_EQ(service_config.io_pattern_cfm.auth_token, "secret"); + EXPECT_EQ(service_config.io_pattern_cfm.producer_auth_token, + "producer-secret"); + EXPECT_EQ(service_config.io_pattern_cfm.timeout_ms, 750); + EXPECT_EQ(service_config.io_pattern_cfm.policy_queue_capacity, 32); +} + } // namespace mooncake::test From 3ee7b4f074ca97a0983702e88de7fdedb691b62c Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 3 Sep 2026 15:48:19 +0800 Subject: [PATCH 05/47] merge cvm --- docs/source/io_pattern_design.md | 93 ++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index 59a7a656bf..6c6a260142 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -965,6 +965,99 @@ metadata. Its handlers use the existing safe quota-eviction and promotion-on-hit queues; HBM stays inference-runtime-owned and is never moved by the Store master. +### Current implementation architecture + +The following diagram is the implementation-level view. It distinguishes the +local Store data path from the optional *remote* central CFM deployment: metric +reporting is asynchronous, while a received CFM command is executed by the +same storage handlers as a locally planned command. The two `MasterService` +boxes are deployment roles, not two mandatory Mooncake service types. A normal +deployment has one active Master (plus an optional HA standby); a separate +central CFM Master is needed only when metrics and policy are centralized +across multiple Masters. The roles may also be co-located for a single-Master +deployment. + +```mermaid +flowchart TB + subgraph producers["Metric producers"] + direction LR + inference["vLLM / SGLang bridge\nInferenceMetrics"] + access["Store Get/Put paths\nAccessRecord"] + storage["Storage and watermark paths\nStorageMetric"] + end + + subgraph local["Reporting / policy-consuming MasterService"] + direction TB + runtime["IoPatternRuntime"] + collector["IoPatternCollectorImpl\nper-tenant/object aggregation\nrolling snapshot"] + reporter["IoPatternReporter\nbounded MetricBatch queue\nadaptive 100/200/500/1000 ms flush"] + analyzer["ResilientAnalyzer\nSlidingWindowAnalyzer\nbudget + timeout fallback"] + policy["DegradingPolicyEngine\nWorkloadPolicyEngine\nper-session templates"] + executor["TierOperationExecutor"] + feedback["PolicyFeedbackWindow +\nAdaptivePolicyTuner"] + admission_worker["Admission worker\nbounded deferred queue"] + + runtime --> collector + collector --> reporter + collector --> analyzer + analyzer --> policy + policy --> executor + executor --> feedback + feedback -. "tune eviction weights" .-> policy + policy --> admission_worker + end + + subgraph local_ops["Store-owned safe execution handlers"] + direction LR + evict["Eviction\ntenant-qualified quota eviction"] + prefetch["Prefetch\nLOCAL_DISK → MEMORY promotion queue"] + admit["Admission\npost-write retention / promotion"] + end + + subgraph transport["Optional authenticated CFM transport"] + direction LR + codec["CfmBinaryCodec\nversioned CFM2 wire format"] + channel["CfmRpcChannel\nauthenticate + encode/decode"] + resilient["ResilientCfmChannel\nbounded retry + degradation state"] + rpc["CoroRpcCfmTransport\nexisting coro_rpc client pool"] + codec --> channel --> resilient + channel --> rpc + end + + subgraph central["Central CFM MasterService"] + direction TB + rpc_service["CfmRpcService\nAuthenticate / Send / Receive /\nAcknowledge / EnqueuePolicy"] + service["CfmService\nauthentication + per-node bounded queues"] + ingress["CfmIngress\ndecode and normalize remote metrics"] + central_runtime["IoPatternRuntime\nCollector → Analyzer → PolicyEngine"] + producer_worker["PolicyProducerWorker\nproduce high-watermark eviction\nand trace-derived prefetch commands"] + policy_queue["Policy queue per stable node_id\ndelivery_id + ACK state"] + rpc_service --> service --> ingress --> central_runtime --> producer_worker --> policy_queue + end + + inference --> runtime + access --> runtime + storage --> runtime + executor --> evict + executor --> prefetch + executor --> admit + + reporter -->|"report_metric_batch"| channel + rpc -->|"authenticated RPC"| rpc_service + policy_queue -->|"poll_policy"| rpc + resilient --> client["CfmClientImpl\nPollAndDispatchPolicy"] + client -->|"PolicyCommand"| runtime + client -->|"ACK success / failure"| resilient + + external_producer["External policy producer\nproducer credential"] -->|"enqueue_policy"| rpc_service + observability["IoPatternObservability\nlatency, hit rate, false positives,\ndegradation, report drops"] -.-> runtime +``` + +`CfmClientImpl` is deliberately not the normal metric-reporting entry point in +the production wiring. `IoPatternReporter` sends metric batches directly +through `CfmRpcChannel`; the client object owns the polling, dispatch and ACK +loop for CFM-issued policy commands. + ## Implemented - `IoPatternCollectorImpl` aggregates inference, access and storage metrics by From ff0ed457606905cc61dbfdd12f511544eaf11395 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 3 Sep 2026 17:10:00 +0800 Subject: [PATCH 06/47] add cvm support --- docs/source/io_pattern_design.md | 3 +- .../include/io_pattern/cfm_service.h | 17 ++++ mooncake-store/src/io_pattern/cfm_service.cpp | 53 +++++++++-- mooncake-store/src/master.cpp | 10 ++- mooncake-store/src/master_service.cpp | 11 ++- .../tests/io_pattern_framework_test.cpp | 87 ++++++++++++++++++- 6 files changed, 167 insertions(+), 14 deletions(-) diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index 6c6a260142..6f62651ad4 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -1214,7 +1214,8 @@ Configure a central CFM receiver with `io_pattern_cfm_auth_token`. Configure eac reporting/policy-consuming Master with: - `io_pattern_cfm_endpoint=host:port` -- `io_pattern_cfm_node_id=` (defaults to `cluster_id`) +- `io_pattern_cfm_node_id=` (defaults to the local CVM + SubMaster `master_id`, with `cluster_id` only as a legacy fallback) - the same `io_pattern_cfm_auth_token` - on the central receiver only, a distinct `io_pattern_cfm_producer_auth_token` for policy producers diff --git a/mooncake-store/include/io_pattern/cfm_service.h b/mooncake-store/include/io_pattern/cfm_service.h index 40a53c2fab..e0a00ebc5d 100644 --- a/mooncake-store/include/io_pattern/cfm_service.h +++ b/mooncake-store/include/io_pattern/cfm_service.h @@ -39,8 +39,22 @@ class CfmService final { bool EnqueuePolicy(std::string node_id, std::string payload, std::string_view token); + // Returns the metrics aggregated for one CFM node. In a CVM deployment a + // node is a stable SubMaster identity, so policy generation for one slot + // owner must never observe keys reported by another SubMaster. + IoPatternSnapshot SnapshotForNode(std::string_view node_id) const; + private: + struct NodeRuntime { + std::shared_ptr runtime; + std::unique_ptr ingress; + }; + bool EnqueueValidated(std::string node_id, std::string payload); + std::shared_ptr GetOrCreateNodeRuntime( + std::string_view node_id); + std::shared_ptr FindNodeRuntime( + std::string_view node_id) const; void SchedulePolicyProduction(std::string node_id, MetricBatch batch); void PolicyProducerWorker(); void ProducePolicies(std::string_view node_id, const MetricBatch& batch); @@ -48,6 +62,9 @@ class CfmService final { std::shared_ptr runtime_; std::shared_ptr codec_; CfmIngress ingress_; + mutable std::mutex node_runtimes_mutex_; + std::unordered_map> + node_runtimes_; const std::string auth_token_; const std::string producer_auth_token_; const size_t policy_queue_capacity_; diff --git a/mooncake-store/src/io_pattern/cfm_service.cpp b/mooncake-store/src/io_pattern/cfm_service.cpp index f0efe6d048..f4792bd894 100644 --- a/mooncake-store/src/io_pattern/cfm_service.cpp +++ b/mooncake-store/src/io_pattern/cfm_service.cpp @@ -55,7 +55,12 @@ bool CfmService::Send(std::string_view node_id, std::string_view method, metric_batch = codec_->DecodeMetricBatch(std::string(payload)); if (!metric_batch) return false; } - if (!ingress_.Handle(method, payload, node_id)) return false; + // Direct producer commands are intentionally executed through the CFM + // service runtime. Metrics and snapshots, on the other hand, must remain + // node-local: a CVM node is a SubMaster with an independent slot/key set. + if (executes_policy) return ingress_.Handle(method, payload, node_id); + auto node_runtime = GetOrCreateNodeRuntime(node_id); + if (!node_runtime->ingress->Handle(method, payload, node_id)) return false; if (metric_batch) { SchedulePolicyProduction(std::string(node_id), std::move(*metric_batch)); @@ -63,6 +68,38 @@ bool CfmService::Send(std::string_view node_id, std::string_view method, return true; } +IoPatternSnapshot CfmService::SnapshotForNode(std::string_view node_id) const { + const auto node_runtime = FindNodeRuntime(node_id); + return node_runtime ? node_runtime->runtime->Snapshot() + : IoPatternSnapshot{}; +} + +std::shared_ptr CfmService::GetOrCreateNodeRuntime( + std::string_view node_id) { + std::lock_guard lock(node_runtimes_mutex_); + const std::string id(node_id); + const auto existing = node_runtimes_.find(id); + if (existing != node_runtimes_.end()) return existing->second; + + auto node_runtime = std::make_shared(); + node_runtime->runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + node_runtime->ingress = + std::make_unique(node_runtime->runtime, codec_); + node_runtimes_.emplace(id, node_runtime); + return node_runtime; +} + +std::shared_ptr CfmService::FindNodeRuntime( + std::string_view node_id) const { + std::lock_guard lock(node_runtimes_mutex_); + const auto it = node_runtimes_.find(std::string(node_id)); + return it == node_runtimes_.end() ? nullptr : it->second; +} + std::optional> CfmService::PollPolicy( std::string_view node_id, std::string_view token) { if (!AuthenticateNode(token) || node_id.empty()) return std::nullopt; @@ -165,7 +202,9 @@ void CfmService::PolicyProducerWorker() { void CfmService::ProducePolicies(std::string_view node_id, const MetricBatch& batch) { - if (!runtime_ || node_id.empty()) return; + const auto node_runtime = FindNodeRuntime(node_id); + if (!node_runtime || node_id.empty()) return; + const auto& runtime = node_runtime->runtime; std::unordered_map match_lengths; std::string session_id; @@ -207,7 +246,7 @@ void CfmService::ProducePolicies(std::string_view node_id, } if (target_bytes == 0) { uint64_t tier_bytes = 0; - for (const auto& key : runtime_->Snapshot().keys) { + for (const auto& key : runtime->Snapshot().keys) { if ((key.replica_tiers & CacheTierBit(storage.tier)) == 0) { continue; } @@ -223,8 +262,8 @@ void CfmService::ProducePolicies(std::string_view node_id, static_cast(tier_bytes) * excess_ratio / std::max(0.01F, used_ratio)); } - auto result = runtime_->Plan(storage.tier, target_bytes, trace, {}, - session_id); + auto result = runtime->Plan(storage.tier, target_bytes, trace, {}, + session_id); if (result.degraded) continue; if (!result.eviction.candidates.empty()) { EnqueueValidated(std::string(node_id), @@ -238,8 +277,8 @@ void CfmService::ProducePolicies(std::string_view node_id, } if (!produced_prefetch && !trace.events.empty()) { - auto result = runtime_->Plan(CacheTier::kL1Host, 0, trace, {}, - session_id); + auto result = runtime->Plan(CacheTier::kL1Host, 0, trace, {}, + session_id); if (!result.degraded && !result.prefetch.candidates.empty()) { EnqueueValidated(std::string(node_id), codec_->EncodePolicy(result.prefetch)); diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index c23e65261f..1909aff1bc 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -164,7 +164,8 @@ DEFINE_string(io_pattern_cfm_endpoint, "", "Central CFM Master RPC endpoint (host:port); empty serves CFM " "requests without outbound reporting"); DEFINE_string(io_pattern_cfm_node_id, "", - "Stable node id used for CFM policy polling; defaults to cluster_id"); + "Stable node id used for CFM policy polling; defaults to " + "the local CVM SubMaster master_id"); DEFINE_string(io_pattern_cfm_auth_token, "", "Authentication token for CFM node report/poll RPCs"); DEFINE_string(io_pattern_cfm_producer_auth_token, "", @@ -1611,7 +1612,12 @@ int main(int argc, char* argv[]) { return 1; } if (master_config.io_pattern_cfm.node_id.empty()) { - master_config.io_pattern_cfm.node_id = master_config.cluster_id; + // CVM policy routing is per SubMaster. master_id is stable across the + // CFM reporting, polling and ACK path; cluster_id would merge every + // SubMaster in the same CVM deployment into one CFM node. + master_config.io_pattern_cfm.node_id = + master_config.master_id.empty() ? master_config.cluster_id + : master_config.master_id; } const char* value = std::getenv("MC_RPC_PROTOCOL"); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 00ee88b62d..e1207869c9 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -454,9 +454,14 @@ MasterService::MasterService(const MasterServiceConfig& config) "io_pattern_cfm_auth_token is required when " "io_pattern_cfm_endpoint is configured"); } - const std::string node_id = config.io_pattern_cfm.node_id.empty() - ? config.cluster_id - : config.io_pattern_cfm.node_id; + // In CVM mode every SubMaster owns a different slot set. Use the + // stable SubMaster id by default so CFM keeps their reports and + // policy queues separate; cluster_id is only a legacy fallback. + const std::string node_id = + config.io_pattern_cfm.node_id.empty() + ? (config.master_id.empty() ? config.cluster_id + : config.master_id) + : config.io_pattern_cfm.node_id; auto transport = std::make_shared( config.io_pattern_cfm.endpoint, node_id, std::chrono::milliseconds(config.io_pattern_cfm.timeout_ms)); diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 51f5329d09..be23ec9277 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1077,6 +1077,41 @@ TEST(IoPatternFrameworkTest, CfmServiceAuthenticatesAndBoundsPolicyQueues) { EXPECT_EQ(admissions, 1); } +TEST(IoPatternFrameworkTest, CfmAggregatesMetricsBySubmasterNode) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + CfmService service(runtime, "secret", 8, "producer"); + CfmBinaryCodec codec; + + MetricBatch submaster_a; + submaster_a.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "key-a"}, + .block_size = 64, + .tier = CacheTier::kL1Host, + .is_hit = true}); + MetricBatch submaster_b; + submaster_b.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "key-b"}, + .block_size = 128, + .tier = CacheTier::kL1Host, + .is_hit = true}); + + ASSERT_TRUE(service.Send("submaster-a", "report_metric_batch", + codec.EncodeMetricBatch(submaster_a), "secret")); + ASSERT_TRUE(service.Send("submaster-b", "report_metric_batch", + codec.EncodeMetricBatch(submaster_b), "secret")); + + const auto snapshot_a = service.SnapshotForNode("submaster-a"); + const auto snapshot_b = service.SnapshotForNode("submaster-b"); + ASSERT_EQ(snapshot_a.keys.size(), 1); + ASSERT_EQ(snapshot_b.keys.size(), 1); + EXPECT_EQ(snapshot_a.keys.front().object.key, "key-a"); + EXPECT_EQ(snapshot_b.keys.front().object.key, "key-b"); +} + TEST(IoPatternFrameworkTest, CoroRpcCfmTransportRunsTheProductionWirePath) { auto runtime = std::make_shared( IoPatternRuntime::Handlers{ @@ -1118,7 +1153,7 @@ TEST(IoPatternFrameworkTest, CoroRpcCfmTransportRunsTheProductionWirePath) { EXPECT_TRUE(transport.Send("report_metric_batch", codec.EncodeMetricBatch(batch), std::chrono::milliseconds(500))); - const auto snapshot = runtime->Snapshot(); + const auto snapshot = service->SnapshotForNode("node-a"); ASSERT_EQ(snapshot.keys.size(), 1); ASSERT_EQ(snapshot.storage.size(), 1); EXPECT_EQ(snapshot.keys.front().object.key, "remote-key"); @@ -1187,6 +1222,56 @@ TEST(IoPatternFrameworkTest, CfmProducesNodePolicyFromHighWatermarkReport) { "node-secret")); } +TEST(IoPatternFrameworkTest, CfmPolicyDoesNotCrossSubmasterKeySets) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + CfmService service(runtime, "node-secret", 8, "producer-secret"); + CfmBinaryCodec codec; + + MetricBatch submaster_b; + submaster_b.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "key-b"}, + .block_size = 4096, + .tier = CacheTier::kL1Host, + .is_hit = false}); + ASSERT_TRUE(service.Send("submaster-b", "report_metric_batch", + codec.EncodeMetricBatch(submaster_b), + "node-secret")); + + MetricBatch submaster_a; + submaster_a.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "key-a"}, + .block_size = 1024, + .tier = CacheTier::kL1Host, + .is_hit = false}); + submaster_a.storage.push_back( + StorageMetric{.tier = CacheTier::kL1Host, + .used_bytes = 950, + .capacity_bytes = 1000, + .memory_used_ratio = 0.95F}); + ASSERT_TRUE(service.Send("submaster-a", "report_metric_batch", + codec.EncodeMetricBatch(submaster_a), + "node-secret")); + + std::optional> delivery; + for (size_t attempt = 0; attempt < 100 && !delivery; ++attempt) { + delivery = service.PollPolicy("submaster-a", "node-secret"); + if (!delivery) std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + ASSERT_TRUE(delivery.has_value()); + const auto command = codec.DecodePolicy(delivery->second); + ASSERT_TRUE(command.has_value()); + const auto* eviction = std::get_if(&*command); + ASSERT_NE(eviction, nullptr); + ASSERT_FALSE(eviction->candidates.empty()); + for (const auto& candidate : eviction->candidates) { + EXPECT_EQ(candidate.object.key, "key-a"); + } +} + TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { size_t batches = 0; IoPatternReporter reporter(4, [&](const MetricBatch&) { From 1fc118cfc0681ee45f15865262f125adf58b8b33 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 3 Sep 2026 20:52:11 +0800 Subject: [PATCH 07/47] add cvm support --- mooncake-store/include/master_config.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index 1bb0b49c99..5db113cd52 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -914,6 +914,9 @@ class MasterServiceConfigBuilder { std::string cxl_path_ = DEFAULT_CXL_PATH; size_t cxl_size_ = DEFAULT_CXL_SIZE; bool enable_cxl_ = false; + IoPatternCfmConfig io_pattern_cfm_; + VChunkConfig vchunk_config_{}; + std::shared_ptr vchunk_metadata_store_; public: MasterServiceConfigBuilder() = default; @@ -1274,6 +1277,8 @@ class MasterServiceConfig { bool kv_events_emit_legacy_compat = true; bool kv_events_emit_object_key = true; uint32_t kv_events_queue_capacity = 65536; + // CFM transport and authentication settings used by this MasterService. + IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; // OpLog store configuration @@ -1329,6 +1334,9 @@ class MasterServiceConfig { std::string cxl_path = DEFAULT_CXL_PATH; size_t cxl_size = DEFAULT_CXL_SIZE; bool enable_cxl = false; + VChunkConfig vchunk_config{}; + std::string vchunk_etcd_endpoints; + std::shared_ptr vchunk_metadata_store; MasterServiceConfig() = default; // From WrappedMasterServiceConfig @@ -1373,6 +1381,7 @@ class MasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; + io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = config.ha_backend_connstring; enable_oplog = config.enable_oplog; @@ -1421,6 +1430,12 @@ class MasterServiceConfig { cxl_path = config.cxl_path; cxl_size = config.cxl_size; enable_cxl = config.enable_cxl; + vchunk_config = config.vchunk_config; + vchunk_etcd_endpoints = config.vchunk_etcd_endpoints; + if (vchunk_config.enabled && !vchunk_etcd_endpoints.empty()) { + vchunk_metadata_store = std::make_shared( + vchunk_etcd_endpoints, vchunk_config, cluster_id); + } } // Static factory method to create a builder @@ -1487,6 +1502,11 @@ inline MasterServiceConfig MasterServiceConfigBuilder::build() const { config.cxl_path = cxl_path_; config.cxl_size = cxl_size_; config.enable_cxl = enable_cxl_; + config.io_pattern_cfm = io_pattern_cfm_; + config.vchunk_config = vchunk_config_; + config.vchunk_metadata_store = vchunk_metadata_store_; + config.vchunk_config = vchunk_config_; + config.vchunk_metadata_store = vchunk_metadata_store_; return config; } From d2dd1b82f23ca3e4a6af59a391f07379160ed343 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Fri, 4 Sep 2026 10:01:24 +0800 Subject: [PATCH 08/47] add cfm client benchmark --- docs/source/io_pattern_design.md | 2 +- mooncake-store/benchmarks/CMakeLists.txt | 7 + .../benchmarks/cfm_client_bench.cpp | 533 ++++++++++++++++++ .../graphify-out/cache/stat-index.json | 1 + .../include/io_pattern/cfm_service.h | 6 + .../include/io_pattern/collector_impl.h | 8 +- .../include/io_pattern/kmeans_analyzer.h | 5 +- mooncake-store/include/io_pattern/runtime.h | 5 +- .../include/io_pattern/tier_executor.h | 1 + mooncake-store/src/io_pattern/cfm_service.cpp | 24 +- .../src/io_pattern/collector_impl.cpp | 9 + mooncake-store/src/io_pattern/runtime.cpp | 7 + mooncake-store/src/master.cpp | 12 +- mooncake-store/src/master_service.cpp | 16 + .../tests/io_pattern_framework_test.cpp | 3 + 15 files changed, 625 insertions(+), 14 deletions(-) create mode 100644 mooncake-store/benchmarks/cfm_client_bench.cpp create mode 100644 mooncake-store/graphify-out/cache/stat-index.json diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index 6f62651ad4..88eac9d220 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -1215,7 +1215,7 @@ reporting/policy-consuming Master with: - `io_pattern_cfm_endpoint=host:port` - `io_pattern_cfm_node_id=` (defaults to the local CVM - SubMaster `master_id`, with `cluster_id` only as a legacy fallback) + SubMaster RPC endpoint, `rpc_address:rpc_port`) - the same `io_pattern_cfm_auth_token` - on the central receiver only, a distinct `io_pattern_cfm_producer_auth_token` for policy producers diff --git a/mooncake-store/benchmarks/CMakeLists.txt b/mooncake-store/benchmarks/CMakeLists.txt index 52cf2d9501..b30d7cba8f 100644 --- a/mooncake-store/benchmarks/CMakeLists.txt +++ b/mooncake-store/benchmarks/CMakeLists.txt @@ -38,6 +38,13 @@ target_link_libraries( stress_cluster_bench PRIVATE mooncake_store transfer_engine asio_shared gflags::gflags glog::glog pthread) +# CFM client benchmark. Simulates vLLM inference requests as KV-cache block +# accesses and reports both client performance and IO Pattern observability. +add_executable(cfm_client_bench cfm_client_bench.cpp) +target_link_libraries( + cfm_client_bench PRIVATE mooncake_store transfer_engine asio_shared + gflags::gflags glog::glog pthread) + # Benchmark for vLLM Store Connector path # Triggers: batch_put_from_multi_buffers / batchIsExist / # batch_get_into_multi_buffers (and setup/register_buffer/tearDownAll). diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp new file mode 100644 index 0000000000..41db1a1a04 --- /dev/null +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -0,0 +1,533 @@ +// CFM client benchmark that models the vLLM KV-cache call path. +// +// One benchmark request consists of prompt_tokens + output_tokens. The KV +// cache is split into tokens_per_block blocks for every transformer layer, +// exactly as a vLLM connector would address its layer/block cache entries. +// Each request records inference and access metrics, reports a snapshot through +// CfmClientImpl, then prints both CFM call latency and IO Pattern metrics. + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "gflags/gflags.h" +#include "glog/logging.h" +#include "io_pattern/cfm_client_impl.h" +#include "io_pattern/cfm_protocol.h" +#include "io_pattern/cfm_service.h" +#include "io_pattern/rpc_transport.h" +#include "io_pattern/runtime.h" + +namespace { + +using Clock = std::chrono::steady_clock; +using mooncake::ErrorCode; +using mooncake::TenantId; +using namespace mooncake::io_pattern; + +DEFINE_uint64(requests, 20, "Number of vLLM-style inference requests"); +DEFINE_uint64(prompt_tokens, 1024, "Input tokens in each inference request"); +DEFINE_uint64(output_tokens, 128, "Decode tokens in each inference request"); +DEFINE_uint64(tokens_per_block, 16, "Tokens represented by one KV block"); +DEFINE_uint64(num_layers, 32, "Transformer layers represented per request"); +DEFINE_uint64(kv_block_bytes, 256 * 1024, + "Bytes in one layer/block KV-cache object"); +DEFINE_uint64(num_sessions, 4, "Independent vLLM request sessions"); +DEFINE_uint64(shared_prefix_tokens, 512, + "Per-session prompt prefix reused by later requests"); +DEFINE_uint64(report_capacity, 262144, + "Maximum queued IO Pattern observations before reporting"); +DEFINE_uint64(policy_queue_capacity, 4096, + "Maximum CFM policy commands queued for the client"); +DEFINE_uint64(report_flush_wait_ms, 1100, + "Maximum wait for remote CFM policy production after a flush"); +DEFINE_double(memory_used_ratio, 0.95, + "Reported L1 memory use ratio; >= 0.90 triggers CFM planning"); +DEFINE_string(tenant, "vllm-benchmark", "Tenant id"); +DEFINE_string(node_id, "vllm-submaster-0", "CFM node/submaster id"); +DEFINE_string(cfm_endpoint, "", + "Remote CFM coro_rpc endpoint; empty uses an embedded CFM service"); +DEFINE_string(cfm_auth_token, "cfm-client-benchmark-node-token", + "Authentication token for the CFM node endpoint"); + +uint64_t SteadyNowNs() { + return static_cast( + std::chrono::duration_cast( + Clock::now().time_since_epoch()) + .count()); +} + +double ToMicroseconds(Clock::duration duration) { + return std::chrono::duration(duration).count(); +} + +size_t BlockCount(uint64_t tokens) { + return static_cast((tokens + FLAGS_tokens_per_block - 1) / + FLAGS_tokens_per_block); +} + +std::string KvKey(size_t session, size_t request, size_t layer, size_t block, + bool is_shared_prefix) { + const auto owner = is_shared_prefix ? std::string("prefix") + : std::string("request-") + + std::to_string(request); + return "vllm/" + FLAGS_node_id + "/session-" + + std::to_string(session) + "/" + owner + "/layer-" + + std::to_string(layer) + "/block-" + std::to_string(block); +} + +class ServiceBackedCfmTransport final : public CfmRpcTransport { + public: + ServiceBackedCfmTransport(std::shared_ptr service, + std::string node_id) + : service_(std::move(service)), node_id_(std::move(node_id)) {} + + bool Authenticate(std::string_view token) override { + if (!service_ || !service_->AuthenticateNode(token)) return false; + token_ = std::string(token); + authenticated_ = true; + return true; + } + + bool Send(std::string_view method, std::string_view payload, + std::chrono::milliseconds) override { + return authenticated_ && service_ && + service_->Send(node_id_, method, payload, token_); + } + + CfmReceiveResult Receive(std::string_view method, + std::chrono::milliseconds) override { + if (!authenticated_ || !service_ || method != "poll_policy") { + return CfmReceiveResult::Error(); + } + const auto delivery = service_->PollPolicy(node_id_, token_); + return delivery + ? CfmReceiveResult::Payload(delivery->second, delivery->first) + : CfmReceiveResult::Empty(); + } + + bool Acknowledge(uint64_t delivery_id, bool success, + std::chrono::milliseconds) override { + return authenticated_ && service_ && + service_->AcknowledgePolicy(node_id_, delivery_id, success, + token_); + } + + private: + std::shared_ptr service_; + std::string node_id_; + std::string token_; + bool authenticated_{false}; +}; + +class LatencyStats final { + public: + void Record(double value_us) { values_us_.push_back(value_us); } + + double Percentile(double percentile) const { + if (values_us_.empty()) return 0.0; + const double rank = percentile / 100.0 * (values_us_.size() - 1); + const auto lower = static_cast(rank); + const auto upper = std::min(lower + 1, values_us_.size() - 1); + const double fraction = rank - lower; + return values_us_[lower] * (1.0 - fraction) + + values_us_[upper] * fraction; + } + + void Finalize() { std::sort(values_us_.begin(), values_us_.end()); } + + double Mean() const { + if (values_us_.empty()) return 0.0; + return std::accumulate(values_us_.begin(), values_us_.end(), 0.0) / + values_us_.size(); + } + + private: + std::vector values_us_; +}; + +struct MetricReportSnapshot { + uint64_t calls{0}; + uint64_t failures{0}; + uint64_t observations{0}; + LatencyStats latency; +}; + +class MetricReportStats final { + public: + void Record(const MetricBatch& batch, double latency_us, bool success) { + std::lock_guard lock(mutex_); + ++calls; + if (!success) ++failures; + observations += + batch.inference.size() + batch.accesses.size() + batch.storage.size(); + latency.Record(latency_us); + } + + MetricReportSnapshot Finalize() { + std::lock_guard lock(mutex_); + latency.Finalize(); + return {.calls = calls, + .failures = failures, + .observations = observations, + .latency = latency}; + } + + private: + std::mutex mutex_; + uint64_t calls{0}; + uint64_t failures{0}; + uint64_t observations{0}; + LatencyStats latency; +}; + +struct RequestData { + IoPatternSnapshot snapshot; + std::vector inference; + std::vector accesses; +}; + +RequestData BuildRequest(size_t request_index) { + const size_t session = request_index % FLAGS_num_sessions; + const uint64_t total_tokens = FLAGS_prompt_tokens + FLAGS_output_tokens; + const size_t blocks = BlockCount(total_tokens); + const size_t shared_blocks = + std::min(blocks, BlockCount(FLAGS_shared_prefix_tokens)); + const bool prefix_is_cached = request_index >= FLAGS_num_sessions; + const uint64_t now_ns = SteadyNowNs(); + + RequestData request; + request.snapshot.generated_at_ns = now_ns; + request.inference.reserve(blocks * FLAGS_num_layers); + request.accesses.reserve(blocks * FLAGS_num_layers); + request.snapshot.keys.reserve(blocks * FLAGS_num_layers); + const auto tenant = TenantId(FLAGS_tenant); + const auto session_id = "vllm-session-" + std::to_string(session); + + for (size_t layer = 0; layer < FLAGS_num_layers; ++layer) { + for (size_t block = 0; block < blocks; ++block) { + const bool is_shared_prefix = block < shared_blocks; + const bool is_hit = is_shared_prefix && prefix_is_cached; + const ObjectRef object{ + .tenant_id = tenant, + .key = KvKey(session, request_index, layer, block, + is_shared_prefix)}; + const auto block_end = std::min( + total_tokens, (block + 1) * FLAGS_tokens_per_block); + const auto block_tokens = static_cast( + block_end - block * FLAGS_tokens_per_block); + + InferenceMetrics inference{ + .object = object, + .session_id = session_id, + .layout = CacheLayout::kLayerFirst, + .layout_group = static_cast(layer), + .prefix_depth = static_cast(shared_blocks), + .prefix_fanout = static_cast(FLAGS_num_sessions), + .match_length = is_hit + ? static_cast( + FLAGS_shared_prefix_tokens) + : 0U, + .continuous_prefix_length = is_hit + ? static_cast( + FLAGS_shared_prefix_tokens) + : 0U, + .token_count = block_tokens, + .recompute_cost = is_hit ? 0.0F : static_cast(block_tokens), + .request_priority = 1}; + AccessRecord access{ + .object = object, + .observed_at_ns = now_ns, + .block_size = FLAGS_kv_block_bytes, + .latency_us = is_hit ? 20U : 200U, + .tier = CacheTier::kL1Host, + .operation = is_hit ? IoOperation::kGet : IoOperation::kPut, + .is_hit = is_hit, + .write_batch_size = is_hit + ? 0U + : static_cast(FLAGS_num_layers), + .overwrite = !is_hit && is_shared_prefix}; + request.inference.push_back(inference); + request.accesses.push_back(access); + request.snapshot.keys.push_back( + KeyMetrics{.object = object, + .session_id = session_id, + .last_access_time_ns = now_ns, + .access_count_window = 1, + .block_size = FLAGS_kv_block_bytes, + .token_count = block_tokens, + .prefix_depth = static_cast(shared_blocks), + .prefix_fanout = static_cast(FLAGS_num_sessions), + .match_length = inference.match_length, + .continuous_prefix_length = + inference.continuous_prefix_length, + .write_batch_size = access.write_batch_size, + .write_frequency = + access.operation == IoOperation::kPut ? 1U : 0U, + .write_object_size = FLAGS_kv_block_bytes, + .recompute_cost = inference.recompute_cost, + .overwrite_ratio = access.overwrite ? 1.0F : 0.0F, + .replica_tiers = CacheTierBit(CacheTier::kL1Host), + .layout = CacheLayout::kLayerFirst, + .layout_group = static_cast(layer), + .request_priority = 1, + .active = is_hit, + .write_burst = !is_hit}); + } + } + request.snapshot.storage.push_back( + StorageMetric{.source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .read_bandwidth_bytes_per_sec = 20ULL * 1024 * 1024 * 1024, + .write_bandwidth_bytes_per_sec = 10ULL * 1024 * 1024 * 1024, + .read_latency_us = 20, + .write_latency_us = 200, + .used_bytes = static_cast( + FLAGS_memory_used_ratio * 1024 * 1024 * 1024), + .capacity_bytes = 1024ULL * 1024 * 1024, + .rpc_latency_us = 100, + .memory_used_ratio = + static_cast(FLAGS_memory_used_ratio)}); + return request; +} + +void PrintObservability(std::string_view name, + const IoPatternObservabilitySnapshot& metrics) { + std::cout << "\n " << name << " IO Pattern metrics\n" + << " collect max latency: " << metrics.collect_latency_us + << " us\n" + << " analyze max latency: " << metrics.analyze_latency_us + << " us\n" + << " policy decisions: " << metrics.policy_decisions + << " (" << std::fixed << std::setprecision(2) + << metrics.policy_decision_qps << " qps)\n" + << " strategy hit rate: " << metrics.strategy_hit_rate * 100 + << "%\n" + << " false positive rate: " + << metrics.false_positive_rate * 100 << "%\n" + << " degraded: " << metrics.degrade_count << "\n" + << " report drops: " << metrics.report_drop_count + << "\n"; +} + +bool ValidateFlags() { + return FLAGS_requests != 0 && FLAGS_prompt_tokens + FLAGS_output_tokens != 0 && + FLAGS_tokens_per_block != 0 && FLAGS_num_layers != 0 && + FLAGS_kv_block_bytes != 0 && FLAGS_num_sessions != 0 && + FLAGS_report_capacity != 0 && FLAGS_policy_queue_capacity != 0 && + FLAGS_memory_used_ratio >= 0.0 && FLAGS_memory_used_ratio <= 1.0; +} + +} // namespace + +int main(int argc, char* argv[]) { + google::InitGoogleLogging(argv[0]); + gflags::ParseCommandLineFlags(&argc, &argv, true); + if (!ValidateFlags()) { + LOG(ERROR) << "All numeric size/count flags must be positive and " + "--memory_used_ratio must be within [0, 1]"; + return 1; + } + + constexpr char kProducerToken[] = "cfm-client-benchmark-producer-token"; + std::atomic eviction_commands{0}; + std::atomic prefetch_commands{0}; + std::atomic admission_commands{0}; + + std::shared_ptr embedded_service; + std::shared_ptr transport; + if (FLAGS_cfm_endpoint.empty()) { + auto cfm_control_runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { + return ErrorCode::OK; + }}); + embedded_service = std::make_shared( + cfm_control_runtime, FLAGS_cfm_auth_token, + FLAGS_policy_queue_capacity, kProducerToken); + transport = std::make_shared( + embedded_service, FLAGS_node_id); + } else { + transport = std::make_shared( + FLAGS_cfm_endpoint, FLAGS_node_id, std::chrono::milliseconds(500)); + } + auto codec = std::make_shared(); + auto channel = std::make_shared( + transport, codec, + CfmRpcConfig{.timeout = std::chrono::milliseconds(500), + .auth_token = FLAGS_cfm_auth_token}); + + IoPatternRuntime::Config source_config; + source_config.report_capacity = FLAGS_report_capacity; + MetricReportStats metric_reports; + source_config.report_sink = [&channel, &metric_reports](const MetricBatch& batch) { + const auto started = Clock::now(); + const bool success = channel->SendMetricBatch(batch); + metric_reports.Record(batch, ToMicroseconds(Clock::now() - started), + success); + return success; + }; + auto source_runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [&eviction_commands](const EvictionPlan&) { + ++eviction_commands; + return ErrorCode::OK; + }, + .prefetch = [&prefetch_commands](const PrefetchPlan&) { + ++prefetch_commands; + return ErrorCode::OK; + }, + .admission = [&admission_commands](const AdmissionResult&) { + ++admission_commands; + return ErrorCode::OK; + }}, + source_config); + CfmClientImpl client(channel, [&source_runtime](const PolicyCommand& command) { + return source_runtime->ExecuteCommand(command); + }); + + LatencyStats report_latency; + uint64_t failed_reports = 0; + uint64_t total_blocks = 0; + const auto benchmark_start = Clock::now(); + for (size_t request_index = 0; request_index < FLAGS_requests; + ++request_index) { + auto request = BuildRequest(request_index); + total_blocks += request.accesses.size(); + for (size_t i = 0; i < request.inference.size(); ++i) { + source_runtime->ReportInferenceMetrics(request.inference[i]); + source_runtime->RecordAccess(request.accesses[i].object.key, + request.accesses[i]); + } + source_runtime->RecordStorageMetric(request.snapshot.storage.front()); + + const auto report_start = Clock::now(); + const auto result = client.ReportSnapshot(request.snapshot); + report_latency.Record(ToMicroseconds(Clock::now() - report_start)); + if (result != ErrorCode::OK) ++failed_reports; + } + const auto submission_seconds = + std::chrono::duration(Clock::now() - benchmark_start).count(); + + // Stop joins the reporter worker and performs its final flush. No new + // metric batch can reach CFM after this returns. + source_runtime->StopReports(); + if (embedded_service) { + if (!embedded_service->WaitForPolicyIdle( + std::chrono::milliseconds(FLAGS_report_flush_wait_ms))) { + LOG(WARNING) << "Timed out waiting for embedded CFM policy production"; + } + } else { + std::this_thread::sleep_for( + std::chrono::milliseconds(FLAGS_report_flush_wait_ms)); + } + uint64_t observed_commands = 0; + bool policy_drain_complete = false; + const auto policy_deadline = + Clock::now() + std::chrono::milliseconds(FLAGS_report_flush_wait_ms); + while (Clock::now() < policy_deadline) { + if (client.PollAndDispatchPolicy() != ErrorCode::OK) break; + const uint64_t executed = eviction_commands + prefetch_commands + + admission_commands; + if (executed == observed_commands) { + policy_drain_complete = embedded_service != nullptr; + break; + } else { + observed_commands = executed; + } + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + + const auto end_to_end_seconds = + std::chrono::duration(Clock::now() - benchmark_start).count(); + + const auto source_snapshot = source_runtime->Snapshot(); + const auto source_metrics = + source_runtime->ObservabilitySnapshot(end_to_end_seconds); + source_runtime.reset(); + report_latency.Finalize(); + const auto metric_report_snapshot = metric_reports.Finalize(); + const auto cfm_snapshot = embedded_service + ? embedded_service->SnapshotForNode(FLAGS_node_id) + : IoPatternSnapshot{}; + const auto cfm_metrics = embedded_service + ? embedded_service->ObservabilityForNode( + FLAGS_node_id, end_to_end_seconds) + : IoPatternObservabilitySnapshot{}; + + std::cout << "\n============================================================\n" + << "CFM CLIENT BENCHMARK (vLLM inference request model)\n" + << "============================================================\n" + << " Requests: " << FLAGS_requests << "\n" + << " Tokens/request: " + << FLAGS_prompt_tokens + FLAGS_output_tokens << " (prompt=" + << FLAGS_prompt_tokens << ", decode=" << FLAGS_output_tokens + << ")\n" + << " KV blocks/request: " << BlockCount( + FLAGS_prompt_tokens + FLAGS_output_tokens) * FLAGS_num_layers + << " (layers=" << FLAGS_num_layers << ")\n" + << " Total KV blocks: " << total_blocks << "\n" + << " Request submission time: " << std::fixed + << std::setprecision(2) << submission_seconds << " s\n" + << " Submission requests/sec: " + << FLAGS_requests / submission_seconds << "\n" + << " End-to-end time: " << end_to_end_seconds << " s\n" + << "\n CFM ReportSnapshot latency\n" + << " failed reports: " << failed_reports << "\n" + << " mean: " << report_latency.Mean() << " us\n" + << " p50 / p90 / p99: " << report_latency.Percentile(50) + << " / " << report_latency.Percentile(90) << " / " + << report_latency.Percentile(99) << " us\n" + << "\n CFM report_metric_batch latency\n" + << " calls / failures: " << metric_report_snapshot.calls + << " / " << metric_report_snapshot.failures << "\n" + << " observations: " + << metric_report_snapshot.observations + << "\n" + << " mean: " + << metric_report_snapshot.latency.Mean() + << " us\n" + << " p50 / p90 / p99: " + << metric_report_snapshot.latency.Percentile(50) << " / " + << metric_report_snapshot.latency.Percentile(90) << " / " + << metric_report_snapshot.latency.Percentile(99) << " us\n" + << "\n CFM policy commands executed\n" + << " evictions: " << eviction_commands << "\n" + << " prefetches: " << prefetch_commands << "\n" + << " admissions: " << admission_commands << "\n" + << " policy drain: " + << (embedded_service + ? (policy_drain_complete ? "complete" : "timed out/error") + : "remote endpoint is not verifiable") + << "\n" + << "\n IO Pattern snapshots\n" + << " client keys / storage: " << source_snapshot.keys.size() << " / " + << source_snapshot.storage.size() << "\n"; + if (embedded_service) { + std::cout << " CFM keys / storage: " << cfm_snapshot.keys.size() + << " / " << cfm_snapshot.storage.size() << "\n"; + } else { + std::cout << " CFM keys / storage: remote endpoint (not exposed to " + "the client)\n"; + } + PrintObservability("Client", source_metrics); + if (embedded_service) PrintObservability("CFM", cfm_metrics); + std::cout << "============================================================\n"; + return failed_reports == 0 ? 0 : 2; +} diff --git a/mooncake-store/graphify-out/cache/stat-index.json b/mooncake-store/graphify-out/cache/stat-index.json new file mode 100644 index 0000000000..26f3fefc57 --- /dev/null +++ b/mooncake-store/graphify-out/cache/stat-index.json @@ -0,0 +1 @@ +{"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\AGENTS.md":{"size":629,"mtime_ns":1786963737729753600,"word_count":89},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\CMakeLists.txt":{"size":3012,"mtime_ns":1786963737730754700,"word_count":175},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\CMakeLists.txt":{"size":3960,"mtime_ns":1788421144221545100,"word_count":247},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\README.md":{"size":3145,"mtime_ns":1786963737730754700,"word_count":367},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\allocation_strategy_bench.cpp":{"size":79226,"mtime_ns":1786963737731754800,"word_count":7249},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\allocator_bench.cpp":{"size":11232,"mtime_ns":1786963737731754800,"word_count":1091},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\batch_evict_bench.cpp":{"size":18064,"mtime_ns":1786963737732754700,"word_count":1293},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\batch_get_replica_bench.cpp":{"size":28446,"mtime_ns":1786963737732754700,"word_count":2261},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\batch_remove_benchmark.py":{"size":14812,"mtime_ns":1781678985149681300,"word_count":1186},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\cluster_mooncake_diag.py":{"size":97859,"mtime_ns":1786976645759111200,"word_count":8162},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\file_interface_bench.cpp":{"size":34867,"mtime_ns":1781678985149681300,"word_count":3178},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\master_bench.cpp":{"size":19966,"mtime_ns":1786963737733754900,"word_count":1578},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\nof_worker_pool_bench.cpp":{"size":40711,"mtime_ns":1786976645759111200,"word_count":3092},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\oplog_batch_bench.cpp":{"size":13726,"mtime_ns":1786963737733754900,"word_count":993},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\report_oplog_batch.py":{"size":7146,"mtime_ns":1786963737733754900,"word_count":512},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\run_oplog_batch_sweep.py":{"size":7127,"mtime_ns":1786963737733754900,"word_count":475},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\storage_backend_bench.cpp":{"size":94331,"mtime_ns":1781678985150683700,"word_count":8403},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\store_connector_bench.cpp":{"size":19331,"mtime_ns":1788421144221545100,"word_count":1730},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\store_kv_bench.md":{"size":5588,"mtime_ns":1781678985150683700,"word_count":695},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\store_kv_bench.py":{"size":58260,"mtime_ns":1786963737735256300,"word_count":4164},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\stress_cluster_bench.cpp":{"size":74873,"mtime_ns":1788421144221545100,"word_count":6667},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\stress_cluster_ranges_bench.cpp":{"size":42576,"mtime_ns":1786976645760111100,"word_count":4052},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\test_report_oplog_batch.py":{"size":3598,"mtime_ns":1786963737736257800,"word_count":197},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\test_run_oplog_batch_sweep.py":{"size":4288,"mtime_ns":1786963737736257800,"word_count":259},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\test_store_kv_bench.py":{"size":6253,"mtime_ns":1786963737737258000,"word_count":404},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\benchmarks\\vchunk_distributed_bench.cpp":{"size":10326,"mtime_ns":1788421144221545100,"word_count":800},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\conf\\master.json":{"size":1323,"mtime_ns":1788421144221545100,"word_count":80},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\conf\\master.yaml":{"size":1516,"mtime_ns":1788421144221545100,"word_count":132},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\go\\build.sh":{"size":2932,"mtime_ns":1786963737737258000,"word_count":309},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\go\\examples\\basic\\main.go":{"size":2404,"mtime_ns":1781678985152689000,"word_count":318},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\go\\go.mod":{"size":68,"mtime_ns":1781678985152689000,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\go\\mooncakestore\\config.go":{"size":1066,"mtime_ns":1781678985153691400,"word_count":141},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\go\\mooncakestore\\errors.go":{"size":1650,"mtime_ns":1786963737738730400,"word_count":180},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\go\\mooncakestore\\store.go":{"size":14159,"mtime_ns":1781678985153691400,"word_count":1791},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\go\\tests\\integration_test.go":{"size":4372,"mtime_ns":1781678985153691400,"word_count":605},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\admission_ops.h":{"size":45,"mtime_ns":1788421144377666800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\aligned_client_buffer.h":{"size":2139,"mtime_ns":1786963737738730400,"word_count":257},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\allocation_strategy.h":{"size":31158,"mtime_ns":1786963737738730400,"word_count":2636},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\allocator.h":{"size":12151,"mtime_ns":1786976645760111100,"word_count":1075},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\AllocationClass.h":{"size":20014,"mtime_ns":1786963737739832900,"word_count":2544},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\MemoryAllocator.h":{"size":20451,"mtime_ns":1786963737739832900,"word_count":2689},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\MemoryPool.h":{"size":15290,"mtime_ns":1786963737740834200,"word_count":2098},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\MemoryPoolManager.h":{"size":5543,"mtime_ns":1781678985155735100,"word_count":791},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\Slab.h":{"size":10734,"mtime_ns":1781678985155735100,"word_count":1341},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\SlabAllocator.h":{"size":8189,"mtime_ns":1781678985156737400,"word_count":1056},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\common\\CompilerUtils.h":{"size":2349,"mtime_ns":1781678985156737400,"word_count":286},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\common\\Exceptions.h":{"size":3014,"mtime_ns":1781678985156737400,"word_count":334},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\common\\Throttler.h":{"size":3643,"mtime_ns":1781678985156737400,"word_count":459},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\common\\Time.h":{"size":3123,"mtime_ns":1781678985156737400,"word_count":346},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\common\\Utils.h":{"size":1068,"mtime_ns":1781678985157739700,"word_count":165},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\fake_include\\folly\\logging\\xlog.h":{"size":30964,"mtime_ns":1781678985157739700,"word_count":3245},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\fake_include\\folly\\portability\\Config.h":{"size":1116,"mtime_ns":1781678985157739700,"word_count":145},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\args.h":{"size":7699,"mtime_ns":1781678985158742500,"word_count":716},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\chrono.h":{"size":43744,"mtime_ns":1781678985158742500,"word_count":4420},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\color.h":{"size":24752,"mtime_ns":1781678985158742500,"word_count":2221},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\compile.h":{"size":22149,"mtime_ns":1781678985159744700,"word_count":2203},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\core.h":{"size":101361,"mtime_ns":1781678985159744700,"word_count":10856},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\format-inl.h":{"size":106523,"mtime_ns":1781678985160746500,"word_count":9714},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\format.h":{"size":106448,"mtime_ns":1781678985160746500,"word_count":11421},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\locale.h":{"size":102,"mtime_ns":1781678985160746500,"word_count":11},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\os.h":{"size":15582,"mtime_ns":1781678985161748200,"word_count":1809},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\ostream.h":{"size":6152,"mtime_ns":1781678985161748200,"word_count":573},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\printf.h":{"size":21091,"mtime_ns":1781678985161748200,"word_count":2222},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\ranges.h":{"size":14927,"mtime_ns":1781678985161748200,"word_count":1478},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\fmt\\xchar.h":{"size":9467,"mtime_ns":1781678985162750600,"word_count":794},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\folly\\CPortability.h":{"size":10773,"mtime_ns":1781678985162750600,"word_count":1051},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\folly\\Likely.h":{"size":2195,"mtime_ns":1781678985162750600,"word_count":314},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\folly\\Portability.h":{"size":16275,"mtime_ns":1781678985162750600,"word_count":1906},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cachelib_memory_allocator\\include\\folly\\lang\\Builtin.h":{"size":2198,"mtime_ns":1781678985162750600,"word_count":237},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cfm_client.h":{"size":48,"mtime_ns":1788421144377666800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cfm_client_impl.h":{"size":55,"mtime_ns":1788421144492018000,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\client_buffer.h":{"size":5113,"mtime_ns":1786963737740834200,"word_count":552},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\client_metric.h":{"size":28657,"mtime_ns":1786976645761558400,"word_count":2319},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\client_service.h":{"size":44657,"mtime_ns":1788421144221545100,"word_count":4178},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\collector_impl.h":{"size":54,"mtime_ns":1788421144492018000,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\config_helper.h":{"size":2756,"mtime_ns":1781678985164755800,"word_count":352},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\count_min_sketch.h":{"size":2847,"mtime_ns":1781678985164755800,"word_count":332},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\crc32c.h":{"size":1348,"mtime_ns":1786963737741834400,"word_count":184},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\cvm_controller.h":{"size":5947,"mtime_ns":1788421144227958400,"word_count":485},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\cvm_http_server.h":{"size":1763,"mtime_ns":1788421144227958400,"word_count":132},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\cvm_keys.h":{"size":3551,"mtime_ns":1788421144227958400,"word_count":298},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\cvm_service_delegate.h":{"size":1329,"mtime_ns":1788421144227958400,"word_count":177},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\cvm_types.h":{"size":3902,"mtime_ns":1788421144229244100,"word_count":369},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\etcd_view_store.h":{"size":7208,"mtime_ns":1788421144229244100,"word_count":546},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\inter_master_rpc.h":{"size":8136,"mtime_ns":1788421144229244100,"word_count":867},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\slot_hash.h":{"size":6206,"mtime_ns":1788421144230337600,"word_count":674},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\slot_migrator.h":{"size":2634,"mtime_ns":1788421144230337600,"word_count":299},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\cvm\\slot_owner_heartbeat.h":{"size":3600,"mtime_ns":1788421144230337600,"word_count":408},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\deadline_scheduler.h":{"size":4366,"mtime_ns":1786963737741834400,"word_count":306},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\degrading_policy_engine.h":{"size":63,"mtime_ns":1788421144492018000,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\device\\accelerator_device.h":{"size":1495,"mtime_ns":1786963737742939400,"word_count":159},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\device\\accelerator_registry.h":{"size":796,"mtime_ns":1786963737742939400,"word_count":72},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\device\\runtime_accelerator.h":{"size":772,"mtime_ns":1786963737742939400,"word_count":74},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\dummy_client.h":{"size":11336,"mtime_ns":1786963737742939400,"word_count":900},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\engram\\engram_store.h":{"size":2660,"mtime_ns":1781678985164755800,"word_count":309},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\engram\\engram_store_config.h":{"size":552,"mtime_ns":1781678985164755800,"word_count":76},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\etcd_helper.h":{"size":12275,"mtime_ns":1788421144230337600,"word_count":1472},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\eviction_ops.h":{"size":45,"mtime_ns":1788421144377666800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\eviction_strategy.h":{"size":3155,"mtime_ns":1781678985165755600,"word_count":278},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\feedback.h":{"size":48,"mtime_ns":1788421144492018000,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\file_interface.h":{"size":9736,"mtime_ns":1786963737744372300,"word_count":1001},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\file_storage.h":{"size":6514,"mtime_ns":1788421144230337600,"word_count":596},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\common\\redis\\redis_connection.h":{"size":1321,"mtime_ns":1781678985166263700,"word_count":112},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\ha_types.h":{"size":5851,"mtime_ns":1788421144231831000,"word_count":493},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\kv\\etcd_ha_kv_backend.h":{"size":545,"mtime_ns":1786963737745374500,"word_count":50},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\kv\\ha_kv_backend.h":{"size":1031,"mtime_ns":1786963737745374500,"word_count":100},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\leadership\\backends\\etcd\\etcd_leader_coordinator.h":{"size":2693,"mtime_ns":1786963737746760700,"word_count":177},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\leadership\\backends\\k8s\\k8s_leader_coordinator.h":{"size":2345,"mtime_ns":1781678985167265600,"word_count":155},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\leadership\\backends\\redis\\redis_leader_coordinator.h":{"size":3305,"mtime_ns":1781678985167265600,"word_count":210},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\leadership\\leader_coordinator.h":{"size":1284,"mtime_ns":1781678985167265600,"word_count":101},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\leadership\\leader_coordinator_factory.h":{"size":353,"mtime_ns":1781678985168265800,"word_count":29},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\leadership\\leader_label_reconciler.h":{"size":2997,"mtime_ns":1786963737746760700,"word_count":291},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\leadership\\master_service_supervisor.h":{"size":356,"mtime_ns":1781678985168265800,"word_count":33},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\master_metrics_reporter.h":{"size":2974,"mtime_ns":1786976645762816900,"word_count":351},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\oplog_applier.h":{"size":2888,"mtime_ns":1786976645764055800,"word_count":312},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\oplog_batch_codec.h":{"size":539,"mtime_ns":1786963737747919400,"word_count":41},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\oplog_batch_standby_reader.h":{"size":1226,"mtime_ns":1788421144231831000,"word_count":81},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\oplog_batch_storage.h":{"size":1301,"mtime_ns":1788421144231831000,"word_count":104},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\oplog_batch_types.h":{"size":2033,"mtime_ns":1788421144231831000,"word_count":168},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\oplog_test_failpoint.h":{"size":179,"mtime_ns":1786963737747919400,"word_count":20},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\oplog_types.h":{"size":1820,"mtime_ns":1786976645764055800,"word_count":150},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\oplog\\ordered_oplog_writer.h":{"size":2033,"mtime_ns":1786963737749323800,"word_count":135},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\catalog\\backends\\embedded\\embedded_snapshot_catalog_store.h":{"size":1058,"mtime_ns":1781678985170791500,"word_count":81},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\catalog\\backends\\redis\\redis_snapshot_catalog_store.h":{"size":1545,"mtime_ns":1781678985170791500,"word_count":110},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\catalog\\snapshot_catalog_store.h":{"size":5587,"mtime_ns":1786976645764055800,"word_count":475},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\catalog_backed_snapshot_provider.h":{"size":584,"mtime_ns":1781678985171793800,"word_count":58},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\master_snapshot_codec.h":{"size":5886,"mtime_ns":1786976645765517900,"word_count":583},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\object\\backends\\local\\local_file_snapshot_object_store.h":{"size":1937,"mtime_ns":1781678985171793800,"word_count":157},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\object\\backends\\s3\\s3_snapshot_object_store.h":{"size":1538,"mtime_ns":1781678985171793800,"word_count":123},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\object\\snapshot_object_store.h":{"size":4680,"mtime_ns":1781678985171793800,"word_count":532},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\snapshot_constants.h":{"size":916,"mtime_ns":1786976645765517900,"word_count":93},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\snapshot_logger.h":{"size":1226,"mtime_ns":1781678985171793800,"word_count":128},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\snapshot\\snapshot_provider.h":{"size":1788,"mtime_ns":1786976645765517900,"word_count":193},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha\\standby_controller.h":{"size":1641,"mtime_ns":1788421144233316700,"word_count":157},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ha_metric_manager.h":{"size":8826,"mtime_ns":1786963737750325300,"word_count":707},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\hf3fs\\hf3fs.h":{"size":3460,"mtime_ns":1781678985172795700,"word_count":342},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\hot_standby_service.h":{"size":12085,"mtime_ns":1788421144233316700,"word_count":1188},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\http_metadata_server.h":{"size":1693,"mtime_ns":1786963737751598000,"word_count":179},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\hybrid_metric.h":{"size":15710,"mtime_ns":1781678985173797700,"word_count":1131},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern.h":{"size":52,"mtime_ns":1788421144378944000,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\analyzer.h":{"size":591,"mtime_ns":1788421144378944000,"word_count":59},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\cfm_channel.h":{"size":1598,"mtime_ns":1788421144627563800,"word_count":147},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\cfm_client_impl.h":{"size":1121,"mtime_ns":1788421144493421600,"word_count":88},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\cfm_ingress.h":{"size":872,"mtime_ns":1788421144627563800,"word_count":67},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\cfm_protocol.h":{"size":923,"mtime_ns":1788421144493421600,"word_count":84},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\cfm_service.h":{"size":4393,"mtime_ns":1788426179828211900,"word_count":350},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\client.h":{"size":520,"mtime_ns":1788421144378944000,"word_count":56},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\collector.h":{"size":692,"mtime_ns":1788421144493421600,"word_count":70},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\collector_impl.h":{"size":3003,"mtime_ns":1788440556672660100,"word_count":229},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\degrading_policy_engine.h":{"size":1509,"mtime_ns":1788421144494781100,"word_count":113},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\feedback.h":{"size":1781,"mtime_ns":1788421144494781100,"word_count":144},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\io_pattern.h":{"size":1060,"mtime_ns":1788421144634371100,"word_count":56},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\kmeans_analyzer.h":{"size":938,"mtime_ns":1788440380997173000,"word_count":84},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\legacy_eviction_ops.h":{"size":635,"mtime_ns":1788421144494781100,"word_count":48},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\observability.h":{"size":1169,"mtime_ns":1788421144496090400,"word_count":84},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\ops.h":{"size":1622,"mtime_ns":1788421144496333600,"word_count":179},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\policy_engine.h":{"size":15132,"mtime_ns":1788421144496333600,"word_count":979},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\policy_strategies.h":{"size":2102,"mtime_ns":1788421144497337700,"word_count":155},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\registry.h":{"size":1850,"mtime_ns":1788421144380824800,"word_count":143},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\reporter.h":{"size":2065,"mtime_ns":1788421144634371100,"word_count":163},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\resilient_analyzer.h":{"size":1397,"mtime_ns":1788421144497337700,"word_count":118},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\resilient_cfm_channel.h":{"size":1256,"mtime_ns":1788421144635449700,"word_count":103},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\rpc_transport.h":{"size":6678,"mtime_ns":1788421144635449700,"word_count":559},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\runtime.h":{"size":5344,"mtime_ns":1788440557542777000,"word_count":394},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\sliding_window_analyzer.h":{"size":1837,"mtime_ns":1788421144636607800,"word_count":134},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\threshold_analyzer.h":{"size":1368,"mtime_ns":1788421144498337300,"word_count":108},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\tier_executor.h":{"size":1215,"mtime_ns":1788440868041497900,"word_count":86},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern\\types.h":{"size":6727,"mtime_ns":1788421144499337400,"word_count":516},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern_analyzer.h":{"size":94,"mtime_ns":1788421144499337400,"word_count":6},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern_collector.h":{"size":51,"mtime_ns":1788421144380824800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern_registry.h":{"size":50,"mtime_ns":1788421144382219800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\io_pattern_types.h":{"size":47,"mtime_ns":1788421144382219800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\k8s_lease_helper.h":{"size":1837,"mtime_ns":1786963737752099900,"word_count":139},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\kv_event\\key_util.h":{"size":992,"mtime_ns":1786963737752099900,"word_count":105},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\kv_event\\kv_event_config.h":{"size":1357,"mtime_ns":1786963737752099900,"word_count":149},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\kv_event\\kv_event_publisher.h":{"size":3873,"mtime_ns":1786963737753101300,"word_count":298},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\legacy_eviction_ops.h":{"size":59,"mtime_ns":1788421144499337400,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\local_hot_cache.h":{"size":11371,"mtime_ns":1786160474197952500,"word_count":1370},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_admin_service.h":{"size":4897,"mtime_ns":1786963737753101300,"word_count":241},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_client.h":{"size":39042,"mtime_ns":1788421144234318200,"word_count":3659},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_config.h":{"size":72889,"mtime_ns":1788439904229877000,"word_count":4717},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_metric_manager.h":{"size":33987,"mtime_ns":1786963737754101400,"word_count":2293},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_perf.h":{"size":508,"mtime_ns":1786976645768275400,"word_count":61},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_service.h":{"size":119181,"mtime_ns":1788421144637865000,"word_count":10005},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_snapshot_manager.h":{"size":4247,"mtime_ns":1786963737755103100,"word_count":298},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\master_snapshot_repository.h":{"size":4236,"mtime_ns":1786963737756103100,"word_count":425},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\memory_alloc.h":{"size":140,"mtime_ns":1781678985175819100,"word_count":12},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\metadata_store.h":{"size":7441,"mtime_ns":1788421144236318000,"word_count":755},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\mmap_arena.h":{"size":3516,"mtime_ns":1781678985175819100,"word_count":391},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\mutex.h":{"size":8930,"mtime_ns":1781678985175819100,"word_count":873},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\observability.h":{"size":53,"mtime_ns":1788421144500336600,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\offset_allocator\\offset_allocator.h":{"size":17218,"mtime_ns":1786963737756103100,"word_count":1553},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\partition\\kv_hash_map.h":{"size":603,"mtime_ns":1788421144236318000,"word_count":62},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\partition\\partition_router.h":{"size":1346,"mtime_ns":1788421144236318000,"word_count":105},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\pinned_buffer_pool.h":{"size":4319,"mtime_ns":1786963737757223600,"word_count":372},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\pinned_host_buffer.h":{"size":1376,"mtime_ns":1786963737757223600,"word_count":131},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\policy_engine.h":{"size":55,"mtime_ns":1788421144382219800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\policy_strategies.h":{"size":59,"mtime_ns":1788421144500336600,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\prefetch_ops.h":{"size":45,"mtime_ns":1788421144382219800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\pyclient.h":{"size":19997,"mtime_ns":1786976645769276700,"word_count":1655},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\random.h":{"size":2660,"mtime_ns":1786963737757223600,"word_count":231},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\real_client.h":{"size":41321,"mtime_ns":1788009798331033800,"word_count":3486},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\replica.h":{"size":25893,"mtime_ns":1786976645770732100,"word_count":2030},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\replica_selection.h":{"size":6874,"mtime_ns":1786963737759606600,"word_count":806},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\reporter.h":{"size":48,"mtime_ns":1788421144500336600,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\resilient_analyzer.h":{"size":58,"mtime_ns":1788421144501826800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\resilient_cfm_channel.h":{"size":61,"mtime_ns":1788421144501826800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\rpc_helper.h":{"size":4652,"mtime_ns":1786976645770732100,"word_count":462},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\rpc_service.h":{"size":19067,"mtime_ns":1788421144637865000,"word_count":1350},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\rpc_transport.h":{"size":53,"mtime_ns":1788421144501826800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\rpc_types.h":{"size":11159,"mtime_ns":1788421144237318100,"word_count":851},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\segment.h":{"size":22925,"mtime_ns":1788421144237318100,"word_count":1999},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\serialize\\serializer.h":{"size":4779,"mtime_ns":1786963737760606500,"word_count":359},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\serializer.h":{"size":12547,"mtime_ns":1781678985178825800,"word_count":1559},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\shm_helper.h":{"size":1423,"mtime_ns":1786963737761968200,"word_count":159},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\sliding_window_analyzer.h":{"size":63,"mtime_ns":1788421144501826800,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\spdk\\spdk_wrapper.h":{"size":3550,"mtime_ns":1781678985179829300,"word_count":258},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\ssd_register_client.h":{"size":1232,"mtime_ns":1781678985179829300,"word_count":115},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\standby_state_machine.h":{"size":11769,"mtime_ns":1781678985179829300,"word_count":971},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\storage\\distributed\\distributed_storage_backend.h":{"size":2253,"mtime_ns":1786963737761968200,"word_count":164},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\storage\\distributed\\fs_adapter.h":{"size":3420,"mtime_ns":1781678985180832500,"word_count":334},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\storage\\distributed\\hf3fs_adapter.h":{"size":1868,"mtime_ns":1781678985180832500,"word_count":141},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\storage_backend.h":{"size":72796,"mtime_ns":1788421144238318300,"word_count":7544},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\store_c.h":{"size":5260,"mtime_ns":1781678985180832500,"word_count":410},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\store_rpc_client_io_context.h":{"size":568,"mtime_ns":1786963737762969600,"word_count":43},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\task_manager.h":{"size":7883,"mtime_ns":1781678985181835400,"word_count":627},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\tenant_id.h":{"size":2667,"mtime_ns":1786963737762969600,"word_count":241},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\tenant_quota.h":{"size":4201,"mtime_ns":1786963737762969600,"word_count":323},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\tenant_quota_policy_store.h":{"size":1984,"mtime_ns":1786963737764195500,"word_count":142},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\tenant_quota_sharded.h":{"size":2772,"mtime_ns":1786963737764195500,"word_count":200},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\tenant_quota_sharded_impl.h":{"size":9132,"mtime_ns":1786963737764195500,"word_count":581},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\thread_pool.h":{"size":2271,"mtime_ns":1781678985181835400,"word_count":264},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\threshold_analyzer.h":{"size":60,"mtime_ns":1788421144502828700,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\tier_executor.h":{"size":53,"mtime_ns":1788421144502828700,"word_count":4},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\transfer_task.h":{"size":21678,"mtime_ns":1786976645774403300,"word_count":1939},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\types.h":{"size":21758,"mtime_ns":1788421144238318300,"word_count":2444},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\uds_transport.h":{"size":2240,"mtime_ns":1786963737765196800,"word_count":203},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\utils.h":{"size":17970,"mtime_ns":1786976645774403300,"word_count":2213},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\utils\\base64.h":{"size":4714,"mtime_ns":1781678985182835000,"word_count":629},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\utils\\file_util.h":{"size":1263,"mtime_ns":1781678985182835000,"word_count":145},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\utils\\s3_helper.h":{"size":2664,"mtime_ns":1781678985183344100,"word_count":204},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\utils\\scoped_vlog_timer.h":{"size":4435,"mtime_ns":1781678985183344100,"word_count":390},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\utils\\type_util.h":{"size":1255,"mtime_ns":1786976645775882200,"word_count":166},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\utils\\zstd_util.h":{"size":8044,"mtime_ns":1781678985183344100,"word_count":564},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_allocation_strategy.h":{"size":1567,"mtime_ns":1788421144239424500,"word_count":111},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_client.h":{"size":3675,"mtime_ns":1788421144239424500,"word_count":271},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_config.h":{"size":1151,"mtime_ns":1788421144239424500,"word_count":90},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_control_plane.h":{"size":3011,"mtime_ns":1788421144239424500,"word_count":255},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_master_manager.h":{"size":3572,"mtime_ns":1788421144239424500,"word_count":267},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_metadata.h":{"size":3019,"mtime_ns":1788421144240919400,"word_count":231},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_metadata_store.h":{"size":2176,"mtime_ns":1788421144240919400,"word_count":192},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_metrics.h":{"size":2270,"mtime_ns":1788421144240919400,"word_count":173},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\include\\vchunk_transfer_engine.h":{"size":1423,"mtime_ns":1788421144240919400,"word_count":92},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\CMakeLists.txt":{"size":2712,"mtime_ns":1786963737766493000,"word_count":189},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\README.md":{"size":4434,"mtime_ns":1786963737767565400,"word_count":494},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\build.rs":{"size":15856,"mtime_ns":1786963737767565400,"word_count":1197},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\examples\\basic_usage.rs":{"size":5645,"mtime_ns":1786963737767565400,"word_count":643},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\examples\\generate_dlopen_bindings.rs":{"size":2078,"mtime_ns":1786963737768775400,"word_count":216},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\examples\\store_benchmark.rs":{"size":4657,"mtime_ns":1781678985185350300,"word_count":385},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\src\\error.rs":{"size":2793,"mtime_ns":1786963737768775400,"word_count":370},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\src\\ffi_dlopen.rs":{"size":9934,"mtime_ns":1786963737768775400,"word_count":1053},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\src\\generated\\ffi_dlopen_bindings.rs":{"size":15653,"mtime_ns":1786963737768775400,"word_count":1065},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\src\\lib.rs":{"size":2322,"mtime_ns":1786963737770142200,"word_count":307},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\src\\store.rs":{"size":27624,"mtime_ns":1786976645775882200,"word_count":2578},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\rust\\tests\\minimal_smoke.rs":{"size":2619,"mtime_ns":1781678985186368700,"word_count":230},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\CMakeLists.txt":{"size":19816,"mtime_ns":1788421144639234500,"word_count":1256},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\aligned_client_buffer.cpp":{"size":6426,"mtime_ns":1786963737771143400,"word_count":544},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\allocator.cpp":{"size":24154,"mtime_ns":1786976645776883700,"word_count":1802},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cachelib_memory_allocator\\AllocationClass.cpp":{"size":24528,"mtime_ns":1786963737771143400,"word_count":2404},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cachelib_memory_allocator\\CMakeLists.txt":{"size":479,"mtime_ns":1781678985187370600,"word_count":17},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cachelib_memory_allocator\\MemoryAllocator.cpp":{"size":10858,"mtime_ns":1786963737772491500,"word_count":1062},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cachelib_memory_allocator\\MemoryPool.cpp":{"size":13939,"mtime_ns":1786963737772992800,"word_count":1481},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cachelib_memory_allocator\\MemoryPoolManager.cpp":{"size":4726,"mtime_ns":1781678985188372500,"word_count":514},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cachelib_memory_allocator\\Slab.cpp":{"size":932,"mtime_ns":1781678985188372500,"word_count":127},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cachelib_memory_allocator\\SlabAllocator.cpp":{"size":5166,"mtime_ns":1781678985188372500,"word_count":520},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\client_buffer.cpp":{"size":6712,"mtime_ns":1786963737772992800,"word_count":527},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\client_metric.cpp":{"size":8798,"mtime_ns":1786976645776883700,"word_count":718},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\client_service.cpp":{"size":209778,"mtime_ns":1788421144242920700,"word_count":16465},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cvm\\cvm_controller.cpp":{"size":21148,"mtime_ns":1788421144242920700,"word_count":1545},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cvm\\cvm_http_server.cpp":{"size":4097,"mtime_ns":1788421144242920700,"word_count":303},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cvm\\etcd_view_store.cpp":{"size":23996,"mtime_ns":1788421144244220900,"word_count":1874},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cvm\\inter_master_rpc.cpp":{"size":16952,"mtime_ns":1788421144244220900,"word_count":1252},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cvm\\slot_migrator.cpp":{"size":11634,"mtime_ns":1788421144244220900,"word_count":849},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\cvm\\slot_owner_heartbeat.cpp":{"size":3042,"mtime_ns":1788421144245222700,"word_count":255},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\device\\accelerator_device.cpp":{"size":696,"mtime_ns":1786963737772992800,"word_count":77},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\device\\accelerator_registry.cpp":{"size":3672,"mtime_ns":1786963737772992800,"word_count":262},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\device\\ascend_accelerator_device.cpp":{"size":3610,"mtime_ns":1786963737772992800,"word_count":287},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\device\\cuda_like_accelerator_device.cpp":{"size":3591,"mtime_ns":1786963737772992800,"word_count":274},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\device\\hip_accelerator_device.cpp":{"size":2709,"mtime_ns":1786963737772992800,"word_count":229},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\device\\runtime_accelerator.cpp":{"size":1851,"mtime_ns":1786963737772992800,"word_count":145},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\device\\sunrise_accelerator_device.cpp":{"size":5723,"mtime_ns":1786963737772992800,"word_count":384},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\dummy_client.cpp":{"size":56320,"mtime_ns":1786963737772992800,"word_count":4231},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\engram\\engram_store.cpp":{"size":12078,"mtime_ns":1781678985190376400,"word_count":1037},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\etcd_helper.cpp":{"size":29363,"mtime_ns":1788421144245222700,"word_count":2212},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\file_storage.cpp":{"size":62632,"mtime_ns":1788421144246222200,"word_count":4961},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\common\\redis\\redis_connection.cpp":{"size":7247,"mtime_ns":1781678985200926800,"word_count":575},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\kv\\etcd_ha_kv_backend.cpp":{"size":2984,"mtime_ns":1786963737772992800,"word_count":225},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\leadership\\backends\\etcd\\etcd_leader_coordinator.cpp":{"size":25917,"mtime_ns":1786976645779883500,"word_count":2261},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\leadership\\backends\\k8s\\k8s_leader_coordinator.cpp":{"size":15668,"mtime_ns":1786976645779883500,"word_count":1079},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\leadership\\backends\\redis\\redis_leader_coordinator.cpp":{"size":31018,"mtime_ns":1786976645780883600,"word_count":2119},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\leadership\\leader_coordinator_factory.cpp":{"size":2006,"mtime_ns":1781678985202984100,"word_count":114},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\leadership\\master_service_supervisor.cpp":{"size":24871,"mtime_ns":1788421144246222200,"word_count":1799},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\master_metrics_reporter.cpp":{"size":7444,"mtime_ns":1786976645780883600,"word_count":670},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\oplog_applier.cpp":{"size":10421,"mtime_ns":1788008058567345700,"word_count":838},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\oplog_batch_codec.cpp":{"size":9183,"mtime_ns":1786963737772992800,"word_count":748},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\oplog_batch_standby_reader.cpp":{"size":5475,"mtime_ns":1788421144247222200,"word_count":360},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\oplog_batch_storage.cpp":{"size":12985,"mtime_ns":1788421144247222200,"word_count":1154},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\oplog_batch_types.cpp":{"size":5538,"mtime_ns":1788421144247222200,"word_count":509},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\oplog_test_failpoint.cpp":{"size":3135,"mtime_ns":1786963737772992800,"word_count":326},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\oplog_types.cpp":{"size":1541,"mtime_ns":1786963737772992800,"word_count":130},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\oplog\\ordered_oplog_writer.cpp":{"size":14881,"mtime_ns":1786976645782157400,"word_count":796},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\snapshot\\catalog\\backends\\embedded\\embedded_snapshot_catalog_store.cpp":{"size":8687,"mtime_ns":1786976645783405900,"word_count":556},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\snapshot\\catalog\\backends\\redis\\redis_snapshot_catalog_store.cpp":{"size":12307,"mtime_ns":1786976645783405900,"word_count":835},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\snapshot\\catalog_backed_snapshot_provider.cpp":{"size":25268,"mtime_ns":1786976645784408300,"word_count":1871},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\snapshot\\master_snapshot_codec.cpp":{"size":6322,"mtime_ns":1786963737772992800,"word_count":489},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\snapshot\\object\\backends\\local\\local_file_snapshot_object_store.cpp":{"size":11665,"mtime_ns":1781678985207015300,"word_count":943},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\snapshot\\object\\backends\\s3\\s3_snapshot_object_store.cpp":{"size":2986,"mtime_ns":1781678985207015300,"word_count":204},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\snapshot\\object\\snapshot_object_store.cpp":{"size":1061,"mtime_ns":1781678985207015300,"word_count":72},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha\\standby_controller.cpp":{"size":13605,"mtime_ns":1788421144248222200,"word_count":816},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ha_metric_manager.cpp":{"size":17497,"mtime_ns":1786963737772992800,"word_count":971},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\hf3fs\\CMakeLists.txt":{"size":214,"mtime_ns":1781678985208017900,"word_count":8},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\hf3fs\\README.md":{"size":1245,"mtime_ns":1781678985208017900,"word_count":147},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\hf3fs\\hf3fs_file.cpp":{"size":11781,"mtime_ns":1781678985208017900,"word_count":1014},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\hf3fs\\hf3fs_resource_manager.cpp":{"size":2635,"mtime_ns":1781678985208017900,"word_count":212},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\hot_standby_service.cpp":{"size":36915,"mtime_ns":1788421144248222200,"word_count":2792},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\http_metadata_server.cpp":{"size":5754,"mtime_ns":1786963737772992800,"word_count":412},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\cfm_client_impl.cpp":{"size":1598,"mtime_ns":1788421144639234500,"word_count":107},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\cfm_ingress.cpp":{"size":2449,"mtime_ns":1788421144640236400,"word_count":207},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\cfm_protocol.cpp":{"size":17532,"mtime_ns":1788421144504174300,"word_count":1327},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\cfm_service.cpp":{"size":13096,"mtime_ns":1788441063931337600,"word_count":933},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\collector_impl.cpp":{"size":11109,"mtime_ns":1788421144640236400,"word_count":755},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\degrading_policy_engine.cpp":{"size":1992,"mtime_ns":1788421144505176000,"word_count":162},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\feedback.cpp":{"size":1962,"mtime_ns":1788421144505176000,"word_count":162},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\kmeans_analyzer.cpp":{"size":6580,"mtime_ns":1788421144505176000,"word_count":533},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\legacy_eviction_ops.cpp":{"size":1468,"mtime_ns":1788421144505176000,"word_count":108},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\observability.cpp":{"size":2154,"mtime_ns":1788421144506176100,"word_count":128},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\policy_strategies.cpp":{"size":9558,"mtime_ns":1788421144641310700,"word_count":705},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\reporter.cpp":{"size":4519,"mtime_ns":1788421144641310700,"word_count":322},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\resilient_analyzer.cpp":{"size":1904,"mtime_ns":1788421144506176100,"word_count":164},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\resilient_cfm_channel.cpp":{"size":2633,"mtime_ns":1788421144641310700,"word_count":205},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\rpc_transport.cpp":{"size":11308,"mtime_ns":1788421144641310700,"word_count":784},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\runtime.cpp":{"size":15196,"mtime_ns":1788440558445802800,"word_count":980},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\sliding_window_analyzer.cpp":{"size":5112,"mtime_ns":1788421144642804600,"word_count":387},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\threshold_analyzer.cpp":{"size":5870,"mtime_ns":1788421144508611900,"word_count":435},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\io_pattern\\tier_executor.cpp":{"size":1423,"mtime_ns":1788421144508611900,"word_count":101},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\k8s_lease_helper.cpp":{"size":9412,"mtime_ns":1786963737772992800,"word_count":701},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\kv_event\\kv_event_publisher.cpp":{"size":12755,"mtime_ns":1786963737785618600,"word_count":875},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\local_hot_cache.cpp":{"size":19884,"mtime_ns":1786963737785618600,"word_count":1857},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\master.cpp":{"size":88503,"mtime_ns":1788441619317981400,"word_count":5518},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\master_admin_service.cpp":{"size":50115,"mtime_ns":1786963737785618600,"word_count":3061},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\master_client.cpp":{"size":88040,"mtime_ns":1788421144249638700,"word_count":6012},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\master_metric_manager.cpp":{"size":111439,"mtime_ns":1786963737785618600,"word_count":6128},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\master_service.cpp":{"size":575007,"mtime_ns":1788441910616988000,"word_count":38297},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\master_snapshot_manager.cpp":{"size":25216,"mtime_ns":1786963737788969700,"word_count":1810},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\master_snapshot_repository.cpp":{"size":13342,"mtime_ns":1786976645788931900,"word_count":883},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\memory_alloc.cpp":{"size":383,"mtime_ns":1781678985212026200,"word_count":31},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\metadata_store.cpp":{"size":1792,"mtime_ns":1786963737788969700,"word_count":108},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\mmap_arena.cpp":{"size":11119,"mtime_ns":1781678985212026200,"word_count":1152},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\offset_allocator.cpp":{"size":26141,"mtime_ns":1786963737788969700,"word_count":2404},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\partition\\partition_router.cpp":{"size":2990,"mtime_ns":1788421144252639700,"word_count":263},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\posix_file.cpp":{"size":4326,"mtime_ns":1786963737788969700,"word_count":398},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\real_client.cpp":{"size":321095,"mtime_ns":1788421144253639800,"word_count":24409},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\real_client_main.cpp":{"size":7930,"mtime_ns":1786976645789945900,"word_count":404},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\registered_pinned_memory.cpp":{"size":8695,"mtime_ns":1786976645791397500,"word_count":776},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\registered_pinned_memory.h":{"size":2274,"mtime_ns":1786963737788969700,"word_count":172},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\rpc_service.cpp":{"size":97438,"mtime_ns":1788421144646385700,"word_count":5584},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\segment.cpp":{"size":69573,"mtime_ns":1788008058575348000,"word_count":4734},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\serialize\\serializer.cpp":{"size":42113,"mtime_ns":1786976645792398900,"word_count":2631},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\shm_helper.cpp":{"size":5767,"mtime_ns":1786963737788969700,"word_count":511},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\spdk\\CMakeLists.txt":{"size":143,"mtime_ns":1781678985215033200,"word_count":6},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\spdk\\spdk_wrapper.cpp":{"size":16105,"mtime_ns":1781678985215033200,"word_count":1297},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\ssd_register_client.cpp":{"size":5600,"mtime_ns":1781678985215033200,"word_count":527},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\standby_state_machine.cpp":{"size":11520,"mtime_ns":1781678985215033200,"word_count":728},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\storage\\distributed\\distributed_storage_backend.cpp":{"size":12704,"mtime_ns":1786976645793399200,"word_count":990},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\storage\\distributed\\hf3fs_adapter.cpp":{"size":12277,"mtime_ns":1781678985216035500,"word_count":1161},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\storage_backend.cpp":{"size":257419,"mtime_ns":1788421144256353600,"word_count":20971},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\store_c.cpp":{"size":11938,"mtime_ns":1781678985217035500,"word_count":1147},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\store_c_shared.cpp":{"size":940,"mtime_ns":1786963737788969700,"word_count":140},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\task_manager.cpp":{"size":16374,"mtime_ns":1781678985217541100,"word_count":1137},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\tenant_quota.cpp":{"size":14767,"mtime_ns":1786963737788969700,"word_count":1115},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\tenant_quota_policy_store.cpp":{"size":15822,"mtime_ns":1786963737788969700,"word_count":1447},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\thread_pool.cpp":{"size":1143,"mtime_ns":1781678985217541100,"word_count":87},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\transfer_task.cpp":{"size":58102,"mtime_ns":1786976645795398900,"word_count":4785},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\types.cpp":{"size":6149,"mtime_ns":1788421144257354700,"word_count":300},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\uds_transport.cpp":{"size":10398,"mtime_ns":1786963737788969700,"word_count":1012},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\uring_file.cpp":{"size":30791,"mtime_ns":1786976645795398900,"word_count":2696},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\utils.cpp":{"size":28970,"mtime_ns":1786976645796544700,"word_count":2897},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\utils\\file_util.cpp":{"size":4547,"mtime_ns":1781678985218544000,"word_count":387},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\utils\\s3_helper.cpp":{"size":31051,"mtime_ns":1786963737788969700,"word_count":2370},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\utils\\type_util.cpp":{"size":1326,"mtime_ns":1786976645796544700,"word_count":159},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_allocation_strategy.cpp":{"size":5666,"mtime_ns":1788421144257354700,"word_count":416},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_client.cpp":{"size":12880,"mtime_ns":1788421144257354700,"word_count":885},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_config.cpp":{"size":866,"mtime_ns":1788421144257354700,"word_count":89},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_control_plane.cpp":{"size":4311,"mtime_ns":1788421144258354800,"word_count":274},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_master_manager.cpp":{"size":14332,"mtime_ns":1788421144258354800,"word_count":1095},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_metadata.cpp":{"size":6723,"mtime_ns":1788421144258354800,"word_count":491},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_metadata_store.cpp":{"size":7636,"mtime_ns":1788421144259354700,"word_count":586},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_metrics.cpp":{"size":2383,"mtime_ns":1788421144259354700,"word_count":175},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\src\\vchunk_transfer_engine.cpp":{"size":5609,"mtime_ns":1788421144259354700,"word_count":401},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\CMakeLists.txt":{"size":12942,"mtime_ns":1788421144307204800,"word_count":432},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\allocation_strategy_test.cpp":{"size":44013,"mtime_ns":1786963737788969700,"word_count":3531},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\batch_remove_test.cpp":{"size":13444,"mtime_ns":1781678985220549600,"word_count":1098},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\buffer_allocator_test.cpp":{"size":24161,"mtime_ns":1786976645797603000,"word_count":1794},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\client_buffer_test.cpp":{"size":15733,"mtime_ns":1786963737788969700,"word_count":1425},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\client_integration_test.cpp":{"size":82614,"mtime_ns":1788421144260421400,"word_count":6516},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\client_local_hot_cache_test.cpp":{"size":51492,"mtime_ns":1786963737802609400,"word_count":4775},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\client_metrics_test.cpp":{"size":21394,"mtime_ns":1786963737802609400,"word_count":1354},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\client_storage_backend_test.cpp":{"size":1748,"mtime_ns":1786963737802609400,"word_count":129},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\client_tcp_local_memcpy_test.cpp":{"size":18159,"mtime_ns":1781678985221553100,"word_count":1172},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\cxl_client_integration_test.cpp":{"size":19315,"mtime_ns":1781678985222557200,"word_count":1433},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\deadline_scheduler_test.cpp":{"size":7378,"mtime_ns":1786963737802609400,"word_count":501},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\distributed_object_store_provider.py":{"size":1083,"mtime_ns":1781678985222557200,"word_count":83},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\dummy_client_get_buffer_test.cpp":{"size":26965,"mtime_ns":1786963737802609400,"word_count":2357},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\CMakeLists.txt":{"size":2928,"mtime_ns":1788421144261423100,"word_count":148},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\chaos_rand_test.cpp":{"size":14954,"mtime_ns":1781678985223561200,"word_count":1385},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\chaos_test.cpp":{"size":13407,"mtime_ns":1781678985224564800,"word_count":1203},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\chaosctl.cpp":{"size":9795,"mtime_ns":1781678985224564800,"word_count":874},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\client_ctl_cases\\case1.txt":{"size":211,"mtime_ns":1781678985224564800,"word_count":43},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\client_ctl_cases\\case2.txt":{"size":617,"mtime_ns":1781678985224564800,"word_count":87},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\client_runner.cpp":{"size":7008,"mtime_ns":1781678985225567500,"word_count":702},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\client_wrapper.cpp":{"size":12085,"mtime_ns":1786963737804326200,"word_count":962},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\client_wrapper.h":{"size":4287,"mtime_ns":1786963737804326200,"word_count":449},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\clientctl.cpp":{"size":9623,"mtime_ns":1786963737804326200,"word_count":913},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\e2e_rand_test.cpp":{"size":6986,"mtime_ns":1781678985226570000,"word_count":648},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\e2e_utils.h":{"size":2801,"mtime_ns":1781678985226570000,"word_count":225},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\gc_e2e_test.cpp":{"size":22595,"mtime_ns":1788421144261423100,"word_count":2181},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\oplog_batch_e2e_test.cpp":{"size":3002,"mtime_ns":1786963737804326200,"word_count":201},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\oplog_fault_ctl.sh":{"size":3224,"mtime_ns":1786963737804326200,"word_count":394},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\oplog_fault_ctl_test.sh":{"size":1949,"mtime_ns":1786963737804326200,"word_count":209},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\oplog_ha_client.cpp":{"size":10776,"mtime_ns":1786963737804326200,"word_count":1031},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\process_handler.cpp":{"size":18424,"mtime_ns":1786963737804326200,"word_count":1758},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\process_handler.h":{"size":6583,"mtime_ns":1786963737804326200,"word_count":714},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\readme.md":{"size":7255,"mtime_ns":1781678985227573000,"word_count":927},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\run_nof_heartbeat_tcp_e2e.sh":{"size":8942,"mtime_ns":1781678985227573000,"word_count":812},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\run_oplog_batch_cluster.sh":{"size":71362,"mtime_ns":1786963737804326200,"word_count":6265},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\run_oplog_batch_cluster_test.sh":{"size":5805,"mtime_ns":1786963737804326200,"word_count":553},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\storage_backend_e2e_test.cpp":{"size":14871,"mtime_ns":1781678985227573000,"word_count":1297},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\e2e\\store_client_e2e.py":{"size":3713,"mtime_ns":1781678985227573000,"word_count":291},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\eviction_strategy_test.cpp":{"size":4268,"mtime_ns":1781678985228575500,"word_count":340},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\file_storage_promotion_test.cpp":{"size":17038,"mtime_ns":1781678985228575500,"word_count":1419},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\file_storage_test.cpp":{"size":32959,"mtime_ns":1786976645798653700,"word_count":1951},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\file_util_test.cpp":{"size":4144,"mtime_ns":1781678985228575500,"word_count":330},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\common\\redis\\redis_test_utils.h":{"size":1443,"mtime_ns":1781678985229577800,"word_count":105},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\leadership\\backends\\k8s\\high_availability_k8s_test.cpp":{"size":12997,"mtime_ns":1781678985229577800,"word_count":854},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\leadership\\backends\\redis\\high_availability_redis_test.cpp":{"size":8369,"mtime_ns":1781678985230577500,"word_count":428},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\leadership\\etcd_leader_hang_e2e.sh":{"size":5061,"mtime_ns":1786963737804326200,"word_count":432},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\leadership\\ha_backend_availability_test.cpp":{"size":1974,"mtime_ns":1786963737804326200,"word_count":94},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\leadership\\high_availability_test.cpp":{"size":32969,"mtime_ns":1786963737804326200,"word_count":2154},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\leadership\\high_availability_test_fixture.h":{"size":289,"mtime_ns":1781678985231083700,"word_count":32},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\leadership\\leader_label_reconciler_test.cpp":{"size":5653,"mtime_ns":1786963737804326200,"word_count":373},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\master_service_ha_test.cpp":{"size":169122,"mtime_ns":1788421144262422900,"word_count":8402},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\mock_metadata_store.h":{"size":3762,"mtime_ns":1786976645799655300,"word_count":328},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\mock_snapshot_provider.h":{"size":1552,"mtime_ns":1786963737804326200,"word_count":122},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\oplog_applier_test.cpp":{"size":21124,"mtime_ns":1788008058577713700,"word_count":1393},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\oplog_batch_auditor_test.cpp":{"size":6698,"mtime_ns":1786963737804326200,"word_count":474},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\oplog_batch_codec_test.cpp":{"size":11961,"mtime_ns":1786963737804326200,"word_count":697},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\oplog_batch_standby_reader_test.cpp":{"size":19562,"mtime_ns":1786963737804326200,"word_count":1081},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\oplog_batch_storage_test.cpp":{"size":19583,"mtime_ns":1786976645800729500,"word_count":1099},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\oplog_test_failpoint_test.cpp":{"size":2343,"mtime_ns":1786963737804326200,"word_count":160},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\oplog_types_test.cpp":{"size":731,"mtime_ns":1786963737804326200,"word_count":46},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\oplog\\ordered_oplog_writer_test.cpp":{"size":34474,"mtime_ns":1786976645800729500,"word_count":1962},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\catalog\\backends\\embedded\\embedded_snapshot_catalog_store_test.cpp":{"size":10555,"mtime_ns":1781678985235623400,"word_count":535},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\catalog\\backends\\redis\\redis_snapshot_catalog_store_test.cpp":{"size":8911,"mtime_ns":1781678985235623400,"word_count":494},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\catalog_backed_snapshot_provider_test.cpp":{"size":10540,"mtime_ns":1786963737804326200,"word_count":669},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\master_service_promotion_test_for_snapshot.cpp":{"size":8793,"mtime_ns":1786963737804326200,"word_count":752},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\master_service_ssd_test_for_snapshot.cpp":{"size":20221,"mtime_ns":1786963737804326200,"word_count":1420},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\master_service_test_for_snapshot.cpp":{"size":180575,"mtime_ns":1786976645801828600,"word_count":13831},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\master_service_test_for_snapshot_base.h":{"size":35000,"mtime_ns":1786976645802828400,"word_count":2922},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\master_snapshot_codec_test.cpp":{"size":8747,"mtime_ns":1786976645802828400,"word_count":707},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\object\\backends\\local\\local_file_snapshot_object_store_test.cpp":{"size":5200,"mtime_ns":1781678985238631600,"word_count":338},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\snapshot_child_process_test.cpp":{"size":57516,"mtime_ns":1786976645802828400,"word_count":3627},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\snapshot\\snapshot_test_utils.h":{"size":14692,"mtime_ns":1786963737804326200,"word_count":1007},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\standby\\ha_metric_manager_test.cpp":{"size":8322,"mtime_ns":1786963737818801700,"word_count":462},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\standby\\hot_standby_service_test.cpp":{"size":35627,"mtime_ns":1788421144263423100,"word_count":2327},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\standby\\hot_standby_snapshot_bootstrap_test.cpp":{"size":9842,"mtime_ns":1788421144263423100,"word_count":520},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ha\\standby\\standby_state_machine_test.cpp":{"size":27128,"mtime_ns":1781678985240635800,"word_count":1396},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\health_check_test.cpp":{"size":8991,"mtime_ns":1781678985240635800,"word_count":668},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\host_port_fix_test.cpp":{"size":1619,"mtime_ns":1786963737819303600,"word_count":148},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\http_metadata_server_test.cpp":{"size":3730,"mtime_ns":1786963737820332700,"word_count":270},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\io_pattern_framework_test.cpp":{"size":71756,"mtime_ns":1788426265819429000,"word_count":4116},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ipv6_client_test.cpp":{"size":16051,"mtime_ns":1781678985240635800,"word_count":1489},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\kv_event_publisher_test.cpp":{"size":7605,"mtime_ns":1786963737820332700,"word_count":546},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\master_admin_server_test.cpp":{"size":48242,"mtime_ns":1786963737820332700,"word_count":2794},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\master_metrics_test.cpp":{"size":46882,"mtime_ns":1786963737820332700,"word_count":2756},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\master_service_config_test.cpp":{"size":2716,"mtime_ns":1788421144647877600,"word_count":116},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\master_service_ssd_test.cpp":{"size":39181,"mtime_ns":1786963737820332700,"word_count":2962},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\master_service_tenant_quota_test.cpp":{"size":36942,"mtime_ns":1788008058578788900,"word_count":1985},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\master_service_test.cpp":{"size":349866,"mtime_ns":1788421144264275800,"word_count":24221},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\mmap_arena_fallback_test.cpp":{"size":7309,"mtime_ns":1786963737820332700,"word_count":624},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\mmap_arena_test.cpp":{"size":25793,"mtime_ns":1786963737820332700,"word_count":2726},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\mutex_test.cpp":{"size":7305,"mtime_ns":1781678985243641600,"word_count":595},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\nof_heartbeat_test.cpp":{"size":10163,"mtime_ns":1781678985243641600,"word_count":522},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\non_ha_reconnect_test.cpp":{"size":6743,"mtime_ns":1781678985243641600,"word_count":464},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\object_data_type_test.cpp":{"size":5636,"mtime_ns":1786963737820332700,"word_count":395},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\offload_on_evict_test.cpp":{"size":17359,"mtime_ns":1786963737820332700,"word_count":1501},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\offset_allocator_test.cpp":{"size":66004,"mtime_ns":1786963737820332700,"word_count":5833},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\partition_router_test.cpp":{"size":3066,"mtime_ns":1788421144264275800,"word_count":170},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\posix_file_test.cpp":{"size":5847,"mtime_ns":1781678985244643700,"word_count":529},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\promotion_on_hit_test.cpp":{"size":119415,"mtime_ns":1788421144264275800,"word_count":10724},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\pybind_client_test.cpp":{"size":76773,"mtime_ns":1786963737820332700,"word_count":5711},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\registered_pinned_memory_test.cpp":{"size":3692,"mtime_ns":1786963737820332700,"word_count":299},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\replica_selection_test.cpp":{"size":13142,"mtime_ns":1786963737820332700,"word_count":1029},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\rpc_timeout_test.cpp":{"size":4399,"mtime_ns":1786963737820332700,"word_count":471},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\runtime_accelerator_test.cpp":{"size":3894,"mtime_ns":1786963737820332700,"word_count":294},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\scripts\\setup_dev.sh":{"size":569,"mtime_ns":1781678985245645900,"word_count":77},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\segment_test.cpp":{"size":30651,"mtime_ns":1788008058579789400,"word_count":2197},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\serializer_test.cpp":{"size":7596,"mtime_ns":1786963737820332700,"word_count":549},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\slot_hash_test.cpp":{"size":5527,"mtime_ns":1788421144264275800,"word_count":474},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\ssd_metrics_test.cpp":{"size":16705,"mtime_ns":1781678985246648300,"word_count":1247},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\storage_backend_test.cpp":{"size":206575,"mtime_ns":1788421144264275800,"word_count":14285},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\stress_cluster_benchmark.py":{"size":27143,"mtime_ns":1781678985247650000,"word_count":2199},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\stress_workload_test.cpp":{"size":15129,"mtime_ns":1781678985247650000,"word_count":1413},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\task_executor_test.cpp":{"size":17776,"mtime_ns":1781678985247650000,"word_count":1297},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\task_integration_test.cpp":{"size":57381,"mtime_ns":1781678985248651700,"word_count":4871},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\task_manager_test.cpp":{"size":17356,"mtime_ns":1781678985248651700,"word_count":1060},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\tenant_id_test.cpp":{"size":2200,"mtime_ns":1786963737820332700,"word_count":129},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\tenant_quota_test.cpp":{"size":24636,"mtime_ns":1786963737820332700,"word_count":1464},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\test_server_helpers.h":{"size":10528,"mtime_ns":1786963737820332700,"word_count":667},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\thread_pool_test.cpp":{"size":3903,"mtime_ns":1781678985248651700,"word_count":324},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\transfer_task_test.cpp":{"size":7130,"mtime_ns":1786976645807825700,"word_count":589},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\uds_transport_test.cpp":{"size":8556,"mtime_ns":1786963737820332700,"word_count":591},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\utils\\common.h":{"size":2599,"mtime_ns":1781678985249670600,"word_count":185},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\utils_test.cpp":{"size":9467,"mtime_ns":1786976645807825700,"word_count":838},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_allocation_strategy_test.cpp":{"size":4432,"mtime_ns":1788421144264275800,"word_count":295},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_client_test.cpp":{"size":16169,"mtime_ns":1788421144264275800,"word_count":1046},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_config_test.cpp":{"size":2160,"mtime_ns":1788421144264275800,"word_count":134},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_master_manager_test.cpp":{"size":8285,"mtime_ns":1788421144264275800,"word_count":488},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_master_service_test.cpp":{"size":6013,"mtime_ns":1788421144264275800,"word_count":314},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_metadata_store_test.cpp":{"size":10982,"mtime_ns":1788421144264275800,"word_count":652},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_metadata_test.cpp":{"size":5231,"mtime_ns":1788421144264275800,"word_count":290},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_test_allocator.h":{"size":1843,"mtime_ns":1788421144264275800,"word_count":156},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\vchunk_transfer_engine_test.cpp":{"size":3477,"mtime_ns":1788421144264275800,"word_count":264},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tests\\zstd_util_test.cpp":{"size":3045,"mtime_ns":1781678985249670600,"word_count":238},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tools\\CMakeLists.txt":{"size":384,"mtime_ns":1786963737820332700,"word_count":10},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tools\\oplog_batch_auditor.cpp":{"size":9284,"mtime_ns":1786963737820332700,"word_count":765},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tools\\oplog_batch_auditor.h":{"size":1369,"mtime_ns":1786963737820332700,"word_count":85},"C:\\workspace\\code\\opensource\\Mooncake\\mooncake-store\\tools\\oplog_batch_inspector.cpp":{"size":8158,"mtime_ns":1786963737820332700,"word_count":731}} \ No newline at end of file diff --git a/mooncake-store/include/io_pattern/cfm_service.h b/mooncake-store/include/io_pattern/cfm_service.h index e0a00ebc5d..62d1764f0e 100644 --- a/mooncake-store/include/io_pattern/cfm_service.h +++ b/mooncake-store/include/io_pattern/cfm_service.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -43,6 +44,9 @@ class CfmService final { // node is a stable SubMaster identity, so policy generation for one slot // owner must never observe keys reported by another SubMaster. IoPatternSnapshot SnapshotForNode(std::string_view node_id) const; + IoPatternObservabilitySnapshot ObservabilityForNode( + std::string_view node_id, double window_seconds = 0.0) const; + bool WaitForPolicyIdle(std::chrono::milliseconds timeout); private: struct NodeRuntime { @@ -77,7 +81,9 @@ class CfmService final { std::mutex producer_mutex_; std::condition_variable producer_cv_; std::deque> pending_metric_batches_; + size_t active_policy_productions_{0}; bool producer_stopping_{false}; + std::condition_variable producer_idle_cv_; std::thread producer_worker_; }; diff --git a/mooncake-store/include/io_pattern/collector_impl.h b/mooncake-store/include/io_pattern/collector_impl.h index fb2a43844d..f3de308dc9 100644 --- a/mooncake-store/include/io_pattern/collector_impl.h +++ b/mooncake-store/include/io_pattern/collector_impl.h @@ -25,9 +25,10 @@ class IoPatternCollectorImpl final : public IoPatternCollector { std::function now_ns; }; - explicit IoPatternCollectorImpl(Config config = {}, - std::shared_ptr reporter = - nullptr) + IoPatternCollectorImpl() + : IoPatternCollectorImpl(Config{}, nullptr) {} + explicit IoPatternCollectorImpl( + Config config, std::shared_ptr reporter = nullptr) : config_(config), reporter_(std::move(reporter)) {} void ReportInferenceMetrics(const InferenceMetrics& metrics) override; void RecordAccess(const std::string& key, @@ -40,6 +41,7 @@ class IoPatternCollectorImpl final : public IoPatternCollector { uint64_t dropped() const; bool degraded() const; bool FlushReports(); + void StopReports(); private: struct StorageMetricKey { diff --git a/mooncake-store/include/io_pattern/kmeans_analyzer.h b/mooncake-store/include/io_pattern/kmeans_analyzer.h index e8962e5346..73073b543f 100644 --- a/mooncake-store/include/io_pattern/kmeans_analyzer.h +++ b/mooncake-store/include/io_pattern/kmeans_analyzer.h @@ -13,7 +13,8 @@ class KMeansWorkloadAnalyzer final : public IoPatternAnalyzer { ThresholdAnalyzerConfig thresholds{}; }; - explicit KMeansWorkloadAnalyzer(Config config = {}) : config_(config) {} + KMeansWorkloadAnalyzer() = default; + explicit KMeansWorkloadAnalyzer(Config config) : config_(config) {} PatternResult Analyze(const IoPatternSnapshot& snapshot) const override; WorkloadType DetectWorkloadType( @@ -22,7 +23,7 @@ class KMeansWorkloadAnalyzer final : public IoPatternAnalyzer { const IoPatternSnapshot& snapshot) const override; private: - Config config_; + Config config_{}; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 7ec9fffc27..8c3baf03f4 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -48,13 +48,16 @@ class IoPatternRuntime final { LegacyFallback legacy_fallback{LegacyFallback::kLru}; }; - explicit IoPatternRuntime(Handlers handlers, Config config = {}); + explicit IoPatternRuntime(Handlers handlers); + IoPatternRuntime(Handlers handlers, Config config); ~IoPatternRuntime(); void ReportInferenceMetrics(const InferenceMetrics& metrics); void RecordAccess(const std::string& key, const AccessRecord& record); void RecordStorageMetric(const StorageMetric& metric); void MergeSnapshot(const IoPatternSnapshot& snapshot); + bool FlushReports(); + void StopReports(); PolicyExecutionStatus Execute( CacheTier eviction_tier, uint64_t eviction_bytes, diff --git a/mooncake-store/include/io_pattern/tier_executor.h b/mooncake-store/include/io_pattern/tier_executor.h index 3ac0265d7f..30a03bb6fd 100644 --- a/mooncake-store/include/io_pattern/tier_executor.h +++ b/mooncake-store/include/io_pattern/tier_executor.h @@ -5,6 +5,7 @@ #include #include "types.h" +#include "../types.h" namespace mooncake::io_pattern { diff --git a/mooncake-store/src/io_pattern/cfm_service.cpp b/mooncake-store/src/io_pattern/cfm_service.cpp index f4792bd894..1792c6b68c 100644 --- a/mooncake-store/src/io_pattern/cfm_service.cpp +++ b/mooncake-store/src/io_pattern/cfm_service.cpp @@ -74,6 +74,21 @@ IoPatternSnapshot CfmService::SnapshotForNode(std::string_view node_id) const { : IoPatternSnapshot{}; } +IoPatternObservabilitySnapshot CfmService::ObservabilityForNode( + std::string_view node_id, double window_seconds) const { + const auto node_runtime = FindNodeRuntime(node_id); + return node_runtime + ? node_runtime->runtime->ObservabilitySnapshot(window_seconds) + : IoPatternObservabilitySnapshot{}; +} + +bool CfmService::WaitForPolicyIdle(std::chrono::milliseconds timeout) { + std::unique_lock lock(producer_mutex_); + return producer_idle_cv_.wait_for(lock, timeout, [this] { + return pending_metric_batches_.empty() && active_policy_productions_ == 0; + }); +} + std::shared_ptr CfmService::GetOrCreateNodeRuntime( std::string_view node_id) { std::lock_guard lock(node_runtimes_mutex_); @@ -86,7 +101,8 @@ std::shared_ptr CfmService::GetOrCreateNodeRuntime( IoPatternRuntime::Handlers{ .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, - .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + IoPatternRuntime::Config{}); node_runtime->ingress = std::make_unique(node_runtime->runtime, codec_); node_runtimes_.emplace(id, node_runtime); @@ -190,6 +206,7 @@ void CfmService::PolicyProducerWorker() { if (producer_stopping_) return; pending = std::move(pending_metric_batches_.front()); pending_metric_batches_.pop_front(); + ++active_policy_productions_; } try { ProducePolicies(pending.first, pending.second); @@ -197,6 +214,11 @@ void CfmService::PolicyProducerWorker() { // Policy production is best effort and must never terminate the // RPC service. The next metric batch will trigger a fresh plan. } + { + std::lock_guard lock(producer_mutex_); + --active_policy_productions_; + } + producer_idle_cv_.notify_all(); } } diff --git a/mooncake-store/src/io_pattern/collector_impl.cpp b/mooncake-store/src/io_pattern/collector_impl.cpp index ff03f54ade..416ef811fa 100644 --- a/mooncake-store/src/io_pattern/collector_impl.cpp +++ b/mooncake-store/src/io_pattern/collector_impl.cpp @@ -266,4 +266,13 @@ bool IoPatternCollectorImpl::FlushReports() { return !reporter || reporter->Flush(); } +void IoPatternCollectorImpl::StopReports() { + std::shared_ptr reporter; + { + std::lock_guard lock(mutex_); + reporter = reporter_; + } + if (reporter) reporter->Stop(); +} + } // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index 1e4bc31d02..964a62ebb8 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -6,6 +6,9 @@ namespace mooncake::io_pattern { +IoPatternRuntime::IoPatternRuntime(Handlers handlers) + : IoPatternRuntime(std::move(handlers), Config{}) {} + IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) : config_(config), executor_(std::move(handlers.eviction), std::move(handlers.prefetch), @@ -134,6 +137,10 @@ void IoPatternRuntime::MergeSnapshot(const IoPatternSnapshot& snapshot) { } } +bool IoPatternRuntime::FlushReports() { return collector_->FlushReports(); } + +void IoPatternRuntime::StopReports() { collector_->StopReports(); } + PatternResult IoPatternRuntime::AnalyzeWithinBudget( const IoPatternSnapshot& snapshot, bool& degraded) { degraded = config_.max_analysis_keys != 0 && diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 1909aff1bc..4b54fb8597 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -165,7 +165,7 @@ DEFINE_string(io_pattern_cfm_endpoint, "", "requests without outbound reporting"); DEFINE_string(io_pattern_cfm_node_id, "", "Stable node id used for CFM policy polling; defaults to " - "the local CVM SubMaster master_id"); + "the local CVM SubMaster RPC endpoint"); DEFINE_string(io_pattern_cfm_auth_token, "", "Authentication token for CFM node report/poll RPCs"); DEFINE_string(io_pattern_cfm_producer_auth_token, "", @@ -1612,12 +1612,12 @@ int main(int argc, char* argv[]) { return 1; } if (master_config.io_pattern_cfm.node_id.empty()) { - // CVM policy routing is per SubMaster. master_id is stable across the - // CFM reporting, polling and ACK path; cluster_id would merge every - // SubMaster in the same CVM deployment into one CFM node. + // CVM registers each SubMaster with the same address used as its + // stable local_hostname. cluster_id would merge every SubMaster in + // the same CVM deployment into one CFM node. master_config.io_pattern_cfm.node_id = - master_config.master_id.empty() ? master_config.cluster_id - : master_config.master_id; + master_config.rpc_address + ":" + + std::to_string(master_config.rpc_port); } const char* value = std::getenv("MC_RPC_PROTOCOL"); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index e1207869c9..013c407661 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -1523,6 +1523,22 @@ ErrorCode MasterService::ImportSlotMetadata(uint16_t slot) { return ErrorCode::OK; } #else +ErrorCode MasterService::StartInterMasterRpc() { return ErrorCode::OK; } + +void MasterService::StopInterMasterRpc() {} + +tl::expected, ErrorCode> +MasterService::TryAllocateReplicasRemotely( + const std::string& /*key*/, const TenantId& /*tenant_id*/, + uint64_t /*value_length*/, size_t /*replica_num*/, + const std::vector& /*preferred_segments*/) { + return tl::make_unexpected(ErrorCode::NO_AVAILABLE_HANDLE); +} + +void MasterService::EnqueueRemoteFreeIfTracked( + const TenantId& /*tenant_id*/, const std::string& /*key*/, + QuotaEraseMode /*quota_mode*/) {} + ErrorCode MasterService::StartSlotOwnerHeartbeat() { return ErrorCode::OK; } ErrorCode MasterService::ExportSlotMetadata(uint16_t /*slot*/) { diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index be23ec9277..d524bcb2ed 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1255,6 +1255,7 @@ TEST(IoPatternFrameworkTest, CfmPolicyDoesNotCrossSubmasterKeySets) { ASSERT_TRUE(service.Send("submaster-a", "report_metric_batch", codec.EncodeMetricBatch(submaster_a), "node-secret")); + ASSERT_TRUE(service.WaitForPolicyIdle(std::chrono::milliseconds(500))); std::optional> delivery; for (size_t attempt = 0; attempt < 100 && !delivery; ++attempt) { @@ -1262,6 +1263,8 @@ TEST(IoPatternFrameworkTest, CfmPolicyDoesNotCrossSubmasterKeySets) { if (!delivery) std::this_thread::sleep_for(std::chrono::milliseconds(5)); } ASSERT_TRUE(delivery.has_value()); + EXPECT_GE(service.ObservabilityForNode("submaster-a").policy_decisions, + 1); const auto command = codec.DecodePolicy(delivery->second); ASSERT_TRUE(command.has_value()); const auto* eviction = std::get_if(&*command); From 1d89e5e2e9676b2de7229f429544a01582ae4a89 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 8 Sep 2026 09:32:30 +0800 Subject: [PATCH 09/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/io_pattern_design.md | 164 ++++--- .../benchmarks/cfm_client_bench.cpp | 204 ++++----- .../include/io_pattern/cfm_channel.h | 38 +- .../include/io_pattern/cfm_client_impl.h | 22 +- .../include/io_pattern/cfm_ownership_client.h | 77 ++++ .../include/io_pattern/cfm_protocol.h | 2 +- .../include/io_pattern/cfm_service.h | 109 ++--- mooncake-store/include/io_pattern/client.h | 7 +- .../include/io_pattern/io_pattern.h | 1 + .../io_pattern/resilient_cfm_channel.h | 4 +- .../include/io_pattern/rpc_transport.h | 89 +--- mooncake-store/include/io_pattern/runtime.h | 10 +- mooncake-store/include/master_config.h | 27 -- mooncake-store/include/master_service.h | 12 +- mooncake-store/src/CMakeLists.txt | 1 + .../src/io_pattern/cfm_client_impl.cpp | 27 +- .../src/io_pattern/cfm_ownership_client.cpp | 104 +++++ mooncake-store/src/io_pattern/cfm_service.cpp | 347 +-------------- .../src/io_pattern/resilient_cfm_channel.cpp | 27 +- .../src/io_pattern/rpc_transport.cpp | 171 +------- mooncake-store/src/master.cpp | 105 ----- mooncake-store/src/master_service.cpp | 86 +--- mooncake-store/src/rpc_service.cpp | 4 - .../tests/io_pattern_framework_test.cpp | 412 +++++++----------- .../tests/master_service_config_test.cpp | 22 - mooncake-wheel/mooncake/io_pattern_bridge.py | 12 +- 26 files changed, 639 insertions(+), 1445 deletions(-) create mode 100644 mooncake-store/include/io_pattern/cfm_ownership_client.h create mode 100644 mooncake-store/src/io_pattern/cfm_ownership_client.cpp diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index 88eac9d220..16b43ae3de 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -957,7 +957,8 @@ Store/Get/Put -> Collector -> bounded Analyzer -> PolicyEngine -> Ops | | |-> Prefetch handler | | `-> Admission handler | `-> per-session K-means fallback - `-> Reporter -> authenticated CFM channel/pool + `-> CfmIngress <- report_metric_batch/report_snapshot (coro_rpc) + (merge into the SubMaster's own runtime) ``` `MasterService` owns the runtime because it owns the authoritative replica @@ -967,15 +968,17 @@ by the Store master. ### Current implementation architecture -The following diagram is the implementation-level view. It distinguishes the -local Store data path from the optional *remote* central CFM deployment: metric -reporting is asynchronous, while a received CFM command is executed by the -same storage handlers as a locally planned command. The two `MasterService` -boxes are deployment roles, not two mandatory Mooncake service types. A normal -deployment has one active Master (plus an optional HA standby); a separate -central CFM Master is needed only when metrics and policy are centralized -across multiple Masters. The roles may also be co-located for a single-Master -deployment. +CFM is an embedded component of every SubMaster: the SubMaster's own +`IoPatternRuntime` collects observations from its Store data path, evaluates +policy and executes through the same storage-safe handlers. There is no +standalone CFM Master deployment, no separate CFM endpoint and no auth token. +A reporting client (inference connector / Store client) observes keys that may +live on many SubMasters, resolves the owning SubMaster for every key through +the CVM key->slot->submaster mapping, aggregates observations per owner and +sends metric batches to each owning SubMaster's regular `coro_rpc` endpoint. +The receiving SubMaster merges the report into its local runtime so +collection, analysis and execution all stay on the SubMaster that owns the +reported keys. ```mermaid flowchart TB @@ -986,10 +989,11 @@ flowchart TB storage["Storage and watermark paths\nStorageMetric"] end - subgraph local["Reporting / policy-consuming MasterService"] + subgraph local["SubMaster MasterService (embedded CFM)"] direction TB runtime["IoPatternRuntime"] collector["IoPatternCollectorImpl\nper-tenant/object aggregation\nrolling snapshot"] + ingress["CfmIngress\nmerge ownership-addressed reports"] reporter["IoPatternReporter\nbounded MetricBatch queue\nadaptive 100/200/500/1000 ms flush"] analyzer["ResilientAnalyzer\nSlidingWindowAnalyzer\nbudget + timeout fallback"] policy["DegradingPolicyEngine\nWorkloadPolicyEngine\nper-session templates"] @@ -997,6 +1001,7 @@ flowchart TB feedback["PolicyFeedbackWindow +\nAdaptivePolicyTuner"] admission_worker["Admission worker\nbounded deferred queue"] + ingress --> runtime runtime --> collector collector --> reporter collector --> analyzer @@ -1014,49 +1019,36 @@ flowchart TB admit["Admission\npost-write retention / promotion"] end - subgraph transport["Optional authenticated CFM transport"] + subgraph client["Reporting client (connector / Store client)"] direction LR + owner["CvmOwnershipClient\nCVM key→slot→submaster bucketing"] codec["CfmBinaryCodec\nversioned CFM2 wire format"] - channel["CfmRpcChannel\nauthenticate + encode/decode"] - resilient["ResilientCfmChannel\nbounded retry + degradation state"] - rpc["CoroRpcCfmTransport\nexisting coro_rpc client pool"] - codec --> channel --> resilient - channel --> rpc + channel["CfmRpcChannel\nencode/decode"] + rpc["CoroRpcCfmTransport\ncoro_rpc to the owning SubMaster"] + owner --> codec --> channel --> rpc end - subgraph central["Central CFM MasterService"] - direction TB - rpc_service["CfmRpcService\nAuthenticate / Send / Receive /\nAcknowledge / EnqueuePolicy"] - service["CfmService\nauthentication + per-node bounded queues"] - ingress["CfmIngress\ndecode and normalize remote metrics"] - central_runtime["IoPatternRuntime\nCollector → Analyzer → PolicyEngine"] - producer_worker["PolicyProducerWorker\nproduce high-watermark eviction\nand trace-derived prefetch commands"] - policy_queue["Policy queue per stable node_id\ndelivery_id + ACK state"] - rpc_service --> service --> ingress --> central_runtime --> producer_worker --> policy_queue - end + rpc_service["CfmRpcService::Send\non the SubMaster's coro_rpc port"] - inference --> runtime + inference -->|"ownership-addressed metric batches"| owner access --> runtime storage --> runtime executor --> evict executor --> prefetch executor --> admit - reporter -->|"report_metric_batch"| channel - rpc -->|"authenticated RPC"| rpc_service - policy_queue -->|"poll_policy"| rpc - resilient --> client["CfmClientImpl\nPollAndDispatchPolicy"] - client -->|"PolicyCommand"| runtime - client -->|"ACK success / failure"| resilient - - external_producer["External policy producer\nproducer credential"] -->|"enqueue_policy"| rpc_service + reporter -->|"report_metric_batch"| ingress + rpc -->|"report_metric_batch / report_snapshot / execute_*"| rpc_service + rpc_service --> ingress observability["IoPatternObservability\nlatency, hit rate, false positives,\ndegradation, report drops"] -.-> runtime ``` -`CfmClientImpl` is deliberately not the normal metric-reporting entry point in -the production wiring. `IoPatternReporter` sends metric batches directly -through `CfmRpcChannel`; the client object owns the polling, dispatch and ACK -loop for CFM-issued policy commands. +`CfmOwnershipClient` (or a single-endpoint `CfmRpcChannel` when one SubMaster +is the only owner) is the normal report path. `CfmClientImpl` wraps a single +channel for connectors; the client owns aggregation per owning SubMaster and +explicitly drops observations whose owner cannot be resolved. The receiver +merges every accepted report into the local runtime — there is no policy +queue, poll, ACK or producer role to configure. ## Implemented @@ -1079,14 +1071,18 @@ loop for CFM-issued policy commands. - `IoPatternRuntime` wires collection, bounded analysis, policy execution, feedback tuning and storage handlers; `MasterService` feeds it from actual Get/Put/watermark paths. -- `CfmClientImpl` dispatches received policy commands through - `IoPatternRuntime::ExecuteCommand`, so CFM-issued plans take the same safe - Store execution route as locally planned ones. -- `CfmIngress` is the CFM-to-Store endpoint: it decodes authenticated snapshot - and metric-batch payloads into the runtime, and executes remote prefetch - plans through the same handlers. +- `CfmClientImpl` wraps a single reporting channel for connectors + (`ReportSnapshot` / `ReportMetricBatch` / `ExecutePrefetch`); policy runs in + the SubMaster's own runtime, so there is no client-side dispatch loop. +- `CfmOwnershipClient` is the multi-SubMaster report path: it resolves the + owning SubMaster of every key through a CVM-backed resolver, aggregates + observations per owner and delivers one batch per owner; unresolvable + observations are counted as drops. +- `CfmIngress` is the CFM-to-Store endpoint on each SubMaster: it decodes + snapshot and metric-batch payloads into that SubMaster's local runtime and + executes prefetch/eviction plans through the same storage-safe handlers. - `ResilientCfmChannel` adds bounded retries and consecutive-failure - degradation state around a concrete transport. + degradation state around a concrete reporting transport. - `PolicyFeedbackWindow` aggregates bounded execution-effect windows, and `AdaptivePolicyTuner` adjusts eviction weights after repeated negative hit-rate deltas. @@ -1109,8 +1105,8 @@ loop for CFM-issued policy commands. (or conservative mixed mode) when analysis throws, with failure tracking. - `CfmBinaryCodec` defines the versioned `CFM2` protocol and fully round-trips snapshots, metric batches and every policy command. `InProcessCfmRpcTransport` - provides authenticated embedded operation, while `CfmChannelPool` reuses and - fails over a bounded set of injected network channels. + provides embedded operation for tests, while `CfmChannelPool` reuses and + fails over a bounded set of reporting channels. - `CfmRpcChannel::SendMetricBatch` and `MakeCfmMetricBatchSink` connect the bounded Reporter to the RPC path; producers only enqueue and Flush performs the transport call outside the data-path critical section. @@ -1140,8 +1136,6 @@ loop for CFM-issued policy commands. promotion evaluation. This is a post-write cache-admission hook, not initial replica placement: the existing `PutStart` contract selects and allocates replicas before write metrics such as batch and overwrite are known. -- CFM polling distinguishes a command, a healthy empty queue and a transport - error. Only transport errors contribute to consecutive-failure degradation. - Reporter intervals follow the documented memory/RPC load thresholds (100/200/500/1000 ms), and in-process transport callbacks execute outside the transport mutex. @@ -1201,46 +1195,34 @@ RPC resources are injected through execution handlers and CFM channels. ## Production CFM wiring -Master registers authenticated CFM handlers on its existing `coro_rpc` port. -`CoroRpcCfmTransport` is the production client: metric batches are delivered to -`CfmIngress`, while policy commands use a bounded per-node queue and are polled -by stable `node_id`. Received commands execute through -`IoPatternRuntime::ExecuteCommand`, preserving the same storage-safe handlers as -local policy decisions. Every report RPC also carries that `node_id`; ingress -uses it as the authoritative storage-metric source so central aggregation does -not merge watermarks from different Masters. - -Configure a central CFM receiver with `io_pattern_cfm_auth_token`. Configure each -reporting/policy-consuming Master with: - -- `io_pattern_cfm_endpoint=host:port` -- `io_pattern_cfm_node_id=` (defaults to the local CVM - SubMaster RPC endpoint, `rpc_address:rpc_port`) -- the same `io_pattern_cfm_auth_token` -- on the central receiver only, a distinct - `io_pattern_cfm_producer_auth_token` for policy producers -- optional `io_pattern_cfm_timeout_ms` and - `io_pattern_cfm_policy_queue_capacity` - -An outbound Master authenticates during construction and fails startup if the -configured CFM endpoint cannot be reached or rejects the token. At runtime the -Reporter sends metric batches over the channel and a resilient poll loop -dispatches queued policies. On the receiver, each accepted metric batch is put -onto a bounded policy-production queue; the central runtime runs -Collector -> Analyzer -> PolicyEngine asynchronously and automatically queues -high-watermark eviction and trace-derived prefetch commands for the reporting -`node_id`. An external policy producer may also call the registered -`CfmRpcService::EnqueuePolicy` RPC with a target node id and an encoded -`PolicyCommand`. - -Node credentials cannot use that explicit enqueue RPC; an external producer -must authenticate with the separately configured producer credential. The -server validates commands before enqueueing them, assigns a delivery id, and -retains each command until the target node acknowledges successful execution. -Its poll response distinguishes an authenticated empty queue from a rejected -or invalid request, so authorization failures enter the normal degradation -path. The configured capacity is enforced for both pending production work and -policy delivery, with policy delivery bounded both per node and globally. +Every Master registers the CFM report handler (`CfmRpcService::Send`) on its +existing `coro_rpc` port — the same endpoint all other Mooncake RPCs use. No +extra `io_pattern_cfm_endpoint`, `io_pattern_cfm_auth_token`, +`io_pattern_cfm_node_id` or producer credential is configured: Mooncake RPCs +run inside the trusted deployment, and CFM is a component of the SubMaster +that owns the reported keys. + +A SubMaster's embedded CFM works as follows: + +- The local Store data path records `AccessRecord` / `StorageMetric` straight + into the SubMaster's own `IoPatternRuntime`; eviction runs through the same + storage-safe quota/promotion handlers (no per-node runtime, policy queue or + poll loop). +- A reporting client (vLLM/SGLang connector or the Store client) observes keys + that may belong to different SubMasters. It resolves the owner of each key + with the CVM mapping (`cvm::KeySlot` over the `/cvm/` master registry), + aggregates observations by owning SubMaster, and delivers one + `report_metric_batch` / `report_snapshot` per owner over coro_rpc. +- The receiving SubMaster's `CfmIngress` merges the batch into its local + runtime and normalizes remote StorageMetric watermarks against the + transport source, so collection, analysis and execution never leave the + SubMaster that owns the keys. + +Reports that fail to resolve to an owner are counted as dropped observations +on the client, so callers degrade explicitly instead of guessing an owner. +Explicit `execute_prefetch` / `execute_policy` RPCs are also accepted by the +same handler and run through the receiving SubMaster's local storage +handlers, preserving the storage-safe execution seam for cross-node commands. The SGLang adapter remains framework-neutral because SGLang source is not vendored in this repository. diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index 41db1a1a04..ff6d9f3171 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -1,10 +1,21 @@ -// CFM client benchmark that models the vLLM KV-cache call path. +// CFM benchmark that models the vLLM KV-cache call path in the embedded CFM +// architecture. // -// One benchmark request consists of prompt_tokens + output_tokens. The KV -// cache is split into tokens_per_block blocks for every transformer layer, -// exactly as a vLLM connector would address its layer/block cache entries. -// Each request records inference and access metrics, reports a snapshot through -// CfmClientImpl, then prints both CFM call latency and IO Pattern metrics. +// CFM is a component of every SubMaster; there is no standalone CFM Master and +// no credential. A reporting client observes keys (KV blocks) and sends metric +// batches over the SubMaster's regular coro_rpc endpoint. The SubMaster merges +// reports into its local runtime, then policy evaluation and execution run +// locally on the keys it owns (high-watermark eviction in the data path, +// trace-derived prefetch through the same storage-safe handlers). +// +// This benchmark exercises that path in two modes: +// - embedded (default): an in-process SubMaster runtime plays the owning +// CFM component. Reports are delivered in-process and policy is evaluated +// and executed locally, so the benchmark prints both report latency and +// the resulting eviction/prefetch/admission handler activity. +// - remote (--cfm_endpoint=host:port): reports go over coro_rpc to a real +// SubMaster CFM receiver; the receiving side is not observable here, so +// only client-side latency is reported. #include #include @@ -25,7 +36,6 @@ #include "gflags/gflags.h" #include "glog/logging.h" -#include "io_pattern/cfm_client_impl.h" #include "io_pattern/cfm_protocol.h" #include "io_pattern/cfm_service.h" #include "io_pattern/rpc_transport.h" @@ -50,18 +60,16 @@ DEFINE_uint64(shared_prefix_tokens, 512, "Per-session prompt prefix reused by later requests"); DEFINE_uint64(report_capacity, 262144, "Maximum queued IO Pattern observations before reporting"); -DEFINE_uint64(policy_queue_capacity, 4096, - "Maximum CFM policy commands queued for the client"); DEFINE_uint64(report_flush_wait_ms, 1100, - "Maximum wait for remote CFM policy production after a flush"); + "Grace period for report drain before local policy evaluation"); DEFINE_double(memory_used_ratio, 0.95, - "Reported L1 memory use ratio; >= 0.90 triggers CFM planning"); + "Reported L1 memory use ratio; >= 0.90 triggers eviction"); DEFINE_string(tenant, "vllm-benchmark", "Tenant id"); -DEFINE_string(node_id, "vllm-submaster-0", "CFM node/submaster id"); +DEFINE_string(node_id, "vllm-submaster-0", + "CFM node/submaster id that owns the reported keys"); DEFINE_string(cfm_endpoint, "", - "Remote CFM coro_rpc endpoint; empty uses an embedded CFM service"); -DEFINE_string(cfm_auth_token, "cfm-client-benchmark-node-token", - "Authentication token for the CFM node endpoint"); + "Remote SubMaster coro_rpc endpoint (host:port); empty uses an " + "embedded in-process SubMaster CFM component"); uint64_t SteadyNowNs() { return static_cast( @@ -89,48 +97,21 @@ std::string KvKey(size_t session, size_t request, size_t layer, size_t block, std::to_string(layer) + "/block-" + std::to_string(block); } -class ServiceBackedCfmTransport final : public CfmRpcTransport { +// Sends reports straight into an embedded SubMaster's CFM component. This is +// the ownership-addressed path collapsed to the single owning SubMaster of a +// benchmark run, exercised without network. +class EmbeddedCfmTransport final : public CfmRpcTransport { public: - ServiceBackedCfmTransport(std::shared_ptr service, - std::string node_id) - : service_(std::move(service)), node_id_(std::move(node_id)) {} - - bool Authenticate(std::string_view token) override { - if (!service_ || !service_->AuthenticateNode(token)) return false; - token_ = std::string(token); - authenticated_ = true; - return true; - } + explicit EmbeddedCfmTransport(std::shared_ptr service) + : service_(std::move(service)) {} bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds) override { - return authenticated_ && service_ && - service_->Send(node_id_, method, payload, token_); - } - - CfmReceiveResult Receive(std::string_view method, - std::chrono::milliseconds) override { - if (!authenticated_ || !service_ || method != "poll_policy") { - return CfmReceiveResult::Error(); - } - const auto delivery = service_->PollPolicy(node_id_, token_); - return delivery - ? CfmReceiveResult::Payload(delivery->second, delivery->first) - : CfmReceiveResult::Empty(); - } - - bool Acknowledge(uint64_t delivery_id, bool success, - std::chrono::milliseconds) override { - return authenticated_ && service_ && - service_->AcknowledgePolicy(node_id_, delivery_id, success, - token_); + return service_ && service_->Send(method, payload, FLAGS_node_id); } private: std::shared_ptr service_; - std::string node_id_; - std::string token_; - bool authenticated_{false}; }; class LatencyStats final { @@ -328,8 +309,8 @@ bool ValidateFlags() { return FLAGS_requests != 0 && FLAGS_prompt_tokens + FLAGS_output_tokens != 0 && FLAGS_tokens_per_block != 0 && FLAGS_num_layers != 0 && FLAGS_kv_block_bytes != 0 && FLAGS_num_sessions != 0 && - FLAGS_report_capacity != 0 && FLAGS_policy_queue_capacity != 0 && - FLAGS_memory_used_ratio >= 0.0 && FLAGS_memory_used_ratio <= 1.0; + FLAGS_report_capacity != 0 && FLAGS_memory_used_ratio >= 0.0 && + FLAGS_memory_used_ratio <= 1.0; } } // namespace @@ -343,35 +324,41 @@ int main(int argc, char* argv[]) { return 1; } - constexpr char kProducerToken[] = "cfm-client-benchmark-producer-token"; std::atomic eviction_commands{0}; std::atomic prefetch_commands{0}; std::atomic admission_commands{0}; + // The SubMaster-side CFM component (embedded mode) or the target of the + // remote coro_rpc receiver. Its runtime aggregates whatever is reported. std::shared_ptr embedded_service; std::shared_ptr transport; + std::shared_ptr cfm_runtime; if (FLAGS_cfm_endpoint.empty()) { - auto cfm_control_runtime = std::make_shared( + cfm_runtime = std::make_shared( IoPatternRuntime::Handlers{ - .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, - .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, - .admission = [](const AdmissionResult&) { + .eviction = [&eviction_commands](const EvictionPlan&) { + ++eviction_commands; + return ErrorCode::OK; + }, + .prefetch = [&prefetch_commands](const PrefetchPlan&) { + ++prefetch_commands; + return ErrorCode::OK; + }, + .admission = [&admission_commands](const AdmissionResult&) { + ++admission_commands; return ErrorCode::OK; }}); - embedded_service = std::make_shared( - cfm_control_runtime, FLAGS_cfm_auth_token, - FLAGS_policy_queue_capacity, kProducerToken); - transport = std::make_shared( - embedded_service, FLAGS_node_id); + embedded_service = std::make_shared(cfm_runtime); + transport = std::make_shared(embedded_service); } else { transport = std::make_shared( - FLAGS_cfm_endpoint, FLAGS_node_id, std::chrono::milliseconds(500)); + FLAGS_cfm_endpoint, std::chrono::milliseconds(500)); } + auto codec = std::make_shared(); auto channel = std::make_shared( transport, codec, - CfmRpcConfig{.timeout = std::chrono::milliseconds(500), - .auth_token = FLAGS_cfm_auth_token}); + CfmRpcConfig{.timeout = std::chrono::milliseconds(500)}); IoPatternRuntime::Config source_config; source_config.report_capacity = FLAGS_report_capacity; @@ -385,22 +372,10 @@ int main(int argc, char* argv[]) { }; auto source_runtime = std::make_shared( IoPatternRuntime::Handlers{ - .eviction = [&eviction_commands](const EvictionPlan&) { - ++eviction_commands; - return ErrorCode::OK; - }, - .prefetch = [&prefetch_commands](const PrefetchPlan&) { - ++prefetch_commands; - return ErrorCode::OK; - }, - .admission = [&admission_commands](const AdmissionResult&) { - ++admission_commands; - return ErrorCode::OK; - }}, + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, source_config); - CfmClientImpl client(channel, [&source_runtime](const PolicyCommand& command) { - return source_runtime->ExecuteCommand(command); - }); LatencyStats report_latency; uint64_t failed_reports = 0; @@ -418,40 +393,34 @@ int main(int argc, char* argv[]) { source_runtime->RecordStorageMetric(request.snapshot.storage.front()); const auto report_start = Clock::now(); - const auto result = client.ReportSnapshot(request.snapshot); + const bool sent = channel->SendSnapshot(request.snapshot); report_latency.Record(ToMicroseconds(Clock::now() - report_start)); - if (result != ErrorCode::OK) ++failed_reports; + if (!sent) ++failed_reports; } const auto submission_seconds = std::chrono::duration(Clock::now() - benchmark_start).count(); // Stop joins the reporter worker and performs its final flush. No new - // metric batch can reach CFM after this returns. + // metric batch can reach the SubMaster after this returns. source_runtime->StopReports(); - if (embedded_service) { - if (!embedded_service->WaitForPolicyIdle( - std::chrono::milliseconds(FLAGS_report_flush_wait_ms))) { - LOG(WARNING) << "Timed out waiting for embedded CFM policy production"; - } - } else { - std::this_thread::sleep_for( - std::chrono::milliseconds(FLAGS_report_flush_wait_ms)); - } - uint64_t observed_commands = 0; - bool policy_drain_complete = false; - const auto policy_deadline = - Clock::now() + std::chrono::milliseconds(FLAGS_report_flush_wait_ms); - while (Clock::now() < policy_deadline) { - if (client.PollAndDispatchPolicy() != ErrorCode::OK) break; - const uint64_t executed = eviction_commands + prefetch_commands + - admission_commands; - if (executed == observed_commands) { - policy_drain_complete = embedded_service != nullptr; - break; - } else { - observed_commands = executed; + std::this_thread::sleep_for( + std::chrono::milliseconds(FLAGS_report_flush_wait_ms)); + + // Embedded mode: evaluate and execute policy locally on the SubMaster + // runtime, exactly as the data-path high-watermark trigger does in + // production. The merged report above is what feeds that evaluation. + if (cfm_runtime && !cfm_runtime->Snapshot().keys.empty()) { + const auto capacity = 1024ULL * 1024 * 1024; + const auto target = + static_cast((FLAGS_memory_used_ratio - 0.80F) * + static_cast(capacity)); + const auto status = cfm_runtime->Execute( + CacheTier::kL1Host, + target > 0 ? target : capacity / 10, TraceHistory{}); + if (status.eviction != ErrorCode::OK && + status.prefetch != ErrorCode::OK && status.degraded) { + LOG(WARNING) << "Local CFM evaluation degraded"; } - std::this_thread::sleep_for(std::chrono::milliseconds(1)); } const auto end_to_end_seconds = @@ -463,17 +432,19 @@ int main(int argc, char* argv[]) { source_runtime.reset(); report_latency.Finalize(); const auto metric_report_snapshot = metric_reports.Finalize(); - const auto cfm_snapshot = embedded_service - ? embedded_service->SnapshotForNode(FLAGS_node_id) - : IoPatternSnapshot{}; - const auto cfm_metrics = embedded_service - ? embedded_service->ObservabilityForNode( - FLAGS_node_id, end_to_end_seconds) - : IoPatternObservabilitySnapshot{}; + const auto cfm_snapshot = + embedded_service ? embedded_service->Snapshot() : IoPatternSnapshot{}; + const auto cfm_metrics = + embedded_service ? embedded_service->Observability(end_to_end_seconds) + : IoPatternObservabilitySnapshot{}; std::cout << "\n============================================================\n" << "CFM CLIENT BENCHMARK (vLLM inference request model)\n" << "============================================================\n" + << " CFM deployment: " + << (embedded_service ? "embedded SubMaster (local CFM)" + : "remote SubMaster coro_rpc endpoint") + << "\n" << " Requests: " << FLAGS_requests << "\n" << " Tokens/request: " << FLAGS_prompt_tokens + FLAGS_output_tokens << " (prompt=" @@ -488,7 +459,7 @@ int main(int argc, char* argv[]) { << " Submission requests/sec: " << FLAGS_requests / submission_seconds << "\n" << " End-to-end time: " << end_to_end_seconds << " s\n" - << "\n CFM ReportSnapshot latency\n" + << "\n CFM SendSnapshot latency\n" << " failed reports: " << failed_reports << "\n" << " mean: " << report_latency.Mean() << " us\n" << " p50 / p90 / p99: " << report_latency.Percentile(50) @@ -507,15 +478,10 @@ int main(int argc, char* argv[]) { << metric_report_snapshot.latency.Percentile(50) << " / " << metric_report_snapshot.latency.Percentile(90) << " / " << metric_report_snapshot.latency.Percentile(99) << " us\n" - << "\n CFM policy commands executed\n" + << "\n Local policy handlers executed\n" << " evictions: " << eviction_commands << "\n" << " prefetches: " << prefetch_commands << "\n" << " admissions: " << admission_commands << "\n" - << " policy drain: " - << (embedded_service - ? (policy_drain_complete ? "complete" : "timed out/error") - : "remote endpoint is not verifiable") - << "\n" << "\n IO Pattern snapshots\n" << " client keys / storage: " << source_snapshot.keys.size() << " / " << source_snapshot.storage.size() << "\n"; diff --git a/mooncake-store/include/io_pattern/cfm_channel.h b/mooncake-store/include/io_pattern/cfm_channel.h index 7d0b84e125..1ba0886a1c 100644 --- a/mooncake-store/include/io_pattern/cfm_channel.h +++ b/mooncake-store/include/io_pattern/cfm_channel.h @@ -3,45 +3,23 @@ #include #include +#include "reporter.h" #include "types.h" #include "../types.h" namespace mooncake::io_pattern { -struct CfmPollResult { - enum class Status { kCommand, kEmpty, kError }; - - static CfmPollResult Command(PolicyCommand command, - uint64_t delivery_id = 0) { - return {.status = Status::kCommand, - .command = std::move(command), - .delivery_id = delivery_id}; - } - static CfmPollResult Empty() { return {.status = Status::kEmpty}; } - static CfmPollResult Error() { return {.status = Status::kError}; } - - Status status{Status::kEmpty}; - std::optional command; - uint64_t delivery_id{0}; -}; - -// Transport-neutral CFM RPC channel. Implementations own serialization, -// retries and connection lifecycle. +// Transport-neutral CFM reporting channel. In the embedded architecture a +// SubMaster owns CFM for the keys in its slots, so a channel delivers reports +// (snapshots and metric batches) and explicit prefetch plans to the owning +// SubMaster endpoint. There is no policy polling or delivery acknowledgement +// loop: the receiver merges reports into its local runtime and policy +// commands execute on the SubMaster that owns the reported keys. class CfmChannel { public: virtual ~CfmChannel() = default; virtual bool SendSnapshot(const IoPatternSnapshot& snapshot) = 0; - virtual CfmPollResult PollPolicyResult() = 0; - std::optional PollPolicy() { - auto result = PollPolicyResult(); - if (result.status != CfmPollResult::Status::kCommand || - !result.command) { - return std::nullopt; - } - if (!AcknowledgePolicy(result.delivery_id, true)) return std::nullopt; - return std::move(result.command); - } - virtual bool AcknowledgePolicy(uint64_t, bool) { return true; } + virtual bool SendMetricBatch(const MetricBatch& batch) = 0; virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; }; diff --git a/mooncake-store/include/io_pattern/cfm_client_impl.h b/mooncake-store/include/io_pattern/cfm_client_impl.h index 64ef8e2078..ed19094fb4 100644 --- a/mooncake-store/include/io_pattern/cfm_client_impl.h +++ b/mooncake-store/include/io_pattern/cfm_client_impl.h @@ -1,34 +1,28 @@ #pragma once #include -#include +#include #include "cfm_channel.h" #include "client.h" namespace mooncake::io_pattern { -// Production CFM client orchestration. Network behavior is delegated to the -// injected channel so this class remains independent of RPC libraries. +// Reporting client used by inference-side connectors and integration tests. +// Network behavior is delegated to the injected channel; the client is what a +// vLLM/SGLang connector sees when it reports IO Pattern observations for keys +// owned by the addressed SubMaster. class CfmClientImpl final : public CfmClient { public: - using PolicyCommandHandler = std::function; - - explicit CfmClientImpl(std::shared_ptr channel, - PolicyCommandHandler policy_handler = {}) - : channel_(std::move(channel)), - policy_handler_(std::move(policy_handler)) {} + explicit CfmClientImpl(std::shared_ptr channel) + : channel_(std::move(channel)) {} ErrorCode ReportSnapshot(const IoPatternSnapshot& snapshot) override; - ErrorCode ReceivePolicy(const PolicyCommand& command) override; + ErrorCode ReportMetricBatch(const MetricBatch& batch) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; - std::optional PollPolicy(); - ErrorCode PollAndDispatchPolicy(); - private: std::shared_ptr channel_; - PolicyCommandHandler policy_handler_; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_ownership_client.h b/mooncake-store/include/io_pattern/cfm_ownership_client.h new file mode 100644 index 0000000000..46b8ca967f --- /dev/null +++ b/mooncake-store/include/io_pattern/cfm_ownership_client.h @@ -0,0 +1,77 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "client.h" +#include "cfm_channel.h" +#include "cfm_protocol.h" +#include "cfm_service.h" +#include "reporter.h" +#include "rpc_transport.h" +#include "types.h" + +namespace mooncake::io_pattern { + +// Resolves the SubMaster that owns a reported object. Production callers feed +// this from the CVM key->slot->submaster mapping (cvm::KeySlot over the etcd +// /cvm/ master registry) inside the Store client / connector layer; +// returns the SubMaster's regular coro_rpc endpoint ("host:port"), the same +// endpoint every other Mooncake RPC of that SubMaster uses. +using SubmasterEndpointResolver = + std::function(const TenantId&, const std::string&)>; + +// Ownership-addressed CFM reporting client. +// +// CFM is a component of every SubMaster; there is no standalone CFM Master +// and no auth token. A caller (inference connector / Store client) observes +// keys that may live on many SubMasters, so reports are bucketed by the CVM +// ownership resolver: observations whose key belongs to the same SubMaster are +// aggregated into one metric batch and delivered to that SubMaster's embedded +// CFM over its ordinary coro_rpc endpoint. Metrics whose owner cannot be +// resolved are dropped and counted so callers can degrade instead of guessing. +class CfmOwnershipClient final : public CfmClient { + public: + // `resolver` maps a reported object to the owning SubMaster endpoint. It + // must be kept up to date with the CVM route (slot rebalance is rare). + // `timeout` applies to every RPC. + explicit CfmOwnershipClient(SubmasterEndpointResolver resolver, + std::chrono::milliseconds timeout = + std::chrono::milliseconds(500)); + + // Sends the snapshot, grouped by the owning SubMaster of each key. + ErrorCode ReportSnapshot(const IoPatternSnapshot& snapshot) override; + + // Sends the metric batch, grouped by the owning SubMaster of each + // inference/access object. Storage observations are deliberately not + // routed: the SubMaster that owns the underlying storage already reports + // its own watermark to its local runtime. + ErrorCode ReportMetricBatch(const MetricBatch& batch) override; + + // Sends an explicit prefetch plan to the SubMaster that owns the first + // candidate; that SubMaster executes it through its local storage-safe + // handlers. + ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; + + // Observations dropped because no owner could be resolved. + uint64_t dropped_observations() const; + + private: + std::shared_ptr ChannelFor(const std::string& endpoint); + std::string OwnerEndpoint(const ObjectRef& object) const; + + SubmasterEndpointResolver resolver_; + std::chrono::milliseconds timeout_; + std::unordered_map> channels_; + mutable std::mutex channels_mutex_; + std::atomic dropped_observations_{0}; +}; + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/cfm_protocol.h b/mooncake-store/include/io_pattern/cfm_protocol.h index 05d079a760..112e2b46cd 100644 --- a/mooncake-store/include/io_pattern/cfm_protocol.h +++ b/mooncake-store/include/io_pattern/cfm_protocol.h @@ -5,7 +5,7 @@ namespace mooncake::io_pattern { // Versioned binary wire codec for the CFM RPC methods. It deliberately owns -// every serialization detail so transports only deal in authenticated bytes. +// every serialization detail so transports only deal in method/payload bytes. class CfmBinaryCodec final : public CfmRpcCodec { public: std::string EncodeSnapshot(const IoPatternSnapshot& snapshot) const override; diff --git a/mooncake-store/include/io_pattern/cfm_service.h b/mooncake-store/include/io_pattern/cfm_service.h index 62d1764f0e..3e57ebbd47 100644 --- a/mooncake-store/include/io_pattern/cfm_service.h +++ b/mooncake-store/include/io_pattern/cfm_service.h @@ -1,111 +1,64 @@ #pragma once #include -#include #include -#include -#include #include -#include #include #include #include -#include -#include #include #include "cfm_ingress.h" +#include "observability.h" +#include "types.h" namespace mooncake::io_pattern { -// Authenticated server-side CFM endpoint. The RPC layer delegates to this -// class, keeping authentication, bounded policy queues and runtime dispatch -// independent of the concrete network transport. +// Embedded CFM endpoint of one SubMaster. There is no standalone CFM Master: +// every SubMaster runs the full IO Pattern pipeline locally and accepts +// ownership-addressed metric reports over its regular coro_rpc port (the same +// endpoint the rest of Mooncake uses). Received batches are merged straight +// into the local runtime so collection, analysis and policy execution all stay +// on the SubMaster that owns the reported keys. +// +// Reports are addressed by key ownership, so a receiving SubMaster only ever +// merges observations for keys that belong to its own slots. A separate node +// identity, delivery queues and a poll/ack loop are therefore unnecessary: the +// reporting client fans each batch out to the owning SubMaster(s) using the +// CVM key->slot->submaster mapping. class CfmService final { public: - CfmService(std::shared_ptr runtime, - std::string auth_token, size_t policy_queue_capacity = 4096, - std::string producer_auth_token = {}); - ~CfmService(); + explicit CfmService(std::shared_ptr runtime); - bool Authenticate(std::string_view token) const; - bool AuthenticateNode(std::string_view token) const; - bool AuthenticateProducer(std::string_view token) const; - bool Send(std::string_view node_id, std::string_view method, - std::string_view payload, std::string_view token); - std::optional> PollPolicy( - std::string_view node_id, std::string_view token); - bool AcknowledgePolicy(std::string_view node_id, uint64_t delivery_id, - bool success, std::string_view token); - bool EnqueuePolicy(std::string node_id, std::string payload, - std::string_view token); + // RPC entry point. Supported methods: + // report_metric_batch / report_snapshot -> merge into the local runtime + // execute_prefetch / execute_policy -> execute through the local + // runtime's storage handlers + // `source_id` identifies the reporting process and is used to normalize + // remote StorageMetric watermarks. + bool Send(std::string_view method, std::string_view payload, + std::string_view source_id = {}); - // Returns the metrics aggregated for one CFM node. In a CVM deployment a - // node is a stable SubMaster identity, so policy generation for one slot - // owner must never observe keys reported by another SubMaster. - IoPatternSnapshot SnapshotForNode(std::string_view node_id) const; - IoPatternObservabilitySnapshot ObservabilityForNode( - std::string_view node_id, double window_seconds = 0.0) const; - bool WaitForPolicyIdle(std::chrono::milliseconds timeout); + IoPatternSnapshot Snapshot() const; + IoPatternObservabilitySnapshot Observability( + double window_seconds = 0.0) const; private: - struct NodeRuntime { - std::shared_ptr runtime; - std::unique_ptr ingress; - }; - - bool EnqueueValidated(std::string node_id, std::string payload); - std::shared_ptr GetOrCreateNodeRuntime( - std::string_view node_id); - std::shared_ptr FindNodeRuntime( - std::string_view node_id) const; - void SchedulePolicyProduction(std::string node_id, MetricBatch batch); - void PolicyProducerWorker(); - void ProducePolicies(std::string_view node_id, const MetricBatch& batch); - std::shared_ptr runtime_; std::shared_ptr codec_; CfmIngress ingress_; - mutable std::mutex node_runtimes_mutex_; - std::unordered_map> - node_runtimes_; - const std::string auth_token_; - const std::string producer_auth_token_; - const size_t policy_queue_capacity_; - std::mutex mutex_; - std::unordered_map>> - policy_queues_; - uint64_t next_delivery_id_{1}; - size_t total_queued_policies_{0}; - std::mutex producer_mutex_; - std::condition_variable producer_cv_; - std::deque> pending_metric_batches_; - size_t active_policy_productions_{0}; - bool producer_stopping_{false}; - std::condition_variable producer_idle_cv_; - std::thread producer_worker_; }; -// coro_rpc-facing adapter. Keeping RPC signatures here lets both the Master -// server and integration tests register the exact production endpoints. +// coro_rpc-facing adapter. Keeping the RPC signature here lets both the +// SubMaster server and integration tests register the exact production +// endpoint. class CfmRpcService final { public: explicit CfmRpcService(std::shared_ptr service) : service_(std::move(service)) {} - bool Authenticate(const std::string& auth_token); - bool Send(const std::string& node_id, const std::string& method, - const std::string& payload, const std::string& auth_token); - // The boolean explicitly distinguishes a rejected request from an - // authenticated queue that currently has no policy. - std::pair>> Receive( - const std::string& method, const std::string& node_id, - const std::string& auth_token); - bool Acknowledge(const std::string& node_id, uint64_t delivery_id, - bool success, const std::string& auth_token); - bool EnqueuePolicy(const std::string& node_id, const std::string& payload, - const std::string& auth_token); + bool Send(const std::string& method, const std::string& payload, + const std::string& source_id = {}); private: std::shared_ptr service_; diff --git a/mooncake-store/include/io_pattern/client.h b/mooncake-store/include/io_pattern/client.h index 1c4e061c9d..150c980797 100644 --- a/mooncake-store/include/io_pattern/client.h +++ b/mooncake-store/include/io_pattern/client.h @@ -5,13 +5,16 @@ namespace mooncake::io_pattern { -// Adapter seam between an inference node and the remote Cache Flow Manager. +// Adapter seam between an inference node and the Cache Flow Manager of the +// SubMaster(s) that own the reported keys. Reports are sent over the same +// coro_rpc endpoint every other Mooncake API uses; there is no separate CFM +// Master and no credential to obtain. class CfmClient { public: virtual ~CfmClient() = default; virtual ErrorCode ReportSnapshot(const IoPatternSnapshot& snapshot) = 0; - virtual ErrorCode ReceivePolicy(const PolicyCommand& command) = 0; + virtual ErrorCode ReportMetricBatch(const MetricBatch& batch) = 0; virtual ErrorCode ExecutePrefetch(const PrefetchPlan& plan) = 0; }; diff --git a/mooncake-store/include/io_pattern/io_pattern.h b/mooncake-store/include/io_pattern/io_pattern.h index 78f3dbef1d..103901016b 100644 --- a/mooncake-store/include/io_pattern/io_pattern.h +++ b/mooncake-store/include/io_pattern/io_pattern.h @@ -5,6 +5,7 @@ #include "io_pattern/cfm_channel.h" #include "io_pattern/cfm_ingress.h" #include "io_pattern/cfm_service.h" +#include "io_pattern/cfm_ownership_client.h" #include "io_pattern/cfm_protocol.h" #include "io_pattern/cfm_client_impl.h" #include "io_pattern/feedback.h" diff --git a/mooncake-store/include/io_pattern/resilient_cfm_channel.h b/mooncake-store/include/io_pattern/resilient_cfm_channel.h index e022de2157..91b6f71dfe 100644 --- a/mooncake-store/include/io_pattern/resilient_cfm_channel.h +++ b/mooncake-store/include/io_pattern/resilient_cfm_channel.h @@ -3,6 +3,7 @@ #include #include #include +#include #include "cfm_channel.h" @@ -21,8 +22,7 @@ class ResilientCfmChannel final : public CfmChannel { : delegate_(std::move(delegate)), config_(config) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - CfmPollResult PollPolicyResult() override; - bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; + bool SendMetricBatch(const MetricBatch& batch) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; bool degraded() const; diff --git a/mooncake-store/include/io_pattern/rpc_transport.h b/mooncake-store/include/io_pattern/rpc_transport.h index 98ff83d830..668d86c1d3 100644 --- a/mooncake-store/include/io_pattern/rpc_transport.h +++ b/mooncake-store/include/io_pattern/rpc_transport.h @@ -1,11 +1,10 @@ #pragma once -#include #include +#include #include #include #include -#include #include #include #include @@ -17,23 +16,6 @@ namespace mooncake::io_pattern { -struct CfmReceiveResult { - enum class Status { kPayload, kEmpty, kError }; - - static CfmReceiveResult Payload(std::string payload, - uint64_t delivery_id = 0) { - return {.status = Status::kPayload, - .payload = std::move(payload), - .delivery_id = delivery_id}; - } - static CfmReceiveResult Empty() { return {.status = Status::kEmpty}; } - static CfmReceiveResult Error() { return {.status = Status::kError}; } - - Status status{Status::kEmpty}; - std::string payload; - uint64_t delivery_id{0}; -}; - class CfmRpcCodec { public: virtual ~CfmRpcCodec() = default; @@ -44,39 +26,32 @@ class CfmRpcCodec { const std::string&) const = 0; }; +// Transport for CFM reports addressed to the SubMaster that owns the reported +// keys. There is no authentication: Mooncake RPCs run inside the trusted +// deployment, and CFM is an embedded component of every SubMaster reached over +// its regular coro_rpc endpoint (the same endpoint all other Mooncake APIs +// use). Only method/payload delivery is needed because the receiver merges +// reports into its local runtime instead of maintaining per-node policy +// queues. class CfmRpcTransport { public: virtual ~CfmRpcTransport() = default; - // Implementations that communicate with a remote CFM should override this - // to bind the connection to the configured service credential. Keeping a - // default preserves compatibility with trusted in-process transports. - virtual bool Authenticate(std::string_view token) { return token.empty(); } virtual bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds timeout) = 0; - virtual CfmReceiveResult Receive( - std::string_view method, std::chrono::milliseconds timeout) = 0; - virtual bool Acknowledge(uint64_t delivery_id, bool success, - std::chrono::milliseconds timeout) = 0; }; // Production CFM transport over Mooncake's existing coro_rpc connection pool. -// It targets the CFM handlers registered on the Master RPC service. +// It targets the CFM handler registered on the Master RPC service of the +// SubMaster that owns the reported keys. class CoroRpcCfmTransport final : public CfmRpcTransport { public: - CoroRpcCfmTransport(std::string endpoint, std::string node_id, + CoroRpcCfmTransport(std::string endpoint, std::chrono::milliseconds default_timeout = std::chrono::milliseconds(500)); ~CoroRpcCfmTransport() override; - bool Authenticate(std::string_view token) override; bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds timeout) override; - CfmReceiveResult Receive( - std::string_view method, std::chrono::milliseconds timeout) override; - bool Acknowledge(uint64_t delivery_id, bool success, - std::chrono::milliseconds timeout) override; - bool EnqueuePolicy(std::string_view node_id, std::string_view payload, - std::chrono::milliseconds timeout); private: class Impl; @@ -85,39 +60,26 @@ class CoroRpcCfmTransport final : public CfmRpcTransport { struct CfmRpcConfig { std::chrono::milliseconds timeout{500}; - std::string auth_token; }; -// A concrete authenticated endpoint for embedded deployments and integration -// tests. It is intentionally transport-agnostic at the codec boundary: a +// An in-process CFM endpoint for embedded deployments and integration tests. +// It is intentionally transport-agnostic at the codec boundary: a // socket/HTTP implementation can expose the same method names and wire bytes. class InProcessCfmRpcTransport final : public CfmRpcTransport { public: using SendHandler = std::function; - explicit InProcessCfmRpcTransport(std::string auth_token, - SendHandler send_handler = {}) - : auth_token_(std::move(auth_token)), send_handler_(std::move(send_handler)) {} + explicit InProcessCfmRpcTransport(SendHandler send_handler = {}) + : send_handler_(std::move(send_handler)) {} - bool Authenticate(std::string_view token) override; bool Send(std::string_view method, std::string_view payload, - std::chrono::milliseconds timeout) override; - CfmReceiveResult Receive( - std::string_view method, std::chrono::milliseconds timeout) override; - bool Acknowledge(uint64_t, bool, - std::chrono::milliseconds) override { - return true; - } - - void EnqueuePolicy(std::string payload); + std::chrono::milliseconds) override; + void SetSendHandler(SendHandler handler); private: mutable std::mutex mutex_; - const std::string auth_token_; - bool authenticated_{false}; SendHandler send_handler_; - std::queue policies_; }; class CfmRpcChannel final : public CfmChannel { @@ -130,32 +92,25 @@ class CfmRpcChannel final : public CfmChannel { config_(config) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - CfmPollResult PollPolicyResult() override; - bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; + bool SendMetricBatch(const MetricBatch& batch) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; - bool SendMetricBatch(const MetricBatch& batch); private: - bool EnsureAuthenticated(); - std::shared_ptr transport_; std::shared_ptr codec_; CfmRpcConfig config_; - std::mutex authentication_mutex_; - bool authenticated_{false}; }; -// Reuses a bounded set of authenticated CFM channels. Requests are selected -// round-robin; an unavailable member is skipped so one failed connection does -// not stall policy reporting. +// Reuses a bounded set of CFM channels. Requests are selected round-robin; an +// unavailable member is skipped so one failed connection does not stall +// metric reporting. class CfmChannelPool final : public CfmChannel { public: explicit CfmChannelPool(std::vector> channels) : channels_(std::move(channels)) {} bool SendSnapshot(const IoPatternSnapshot& snapshot) override; - CfmPollResult PollPolicyResult() override; - bool AcknowledgePolicy(uint64_t delivery_id, bool success) override; + bool SendMetricBatch(const MetricBatch& batch) override; ErrorCode ExecutePrefetch(const PrefetchPlan& plan) override; private: diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 8c3baf03f4..8d64543e59 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -64,15 +64,17 @@ class IoPatternRuntime final { const TraceHistory& trace, const std::vector& admissions = {}, const std::string& session_id = {}); - // Runs Collector -> Analyzer -> PolicyEngine without invoking local - // storage handlers. Central CFM uses this to produce commands for a - // target node; Store data paths continue to use Execute(). + // Runs Collector -> Analyzer -> PolicyEngine without invoking the local + // storage handlers. Callers use Plan when they need the raw policy result + // (for example the local eviction watermark path, observability or tests); + // Store data paths continue to use Execute(). PolicyResult Plan(CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, const std::vector& admissions = {}, const std::string& session_id = {}); // Applies a CFM-issued command through the same storage handlers as a - // locally planned policy. This is the CFM-to-Store execution endpoint. + // locally planned policy. This is the CFM-to-Store execution endpoint used + // by the embedded SubMaster CFM receiver. ErrorCode ExecuteCommand(const PolicyCommand& command); bool ScheduleAdmission(ObjectRef object, CacheTier target_tier, std::string session_id = {}); diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index 5db113cd52..d0ad2c96e7 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -18,17 +18,6 @@ namespace mooncake { // Forwarded to the HA serve phase via MasterServiceSupervisorConfig. class HttpMetadataServer; -struct IoPatternCfmConfig { - // Empty endpoint means this Master only serves the CFM RPC endpoints. - // Set host:port to report to and poll policies from a central CFM Master. - std::string endpoint; - std::string node_id; - std::string auth_token; - std::string producer_auth_token; - uint32_t timeout_ms{500}; - uint32_t policy_queue_capacity{4096}; -}; - inline std::string ResolveConfiguredHABackendConnstring( std::string_view ha_backend_type, std::string_view ha_backend_connstring, std::string_view etcd_endpoints) { @@ -51,7 +40,6 @@ struct MasterConfig { std::string rpc_interface; int32_t rpc_conn_timeout_seconds; bool rpc_enable_tcp_no_delay; - IoPatternCfmConfig io_pattern_cfm; uint64_t default_kv_lease_ttl; uint64_t default_kv_soft_pin_ttl; @@ -227,7 +215,6 @@ class MasterServiceSupervisorConfig { std::chrono::steady_clock::duration rpc_conn_timeout = std::chrono::seconds( 0); // Client connection timeout. 0 = no timeout (infinite) bool rpc_enable_tcp_no_delay = true; - IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; std::string etcd_endpoints = "0.0.0.0:2379"; @@ -373,7 +360,6 @@ class MasterServiceSupervisorConfig { rpc_conn_timeout = std::chrono::seconds(config.rpc_conn_timeout_seconds); rpc_enable_tcp_no_delay = config.rpc_enable_tcp_no_delay; - io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; etcd_endpoints = config.etcd_endpoints; ha_backend_connstring = ResolveConfiguredHABackendConnstring( @@ -568,7 +554,6 @@ class WrappedMasterServiceConfig { bool kv_events_emit_legacy_compat = true; bool kv_events_emit_object_key = true; uint32_t kv_events_queue_capacity = 65536; - IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; // OpLog store configuration @@ -674,7 +659,6 @@ class WrappedMasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; - io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = ResolveConfiguredHABackendConnstring( ha_backend_type, config.ha_backend_connstring, @@ -799,7 +783,6 @@ class WrappedMasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; - io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = ResolveConfiguredHABackendConnstring( ha_backend_type, config.ha_backend_connstring, @@ -914,7 +897,6 @@ class MasterServiceConfigBuilder { std::string cxl_path_ = DEFAULT_CXL_PATH; size_t cxl_size_ = DEFAULT_CXL_SIZE; bool enable_cxl_ = false; - IoPatternCfmConfig io_pattern_cfm_; VChunkConfig vchunk_config_{}; std::shared_ptr vchunk_metadata_store_; @@ -1217,11 +1199,6 @@ class MasterServiceConfigBuilder { return *this; } - MasterServiceConfigBuilder& set_io_pattern_cfm(IoPatternCfmConfig config) { - io_pattern_cfm_ = std::move(config); - return *this; - } - MasterServiceConfig build() const; }; @@ -1277,8 +1254,6 @@ class MasterServiceConfig { bool kv_events_emit_legacy_compat = true; bool kv_events_emit_object_key = true; uint32_t kv_events_queue_capacity = 65536; - // CFM transport and authentication settings used by this MasterService. - IoPatternCfmConfig io_pattern_cfm; std::string ha_backend_type = "etcd"; std::string ha_backend_connstring; // OpLog store configuration @@ -1381,7 +1356,6 @@ class MasterServiceConfig { kv_events_emit_legacy_compat = config.kv_events_emit_legacy_compat; kv_events_emit_object_key = config.kv_events_emit_object_key; kv_events_queue_capacity = config.kv_events_queue_capacity; - io_pattern_cfm = config.io_pattern_cfm; ha_backend_type = config.ha_backend_type; ha_backend_connstring = config.ha_backend_connstring; enable_oplog = config.enable_oplog; @@ -1502,7 +1476,6 @@ inline MasterServiceConfig MasterServiceConfigBuilder::build() const { config.cxl_path = cxl_path_; config.cxl_size = cxl_size_; config.enable_cxl = enable_cxl_; - config.io_pattern_cfm = io_pattern_cfm_; config.vchunk_config = vchunk_config_; config.vchunk_metadata_store = vchunk_metadata_store_; config.vchunk_config = vchunk_config_; diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 9f2c720ecd..19f62d4067 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -53,8 +53,6 @@ namespace mooncake { namespace io_pattern { -class CfmChannel; -class CfmClientImpl; class CfmService; class IoPatternRuntime; } @@ -2224,14 +2222,12 @@ class MasterService { // The IO-pattern pipeline is deliberately owned by MasterService: the // master has the authoritative replica map and is the only component that // can safely translate a policy plan into promotion/eviction operations. + // CFM is embedded here as a component of this SubMaster: reports addressed + // to the keys this master owns are merged into the local runtime over the + // regular coro_rpc endpoint, so no remote reporting channel, policy poller + // or credential is needed. std::shared_ptr io_pattern_runtime_; std::shared_ptr io_pattern_cfm_service_; - std::shared_ptr io_pattern_cfm_channel_; - std::unique_ptr io_pattern_cfm_client_; - std::atomic io_pattern_cfm_polling_{false}; - std::mutex io_pattern_cfm_poll_mutex_; - std::condition_variable io_pattern_cfm_poll_cv_; - std::thread io_pattern_cfm_poll_thread_; const std::string ha_backend_type_; diff --git a/mooncake-store/src/CMakeLists.txt b/mooncake-store/src/CMakeLists.txt index f1f153aef9..0931868306 100644 --- a/mooncake-store/src/CMakeLists.txt +++ b/mooncake-store/src/CMakeLists.txt @@ -73,6 +73,7 @@ set(MOONCAKE_STORE_SOURCES io_pattern/cfm_client_impl.cpp io_pattern/cfm_ingress.cpp io_pattern/cfm_service.cpp + io_pattern/cfm_ownership_client.cpp io_pattern/cfm_protocol.cpp io_pattern/resilient_cfm_channel.cpp io_pattern/feedback.cpp diff --git a/mooncake-store/src/io_pattern/cfm_client_impl.cpp b/mooncake-store/src/io_pattern/cfm_client_impl.cpp index 88425020fd..b9973a5351 100644 --- a/mooncake-store/src/io_pattern/cfm_client_impl.cpp +++ b/mooncake-store/src/io_pattern/cfm_client_impl.cpp @@ -5,13 +5,13 @@ namespace mooncake::io_pattern { ErrorCode CfmClientImpl::ReportSnapshot(const IoPatternSnapshot& snapshot) { if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; return channel_->SendSnapshot(snapshot) ? ErrorCode::OK - : ErrorCode::RPC_FAIL; + : ErrorCode::RPC_FAIL; } -ErrorCode CfmClientImpl::ReceivePolicy(const PolicyCommand& command) { +ErrorCode CfmClientImpl::ReportMetricBatch(const MetricBatch& batch) { if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; - if (!policy_handler_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; - return policy_handler_(command); + return channel_->SendMetricBatch(batch) ? ErrorCode::OK + : ErrorCode::RPC_FAIL; } ErrorCode CfmClientImpl::ExecutePrefetch(const PrefetchPlan& plan) { @@ -19,23 +19,4 @@ ErrorCode CfmClientImpl::ExecutePrefetch(const PrefetchPlan& plan) { return channel_->ExecutePrefetch(plan); } -std::optional CfmClientImpl::PollPolicy() { - return channel_ ? channel_->PollPolicy() : std::nullopt; -} - -ErrorCode CfmClientImpl::PollAndDispatchPolicy() { - if (!channel_) return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; - auto result = channel_->PollPolicyResult(); - if (result.status == CfmPollResult::Status::kEmpty) return ErrorCode::OK; - if (result.status == CfmPollResult::Status::kError || !result.command) { - return ErrorCode::RPC_TIMEOUT; - } - const auto execution = ReceivePolicy(*result.command); - if (!channel_->AcknowledgePolicy(result.delivery_id, - execution == ErrorCode::OK)) { - return ErrorCode::RPC_FAIL; - } - return execution; -} - } // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_ownership_client.cpp b/mooncake-store/src/io_pattern/cfm_ownership_client.cpp new file mode 100644 index 0000000000..943c5f34cb --- /dev/null +++ b/mooncake-store/src/io_pattern/cfm_ownership_client.cpp @@ -0,0 +1,104 @@ +#include "io_pattern/cfm_ownership_client.h" + +#include + +namespace mooncake::io_pattern { + +CfmOwnershipClient::CfmOwnershipClient(SubmasterEndpointResolver resolver, + std::chrono::milliseconds timeout) + : resolver_(std::move(resolver)), timeout_(timeout) {} + +std::string CfmOwnershipClient::OwnerEndpoint(const ObjectRef& object) const { + if (!resolver_) return {}; + const auto endpoint = resolver_(object.tenant_id, object.key); + return endpoint ? *endpoint : std::string{}; +} + +std::shared_ptr CfmOwnershipClient::ChannelFor( + const std::string& endpoint) { + if (endpoint.empty()) return nullptr; + { + std::lock_guard lock(channels_mutex_); + const auto existing = channels_.find(endpoint); + if (existing != channels_.end()) return existing->second; + } + auto transport = std::make_shared(endpoint, timeout_); + auto channel = std::make_shared( + std::move(transport), std::make_shared(), + CfmRpcConfig{.timeout = timeout_}); + std::lock_guard lock(channels_mutex_); + const auto inserted = channels_.emplace(endpoint, std::move(channel)); + return inserted.first->second; +} + +ErrorCode CfmOwnershipClient::ReportSnapshot(const IoPatternSnapshot& snapshot) { + // Bucket keys by their owning SubMaster and report one snapshot per owner. + // Storage observations are not routed here: the SubMaster that owns the + // storage already reports its own watermark into its local runtime. + std::unordered_map by_owner; + for (const auto& key : snapshot.keys) { + const auto endpoint = OwnerEndpoint(key.object); + if (endpoint.empty()) { + ++dropped_observations_; + continue; + } + auto& owned = by_owner[endpoint]; + owned.generated_at_ns = snapshot.generated_at_ns; + owned.keys.push_back(key); + } + bool all_ok = true; + for (const auto& [endpoint, owned] : by_owner) { + auto channel = ChannelFor(endpoint); + if (!channel || !channel->SendSnapshot(owned)) all_ok = false; + } + return all_ok ? ErrorCode::OK : ErrorCode::RPC_FAIL; +} + +ErrorCode CfmOwnershipClient::ReportMetricBatch(const MetricBatch& batch) { + std::unordered_map by_owner; + for (const auto& metric : batch.inference) { + const auto endpoint = OwnerEndpoint(metric.object); + if (endpoint.empty()) { + ++dropped_observations_; + continue; + } + by_owner[endpoint].inference.push_back(metric); + } + for (const auto& access : batch.accesses) { + const auto endpoint = OwnerEndpoint(access.object); + if (endpoint.empty()) { + ++dropped_observations_; + continue; + } + by_owner[endpoint].accesses.push_back(access); + } + // batch.storage is intentionally not forwarded (see header/ReportSnapshot). + bool all_ok = true; + for (const auto& [endpoint, owned] : by_owner) { + auto channel = ChannelFor(endpoint); + if (!channel || !channel->SendMetricBatch(owned)) all_ok = false; + } + return all_ok ? ErrorCode::OK : ErrorCode::RPC_FAIL; +} + +ErrorCode CfmOwnershipClient::ExecutePrefetch(const PrefetchPlan& plan) { + if (plan.candidates.empty()) return ErrorCode::OK; + // Address the plan to the SubMaster owning the first candidate object; + // a plan produced by one SubMaster concerns keys it owns, so routing by + // the primary candidate keeps a single endpoint target in practice. + const auto endpoint = OwnerEndpoint(plan.candidates.front().object); + if (endpoint.empty()) { + ++dropped_observations_; + return ErrorCode::OBJECT_NOT_FOUND; + } + auto channel = ChannelFor(endpoint); + if (!channel) return ErrorCode::RPC_FAIL; + const auto code = channel->ExecutePrefetch(plan); + return code == ErrorCode::OK ? ErrorCode::OK : ErrorCode::RPC_FAIL; +} + +uint64_t CfmOwnershipClient::dropped_observations() const { + return dropped_observations_.load(std::memory_order_relaxed); +} + +} // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/cfm_service.cpp b/mooncake-store/src/io_pattern/cfm_service.cpp index 1792c6b68c..132bc9231a 100644 --- a/mooncake-store/src/io_pattern/cfm_service.cpp +++ b/mooncake-store/src/io_pattern/cfm_service.cpp @@ -1,345 +1,36 @@ #include "io_pattern/cfm_service.h" -#include -#include -#include +#include namespace mooncake::io_pattern { -CfmService::CfmService(std::shared_ptr runtime, - std::string auth_token, - size_t policy_queue_capacity, - std::string producer_auth_token) +CfmService::CfmService(std::shared_ptr runtime) : runtime_(std::move(runtime)), codec_(std::make_shared()), - ingress_(runtime_, codec_), - auth_token_(std::move(auth_token)), - producer_auth_token_(std::move(producer_auth_token)), - policy_queue_capacity_(policy_queue_capacity), - producer_worker_(&CfmService::PolicyProducerWorker, this) {} + ingress_(runtime_, codec_) {} -CfmService::~CfmService() { - { - std::lock_guard lock(producer_mutex_); - producer_stopping_ = true; - pending_metric_batches_.clear(); - } - producer_cv_.notify_all(); - if (producer_worker_.joinable()) producer_worker_.join(); +bool CfmService::Send(std::string_view method, std::string_view payload, + std::string_view source_id) { + // CFM is a component of this SubMaster: there is no per-reporting-node + // runtime and no policy delivery queue. Every accepted method is handled + // by the ingress against the local runtime so that collection, analysis + // and execution all observe the keys this SubMaster owns. + return ingress_.Handle(method, payload, source_id); } -bool CfmService::Authenticate(std::string_view token) const { - return AuthenticateNode(token) || AuthenticateProducer(token); +IoPatternSnapshot CfmService::Snapshot() const { + return runtime_ ? runtime_->Snapshot() : IoPatternSnapshot{}; } -bool CfmService::AuthenticateNode(std::string_view token) const { - return !auth_token_.empty() && token == auth_token_; +IoPatternObservabilitySnapshot CfmService::Observability( + double window_seconds) const { + return runtime_ ? runtime_->ObservabilitySnapshot(window_seconds) + : IoPatternObservabilitySnapshot{}; } -bool CfmService::AuthenticateProducer(std::string_view token) const { - return !producer_auth_token_.empty() && producer_auth_token_ != auth_token_ && - token == producer_auth_token_; -} - -bool CfmService::Send(std::string_view node_id, std::string_view method, - std::string_view payload, std::string_view token) { - const bool executes_policy = - method == "execute_policy" || method == "execute_prefetch"; - if (node_id.empty() || - (executes_policy ? !AuthenticateProducer(token) - : !AuthenticateNode(token))) { - return false; - } - std::optional metric_batch; - if (method == "report_metric_batch") { - metric_batch = codec_->DecodeMetricBatch(std::string(payload)); - if (!metric_batch) return false; - } - // Direct producer commands are intentionally executed through the CFM - // service runtime. Metrics and snapshots, on the other hand, must remain - // node-local: a CVM node is a SubMaster with an independent slot/key set. - if (executes_policy) return ingress_.Handle(method, payload, node_id); - auto node_runtime = GetOrCreateNodeRuntime(node_id); - if (!node_runtime->ingress->Handle(method, payload, node_id)) return false; - if (metric_batch) { - SchedulePolicyProduction(std::string(node_id), - std::move(*metric_batch)); - } - return true; -} - -IoPatternSnapshot CfmService::SnapshotForNode(std::string_view node_id) const { - const auto node_runtime = FindNodeRuntime(node_id); - return node_runtime ? node_runtime->runtime->Snapshot() - : IoPatternSnapshot{}; -} - -IoPatternObservabilitySnapshot CfmService::ObservabilityForNode( - std::string_view node_id, double window_seconds) const { - const auto node_runtime = FindNodeRuntime(node_id); - return node_runtime - ? node_runtime->runtime->ObservabilitySnapshot(window_seconds) - : IoPatternObservabilitySnapshot{}; -} - -bool CfmService::WaitForPolicyIdle(std::chrono::milliseconds timeout) { - std::unique_lock lock(producer_mutex_); - return producer_idle_cv_.wait_for(lock, timeout, [this] { - return pending_metric_batches_.empty() && active_policy_productions_ == 0; - }); -} - -std::shared_ptr CfmService::GetOrCreateNodeRuntime( - std::string_view node_id) { - std::lock_guard lock(node_runtimes_mutex_); - const std::string id(node_id); - const auto existing = node_runtimes_.find(id); - if (existing != node_runtimes_.end()) return existing->second; - - auto node_runtime = std::make_shared(); - node_runtime->runtime = std::make_shared( - IoPatternRuntime::Handlers{ - .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, - .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, - .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, - IoPatternRuntime::Config{}); - node_runtime->ingress = - std::make_unique(node_runtime->runtime, codec_); - node_runtimes_.emplace(id, node_runtime); - return node_runtime; -} - -std::shared_ptr CfmService::FindNodeRuntime( - std::string_view node_id) const { - std::lock_guard lock(node_runtimes_mutex_); - const auto it = node_runtimes_.find(std::string(node_id)); - return it == node_runtimes_.end() ? nullptr : it->second; -} - -std::optional> CfmService::PollPolicy( - std::string_view node_id, std::string_view token) { - if (!AuthenticateNode(token) || node_id.empty()) return std::nullopt; - std::lock_guard lock(mutex_); - auto it = policy_queues_.find(std::string(node_id)); - if (it == policy_queues_.end() || it->second.empty()) return std::nullopt; - return it->second.front(); -} - -bool CfmService::AcknowledgePolicy(std::string_view node_id, - uint64_t delivery_id, bool success, - std::string_view token) { - if (!AuthenticateNode(token) || node_id.empty() || delivery_id == 0) { - return false; - } - std::lock_guard lock(mutex_); - auto it = policy_queues_.find(std::string(node_id)); - if (it == policy_queues_.end() || it->second.empty() || - it->second.front().first != delivery_id) { - return false; - } - if (!success) { - if (it->second.size() > 1) { - auto failed = std::move(it->second.front()); - it->second.pop_front(); - it->second.push_back(std::move(failed)); - } - return true; - } - it->second.pop_front(); - --total_queued_policies_; - if (it->second.empty()) policy_queues_.erase(it); - return true; -} - -bool CfmService::EnqueuePolicy(std::string node_id, std::string payload, - std::string_view token) { - if (!AuthenticateProducer(token) || node_id.empty() || payload.empty() || - !codec_->DecodePolicy(payload)) { - return false; - } - return EnqueueValidated(std::move(node_id), std::move(payload)); -} - -bool CfmService::EnqueueValidated(std::string node_id, std::string payload) { - std::lock_guard lock(mutex_); - auto& queue = policy_queues_[node_id]; - if (std::any_of(queue.begin(), queue.end(), [&](const auto& queued) { - return queued.second == payload; - })) { - return true; - } - if (policy_queue_capacity_ == 0 || - total_queued_policies_ >= policy_queue_capacity_ || - queue.size() >= policy_queue_capacity_) { - if (queue.empty()) policy_queues_.erase(node_id); - return false; - } - if (next_delivery_id_ == 0) next_delivery_id_ = 1; - const uint64_t delivery_id = next_delivery_id_++; - queue.emplace_back(delivery_id, std::move(payload)); - ++total_queued_policies_; - return true; -} - -void CfmService::SchedulePolicyProduction(std::string node_id, - MetricBatch batch) { - { - std::lock_guard lock(producer_mutex_); - if (producer_stopping_ || policy_queue_capacity_ == 0 || - pending_metric_batches_.size() >= policy_queue_capacity_) { - return; - } - pending_metric_batches_.emplace_back(std::move(node_id), - std::move(batch)); - } - producer_cv_.notify_one(); -} - -void CfmService::PolicyProducerWorker() { - while (true) { - std::pair pending; - { - std::unique_lock lock(producer_mutex_); - producer_cv_.wait(lock, [this] { - return producer_stopping_ || !pending_metric_batches_.empty(); - }); - if (producer_stopping_) return; - pending = std::move(pending_metric_batches_.front()); - pending_metric_batches_.pop_front(); - ++active_policy_productions_; - } - try { - ProducePolicies(pending.first, pending.second); - } catch (...) { - // Policy production is best effort and must never terminate the - // RPC service. The next metric batch will trigger a fresh plan. - } - { - std::lock_guard lock(producer_mutex_); - --active_policy_productions_; - } - producer_idle_cv_.notify_all(); - } -} - -void CfmService::ProducePolicies(std::string_view node_id, - const MetricBatch& batch) { - const auto node_runtime = FindNodeRuntime(node_id); - if (!node_runtime || node_id.empty()) return; - const auto& runtime = node_runtime->runtime; - - std::unordered_map match_lengths; - std::string session_id; - for (const auto& metric : batch.inference) { - match_lengths[metric.object] = metric.match_length; - if (session_id.empty()) session_id = metric.session_id; - } - TraceHistory trace; - trace.events.reserve(batch.accesses.size()); - for (const auto& access : batch.accesses) { - const auto match = match_lengths.find(access.object); - trace.events.push_back( - {.object = access.object, - .observed_at_ns = access.observed_at_ns, - .match_length = match == match_lengths.end() ? 0U - : match->second, - .is_hit = access.is_hit}); - } - - bool produced_prefetch = false; - for (const auto& storage : batch.storage) { - if (static_cast(storage.tier) > - static_cast(CacheTier::kL3NofSsd)) { - continue; - } - if (!std::isfinite(storage.memory_used_ratio) || - storage.memory_used_ratio < 0.90F) { - continue; - } - const float used_ratio = - std::clamp(storage.memory_used_ratio, 0.0F, 1.0F); - uint64_t target_bytes = 0; - if (storage.capacity_bytes != 0) { - const uint64_t low_watermark = - storage.capacity_bytes - storage.capacity_bytes / 5; - if (storage.used_bytes > low_watermark) { - target_bytes = storage.used_bytes - low_watermark; - } - } - if (target_bytes == 0) { - uint64_t tier_bytes = 0; - for (const auto& key : runtime->Snapshot().keys) { - if ((key.replica_tiers & CacheTierBit(storage.tier)) == 0) { - continue; - } - tier_bytes = - key.block_size > - std::numeric_limits::max() - tier_bytes - ? std::numeric_limits::max() - : tier_bytes + key.block_size; - } - const auto excess_ratio = std::max( - 0.0F, used_ratio - 0.80F); - target_bytes = static_cast( - static_cast(tier_bytes) * excess_ratio / - std::max(0.01F, used_ratio)); - } - auto result = runtime->Plan(storage.tier, target_bytes, trace, {}, - session_id); - if (result.degraded) continue; - if (!result.eviction.candidates.empty()) { - EnqueueValidated(std::string(node_id), - codec_->EncodePolicy(result.eviction)); - } - if (!produced_prefetch && !result.prefetch.candidates.empty()) { - EnqueueValidated(std::string(node_id), - codec_->EncodePolicy(result.prefetch)); - produced_prefetch = true; - } - } - - if (!produced_prefetch && !trace.events.empty()) { - auto result = runtime->Plan(CacheTier::kL1Host, 0, trace, {}, - session_id); - if (!result.degraded && !result.prefetch.candidates.empty()) { - EnqueueValidated(std::string(node_id), - codec_->EncodePolicy(result.prefetch)); - } - } -} - -bool CfmRpcService::Authenticate(const std::string& auth_token) { - return service_ && service_->Authenticate(auth_token); -} - -bool CfmRpcService::Send(const std::string& node_id, const std::string& method, - const std::string& payload, - const std::string& auth_token) { - return service_ && service_->Send(node_id, method, payload, auth_token); -} - -std::pair>> -CfmRpcService::Receive( - const std::string& method, const std::string& node_id, - const std::string& auth_token) { - if (!service_ || method != "poll_policy" || - !service_->AuthenticateNode(auth_token) || node_id.empty()) { - return {false, std::nullopt}; - } - return {true, service_->PollPolicy(node_id, auth_token)}; -} - -bool CfmRpcService::Acknowledge(const std::string& node_id, - uint64_t delivery_id, bool success, - const std::string& auth_token) { - return service_ && service_->AcknowledgePolicy( - node_id, delivery_id, success, auth_token); -} - -bool CfmRpcService::EnqueuePolicy(const std::string& node_id, - const std::string& payload, - const std::string& auth_token) { - return service_ && service_->EnqueuePolicy(node_id, payload, auth_token); +bool CfmRpcService::Send(const std::string& method, const std::string& payload, + const std::string& source_id) { + return service_ && service_->Send(method, payload, source_id); } } // namespace mooncake::io_pattern diff --git a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp index 013f7e725d..5b1f5ce69e 100644 --- a/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp +++ b/mooncake-store/src/io_pattern/resilient_cfm_channel.cpp @@ -22,31 +22,8 @@ bool ResilientCfmChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { return Retry([&] { return delegate_->SendSnapshot(snapshot); }); } -CfmPollResult ResilientCfmChannel::PollPolicyResult() { - if (!delegate_) { - RecordFailure(); - return CfmPollResult::Error(); - } - for (uint32_t attempt = 0; attempt <= config_.max_retries; ++attempt) { - auto result = delegate_->PollPolicyResult(); - if (result.status == CfmPollResult::Status::kCommand) { - RecordSuccess(); - return result; - } - if (result.status == CfmPollResult::Status::kEmpty) { - RecordSuccess(); - return result; - } - } - RecordFailure(); - return CfmPollResult::Error(); -} - -bool ResilientCfmChannel::AcknowledgePolicy(uint64_t delivery_id, - bool success) { - return Retry([&] { - return delegate_->AcknowledgePolicy(delivery_id, success); - }); +bool ResilientCfmChannel::SendMetricBatch(const MetricBatch& batch) { + return Retry([&] { return delegate_->SendMetricBatch(batch); }); } ErrorCode ResilientCfmChannel::ExecutePrefetch(const PrefetchPlan& plan) { diff --git a/mooncake-store/src/io_pattern/rpc_transport.cpp b/mooncake-store/src/io_pattern/rpc_transport.cpp index 8f9e992890..14521ba949 100644 --- a/mooncake-store/src/io_pattern/rpc_transport.cpp +++ b/mooncake-store/src/io_pattern/rpc_transport.cpp @@ -13,11 +13,8 @@ namespace mooncake::io_pattern { class CoroRpcCfmTransport::Impl { public: - Impl(std::string endpoint, std::string node_id, - std::chrono::milliseconds default_timeout) - : endpoint_(std::move(endpoint)), - node_id_(std::move(node_id)), - default_timeout_(default_timeout) {} + Impl(std::string endpoint, std::chrono::milliseconds default_timeout) + : endpoint_(std::move(endpoint)), default_timeout_(default_timeout) {} template std::optional Invoke(std::chrono::milliseconds timeout, @@ -54,10 +51,8 @@ class CoroRpcCfmTransport::Impl { } std::string endpoint_; - std::string node_id_; std::chrono::milliseconds default_timeout_; std::mutex mutex_; - std::string auth_token_; std::unordered_map< int64_t, std::shared_ptr>> @@ -65,168 +60,52 @@ class CoroRpcCfmTransport::Impl { }; CoroRpcCfmTransport::CoroRpcCfmTransport( - std::string endpoint, std::string node_id, - std::chrono::milliseconds default_timeout) - : impl_(std::make_unique(std::move(endpoint), std::move(node_id), - default_timeout)) {} + std::string endpoint, std::chrono::milliseconds default_timeout) + : impl_(std::make_unique(std::move(endpoint), default_timeout)) {} CoroRpcCfmTransport::~CoroRpcCfmTransport() = default; -bool CoroRpcCfmTransport::Authenticate(std::string_view token) { - if (!impl_ || token.empty()) return false; - const std::string wire_token(token); - const auto result = impl_->Invoke<&CfmRpcService::Authenticate, bool>( - impl_->default_timeout_, wire_token); - if (!result || !*result) return false; - std::lock_guard lock(impl_->mutex_); - impl_->auth_token_ = wire_token; - return true; -} - bool CoroRpcCfmTransport::Send(std::string_view method, std::string_view payload, std::chrono::milliseconds timeout) { if (!impl_) return false; - std::string auth_token; - { - std::lock_guard lock(impl_->mutex_); - auth_token = impl_->auth_token_; - } - if (auth_token.empty()) return false; + // Reports are addressed to the SubMaster endpoint this transport was + // created for; there is no separate node identity or credential. const auto result = impl_->Invoke<&CfmRpcService::Send, bool>( - timeout, impl_->node_id_, std::string(method), std::string(payload), - auth_token); - return result && *result; -} - -CfmReceiveResult CoroRpcCfmTransport::Receive( - std::string_view method, std::chrono::milliseconds timeout) { - if (!impl_) return CfmReceiveResult::Error(); - std::string auth_token; - { - std::lock_guard lock(impl_->mutex_); - auth_token = impl_->auth_token_; - } - if (auth_token.empty()) return CfmReceiveResult::Error(); - const auto result = - impl_->Invoke<&CfmRpcService::Receive, - std::pair< - bool, - std::optional>>>( - timeout, std::string(method), impl_->node_id_, auth_token); - if (!result || !result->first) return CfmReceiveResult::Error(); - if (!result->second) return CfmReceiveResult::Empty(); - return CfmReceiveResult::Payload(std::move(result->second->second), - result->second->first); -} - -bool CoroRpcCfmTransport::Acknowledge( - uint64_t delivery_id, bool success, std::chrono::milliseconds timeout) { - if (!impl_ || delivery_id == 0) return false; - std::string auth_token; - { - std::lock_guard lock(impl_->mutex_); - auth_token = impl_->auth_token_; - } - if (auth_token.empty()) return false; - const auto result = impl_->Invoke<&CfmRpcService::Acknowledge, bool>( - timeout, impl_->node_id_, delivery_id, success, auth_token); + timeout, std::string(method), std::string(payload), std::string{}); return result && *result; } -bool CoroRpcCfmTransport::EnqueuePolicy( - std::string_view node_id, std::string_view payload, - std::chrono::milliseconds timeout) { - if (!impl_) return false; - std::string auth_token; - { - std::lock_guard lock(impl_->mutex_); - auth_token = impl_->auth_token_; - } - if (auth_token.empty()) return false; - const auto result = - impl_->Invoke<&CfmRpcService::EnqueuePolicy, bool>( - timeout, std::string(node_id), std::string(payload), auth_token); - return result && *result; -} - -bool InProcessCfmRpcTransport::Authenticate(std::string_view token) { - std::lock_guard lock(mutex_); - authenticated_ = token == auth_token_; - return authenticated_; -} - bool InProcessCfmRpcTransport::Send(std::string_view method, std::string_view payload, std::chrono::milliseconds) { SendHandler handler; { std::lock_guard lock(mutex_); - if (!authenticated_) return false; handler = send_handler_; } return !handler || handler(method, payload); } -CfmReceiveResult InProcessCfmRpcTransport::Receive( - std::string_view method, std::chrono::milliseconds) { - std::lock_guard lock(mutex_); - if (!authenticated_ || method != "poll_policy") return CfmReceiveResult::Error(); - if (policies_.empty()) return CfmReceiveResult::Empty(); - auto payload = std::move(policies_.front()); - policies_.pop(); - return CfmReceiveResult::Payload(std::move(payload)); -} - -void InProcessCfmRpcTransport::EnqueuePolicy(std::string payload) { - std::lock_guard lock(mutex_); - policies_.push(std::move(payload)); -} - void InProcessCfmRpcTransport::SetSendHandler(SendHandler handler) { std::lock_guard lock(mutex_); send_handler_ = std::move(handler); } -bool CfmRpcChannel::EnsureAuthenticated() { - std::lock_guard lock(authentication_mutex_); - if (authenticated_) return true; - authenticated_ = transport_ && transport_->Authenticate(config_.auth_token); - return authenticated_; -} - bool CfmRpcChannel::SendSnapshot(const IoPatternSnapshot& snapshot) { - if (!transport_ || !codec_ || !EnsureAuthenticated()) return false; + if (!transport_ || !codec_) return false; return transport_->Send("report_snapshot", codec_->EncodeSnapshot(snapshot), config_.timeout); } -CfmPollResult CfmRpcChannel::PollPolicyResult() { - if (!transport_ || !codec_ || !EnsureAuthenticated()) { - return CfmPollResult::Error(); - } - auto received = transport_->Receive("poll_policy", config_.timeout); - if (received.status == CfmReceiveResult::Status::kEmpty) { - return CfmPollResult::Empty(); - } - if (received.status == CfmReceiveResult::Status::kError) { - return CfmPollResult::Error(); - } - auto command = codec_->DecodePolicy(received.payload); - if (!command) { - transport_->Acknowledge(received.delivery_id, false, config_.timeout); - return CfmPollResult::Error(); - } - return CfmPollResult::Command(std::move(*command), received.delivery_id); -} - -bool CfmRpcChannel::AcknowledgePolicy(uint64_t delivery_id, bool success) { - return transport_ && delivery_id != 0 && EnsureAuthenticated() && - transport_->Acknowledge(delivery_id, success, config_.timeout); +bool CfmRpcChannel::SendMetricBatch(const MetricBatch& batch) { + if (!transport_ || !codec_) return false; + return transport_->Send("report_metric_batch", codec_->EncodeMetricBatch(batch), + config_.timeout); } ErrorCode CfmRpcChannel::ExecutePrefetch(const PrefetchPlan& plan) { - if (!transport_ || !codec_ || !EnsureAuthenticated()) { + if (!transport_ || !codec_) { return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; } return transport_->Send("execute_prefetch", codec_->EncodePrefetch(plan), @@ -235,12 +114,6 @@ ErrorCode CfmRpcChannel::ExecutePrefetch(const PrefetchPlan& plan) { : ErrorCode::RPC_TIMEOUT; } -bool CfmRpcChannel::SendMetricBatch(const MetricBatch& batch) { - if (!transport_ || !codec_ || !EnsureAuthenticated()) return false; - return transport_->Send("report_metric_batch", codec_->EncodeMetricBatch(batch), - config_.timeout); -} - std::shared_ptr CfmChannelPool::Next() const { if (channels_.empty()) return nullptr; const auto index = next_.fetch_add(1, std::memory_order_relaxed) % @@ -256,24 +129,10 @@ bool CfmChannelPool::SendSnapshot(const IoPatternSnapshot& snapshot) { return false; } -CfmPollResult CfmChannelPool::PollPolicyResult() { - bool saw_empty = false; - for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { - auto channel = Next(); - if (!channel) continue; - auto result = channel->PollPolicyResult(); - if (result.status == CfmPollResult::Status::kCommand) return result; - saw_empty = saw_empty || result.status == CfmPollResult::Status::kEmpty; - } - return saw_empty ? CfmPollResult::Empty() : CfmPollResult::Error(); -} - -bool CfmChannelPool::AcknowledgePolicy(uint64_t delivery_id, bool success) { +bool CfmChannelPool::SendMetricBatch(const MetricBatch& batch) { for (size_t attempt = 0; attempt < channels_.size(); ++attempt) { auto channel = Next(); - if (channel && channel->AcknowledgePolicy(delivery_id, success)) { - return true; - } + if (channel && channel->SendMetricBatch(batch)) return true; } return false; } diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 4b54fb8597..b6176b11e7 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -160,20 +160,6 @@ DEFINE_int32(rpc_conn_timeout_seconds, 0, "Connection timeout in seconds (0 = no timeout)"); DEFINE_bool(rpc_enable_tcp_no_delay, true, "Enable TCP_NODELAY for RPC connections"); -DEFINE_string(io_pattern_cfm_endpoint, "", - "Central CFM Master RPC endpoint (host:port); empty serves CFM " - "requests without outbound reporting"); -DEFINE_string(io_pattern_cfm_node_id, "", - "Stable node id used for CFM policy polling; defaults to " - "the local CVM SubMaster RPC endpoint"); -DEFINE_string(io_pattern_cfm_auth_token, "", - "Authentication token for CFM node report/poll RPCs"); -DEFINE_string(io_pattern_cfm_producer_auth_token, "", - "Separate token authorized to enqueue CFM policies"); -DEFINE_uint32(io_pattern_cfm_timeout_ms, 500, - "CFM RPC request timeout in milliseconds"); -DEFINE_uint32(io_pattern_cfm_policy_queue_capacity, 4096, - "Maximum queued CFM policies per node"); DEFINE_validator(eviction_ratio, [](const char* flagname, double value) { if (value < 0.0 || value > 1.0) { LOG(FATAL) << "Mem eviction ratio must be between 0.0 and 1.0"; @@ -527,26 +513,6 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, default_config.GetBool("rpc_enable_tcp_no_delay", &master_config.rpc_enable_tcp_no_delay, FLAGS_rpc_enable_tcp_no_delay); - default_config.GetString("io_pattern_cfm_endpoint", - &master_config.io_pattern_cfm.endpoint, - FLAGS_io_pattern_cfm_endpoint); - default_config.GetString("io_pattern_cfm_node_id", - &master_config.io_pattern_cfm.node_id, - FLAGS_io_pattern_cfm_node_id); - default_config.GetString("io_pattern_cfm_auth_token", - &master_config.io_pattern_cfm.auth_token, - FLAGS_io_pattern_cfm_auth_token); - default_config.GetString( - "io_pattern_cfm_producer_auth_token", - &master_config.io_pattern_cfm.producer_auth_token, - FLAGS_io_pattern_cfm_producer_auth_token); - default_config.GetUInt32("io_pattern_cfm_timeout_ms", - &master_config.io_pattern_cfm.timeout_ms, - FLAGS_io_pattern_cfm_timeout_ms); - default_config.GetUInt32( - "io_pattern_cfm_policy_queue_capacity", - &master_config.io_pattern_cfm.policy_queue_capacity, - FLAGS_io_pattern_cfm_policy_queue_capacity); default_config.GetDurationMs("default_kv_lease_ttl", &master_config.default_kv_lease_ttl, mooncake::DEFAULT_DEFAULT_KV_LEASE_TTL); @@ -843,41 +809,6 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, } google::CommandLineFlagInfo info; - if ((google::GetCommandLineFlagInfo("io_pattern_cfm_endpoint", &info) && - !info.is_default) || - !conf_set) { - master_config.io_pattern_cfm.endpoint = FLAGS_io_pattern_cfm_endpoint; - } - if ((google::GetCommandLineFlagInfo("io_pattern_cfm_node_id", &info) && - !info.is_default) || - !conf_set) { - master_config.io_pattern_cfm.node_id = FLAGS_io_pattern_cfm_node_id; - } - if ((google::GetCommandLineFlagInfo("io_pattern_cfm_auth_token", &info) && - !info.is_default) || - !conf_set) { - master_config.io_pattern_cfm.auth_token = - FLAGS_io_pattern_cfm_auth_token; - } - if ((google::GetCommandLineFlagInfo( - "io_pattern_cfm_producer_auth_token", &info) && - !info.is_default) || - !conf_set) { - master_config.io_pattern_cfm.producer_auth_token = - FLAGS_io_pattern_cfm_producer_auth_token; - } - if ((google::GetCommandLineFlagInfo("io_pattern_cfm_timeout_ms", &info) && - !info.is_default) || - !conf_set) { - master_config.io_pattern_cfm.timeout_ms = FLAGS_io_pattern_cfm_timeout_ms; - } - if ((google::GetCommandLineFlagInfo( - "io_pattern_cfm_policy_queue_capacity", &info) && - !info.is_default) || - !conf_set) { - master_config.io_pattern_cfm.policy_queue_capacity = - FLAGS_io_pattern_cfm_policy_queue_capacity; - } if ((google::GetCommandLineFlagInfo("enable_cxl", &info) && !info.is_default) || !conf_set) { @@ -1591,34 +1522,6 @@ int main(int argc, char* argv[]) { << ", must be 'cachelib' or 'offset'"; return 1; } - if (!master_config.io_pattern_cfm.endpoint.empty() && - master_config.io_pattern_cfm.auth_token.empty()) { - LOG(FATAL) << "io_pattern_cfm_auth_token is required when " - "io_pattern_cfm_endpoint is configured"; - return 1; - } - if (!master_config.io_pattern_cfm.producer_auth_token.empty() && - master_config.io_pattern_cfm.producer_auth_token == - master_config.io_pattern_cfm.auth_token) { - LOG(FATAL) << "io_pattern_cfm_producer_auth_token must differ from " - "io_pattern_cfm_auth_token"; - return 1; - } - if (master_config.io_pattern_cfm.timeout_ms == 0 || - master_config.io_pattern_cfm.timeout_ms > 10'000 || - master_config.io_pattern_cfm.policy_queue_capacity == 0) { - LOG(FATAL) << "io_pattern_cfm_timeout_ms must be in [1, 10000] and " - "io_pattern_cfm_policy_queue_capacity must be non-zero"; - return 1; - } - if (master_config.io_pattern_cfm.node_id.empty()) { - // CVM registers each SubMaster with the same address used as its - // stable local_hostname. cluster_id would merge every SubMaster in - // the same CVM deployment into one CFM node. - master_config.io_pattern_cfm.node_id = - master_config.rpc_address + ":" + - std::to_string(master_config.rpc_port); - } const char* value = std::getenv("MC_RPC_PROTOCOL"); std::string protocol = "tcp"; @@ -1688,14 +1591,6 @@ int main(int argc, char* argv[]) { << ", client_ttl=" << master_config.client_live_ttl_sec << ", rpc_thread_num=" << master_config.rpc_thread_num << ", rpc_port=" << master_config.rpc_port - << ", io_pattern_cfm_endpoint=" - << (master_config.io_pattern_cfm.endpoint.empty() - ? "" - : master_config.io_pattern_cfm.endpoint) - << ", io_pattern_cfm_node_id=" - << master_config.io_pattern_cfm.node_id - << ", io_pattern_cfm_server_enabled=" - << !master_config.io_pattern_cfm.auth_token.empty() << ", rpc_address=" << master_config.rpc_address << ", rpc_interface=" << master_config.rpc_interface << ", rpc_conn_timeout_seconds=" diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 013c407661..80b80bc940 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -60,11 +60,7 @@ #include "ha_metric_manager.h" #include "metadata_store.h" #include "io_pattern/runtime.h" -#include "io_pattern/cfm_client_impl.h" -#include "io_pattern/cfm_protocol.h" #include "io_pattern/cfm_service.h" -#include "io_pattern/resilient_cfm_channel.h" -#include "io_pattern/rpc_transport.h" namespace mooncake { @@ -442,48 +438,6 @@ MasterService::MasterService(const MasterServiceConfig& config) } io_pattern::IoPatternRuntime::Config io_pattern_config; - std::shared_ptr cfm_rpc_channel; - if (!config.io_pattern_cfm.endpoint.empty()) { - if (config.io_pattern_cfm.timeout_ms == 0 || - config.io_pattern_cfm.timeout_ms > 10'000) { - throw std::invalid_argument( - "io_pattern_cfm_timeout_ms must be in [1, 10000]"); - } - if (config.io_pattern_cfm.auth_token.empty()) { - throw std::invalid_argument( - "io_pattern_cfm_auth_token is required when " - "io_pattern_cfm_endpoint is configured"); - } - // In CVM mode every SubMaster owns a different slot set. Use the - // stable SubMaster id by default so CFM keeps their reports and - // policy queues separate; cluster_id is only a legacy fallback. - const std::string node_id = - config.io_pattern_cfm.node_id.empty() - ? (config.master_id.empty() ? config.cluster_id - : config.master_id) - : config.io_pattern_cfm.node_id; - auto transport = std::make_shared( - config.io_pattern_cfm.endpoint, node_id, - std::chrono::milliseconds(config.io_pattern_cfm.timeout_ms)); - if (!transport->Authenticate(config.io_pattern_cfm.auth_token)) { - throw std::runtime_error( - "failed to authenticate with configured IO-pattern CFM " - "endpoint " + - config.io_pattern_cfm.endpoint); - } - cfm_rpc_channel = std::make_shared( - std::move(transport), - std::make_shared(), - io_pattern::CfmRpcConfig{ - .timeout = - std::chrono::milliseconds(config.io_pattern_cfm.timeout_ms), - .auth_token = config.io_pattern_cfm.auth_token}); - io_pattern_config.report_sink = - io_pattern::MakeCfmMetricBatchSink(cfm_rpc_channel); - io_pattern_cfm_channel_ = - std::make_shared(cfm_rpc_channel); - } - io_pattern_runtime_ = std::make_shared( io_pattern::IoPatternRuntime::Handlers{ .eviction = @@ -552,9 +506,7 @@ MasterService::MasterService(const MasterServiceConfig& config) }}, std::move(io_pattern_config)); io_pattern_cfm_service_ = std::make_shared( - io_pattern_runtime_, config.io_pattern_cfm.auth_token, - config.io_pattern_cfm.policy_queue_capacity, - config.io_pattern_cfm.producer_auth_token); + io_pattern_runtime_); kv_event_publisher_ = std::make_unique(BuildKvEventConfig(config)); @@ -684,34 +636,6 @@ MasterService::MasterService(const MasterServiceConfig& config) if (vchunk_enabled_ && !vchunk_recovery_pending_) { StartVChunkReaper(); } - - // Start the CFM consumer last. If any preceding initialization throws, - // constructor unwinding must not encounter a joinable std::thread. - if (io_pattern_cfm_channel_) { - io_pattern_cfm_client_ = std::make_unique( - io_pattern_cfm_channel_, - [this](const io_pattern::PolicyCommand& command) { - return io_pattern_runtime_ - ? io_pattern_runtime_->ExecuteCommand(command) - : ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; - }); - io_pattern_cfm_polling_ = true; - io_pattern_cfm_poll_thread_ = std::thread([this] { - while (io_pattern_cfm_polling_.load(std::memory_order_acquire)) { - const auto result = - io_pattern_cfm_client_->PollAndDispatchPolicy(); - std::unique_lock lock(io_pattern_cfm_poll_mutex_); - io_pattern_cfm_poll_cv_.wait_for( - lock, - result == ErrorCode::OK ? std::chrono::milliseconds(100) - : std::chrono::seconds(1), - [this] { - return !io_pattern_cfm_polling_.load( - std::memory_order_acquire); - }); - } - }); - } } tl::expected MasterService::VChunkPutStart( @@ -1726,14 +1650,6 @@ bool MasterService::OwnsVChunkSlot(uint16_t slot) const { } MasterService::~MasterService() { - io_pattern_cfm_polling_.store(false, std::memory_order_release); - io_pattern_cfm_poll_cv_.notify_all(); - if (io_pattern_cfm_poll_thread_.joinable()) { - io_pattern_cfm_poll_thread_.join(); - } - io_pattern_cfm_client_.reset(); - io_pattern_cfm_channel_.reset(); - if (ordered_oplog_writer_) { ordered_oplog_writer_->Stop(); } diff --git a/mooncake-store/src/rpc_service.cpp b/mooncake-store/src/rpc_service.cpp index d8c1666976..070269012f 100644 --- a/mooncake-store/src/rpc_service.cpp +++ b/mooncake-store/src/rpc_service.cpp @@ -2070,11 +2070,7 @@ void RegisterRpcService( server.register_handler<&mooncake::WrappedMasterService::PutStart>( &wrapped_master_service); auto& cfm = wrapped_master_service.CfmRpcEndpoint(); - server.register_handler<&io_pattern::CfmRpcService::Authenticate>(&cfm); server.register_handler<&io_pattern::CfmRpcService::Send>(&cfm); - server.register_handler<&io_pattern::CfmRpcService::Receive>(&cfm); - server.register_handler<&io_pattern::CfmRpcService::Acknowledge>(&cfm); - server.register_handler<&io_pattern::CfmRpcService::EnqueuePolicy>(&cfm); server.register_handler<&mooncake::WrappedMasterService::PutEnd>( &wrapped_master_service); server.register_handler<&mooncake::WrappedMasterService::PutRevoke>( diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index d524bcb2ed..d4b0ca2288 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -113,26 +113,18 @@ class TestCfmChannel final : public CfmChannel { snapshot = value; return send_ok; } - CfmPollResult PollPolicyResult() override { - return policy ? CfmPollResult::Command(*policy, 42) - : CfmPollResult::Empty(); - } - bool AcknowledgePolicy(uint64_t delivery_id, bool success) override { - acknowledged_delivery_id = delivery_id; - acknowledged_success = success; - return acknowledge_ok; + bool SendMetricBatch(const MetricBatch& value) override { + batch = value; + return send_ok; } ErrorCode ExecutePrefetch(const PrefetchPlan& value) override { plan = value; return execute_code; } bool send_ok{true}; - bool acknowledge_ok{true}; - bool acknowledged_success{false}; - uint64_t acknowledged_delivery_id{0}; ErrorCode execute_code{ErrorCode::OK}; IoPatternSnapshot snapshot; - std::optional policy; + MetricBatch batch; PrefetchPlan plan; }; @@ -141,8 +133,8 @@ class FlakyCfmChannel final : public CfmChannel { bool SendSnapshot(const IoPatternSnapshot&) override { return send_failures-- <= 0; } - CfmPollResult PollPolicyResult() override { - return CfmPollResult::Command(PrefetchPlan{}); + bool SendMetricBatch(const MetricBatch&) override { + return send_failures-- <= 0; } ErrorCode ExecutePrefetch(const PrefetchPlan&) override { return ErrorCode::RPC_FAIL; @@ -159,25 +151,7 @@ class TestRpcTransport final : public CfmRpcTransport { last_timeout = timeout; return send_ok; } - CfmReceiveResult Receive(std::string_view method, - std::chrono::milliseconds timeout) override { - last_method = std::string(method); - last_timeout = timeout; - return response ? CfmReceiveResult::Payload(*response) - : CfmReceiveResult::Empty(); - } - bool Acknowledge(uint64_t delivery_id, bool success, - std::chrono::milliseconds timeout) override { - acknowledged_delivery_id = delivery_id; - acknowledged_success = success; - last_timeout = timeout; - return acknowledge_ok; - } bool send_ok{true}; - bool acknowledge_ok{true}; - bool acknowledged_success{false}; - uint64_t acknowledged_delivery_id{0}; - std::optional response; std::string last_method; std::string last_payload; std::chrono::milliseconds last_timeout{0}; @@ -803,35 +777,19 @@ TEST(IoPatternFrameworkTest, CfmClientDelegatesToTransportChannel) { snapshot.generated_at_ns = 42; EXPECT_EQ(client.ReportSnapshot(snapshot), ErrorCode::OK); EXPECT_EQ(channel->snapshot.generated_at_ns, 42); - channel->policy = PrefetchPlan{}; - EXPECT_TRUE(client.PollPolicy().has_value()); EXPECT_EQ(client.ExecutePrefetch(PrefetchPlan{}), ErrorCode::OK); + MetricBatch batch; + batch.inference.push_back(InferenceMetrics{}); + EXPECT_EQ(client.ReportMetricBatch(batch), ErrorCode::OK); + EXPECT_EQ(channel->batch.inference.size(), 1); channel->send_ok = false; EXPECT_EQ(client.ReportSnapshot(snapshot), ErrorCode::RPC_FAIL); + EXPECT_EQ(client.ReportMetricBatch(batch), ErrorCode::RPC_FAIL); CfmClientImpl unavailable(nullptr); EXPECT_EQ(unavailable.ReportSnapshot(snapshot), ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); -} - -TEST(IoPatternFrameworkTest, CfmClientDispatchesReceivedPolicyCommands) { - auto channel = std::make_shared(); - int dispatched = 0; - CfmClientImpl client(channel, [&](const PolicyCommand& command) { - EXPECT_TRUE(std::holds_alternative(command)); - ++dispatched; - return ErrorCode::OK; - }); - - EXPECT_EQ(client.ReceivePolicy(PolicyCommand{PrefetchPlan{}}), ErrorCode::OK); - EXPECT_EQ(dispatched, 1); - channel->policy = PolicyCommand{AdmissionResult{}}; - EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); - EXPECT_EQ(dispatched, 2); - EXPECT_EQ(channel->acknowledged_delivery_id, 42); - EXPECT_TRUE(channel->acknowledged_success); - channel->policy.reset(); - EXPECT_EQ(client.PollAndDispatchPolicy(), ErrorCode::OK); - EXPECT_EQ(dispatched, 2); + EXPECT_EQ(unavailable.ReportMetricBatch(batch), + ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); } TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { @@ -849,15 +807,14 @@ TEST(IoPatternFrameworkTest, ResilientChannelRetriesAndTracksDegrade) { EXPECT_TRUE(channel.degraded()); } -TEST(IoPatternFrameworkTest, EmptyPolicyPollKeepsChannelHealthy) { - auto idle = std::make_shared(); - ResilientCfmChannel channel( - idle, CfmRetryConfig{.max_retries = 2, .degrade_after_failures = 2}); - - EXPECT_FALSE(channel.PollPolicy().has_value()); - EXPECT_FALSE(channel.PollPolicy().has_value()); - EXPECT_EQ(channel.consecutive_failures(), 0); +TEST(IoPatternFrameworkTest, ResilientChannelRecoversAfterSuccess) { + auto flaky = std::make_shared(); + ResilientCfmChannel channel(flaky, CfmRetryConfig{.max_retries = 2, + .degrade_after_failures = 1}); + MetricBatch batch; + EXPECT_TRUE(channel.SendMetricBatch(batch)); EXPECT_FALSE(channel.degraded()); + EXPECT_EQ(channel.consecutive_failures(), 0); } TEST(IoPatternFrameworkTest, ResilientAnalyzerFallsBackAfterFailure) { @@ -877,9 +834,8 @@ TEST(IoPatternFrameworkTest, RpcChannelUsesCodecTransportAndTimeout) { EXPECT_EQ(transport->last_method, "report_snapshot"); EXPECT_EQ(transport->last_payload, "snapshot"); EXPECT_EQ(transport->last_timeout, std::chrono::milliseconds(25)); - transport->response = "policy"; - EXPECT_TRUE(channel.PollPolicy().has_value()); EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::OK); + EXPECT_EQ(transport->last_method, "execute_prefetch"); auto rpc_channel = std::make_shared(transport, codec); IoPatternReporter reporter(2, MakeCfmMetricBatchSink(rpc_channel)); reporter.Enqueue(InferenceMetrics{}); @@ -888,6 +844,7 @@ TEST(IoPatternFrameworkTest, RpcChannelUsesCodecTransportAndTimeout) { EXPECT_EQ(transport->last_payload, "batch"); transport->send_ok = false; EXPECT_EQ(channel.ExecutePrefetch({}), ErrorCode::RPC_TIMEOUT); + EXPECT_FALSE(channel.SendSnapshot({})); } TEST(IoPatternFrameworkTest, BinaryCfmCodecRoundTripsAllPolicyCommands) { @@ -949,29 +906,23 @@ TEST(IoPatternFrameworkTest, BinaryCfmCodecRoundTripsAllPolicyCommands) { EXPECT_TRUE(decoded_batch->accesses.front().is_hit); } -TEST(IoPatternFrameworkTest, InProcessCfmTransportAuthenticatesAndDispatches) { +TEST(IoPatternFrameworkTest, InProcessCfmTransportDispatchesReports) { CfmBinaryCodec codec; bool received_snapshot = false; auto transport = std::make_shared( - "shared-secret", [&received_snapshot](std::string_view method, - std::string_view) { + [&received_snapshot](std::string_view method, std::string_view) { received_snapshot = method == "report_snapshot"; return received_snapshot; }); - CfmRpcChannel authorized(transport, std::make_shared(), - {.auth_token = "shared-secret"}); - EXPECT_TRUE(authorized.SendSnapshot({})); + CfmRpcChannel channel(transport, std::make_shared()); + EXPECT_TRUE(channel.SendSnapshot({})); EXPECT_TRUE(received_snapshot); - transport->EnqueuePolicy(codec.EncodePolicy( - AdmissionResult{.object = {TenantId("tenant"), "key"}, - .decision = AdmissionDecision::kAdmit})); - ASSERT_TRUE(authorized.PollPolicy().has_value()); - - auto rejected = std::make_shared("secret"); - CfmRpcChannel unauthorized(rejected, std::make_shared(), - {.auth_token = "wrong"}); - EXPECT_FALSE(unauthorized.SendSnapshot({})); + // Without a bound handler the transport has no receiver; the embedded + // receiver path is exercised through CfmService/CfmIngress instead. + auto unbound = std::make_shared(); + CfmRpcChannel unbound_channel(unbound, std::make_shared()); + EXPECT_TRUE(unbound_channel.SendSnapshot({})); } TEST(IoPatternFrameworkTest, InProcessTransportDoesNotHoldLockAcrossHandler) { @@ -979,12 +930,11 @@ TEST(IoPatternFrameworkTest, InProcessTransportDoesNotHoldLockAcrossHandler) { std::promise release_handler; auto release = release_handler.get_future().share(); auto transport = std::make_shared( - "shared-secret", [&](std::string_view, std::string_view) { + [&](std::string_view, std::string_view) { handler_entered.set_value(); release.wait(); return true; }); - ASSERT_TRUE(transport->Authenticate("shared-secret")); std::thread sender([&] { EXPECT_TRUE(transport->Send("report_snapshot", {}, @@ -997,12 +947,13 @@ TEST(IoPatternFrameworkTest, InProcessTransportDoesNotHoldLockAcrossHandler) { FAIL() << "send handler did not start"; return; } - auto enqueue = std::async(std::launch::async, [&] { - transport->EnqueuePolicy("policy"); - return true; + // A concurrent Send must not deadlock on the transport mutex while the + // first handler is still executing. + std::thread second([&] { + EXPECT_TRUE(transport->Send("report_snapshot", {}, + std::chrono::milliseconds(10))); }); - EXPECT_EQ(enqueue.wait_for(std::chrono::milliseconds(100)), - std::future_status::ready); + second.join(); release_handler.set_value(); sender.join(); } @@ -1035,7 +986,7 @@ TEST(IoPatternFrameworkTest, CfmIngressFeedsRuntimeFromMetricBatches) { EXPECT_EQ(snapshot.keys.front().access_count_window, 1); } -TEST(IoPatternFrameworkTest, CfmServiceAuthenticatesAndBoundsPolicyQueues) { +TEST(IoPatternFrameworkTest, CfmServiceMergesReportsAndExecutesLocally) { int admissions = 0; auto runtime = std::make_shared( IoPatternRuntime::Handlers{ @@ -1045,71 +996,66 @@ TEST(IoPatternFrameworkTest, CfmServiceAuthenticatesAndBoundsPolicyQueues) { ++admissions; return ErrorCode::OK; }}); - CfmService service(runtime, "secret", 1, "producer"); + CfmService service(runtime); CfmBinaryCodec codec; - EXPECT_FALSE(service.Authenticate("wrong")); - EXPECT_TRUE(service.Authenticate("secret")); - EXPECT_FALSE(service.EnqueuePolicy("node-a", "first", "secret")); + // Reports are merged into the single local runtime: no per-node runtime, + // no policy queue and no credential gate. + MetricBatch batch; + batch.inference.push_back( + InferenceMetrics{.object = {TenantId("tenant"), "key"}, + .session_id = "session", + .token_count = 32}); + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + const auto merged = service.Snapshot(); + ASSERT_EQ(merged.keys.size(), 1); + EXPECT_EQ(merged.keys.front().object.key, "key"); + + // A delivered policy command executes through the local runtime handlers. const auto admission = codec.EncodePolicy(AdmissionResult{ .object = {TenantId("tenant"), "key"}, .target_tier = CacheTier::kL1Host, .decision = AdmissionDecision::kAdmit}); - const auto second = codec.EncodePolicy(PrefetchPlan{}); - EXPECT_FALSE(service.EnqueuePolicy("node-a", "malformed", "producer")); - EXPECT_TRUE(service.EnqueuePolicy("node-a", admission, "producer")); - EXPECT_FALSE(service.EnqueuePolicy("node-a", second, "producer")); - const auto delivery = service.PollPolicy("node-a", "secret"); - ASSERT_TRUE(delivery.has_value()); - EXPECT_EQ(delivery->second, admission); - EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, false, - "secret")); - EXPECT_TRUE(service.PollPolicy("node-a", "secret").has_value()); - EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, true, - "secret")); - EXPECT_FALSE(service.PollPolicy("node-a", "secret").has_value()); - - EXPECT_FALSE(service.Send("", "execute_policy", admission, "secret")); - EXPECT_FALSE( - service.Send("node-a", "execute_policy", admission, "secret")); - EXPECT_TRUE( - service.Send("node-a", "execute_policy", admission, "producer")); + ASSERT_TRUE(service.Send("execute_policy", admission)); + EXPECT_EQ(admissions, 1); + EXPECT_FALSE(service.Send("execute_policy", "malformed")); EXPECT_EQ(admissions, 1); } -TEST(IoPatternFrameworkTest, CfmAggregatesMetricsBySubmasterNode) { +TEST(IoPatternFrameworkTest, CfmServiceMergesAllReportsIntoLocalRuntime) { auto runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); - CfmService service(runtime, "secret", 8, "producer"); + CfmService service(runtime); CfmBinaryCodec codec; - MetricBatch submaster_a; - submaster_a.accesses.push_back( + MetricBatch first; + first.accesses.push_back( AccessRecord{.object = {TenantId("tenant"), "key-a"}, .block_size = 64, .tier = CacheTier::kL1Host, .is_hit = true}); - MetricBatch submaster_b; - submaster_b.accesses.push_back( + MetricBatch second; + second.accesses.push_back( AccessRecord{.object = {TenantId("tenant"), "key-b"}, .block_size = 128, .tier = CacheTier::kL1Host, .is_hit = true}); - ASSERT_TRUE(service.Send("submaster-a", "report_metric_batch", - codec.EncodeMetricBatch(submaster_a), "secret")); - ASSERT_TRUE(service.Send("submaster-b", "report_metric_batch", - codec.EncodeMetricBatch(submaster_b), "secret")); + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(first))); + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(second))); - const auto snapshot_a = service.SnapshotForNode("submaster-a"); - const auto snapshot_b = service.SnapshotForNode("submaster-b"); - ASSERT_EQ(snapshot_a.keys.size(), 1); - ASSERT_EQ(snapshot_b.keys.size(), 1); - EXPECT_EQ(snapshot_a.keys.front().object.key, "key-a"); - EXPECT_EQ(snapshot_b.keys.front().object.key, "key-b"); + // Ownership-addressed reports all land on the receiving SubMaster, whose + // CFM component owns a single local runtime. + const auto snapshot = service.Snapshot(); + ASSERT_EQ(snapshot.keys.size(), 2); + EXPECT_EQ(snapshot.keys[0].object.key, "key-a"); + EXPECT_EQ(snapshot.keys[1].object.key, "key-b"); } TEST(IoPatternFrameworkTest, CoroRpcCfmTransportRunsTheProductionWirePath) { @@ -1118,80 +1064,57 @@ TEST(IoPatternFrameworkTest, CoroRpcCfmTransportRunsTheProductionWirePath) { .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); - auto service = - std::make_shared(runtime, "secret", 2, "producer"); + auto service = std::make_shared(runtime); CfmRpcService endpoint(service); coro_rpc::coro_rpc_server server(1, 0, "127.0.0.1"); - server.register_handler<&CfmRpcService::Authenticate>(&endpoint); server.register_handler<&CfmRpcService::Send>(&endpoint); - server.register_handler<&CfmRpcService::Receive>(&endpoint); - server.register_handler<&CfmRpcService::Acknowledge>(&endpoint); - server.register_handler<&CfmRpcService::EnqueuePolicy>(&endpoint); ASSERT_FALSE(server.async_start().hasResult()); - const auto rejected_poll = - endpoint.Receive("poll_policy", "node-a", "wrong"); - EXPECT_FALSE(rejected_poll.first); - const auto empty_poll = endpoint.Receive("poll_policy", "node-a", "secret"); - EXPECT_TRUE(empty_poll.first); - EXPECT_FALSE(empty_poll.second.has_value()); - CoroRpcCfmTransport transport( - "127.0.0.1:" + std::to_string(server.port()), "node-a", + "127.0.0.1:" + std::to_string(server.port()), std::chrono::milliseconds(500)); - EXPECT_FALSE(transport.Authenticate("wrong")); - ASSERT_TRUE(transport.Authenticate("secret")); CfmBinaryCodec codec; MetricBatch batch; batch.accesses.push_back( AccessRecord{.object = {TenantId("tenant"), "remote-key"}, .is_hit = true}); - batch.storage.push_back(StorageMetric{.source_id = "spoofed", + batch.storage.push_back(StorageMetric{.source_id = "reporter", .tier = CacheTier::kL1Host, .memory_used_ratio = 0.75F}); EXPECT_TRUE(transport.Send("report_metric_batch", codec.EncodeMetricBatch(batch), std::chrono::milliseconds(500))); - const auto snapshot = service->SnapshotForNode("node-a"); + // No authentication: the SubMaster merges the report into its local + // runtime and treats the transport source as the metric origin. + const auto snapshot = service->Snapshot(); ASSERT_EQ(snapshot.keys.size(), 1); ASSERT_EQ(snapshot.storage.size(), 1); EXPECT_EQ(snapshot.keys.front().object.key, "remote-key"); - EXPECT_EQ(snapshot.storage.front().source_id, "node-a"); - - const auto policy = codec.EncodePolicy(PrefetchPlan{}); - CoroRpcCfmTransport producer( - "127.0.0.1:" + std::to_string(server.port()), "producer", - std::chrono::milliseconds(500)); - ASSERT_TRUE(producer.Authenticate("producer")); - EXPECT_TRUE(producer.EnqueuePolicy("node-a", policy, - std::chrono::milliseconds(500))); - const auto received = - transport.Receive("poll_policy", std::chrono::milliseconds(500)); - EXPECT_EQ(received.status, CfmReceiveResult::Status::kPayload); - EXPECT_EQ(received.payload, policy); - EXPECT_NE(received.delivery_id, 0); - EXPECT_TRUE(transport.Acknowledge(received.delivery_id, false, - std::chrono::milliseconds(500))); - const auto redelivered = - transport.Receive("poll_policy", std::chrono::milliseconds(500)); - EXPECT_EQ(redelivered.delivery_id, received.delivery_id); - EXPECT_EQ(redelivered.payload, policy); - EXPECT_TRUE(transport.Acknowledge(redelivered.delivery_id, true, - std::chrono::milliseconds(500))); - EXPECT_EQ(transport.Receive("poll_policy", std::chrono::milliseconds(500)) - .status, - CfmReceiveResult::Status::kEmpty); + EXPECT_EQ(snapshot.storage.front().source_id, "reporter"); + + // Snapshot reports follow the same unauthenticated path. + IoPatternSnapshot snapshot_report; + snapshot_report.keys.push_back( + KeyMetrics{.object = {TenantId("tenant"), "snap-key"}}); + EXPECT_TRUE(transport.Send("report_snapshot", + codec.EncodeSnapshot(snapshot_report), + std::chrono::milliseconds(500))); + EXPECT_EQ(service->Snapshot().keys.back().object.key, "snap-key"); server.stop(); } -TEST(IoPatternFrameworkTest, CfmProducesNodePolicyFromHighWatermarkReport) { +TEST(IoPatternFrameworkTest, LocalCfmExecutesEvictionOnHighWatermarkSnapshot) { + // High-watermark policy now runs in the SubMaster's own runtime rather + // than in a separate central CFM process. Reports feed that runtime; a + // storage observation at >= 0.90 memory ratio then triggers a local + // eviction plan executed through the SubMaster handlers. auto runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); - CfmService service(runtime, "node-secret", 8, "producer-secret"); + CfmService service(runtime); CfmBinaryCodec codec; MetricBatch batch; batch.accesses.push_back( @@ -1199,80 +1122,16 @@ TEST(IoPatternFrameworkTest, CfmProducesNodePolicyFromHighWatermarkReport) { .block_size = 1024, .tier = CacheTier::kL1Host, .is_hit = false}); - batch.storage.push_back(StorageMetric{.tier = CacheTier::kL1Host, - .used_bytes = 950, - .capacity_bytes = 1000, - .memory_used_ratio = 0.95F}); - ASSERT_TRUE(service.Send("node-a", "report_metric_batch", - codec.EncodeMetricBatch(batch), "node-secret")); - - std::optional> delivery; - for (size_t attempt = 0; attempt < 100 && !delivery; ++attempt) { - delivery = service.PollPolicy("node-a", "node-secret"); - if (!delivery) std::this_thread::sleep_for(std::chrono::milliseconds(5)); - } - ASSERT_TRUE(delivery.has_value()); - const auto command = codec.DecodePolicy(delivery->second); - ASSERT_TRUE(command.has_value()); - const auto* eviction = std::get_if(&*command); - ASSERT_NE(eviction, nullptr); - ASSERT_EQ(eviction->candidates.size(), 1); - EXPECT_EQ(eviction->candidates.front().object.key, "cold-key"); - EXPECT_TRUE(service.AcknowledgePolicy("node-a", delivery->first, true, - "node-secret")); -} - -TEST(IoPatternFrameworkTest, CfmPolicyDoesNotCrossSubmasterKeySets) { - auto runtime = std::make_shared( - IoPatternRuntime::Handlers{ - .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, - .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, - .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); - CfmService service(runtime, "node-secret", 8, "producer-secret"); - CfmBinaryCodec codec; - - MetricBatch submaster_b; - submaster_b.accesses.push_back( - AccessRecord{.object = {TenantId("tenant"), "key-b"}, - .block_size = 4096, - .tier = CacheTier::kL1Host, - .is_hit = false}); - ASSERT_TRUE(service.Send("submaster-b", "report_metric_batch", - codec.EncodeMetricBatch(submaster_b), - "node-secret")); - - MetricBatch submaster_a; - submaster_a.accesses.push_back( - AccessRecord{.object = {TenantId("tenant"), "key-a"}, - .block_size = 1024, - .tier = CacheTier::kL1Host, - .is_hit = false}); - submaster_a.storage.push_back( - StorageMetric{.tier = CacheTier::kL1Host, - .used_bytes = 950, - .capacity_bytes = 1000, - .memory_used_ratio = 0.95F}); - ASSERT_TRUE(service.Send("submaster-a", "report_metric_batch", - codec.EncodeMetricBatch(submaster_a), - "node-secret")); - ASSERT_TRUE(service.WaitForPolicyIdle(std::chrono::milliseconds(500))); - - std::optional> delivery; - for (size_t attempt = 0; attempt < 100 && !delivery; ++attempt) { - delivery = service.PollPolicy("submaster-a", "node-secret"); - if (!delivery) std::this_thread::sleep_for(std::chrono::milliseconds(5)); - } - ASSERT_TRUE(delivery.has_value()); - EXPECT_GE(service.ObservabilityForNode("submaster-a").policy_decisions, - 1); - const auto command = codec.DecodePolicy(delivery->second); - ASSERT_TRUE(command.has_value()); - const auto* eviction = std::get_if(&*command); - ASSERT_NE(eviction, nullptr); - ASSERT_FALSE(eviction->candidates.empty()); - for (const auto& candidate : eviction->candidates) { - EXPECT_EQ(candidate.object.key, "key-a"); - } + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + ASSERT_EQ(runtime->Snapshot().keys.size(), 1); + + // Eviction is local: Execute() plans against the merged snapshot and + // invokes the storage handler directly (no policy delivery round trip). + const auto status = runtime->Execute(CacheTier::kL1Host, 1024, + TraceHistory{}); + EXPECT_EQ(status.eviction, ErrorCode::OK); + EXPECT_GE(runtime->ObservabilitySnapshot().policy_decisions, 1); } TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { @@ -1625,12 +1484,10 @@ TEST(IoPatternFrameworkTest, RuntimeExecutesCfmCommandsThroughStorageHandlers) { ++admissions; return ErrorCode::OK; }}); - CfmClientImpl client( - std::make_shared(), - [&runtime](const PolicyCommand& command) { - return runtime.ExecuteCommand(command); - }); - EXPECT_EQ(client.ReceivePolicy( + // In the embedded architecture the receiving SubMaster executes delivered + // policy commands through its own runtime; there is no client-side + // dispatch loop any more. + EXPECT_EQ(runtime.ExecuteCommand( AdmissionResult{.object = {TenantId("tenant"), "key"}, .decision = AdmissionDecision::kAdmit}), ErrorCode::OK); @@ -1658,5 +1515,56 @@ TEST(IoPatternFrameworkTest, RuntimeSchedulesAdmissionOffTheProducerPath) { EXPECT_EQ(result.get().object, access.object); } +TEST(IoPatternFrameworkTest, OwnershipClientBucketsReportsByResolvedOwner) { + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + auto service = std::make_shared(runtime); + CfmRpcService endpoint(service); + coro_rpc::coro_rpc_server server(1, 0, "127.0.0.1"); + server.register_handler<&CfmRpcService::Send>(&endpoint); + ASSERT_FALSE(server.async_start().hasResult()); + const std::string owner_endpoint = + "127.0.0.1:" + std::to_string(server.port()); + + // Only keys starting with "owned/" resolve to the single SubMaster under + // test; "foreign/" and empty resolver results are dropped. + CfmOwnershipClient client( + [&](const TenantId&, const std::string& key) + -> std::optional { + return key.rfind("owned/", 0) == 0 + ? std::optional(owner_endpoint) + : std::nullopt; + }, + std::chrono::milliseconds(500)); + + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "owned/key-a"}, + .block_size = 64, + .tier = CacheTier::kL1Host, + .is_hit = true}); + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "owned/key-b"}, + .block_size = 64, + .tier = CacheTier::kL1Host, + .is_hit = true}); + batch.inference.push_back(InferenceMetrics{ + .object = {TenantId("tenant"), "foreign/key"}, .session_id = "s"}); + + EXPECT_EQ(client.ReportMetricBatch(batch), ErrorCode::OK); + EXPECT_EQ(client.dropped_observations(), 1); + + // Both owned observations were aggregated into one batch for the resolved + // owner and merged into that SubMaster's local runtime. + const auto snapshot = service->Snapshot(); + ASSERT_EQ(snapshot.keys.size(), 2); + EXPECT_EQ(snapshot.keys[0].object.key, "owned/key-a"); + EXPECT_EQ(snapshot.keys[1].object.key, "owned/key-b"); + server.stop(); +} + } // namespace } // namespace mooncake::io_pattern diff --git a/mooncake-store/tests/master_service_config_test.cpp b/mooncake-store/tests/master_service_config_test.cpp index ed50c77325..4b69bcf5f0 100644 --- a/mooncake-store/tests/master_service_config_test.cpp +++ b/mooncake-store/tests/master_service_config_test.cpp @@ -46,26 +46,4 @@ TEST(MasterServiceConfigTest, OplogBatchMaxEntriesBuilderOverrideRespected) { EXPECT_EQ(17u, config.oplog_batch_max_entries); } -TEST(MasterServiceConfigTest, IoPatternCfmPropagatesToServingConfig) { - MasterConfig master_config{}; - master_config.io_pattern_cfm = {.endpoint = "cfm.example:50051", - .node_id = "master-a", - .auth_token = "secret", - .producer_auth_token = "producer-secret", - .timeout_ms = 750, - .policy_queue_capacity = 32}; - - MasterServiceSupervisorConfig supervisor_config(master_config); - WrappedMasterServiceConfig wrapped_config(supervisor_config, 1); - MasterServiceConfig service_config(wrapped_config); - - EXPECT_EQ(service_config.io_pattern_cfm.endpoint, "cfm.example:50051"); - EXPECT_EQ(service_config.io_pattern_cfm.node_id, "master-a"); - EXPECT_EQ(service_config.io_pattern_cfm.auth_token, "secret"); - EXPECT_EQ(service_config.io_pattern_cfm.producer_auth_token, - "producer-secret"); - EXPECT_EQ(service_config.io_pattern_cfm.timeout_ms, 750); - EXPECT_EQ(service_config.io_pattern_cfm.policy_queue_capacity, 32); -} - } // namespace mooncake::test diff --git a/mooncake-wheel/mooncake/io_pattern_bridge.py b/mooncake-wheel/mooncake/io_pattern_bridge.py index cd0c46d729..bdce2f7da0 100644 --- a/mooncake-wheel/mooncake/io_pattern_bridge.py +++ b/mooncake-wheel/mooncake/io_pattern_bridge.py @@ -1,5 +1,12 @@ """Framework-neutral, non-blocking CFM metric bridges. +CFM is an embedded component of every SubMaster and is reached over the +SubMaster's ordinary Mooncake RPC endpoint; there is no separate CFM Master +endpoint and no auth token to obtain. A deployment wires ``report`` to a +client that resolves the owning SubMaster of each observed key through the CVM +mapping, aggregates observations per SubMaster, and delivers them to that +SubMaster's CFM receiver. + The vLLM connector accepts these objects through ``vllm_config``. SGLang's HiCache integration can instantiate :class:`SglangHiCacheIoPatternBridge` at its request-finished and prefix-match hooks without depending on vLLM. @@ -19,8 +26,9 @@ class BatchedIoPatternBridge: """Bounded asynchronous bridge to a CFM metric reporter. - ``report`` receives complete records (for example, a CFM RPC client - method). Back pressure drops metrics instead of delaying inference. + ``report`` receives complete records (for example, a CVM ownership-aware + CFM client method such as ``CfmOwnershipClient.ReportMetricBatch``). + Back pressure drops metrics instead of delaying inference. """ def __init__(self, report: MetricSink, capacity: int = 4096) -> None: From 15e1ee95043e90e1f0dc0fc1dd22c7720ae2b130 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 8 Sep 2026 11:31:36 +0800 Subject: [PATCH 10/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/io_pattern/client.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mooncake-store/include/io_pattern/client.h b/mooncake-store/include/io_pattern/client.h index 150c980797..bb33442798 100644 --- a/mooncake-store/include/io_pattern/client.h +++ b/mooncake-store/include/io_pattern/client.h @@ -1,7 +1,8 @@ #pragma once #include "../types.h" -#include "io_pattern/types.h" +#include "reporter.h" +#include "types.h" namespace mooncake::io_pattern { From fed6d9ec9facdd2ff6f758e09c70792e5133089c Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 8 Sep 2026 17:10:23 +0800 Subject: [PATCH 11/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/benchmarks/CMakeLists.txt | 3 + .../benchmarks/cfm_client_bench.cpp | 235 ++++++++++++++++-- 2 files changed, 214 insertions(+), 24 deletions(-) diff --git a/mooncake-store/benchmarks/CMakeLists.txt b/mooncake-store/benchmarks/CMakeLists.txt index b30d7cba8f..f851e0b2be 100644 --- a/mooncake-store/benchmarks/CMakeLists.txt +++ b/mooncake-store/benchmarks/CMakeLists.txt @@ -44,6 +44,9 @@ add_executable(cfm_client_bench cfm_client_bench.cpp) target_link_libraries( cfm_client_bench PRIVATE mooncake_store transfer_engine asio_shared gflags::gflags glog::glog pthread) +if(STORE_USE_ETCD) + target_link_libraries(cfm_client_bench PRIVATE ${ETCD_WRAPPER_LIB}) +endif() # Benchmark for vLLM Store Connector path # Triggers: batch_put_from_multi_buffers / batchIsExist / diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index ff6d9f3171..066a1b3368 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -8,44 +8,61 @@ // locally on the keys it owns (high-watermark eviction in the data path, // trace-derived prefetch through the same storage-safe handlers). // -// This benchmark exercises that path in two modes: +// This benchmark exercises that path in the following modes: // - embedded (default): an in-process SubMaster runtime plays the owning // CFM component. Reports are delivered in-process and policy is evaluated // and executed locally, so the benchmark prints both report latency and // the resulting eviction/prefetch/admission handler activity. -// - remote (--cfm_endpoint=host:port): reports go over coro_rpc to a real -// SubMaster CFM receiver; the receiving side is not observable here, so -// only client-side latency is reported. +// - remote (--cfm_endpoint=host:port): reports go over coro_rpc to a single +// SubMaster CFM receiver. +// - remote via etcd (--cfm_endpoint=etcd://connstring): resolves the cluster +// like a Store client, then buckets each key to its owning SubMaster. +// The receiving side is not observable here, so only client-side latency +// is reported. #include #include #include #include #include +#include +#include #include #include +#include #include #include #include #include +#include #include #include #include +#include #include #include #include "gflags/gflags.h" #include "glog/logging.h" +#include "cvm/cvm_types.h" +#include "cvm/etcd_view_store.h" +#include "cvm/slot_hash.h" +#include "io_pattern/cfm_ownership_client.h" #include "io_pattern/cfm_protocol.h" #include "io_pattern/cfm_service.h" #include "io_pattern/rpc_transport.h" #include "io_pattern/runtime.h" +#include "types.h" +#ifdef STORE_USE_ETCD +#include "etcd_helper.h" +#endif namespace { using Clock = std::chrono::steady_clock; using mooncake::ErrorCode; using mooncake::TenantId; +using mooncake::toString; using namespace mooncake::io_pattern; DEFINE_uint64(requests, 20, "Number of vLLM-style inference requests"); @@ -68,8 +85,14 @@ DEFINE_string(tenant, "vllm-benchmark", "Tenant id"); DEFINE_string(node_id, "vllm-submaster-0", "CFM node/submaster id that owns the reported keys"); DEFINE_string(cfm_endpoint, "", - "Remote SubMaster coro_rpc endpoint (host:port); empty uses an " - "embedded in-process SubMaster CFM component"); + "Remote SubMaster endpoint: either host:port or an HA entry " + "(e.g. etcd://host:2379;host2:2379). Empty uses an embedded " + "in-process SubMaster CFM component"); +DEFINE_string(cfm_cluster_namespace, "", + "CVM cluster namespace for etcd entry resolution; defaults to " + "MC_STORE_CLUSTER_ID or mooncake_cluster (same rule as the " + "etcd leader coordinator). When the cluster was started with a " + "non-default cluster_id, pass the same value here"); uint64_t SteadyNowNs() { return static_cast( @@ -114,6 +137,160 @@ class EmbeddedCfmTransport final : public CfmRpcTransport { std::shared_ptr service_; }; +#ifdef STORE_USE_ETCD +// Resolves the CVM cluster namespace used by --cfm_endpoint when it carries an +// etcd:// backend. Mirrors EtcdLeaderCoordinator::ResolveClusterNamespace: +// explicit flag wins, then MC_STORE_CLUSTER_ID, then mooncake_cluster. +std::string ResolveCvmNamespace() { + if (!FLAGS_cfm_cluster_namespace.empty()) { + return FLAGS_cfm_cluster_namespace; + } + const char* env_cluster_id = std::getenv("MC_STORE_CLUSTER_ID"); + if (env_cluster_id != nullptr && std::strlen(env_cluster_id) > 0) { + return env_cluster_id; + } + return mooncake::DEFAULT_CLUSTER_ID; +} + +// Key that stores the leader address for single-leader HA. +// Mirrors EtcdLeaderCoordinator::BuildMasterViewKey. +std::string BuildMasterViewKey(const std::string& cluster_namespace) { + std::string normalized = cluster_namespace; + if (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + return "mooncake-store/" + normalized + "/master_view"; +} +#endif // STORE_USE_ETCD + +// If --cfm_endpoint names a single SubMaster directly ("host:port") this +// returns an ownership resolver that routes every key to it. If it is an +// etcd:// entry, it resolves the cluster like a Store client: a present +// leader master_view yields a single target; otherwise the CVM +// /cvm//masters registry plus slot ownership is used to bucket keys to +// their owning SubMaster. Returns an empty resolver on any resolution failure +// (the caller aborts instead of hanging). +SubmasterEndpointResolver ResolveCfmEndpointOwnership() { + const std::string entry = FLAGS_cfm_endpoint; + const size_t scheme_pos = entry.find("://"); + if (scheme_pos == std::string::npos) { + // Plain host:port -> every observed key belongs to this single + // SubMaster (the equivalent of the old single-endpoint remote mode). + const std::string endpoint = entry; + return [endpoint](const TenantId&, const std::string&) + -> std::optional { return endpoint; }; + } +#ifndef STORE_USE_ETCD + LOG(FATAL) << "cfm_endpoint entry '" << entry + << "' requires a build with STORE_USE_ETCD; pass host:port " + "instead"; + return {}; +#else + const std::string scheme = entry.substr(0, scheme_pos); + if (scheme != "etcd") { + LOG(FATAL) << "cfm_endpoint backend '" << scheme + << "' is not supported; use host:port or etcd://connstring"; + return {}; + } + const std::string connstring = entry.substr(scheme_pos + 3); + const std::string cluster_namespace = ResolveCvmNamespace(); + + ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(connstring); + if (err != ErrorCode::OK) { + LOG(FATAL) << "cfm_endpoint: failed to connect etcd '" << connstring + << "': " << toString(err); + return {}; + } + + // Single-leader HA: leader master_view holds the master address. + const std::string view_key = BuildMasterViewKey(cluster_namespace); + std::string leader_address; + mooncake::EtcdRevisionId revision = 0; + err = EtcdHelper::Get(view_key.data(), view_key.size(), leader_address, + revision); + if (err == ErrorCode::OK && !leader_address.empty()) { + LOG(INFO) << "cfm_endpoint: single-leader HA via " << view_key + << " -> " << leader_address; + const std::string endpoint = std::move(leader_address); + return [endpoint](const TenantId&, const std::string&) + -> std::optional { return endpoint; }; + } + if (err != ErrorCode::OK && err != ErrorCode::ETCD_KEY_NOT_EXIST) { + LOG(FATAL) << "cfm_endpoint: failed to read " << view_key << ": " + << toString(err); + return {}; + } + + // CVM multi-submaster: masters registry + slot ownership. + std::vector masters; + mooncake::ViewVersionId version = 0; + err = cvm::EtcdViewStore::LoadAllMasters(cluster_namespace, masters, + version); + if (err != ErrorCode::OK) { + LOG(FATAL) << "cfm_endpoint: LoadAllMasters failed for namespace '" + << cluster_namespace << "': " << toString(err); + return {}; + } + + std::map address_by_master; // id -> host:port + std::vector primary_ids; + for (const auto& reg : masters) { + if (reg.role == static_cast(cvm::MasterRole::kPrimary) && + !reg.address.empty()) { + address_by_master[reg.master_id] = reg.address; + primary_ids.push_back(reg.master_id); + } + } + if (primary_ids.empty()) { + LOG(FATAL) << "cfm_endpoint: no primary SubMaster registered under " + "/cvm/" + << cluster_namespace << "/masters"; + return {}; + } + std::sort(primary_ids.begin(), primary_ids.end()); + + // Prefer the authoritative slot owner table published by CvmController; + // fall back to the consistent-hash ring used by the masters themselves. + std::unordered_map owner_by_slot; + std::vector slot_owners; + const ErrorCode slot_err = cvm::EtcdViewStore::LoadAllSlotOwners( + cluster_namespace, slot_owners, version); + if (slot_err == ErrorCode::OK) { + for (const auto& owner : slot_owners) { + if (owner.state == static_cast(cvm::SlotState::kStable) && + !owner.primary_master_id.empty()) { + owner_by_slot[owner.slot] = owner.primary_master_id; + } + } + } + const bool has_owner_table = !owner_by_slot.empty(); + LOG(INFO) << "cfm_endpoint: CVM namespace '" << cluster_namespace + << "' has " << primary_ids.size() << " primary submaster(s), " + << (has_owner_table ? owner_by_slot.size() : 0) + << " slot owners" + << (has_owner_table ? "" : " (falling back to hash ring)"); + + return [address_by_master = std::move(address_by_master), + primary_ids = std::move(primary_ids), + owner_by_slot = std::move(owner_by_slot), has_owner_table]( + const TenantId& tenant, + const std::string& key) -> std::optional { + const uint16_t slot = cvm::KeySlot(tenant, key); + std::string owner; + if (has_owner_table) { + const auto it = owner_by_slot.find(slot); + if (it != owner_by_slot.end()) owner = it->second; + } + if (owner.empty()) { + owner = cvm::ResolveSlotOwnerOnRing(primary_ids, slot); + } + const auto address = address_by_master.find(owner); + if (address == address_by_master.end()) return std::nullopt; + return address->second; + }; +#endif +} + class LatencyStats final { public: void Record(double value_us) { values_us_.push_back(value_us); } @@ -328,12 +505,15 @@ int main(int argc, char* argv[]) { std::atomic prefetch_commands{0}; std::atomic admission_commands{0}; - // The SubMaster-side CFM component (embedded mode) or the target of the - // remote coro_rpc receiver. Its runtime aggregates whatever is reported. + // The SubMaster-side CFM component (embedded mode) or the ownership + // resolver used by the remote reporter. std::shared_ptr embedded_service; - std::shared_ptr transport; std::shared_ptr cfm_runtime; + std::shared_ptr ownership_client; + std::shared_ptr embedded_channel; + std::string deployment_description; if (FLAGS_cfm_endpoint.empty()) { + deployment_description = "embedded SubMaster (local CFM)"; cfm_runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [&eviction_commands](const EvictionPlan&) { @@ -349,27 +529,31 @@ int main(int argc, char* argv[]) { return ErrorCode::OK; }}); embedded_service = std::make_shared(cfm_runtime); - transport = std::make_shared(embedded_service); + auto transport = + std::make_shared(embedded_service); + embedded_channel = std::make_shared( + std::move(transport), std::make_shared(), + CfmRpcConfig{.timeout = std::chrono::milliseconds(500)}); } else { - transport = std::make_shared( - FLAGS_cfm_endpoint, std::chrono::milliseconds(500)); + const auto resolver = ResolveCfmEndpointOwnership(); + ownership_client = + std::make_shared(resolver, std::chrono::milliseconds(500)); + deployment_description = "remote SubMaster(s) via CFM coro_rpc"; } - auto codec = std::make_shared(); - auto channel = std::make_shared( - transport, codec, - CfmRpcConfig{.timeout = std::chrono::milliseconds(500)}); - IoPatternRuntime::Config source_config; source_config.report_capacity = FLAGS_report_capacity; MetricReportStats metric_reports; - source_config.report_sink = [&channel, &metric_reports](const MetricBatch& batch) { + const auto report_metric_batch = [&](const MetricBatch& batch) -> bool { const auto started = Clock::now(); - const bool success = channel->SendMetricBatch(batch); + const bool success = + ownership_client ? ownership_client->ReportMetricBatch(batch) == ErrorCode::OK + : (embedded_channel && embedded_channel->SendMetricBatch(batch)); metric_reports.Record(batch, ToMicroseconds(Clock::now() - started), success); return success; }; + source_config.report_sink = report_metric_batch; auto source_runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, @@ -377,6 +561,12 @@ int main(int argc, char* argv[]) { .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, source_config); + const auto send_snapshot = [&](const IoPatternSnapshot& snapshot) -> bool { + return ownership_client + ? ownership_client->ReportSnapshot(snapshot) == ErrorCode::OK + : (embedded_channel && embedded_channel->SendSnapshot(snapshot)); + }; + LatencyStats report_latency; uint64_t failed_reports = 0; uint64_t total_blocks = 0; @@ -393,7 +583,7 @@ int main(int argc, char* argv[]) { source_runtime->RecordStorageMetric(request.snapshot.storage.front()); const auto report_start = Clock::now(); - const bool sent = channel->SendSnapshot(request.snapshot); + const bool sent = send_snapshot(request.snapshot); report_latency.Record(ToMicroseconds(Clock::now() - report_start)); if (!sent) ++failed_reports; } @@ -441,10 +631,7 @@ int main(int argc, char* argv[]) { std::cout << "\n============================================================\n" << "CFM CLIENT BENCHMARK (vLLM inference request model)\n" << "============================================================\n" - << " CFM deployment: " - << (embedded_service ? "embedded SubMaster (local CFM)" - : "remote SubMaster coro_rpc endpoint") - << "\n" + << " CFM deployment: " << deployment_description << "\n" << " Requests: " << FLAGS_requests << "\n" << " Tokens/request: " << FLAGS_prompt_tokens + FLAGS_output_tokens << " (prompt=" From 89293b478ee67381d92bdb2c32d6dcf8a91a6ff4 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 8 Sep 2026 17:30:38 +0800 Subject: [PATCH 12/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=981?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../benchmarks/cfm_client_bench.cpp | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index 066a1b3368..e6964c1046 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -195,7 +195,7 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { const std::string connstring = entry.substr(scheme_pos + 3); const std::string cluster_namespace = ResolveCvmNamespace(); - ErrorCode err = EtcdHelper::ConnectToEtcdStoreClient(connstring); + ErrorCode err = mooncake::EtcdHelper::ConnectToEtcdStoreClient(connstring); if (err != ErrorCode::OK) { LOG(FATAL) << "cfm_endpoint: failed to connect etcd '" << connstring << "': " << toString(err); @@ -206,8 +206,8 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { const std::string view_key = BuildMasterViewKey(cluster_namespace); std::string leader_address; mooncake::EtcdRevisionId revision = 0; - err = EtcdHelper::Get(view_key.data(), view_key.size(), leader_address, - revision); + err = mooncake::EtcdHelper::Get(view_key.data(), view_key.size(), + leader_address, revision); if (err == ErrorCode::OK && !leader_address.empty()) { LOG(INFO) << "cfm_endpoint: single-leader HA via " << view_key << " -> " << leader_address; @@ -222,10 +222,10 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { } // CVM multi-submaster: masters registry + slot ownership. - std::vector masters; + std::vector masters; mooncake::ViewVersionId version = 0; - err = cvm::EtcdViewStore::LoadAllMasters(cluster_namespace, masters, - version); + err = mooncake::cvm::EtcdViewStore::LoadAllMasters(cluster_namespace, + masters, version); if (err != ErrorCode::OK) { LOG(FATAL) << "cfm_endpoint: LoadAllMasters failed for namespace '" << cluster_namespace << "': " << toString(err); @@ -235,7 +235,8 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { std::map address_by_master; // id -> host:port std::vector primary_ids; for (const auto& reg : masters) { - if (reg.role == static_cast(cvm::MasterRole::kPrimary) && + if (reg.role == + static_cast(mooncake::cvm::MasterRole::kPrimary) && !reg.address.empty()) { address_by_master[reg.master_id] = reg.address; primary_ids.push_back(reg.master_id); @@ -252,12 +253,13 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { // Prefer the authoritative slot owner table published by CvmController; // fall back to the consistent-hash ring used by the masters themselves. std::unordered_map owner_by_slot; - std::vector slot_owners; - const ErrorCode slot_err = cvm::EtcdViewStore::LoadAllSlotOwners( + std::vector slot_owners; + const ErrorCode slot_err = mooncake::cvm::EtcdViewStore::LoadAllSlotOwners( cluster_namespace, slot_owners, version); if (slot_err == ErrorCode::OK) { for (const auto& owner : slot_owners) { - if (owner.state == static_cast(cvm::SlotState::kStable) && + if (owner.state == + static_cast(mooncake::cvm::SlotState::kStable) && !owner.primary_master_id.empty()) { owner_by_slot[owner.slot] = owner.primary_master_id; } @@ -275,14 +277,14 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { owner_by_slot = std::move(owner_by_slot), has_owner_table]( const TenantId& tenant, const std::string& key) -> std::optional { - const uint16_t slot = cvm::KeySlot(tenant, key); + const uint16_t slot = mooncake::cvm::KeySlot(tenant, key); std::string owner; if (has_owner_table) { const auto it = owner_by_slot.find(slot); if (it != owner_by_slot.end()) owner = it->second; } if (owner.empty()) { - owner = cvm::ResolveSlotOwnerOnRing(primary_ids, slot); + owner = mooncake::cvm::ResolveSlotOwnerOnRing(primary_ids, slot); } const auto address = address_by_master.find(owner); if (address == address_by_master.end()) return std::nullopt; From 17d2074854ef4aeb289321c821dec63ba6526f9b Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Tue, 8 Sep 2026 20:42:29 +0800 Subject: [PATCH 13/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=981?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/io_pattern_design.md | 33 +++ .../benchmarks/cfm_client_bench.cpp | 50 +++- mooncake-store/include/io_pattern/runtime.h | 89 +++++++ .../include/master_metric_manager.h | 35 ++- mooncake-store/src/io_pattern/cfm_ingress.cpp | 10 + mooncake-store/src/io_pattern/runtime.cpp | 222 ++++++++++++++++++ mooncake-store/src/master_metric_manager.cpp | 128 ++++++++++ mooncake-store/src/master_service.cpp | 61 +++++ .../tests/io_pattern_framework_test.cpp | 122 ++++++++++ 9 files changed, 735 insertions(+), 15 deletions(-) diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index 16b43ae3de..cb7109cad8 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -1050,6 +1050,24 @@ explicitly drops observations whose owner cannot be resolved. The receiver merges every accepted report into the local runtime — there is no policy queue, poll, ACK or producer role to configure. +Every accepted report also drives the local policy pipeline. The receiving +SubMaster runs a background, coalescing `IoPatternRuntime` cycle after each +`report_metric_batch` / `report_snapshot` merge. The cycle derives its +decision inputs from the freshly aggregated snapshot: when the merged L1 +host-memory storage watermark (peak `memory_used_ratio` for `kL1Host` entries) +exceeds the configured high ratio the cycle requests an eviction plan for host +memory (target bytes = excess over the target ratio × reported capacity, only +when the snapshot also contains L1 keys); keys that were recently served as +hits and still carry L2/L3 replicas are fed to the prefetch ops, and hot +lower-tier keys (not pinned, no L1 replica) are offered to the admission ops. +Each cycle executes through the same storage-safe handlers, logs one +`[IO-PATTERN-REPORT-CYCLE]` summary line and reports per-dimension outcomes to +`MasterMetricManager` (`io_pattern_report_*` counters, visible on the master +`/metrics` endpoint and in the periodic "Master Admin Metrics" log). This is +what makes remote-mode policy execution observable: policy is no longer only +run by the local memory-watermark thread, the local Put admission path or +explicit `execute_*` RPCs. + ## Implemented - `IoPatternCollectorImpl` aggregates inference, access and storage metrics by @@ -1071,6 +1089,21 @@ queue, poll, ACK or producer role to configure. - `IoPatternRuntime` wires collection, bounded analysis, policy execution, feedback tuning and storage handlers; `MasterService` feeds it from actual Get/Put/watermark paths. +- A coalescing report-driven execution worker (`report_driven_execution` + runtime config, enabled by `MasterService`) runs one full + Collector -> Analyzer -> PolicyEngine -> execution cycle after every merged + `report_metric_batch` / `report_snapshot`. The cycle derives an eviction + request only when the merged L1 host-memory storage watermark is above the + configured high ratio and the snapshot contains L1 keys, rebuilds a + prefetch trace from recently hit lower-tier keys and offers hot non-head + keys to the admission ops; an empty-candidate eviction is a clean no-op, not + a failure. Every cycle logs one `[IO-PATTERN-REPORT-CYCLE]` line and reports + its per-dimension outcome (eviction/prefetch/admission candidates, handler + statuses, degradation) to the process observer (`MasterMetricManager` + `io_pattern_report_*` counters on the master, visible in `/metrics` and the + "Master Admin Metrics" log). This keeps remote-mode policy execution + observable and data-driven instead of relying only on the local eviction + thread, the Put admission hook or explicit `execute_*` RPCs. - `CfmClientImpl` wraps a single reporting channel for connectors (`ReportSnapshot` / `ReportMetricBatch` / `ExecutePrefetch`); policy runs in the SubMaster's own runtime, so there is no client-side dispatch loop. diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index e6964c1046..42be726ba0 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -4,15 +4,17 @@ // CFM is a component of every SubMaster; there is no standalone CFM Master and // no credential. A reporting client observes keys (KV blocks) and sends metric // batches over the SubMaster's regular coro_rpc endpoint. The SubMaster merges -// reports into its local runtime, then policy evaluation and execution run -// locally on the keys it owns (high-watermark eviction in the data path, -// trace-derived prefetch through the same storage-safe handlers). +// reports into its local runtime, then every merged report drives a local +// analysis -> decision -> execution cycle (eviction/prefetch/promotion/ +// admission through the storage-safe handlers) on the keys it owns. // // This benchmark exercises that path in the following modes: // - embedded (default): an in-process SubMaster runtime plays the owning -// CFM component. Reports are delivered in-process and policy is evaluated -// and executed locally, so the benchmark prints both report latency and -// the resulting eviction/prefetch/admission handler activity. +// CFM component. Reports are delivered in-process; the runtime's +// report-driven worker executes policy per report, and a final manual +// Execute emulates the production high-watermark trigger, so the benchmark +// prints report latency and the resulting eviction/prefetch/admission +// handler activity. // - remote (--cfm_endpoint=host:port): reports go over coro_rpc to a single // SubMaster CFM receiver. // - remote via etcd (--cfm_endpoint=etcd://connstring): resolves the cluster @@ -516,6 +518,12 @@ int main(int argc, char* argv[]) { std::string deployment_description; if (FLAGS_cfm_endpoint.empty()) { deployment_description = "embedded SubMaster (local CFM)"; + IoPatternRuntime::Config cfm_config; + // Merged reports drive local analysis -> decision -> execution (same + // worker the production SubMaster runs), so handler counters below + // reflect report-triggered policy execution, not only the manual + // watermark evaluation at the end of the run. + cfm_config.report_driven_execution = true; cfm_runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [&eviction_commands](const EvictionPlan&) { @@ -529,7 +537,8 @@ int main(int argc, char* argv[]) { .admission = [&admission_commands](const AdmissionResult&) { ++admission_commands; return ErrorCode::OK; - }}); + }}, + std::move(cfm_config)); embedded_service = std::make_shared(cfm_runtime); auto transport = std::make_shared(embedded_service); @@ -598,10 +607,21 @@ int main(int argc, char* argv[]) { std::this_thread::sleep_for( std::chrono::milliseconds(FLAGS_report_flush_wait_ms)); - // Embedded mode: evaluate and execute policy locally on the SubMaster - // runtime, exactly as the data-path high-watermark trigger does in - // production. The merged report above is what feeds that evaluation. - if (cfm_runtime && !cfm_runtime->Snapshot().keys.empty()) { + // The report-driven worker executes one cycle per merged report. Wait for + // it to drain before reading handler counters / snapshots so the printed + // numbers are deterministic. + if (cfm_runtime && cfm_runtime->report_driven_execution()) { + cfm_runtime->WaitForReportDrivenIdle(); + } + + // Embedded mode: when the report-driven worker is disabled, evaluate and + // execute policy once locally (the pre-worker high-watermark trigger that + // the production EvictionThreadFunc runs). With report_driven_execution + // enabled the runtime already executes a full cycle per merged report, so + // this extra pass is skipped to keep the printed handler counters equal to + // report-triggered executions only. + if (cfm_runtime && !cfm_runtime->report_driven_execution() && + !cfm_runtime->Snapshot().keys.empty()) { const auto capacity = 1024ULL * 1024 * 1024; const auto target = static_cast((FLAGS_memory_used_ratio - 0.80F) * @@ -678,8 +698,12 @@ int main(int argc, char* argv[]) { std::cout << " CFM keys / storage: " << cfm_snapshot.keys.size() << " / " << cfm_snapshot.storage.size() << "\n"; } else { - std::cout << " CFM keys / storage: remote endpoint (not exposed to " - "the client)\n"; + // Remote execution cannot be read back by the client; observe the + // receiving SubMaster's own master admin metrics (`io_pattern_report_*` + // in `/metrics` and the periodic "Master Admin Metrics" log) and its + // [IO-PATTERN-REPORT-CYCLE] log lines. + std::cout << " CFM keys / storage: remote endpoint (see SubMaster " + "master admin metrics)\n"; } PrintObservability("Client", source_metrics); if (embedded_service) PrintObservability("CFM", cfm_metrics); diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 8d64543e59..76a745eacc 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -2,7 +2,10 @@ #include #include +#include +#include #include +#include #include #include #include @@ -34,6 +37,31 @@ class IoPatternRuntime final { AdmissionHandler admission; }; + // Outcome of one report-driven policy cycle. The cycle aggregates the + // merged collector snapshot, runs analysis -> decision and executes the + // three storage-safe flows (eviction, prefetch, admission); this report + // lets the owning process surface each execution in its own metrics. + struct ReportDrivenCycleReport { + uint64_t cycle_id{0}; + size_t keys_analyzed{0}; + uint64_t analysis_elapsed_us{0}; + bool degraded{false}; + // Eviction dimension (derived from merged storage watermarks). + CacheTier eviction_tier{CacheTier::kL1Host}; + uint64_t eviction_target_bytes{0}; + size_t eviction_candidates{0}; + ErrorCode eviction_status{ErrorCode::OK}; + // Prefetch dimension (derived from merged prefix-affinity keys). + size_t prefetch_candidates{0}; + ErrorCode prefetch_status{ErrorCode::OK}; + // Admission dimension (derived from merged lower-tier hot keys). + size_t admission_candidates{0}; + size_t admissions_admitted{0}; + ErrorCode admission_status{ErrorCode::OK}; + }; + using ReportDrivenObserver = + std::function; + struct Config { IoPatternCollectorImpl::Config collector; uint64_t analysis_window_ns{60'000'000'000ULL}; @@ -46,6 +74,27 @@ class IoPatternRuntime final { size_t max_pending_admissions{4096}; MetricBatchSink report_sink; LegacyFallback legacy_fallback{LegacyFallback::kLru}; + // Report-driven execution: after each client report (snapshot or + // metric batch) is merged, the runtime runs its own full + // Collector -> Analyzer -> PolicyEngine -> execution cycle. The + // eviction dimension is triggered by merged storage watermarks; the + // prefetch and admission dimensions are derived from the merged key + // set. Defaults keep the worker off for pure collector/reporter + // runtimes; MasterService enables it on the SubMaster that owns the + // reported keys. + bool report_driven_execution{false}; + // Storage metric ratio (L1Host memory) at or above which the merged + // snapshot is considered under pressure and an eviction cycle is + // executed. Mirrors the master's own high-watermark trigger. + float report_eviction_high_ratio{0.80F}; + // After an eviction cycle the tier is considered relieved once this + // ratio is reached; eviction target bytes are derived as + // (peak_ratio - report_eviction_target_ratio) * capacity_bytes. + float report_eviction_target_ratio{0.70F}; + // Optional per-cycle observer used to surface executions in process + // metrics (e.g. MasterMetricManager). Never called from the report + // data path; only from the background cycle worker. + ReportDrivenObserver report_driven_observer; }; explicit IoPatternRuntime(Handlers handlers); @@ -64,6 +113,19 @@ class IoPatternRuntime final { const TraceHistory& trace, const std::vector& admissions = {}, const std::string& session_id = {}); + // Requests one report-driven cycle after merged report data. Coalesces: + // reports that arrive while a cycle is pending or running only mark the + // cycle dirty; the single background worker runs at most one full cycle + // per drain. Non-blocking for the report path. + void RequestReportDrivenExecution(); + // Blocks until the background report-driven worker has drained all + // currently pending reports (no pending flag and no cycle in flight). + // Used by benchmarks/tests that must read deterministic counters after a + // known report burst. No-op when report-driven execution is disabled. + void WaitForReportDrivenIdle(); + bool report_driven_execution() const { + return config_.report_driven_execution; + } // Runs Collector -> Analyzer -> PolicyEngine without invoking the local // storage handlers. Callers use Plan when they need the raw policy result // (for example the local eviction watermark path, observability or tests); @@ -103,6 +165,22 @@ class IoPatternRuntime final { ErrorCode ExecuteAdmission(const ObjectRef& object, CacheTier target_tier, const std::string& session_id); + // Report-driven cycle internals (single background worker). + void ReportDrivenWorker(); + void RunReportDrivenCycle(); + // Runs the executor over an already planned policy and records the shared + // outcome bookkeeping (policy failure/success, degradation, pending + // prefetch set and feedback). Used by both Execute() and the + // report-driven cycle so the two paths stay semantically identical. + PolicyExecutionStatus CommitPolicy(PlannedPolicy& planned); + static void DeriveEvictionRequest(const IoPatternSnapshot& snapshot, + float high_ratio, float target_ratio, + CacheTier& eviction_tier, + uint64_t& eviction_bytes); + static TraceHistory DeriveTraceHistory(const IoPatternSnapshot& snapshot); + static std::vector DeriveAdmissionCandidates( + const IoPatternSnapshot& snapshot); + struct PendingAdmission { ObjectRef object; CacheTier target_tier{CacheTier::kL1Host}; @@ -133,6 +211,17 @@ class IoPatternRuntime final { std::deque pending_admissions_; std::thread admission_worker_; bool admission_stopping_{false}; + + // Report-driven cycle worker state. Guarded by report_mutex_; the worker + // drains the pending flag and runs one cycle, then loops so reports that + // arrived during the cycle coalesce into the next drain. + std::mutex report_mutex_; + std::condition_variable report_condition_; + std::thread report_worker_; + bool report_pending_{false}; + bool report_stopping_{false}; + bool report_worker_busy_{false}; + uint64_t report_cycle_id_{0}; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/master_metric_manager.h b/mooncake-store/include/master_metric_manager.h index 07e565d542..83d2f77385 100644 --- a/mooncake-store/include/master_metric_manager.h +++ b/mooncake-store/include/master_metric_manager.h @@ -277,8 +277,6 @@ class MasterMetricManager { // nof eviction metrics void inc_nof_eviction_success(int64_t key_count, int64_t size); void inc_nof_eviction_fail(); // not a single object is evicted - - // Eviction Metrics Getters // total eviction metrics int64_t get_eviction_success(); int64_t get_eviction_attempts(); @@ -295,6 +293,29 @@ class MasterMetricManager { int64_t get_nof_evicted_key_count(); int64_t get_nof_evicted_size(); + // Report-driven IO Pattern policy execution metrics. These count the + // SubMaster-local policy cycles that run when merged client reports + // (report_snapshot / report_metric_batch) indicate storage pressure, so + // remote-mode eviction/prefetch/admission execution is observable in + // master admin metrics (not only the embedded watermark path). + void inc_io_pattern_report_cycles(int64_t val = 1); + void inc_io_pattern_report_evictions(int64_t val = 1); + void inc_io_pattern_report_eviction_failures(int64_t val = 1); + void inc_io_pattern_report_prefetches(int64_t val = 1); + void inc_io_pattern_report_prefetch_failures(int64_t val = 1); + void inc_io_pattern_report_admissions(int64_t val = 1); + void inc_io_pattern_report_admission_failures(int64_t val = 1); + void inc_io_pattern_report_degraded(int64_t val = 1); + // Report-driven IO Pattern execution metrics getters + int64_t get_io_pattern_report_cycles(); + int64_t get_io_pattern_report_evictions(); + int64_t get_io_pattern_report_eviction_failures(); + int64_t get_io_pattern_report_prefetches(); + int64_t get_io_pattern_report_prefetch_failures(); + int64_t get_io_pattern_report_admissions(); + int64_t get_io_pattern_report_admission_failures(); + int64_t get_io_pattern_report_degraded(); + // PutStart Discard Metrics void inc_put_start_discard_cnt(int64_t count, int64_t size); void inc_put_start_release_cnt(int64_t count, int64_t size); @@ -684,6 +705,16 @@ class MasterMetricManager { ylt::metric::counter_t nof_evicted_key_count_; ylt::metric::counter_t nof_evicted_size_; + // Report-driven IO Pattern policy execution metrics + ylt::metric::counter_t io_pattern_report_cycles_; + ylt::metric::counter_t io_pattern_report_evictions_; + ylt::metric::counter_t io_pattern_report_eviction_failures_; + ylt::metric::counter_t io_pattern_report_prefetches_; + ylt::metric::counter_t io_pattern_report_prefetch_failures_; + ylt::metric::counter_t io_pattern_report_admissions_; + ylt::metric::counter_t io_pattern_report_admission_failures_; + ylt::metric::counter_t io_pattern_report_degraded_; + // PutStart Discard Metrics ylt::metric::counter_t put_start_discard_cnt_; ylt::metric::counter_t put_start_release_cnt_; diff --git a/mooncake-store/src/io_pattern/cfm_ingress.cpp b/mooncake-store/src/io_pattern/cfm_ingress.cpp index 3ce568c6c3..b281283d88 100644 --- a/mooncake-store/src/io_pattern/cfm_ingress.cpp +++ b/mooncake-store/src/io_pattern/cfm_ingress.cpp @@ -12,6 +12,7 @@ bool CfmIngress::Handle(std::string_view method, std::string_view payload, const auto snapshot = codec_->DecodeSnapshot(wire); if (!snapshot) return false; runtime_->MergeSnapshot(*snapshot); + runtime_->RequestReportDrivenExecution(); return true; } if (method == "report_metric_batch") { @@ -40,6 +41,15 @@ bool CfmIngress::Handle(std::string_view method, std::string_view payload, normalized.observed_at_ns = received_at_ns; runtime_->RecordStorageMetric(normalized); } + // A merged report is a fresh aggregate: run the local policy cycle so + // eviction/prefetch/promotion/admission execution is driven by remote + // reports, not only by the local watermark/admission threads or + // explicit execute_* RPCs. RequestReportDrivenExecution is + // non-blocking and coalesces bursts on the runtime's worker. + if (!batch->inference.empty() || !batch->accesses.empty() || + !batch->storage.empty()) { + runtime_->RequestReportDrivenExecution(); + } return true; } if (method == "execute_prefetch") { diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index 964a62ebb8..6b6c0e82cd 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -42,6 +42,10 @@ IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) nullptr, std::make_shared()); policy_ = std::make_shared(workload_policy_, fallback); admission_worker_ = std::thread(&IoPatternRuntime::AdmissionWorker, this); + if (config_.report_driven_execution) { + report_worker_ = + std::thread(&IoPatternRuntime::ReportDrivenWorker, this); + } } IoPatternRuntime::~IoPatternRuntime() { @@ -52,6 +56,13 @@ IoPatternRuntime::~IoPatternRuntime() { } admission_condition_.notify_all(); if (admission_worker_.joinable()) admission_worker_.join(); + { + std::lock_guard lock(report_mutex_); + report_stopping_ = true; + report_pending_ = false; + } + report_condition_.notify_all(); + if (report_worker_.joinable()) report_worker_.join(); if (reporter_) reporter_->Stop(); } @@ -177,6 +188,10 @@ PolicyExecutionStatus IoPatternRuntime::Execute( const std::vector& admissions, const std::string& session_id) { auto planned = BuildPolicy(eviction_tier, eviction_bytes, trace, admissions, session_id); + return CommitPolicy(planned); +} + +PolicyExecutionStatus IoPatternRuntime::CommitPolicy(PlannedPolicy& planned) { const auto& snapshot = planned.snapshot; const auto& result = planned.result; auto status = executor_.Execute(result); @@ -278,6 +293,208 @@ ErrorCode IoPatternRuntime::ExecuteCommand(const PolicyCommand& command) { return status.admissions.empty() ? ErrorCode::OK : status.admissions.front(); } +void IoPatternRuntime::RequestReportDrivenExecution() { + if (!config_.report_driven_execution) return; + { + std::lock_guard lock(report_mutex_); + if (report_stopping_) return; + report_pending_ = true; + } + // notify_all: a concurrent WaitForReportDrivenIdle() must not swallow the + // worker's wakeup (predicates re-check under the mutex either way). + report_condition_.notify_all(); +} + +void IoPatternRuntime::WaitForReportDrivenIdle() { + if (!config_.report_driven_execution) return; + std::unique_lock lock(report_mutex_); + report_condition_.wait(lock, [this] { + return report_stopping_ || (!report_pending_ && !report_worker_busy_); + }); +} + +void IoPatternRuntime::ReportDrivenWorker() { + while (true) { + { + std::unique_lock lock(report_mutex_); + report_condition_.wait(lock, [this] { + return report_stopping_ || report_pending_; + }); + if (report_stopping_) return; + report_pending_ = false; + report_worker_busy_ = true; + } + try { + RunReportDrivenCycle(); + } catch (...) { + policy_->RecordFailure(); + observability_.RecordDegrade(); + } + { + std::lock_guard lock(report_mutex_); + report_worker_busy_ = false; + } + report_condition_.notify_all(); + } +} + +void IoPatternRuntime::DeriveEvictionRequest(const IoPatternSnapshot& snapshot, + float high_ratio, + float target_ratio, + CacheTier& eviction_tier, + uint64_t& eviction_bytes) { + // The Store-side eviction handler is tenant-qualified MEMORY (L1) quota + // eviction; L2/L3 pressure is handled by the legacy NoF/SSD paths outside + // the IO Pattern runtime. Restrict the report-driven eviction dimension to + // host-memory watermarks so L2/L3 reports never route lower-tier keys into + // the memory quota eviction handler. + eviction_tier = CacheTier::kL1Host; + eviction_bytes = 0; + float peak_ratio = 0.0F; + uint64_t capacity_bytes = 0; + for (const auto& metric : snapshot.storage) { + if (metric.tier != CacheTier::kL1Host) continue; + if (metric.memory_used_ratio > peak_ratio) { + peak_ratio = metric.memory_used_ratio; + capacity_bytes = metric.capacity_bytes; + } + } + // No merged host-memory watermark, or below the high watermark: the merged + // snapshot does not indicate pressure, so the eviction dimension produces + // no action (the prefetch/admission dimensions are still evaluated). + if (peak_ratio < high_ratio) return; + // An eviction plan needs L1 keys to select candidates from. A report that + // only carries storage watermarks (no key observations for this tier) + // would otherwise execute an empty eviction plan through the handler on + // every cycle. + bool any_l1_keys = false; + for (const auto& key : snapshot.keys) { + if ((key.replica_tiers & CacheTierBit(CacheTier::kL1Host)) != 0) { + any_l1_keys = true; + break; + } + } + if (!any_l1_keys) return; + if (capacity_bytes == 0) { + for (const auto& metric : snapshot.storage) { + if (metric.tier == CacheTier::kL1Host && + metric.memory_used_ratio == peak_ratio && + metric.used_bytes != 0) { + capacity_bytes = static_cast( + static_cast(metric.used_bytes) / + static_cast(metric.memory_used_ratio)); + break; + } + } + } + const double reclaim_fraction = + static_cast(peak_ratio) - static_cast(target_ratio); + if (reclaim_fraction <= 0.0) return; + const double target = reclaim_fraction * static_cast(capacity_bytes); + eviction_bytes = target > 0.0 + ? static_cast(target) + : (capacity_bytes > 0 ? capacity_bytes / 10 : 0); +} + +TraceHistory IoPatternRuntime::DeriveTraceHistory( + const IoPatternSnapshot& snapshot) { + // Report-driven prefetch input: keys that were recently served as hits and + // still have a lower-tier replica are the ones a promotion should bring + // closer to the head. TraceBasedPrefetchOps re-applies its own + // match-length / confidence gates against this candidate trace. + TraceHistory trace; + const auto now_ns = static_cast( + std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); + for (const auto& key : snapshot.keys) { + if (!key.active || key.access_count_window == 0) continue; + if ((key.replica_tiers & CacheTierBit(CacheTier::kL2Segment)) == 0 && + (key.replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) == 0) { + continue; + } + trace.events.push_back( + TraceEvent{.object = key.object, + .observed_at_ns = now_ns, + .match_length = key.match_length, + .is_hit = true}); + } + return trace; +} + +std::vector IoPatternRuntime::DeriveAdmissionCandidates( + const IoPatternSnapshot& snapshot) { + // Report-driven admission input: lower-tier objects that the merged + // reports show as hot (frequent reads) and that are not already in the + // head tier are promotion candidates. PrefixMatchAdmissionOps re-applies + // its frequency/watermark gates per object before execution. + std::vector candidates; + for (const auto& key : snapshot.keys) { + if (key.pinned || key.access_count_window == 0) continue; + const bool in_head = + (key.replica_tiers & CacheTierBit(CacheTier::kL1Host)) != 0; + const bool lower_tier = + (key.replica_tiers & CacheTierBit(CacheTier::kL2Segment)) != 0 || + (key.replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) != 0; + if (!in_head && lower_tier) candidates.push_back(key.object); + } + return candidates; +} + +void IoPatternRuntime::RunReportDrivenCycle() { + IoPatternSnapshot snapshot = collector_->GetSnapshot(); + ReportDrivenCycleReport report; + report.cycle_id = ++report_cycle_id_; + report.keys_analyzed = snapshot.keys.size(); + if (snapshot.keys.empty() && snapshot.storage.empty()) { + if (config_.report_driven_observer) { + config_.report_driven_observer(report); + } + return; + } + CacheTier eviction_tier = CacheTier::kL1Host; + uint64_t eviction_bytes = 0; + DeriveEvictionRequest(snapshot, config_.report_eviction_high_ratio, + config_.report_eviction_target_ratio, eviction_tier, + eviction_bytes); + report.eviction_tier = eviction_tier; + report.eviction_target_bytes = eviction_bytes; + const auto trace = DeriveTraceHistory(snapshot); + const auto admissions = DeriveAdmissionCandidates(snapshot); + report.admission_candidates = admissions.size(); + + auto planned = BuildPolicy(eviction_tier, eviction_bytes, trace, + admissions, "report-driven"); + report.analysis_elapsed_us = planned.analysis_elapsed_us; + const auto& result = planned.result; + report.eviction_candidates = result.eviction.candidates.size(); + report.prefetch_candidates = result.prefetch.candidates.size(); + size_t admitted = 0; + for (const auto& admission : result.admissions) { + if (admission.decision == AdmissionDecision::kAdmit) ++admitted; + } + report.admissions_admitted = admitted; + // A pressure report with no evictable L1 keys is a clean no-op for the + // eviction dimension, not a policy failure: do not invoke the storage + // handler with an empty candidate list (TierOperationExecutor otherwise + // forwards any non-zero target). Keep the derived target visible in the + // report for observability. + if (report.eviction_candidates == 0) { + planned.result.eviction.target_bytes = 0; + planned.result.eviction.candidates.clear(); + } + const auto status = CommitPolicy(planned); + report.degraded = status.degraded || planned.result.degraded; + report.eviction_status = status.eviction; + report.prefetch_status = status.prefetch; + if (!status.admissions.empty()) { + report.admission_status = status.admissions.front(); + } + if (config_.report_driven_observer) { + config_.report_driven_observer(report); + } +} + bool IoPatternRuntime::ScheduleAdmission(ObjectRef object, CacheTier target_tier, std::string session_id) { { @@ -343,6 +560,11 @@ ErrorCode IoPatternRuntime::ExecuteAdmission(const ObjectRef& object, } void IoPatternRuntime::RecordFeedback(PolicyFeedbackSample sample) { + // Tuner state is not internally synchronized; the report-driven worker, + // the eviction thread and the data path can all reach RecordFeedback. + // feedback_state_mutex_ serializes them (all callers invoke this method + // outside that lock, so there is no recursive acquisition). + std::lock_guard lock(feedback_state_mutex_); feedback_.Record(sample); auto config = workload_policy_->CurrentEvictionConfig(); if (tuner_.Tune(feedback_.Snapshot(), config)) { diff --git a/mooncake-store/src/master_metric_manager.cpp b/mooncake-store/src/master_metric_manager.cpp index ea52ec91df..ca45ae5b22 100644 --- a/mooncake-store/src/master_metric_manager.cpp +++ b/mooncake-store/src/master_metric_manager.cpp @@ -330,6 +330,36 @@ MasterMetricManager::MasterMetricManager() nof_evicted_size_("master_evicted_size_bytes_nof", "Total bytes of evicted objects in nof"), + // Report-driven IO Pattern policy execution metrics + io_pattern_report_cycles_( + "master_io_pattern_report_cycles_total", + "Total report-driven IO Pattern policy cycles executed after " + "merged client reports"), + io_pattern_report_evictions_( + "master_io_pattern_report_evictions_total", + "Total eviction executions from report-driven IO Pattern cycles"), + io_pattern_report_eviction_failures_( + "master_io_pattern_report_eviction_failures_total", + "Total failed eviction executions from report-driven IO Pattern " + "cycles"), + io_pattern_report_prefetches_( + "master_io_pattern_report_prefetches_total", + "Total prefetch executions from report-driven IO Pattern cycles"), + io_pattern_report_prefetch_failures_( + "master_io_pattern_report_prefetch_failures_total", + "Total failed prefetch executions from report-driven IO Pattern " + "cycles"), + io_pattern_report_admissions_( + "master_io_pattern_report_admissions_total", + "Total admission executions from report-driven IO Pattern cycles"), + io_pattern_report_admission_failures_( + "master_io_pattern_report_admission_failures_total", + "Total failed admission executions from report-driven IO Pattern " + "cycles"), + io_pattern_report_degraded_( + "master_io_pattern_report_degraded_total", + "Total degraded report-driven IO Pattern policy cycles"), + // Initialize Discarded Replicas Counters put_start_discard_cnt_("master_put_start_discard_cnt", "Total number of discarded PutStart operations"), @@ -621,6 +651,16 @@ void MasterMetricManager::update_metrics_for_zero_output() { evicted_key_count_.inc(0); evicted_size_.inc(0); + // Update report-driven IO Pattern policy execution counters + io_pattern_report_cycles_.inc(0); + io_pattern_report_evictions_.inc(0); + io_pattern_report_eviction_failures_.inc(0); + io_pattern_report_prefetches_.inc(0); + io_pattern_report_prefetch_failures_.inc(0); + io_pattern_report_admissions_.inc(0); + io_pattern_report_admission_failures_.inc(0); + io_pattern_report_degraded_.inc(0); + // Update PutStart Discard Metrics put_start_discard_cnt_.inc(0); put_start_release_cnt_.inc(0); @@ -1557,6 +1597,73 @@ int64_t MasterMetricManager::get_nof_evicted_size() { return nof_evicted_size_.value(); } +void MasterMetricManager::inc_io_pattern_report_cycles(int64_t val) { + io_pattern_report_cycles_.inc(val); +} + +void MasterMetricManager::inc_io_pattern_report_evictions(int64_t val) { + io_pattern_report_evictions_.inc(val); +} + +void MasterMetricManager::inc_io_pattern_report_eviction_failures( + int64_t val) { + io_pattern_report_eviction_failures_.inc(val); +} + +void MasterMetricManager::inc_io_pattern_report_prefetches(int64_t val) { + io_pattern_report_prefetches_.inc(val); +} + +void MasterMetricManager::inc_io_pattern_report_prefetch_failures( + int64_t val) { + io_pattern_report_prefetch_failures_.inc(val); +} + +void MasterMetricManager::inc_io_pattern_report_admissions(int64_t val) { + io_pattern_report_admissions_.inc(val); +} + +void MasterMetricManager::inc_io_pattern_report_admission_failures( + int64_t val) { + io_pattern_report_admission_failures_.inc(val); +} + +void MasterMetricManager::inc_io_pattern_report_degraded(int64_t val) { + io_pattern_report_degraded_.inc(val); +} + +int64_t MasterMetricManager::get_io_pattern_report_cycles() { + return io_pattern_report_cycles_.value(); +} + +int64_t MasterMetricManager::get_io_pattern_report_evictions() { + return io_pattern_report_evictions_.value(); +} + +int64_t MasterMetricManager::get_io_pattern_report_eviction_failures() { + return io_pattern_report_eviction_failures_.value(); +} + +int64_t MasterMetricManager::get_io_pattern_report_prefetches() { + return io_pattern_report_prefetches_.value(); +} + +int64_t MasterMetricManager::get_io_pattern_report_prefetch_failures() { + return io_pattern_report_prefetch_failures_.value(); +} + +int64_t MasterMetricManager::get_io_pattern_report_admissions() { + return io_pattern_report_admissions_.value(); +} + +int64_t MasterMetricManager::get_io_pattern_report_admission_failures() { + return io_pattern_report_admission_failures_.value(); +} + +int64_t MasterMetricManager::get_io_pattern_report_degraded() { + return io_pattern_report_degraded_.value(); +} + // PutStart Discard Metrics Getters int64_t MasterMetricManager::get_put_start_discard_cnt() { return put_start_discard_cnt_.value(); @@ -1937,6 +2044,16 @@ std::string MasterMetricManager::serialize_metrics() { serialize_metric(nof_evicted_key_count_); serialize_metric(nof_evicted_size_); + // Serialize report-driven IO Pattern policy execution metrics + serialize_metric(io_pattern_report_cycles_); + serialize_metric(io_pattern_report_evictions_); + serialize_metric(io_pattern_report_eviction_failures_); + serialize_metric(io_pattern_report_prefetches_); + serialize_metric(io_pattern_report_prefetch_failures_); + serialize_metric(io_pattern_report_admissions_); + serialize_metric(io_pattern_report_admission_failures_); + serialize_metric(io_pattern_report_degraded_); + // Serialize PutStart Discard Metrics serialize_metric(put_start_discard_cnt_); serialize_metric(put_start_release_cnt_); @@ -2591,6 +2708,17 @@ std::string MasterMetricManager::get_summary_string( << "keys=" << nof_evicted_key_count << ", " << "size=" << byte_size_to_string(nof_evicted_size); + // Report-driven IO Pattern policy execution summary (cumulative) + ss << " | IO Pattern (report-driven): " + << "cycles=" << io_pattern_report_cycles_.value() << ", " + << "evict=" << io_pattern_report_evictions_.value() << "/" + << io_pattern_report_eviction_failures_.value() << ", " + << "prefetch=" << io_pattern_report_prefetches_.value() << "/" + << io_pattern_report_prefetch_failures_.value() << ", " + << "admit=" << io_pattern_report_admissions_.value() << "/" + << io_pattern_report_admission_failures_.value() << ", " + << "degraded=" << io_pattern_report_degraded_.value(); + // Discard summary ss << " | Discard: " << "Released/Total=" << put_start_release_cnt << "/" diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 80b80bc940..b49f56818e 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -438,6 +438,67 @@ MasterService::MasterService(const MasterServiceConfig& config) } io_pattern::IoPatternRuntime::Config io_pattern_config; + // Report-driven execution: a SubMaster that merges ownership-addressed + // client reports runs its own analysis -> decision -> execution cycle on + // every accepted report (report_snapshot / report_metric_batch). This is + // the driver that makes remote-mode CFM execution visible: without it + // policy only runs on the local watermark thread, the local Put admission + // path or explicit execute_* RPCs. + io_pattern_config.report_driven_execution = true; + // Mirror the master's own high/low watermark semantics so a merged-report + // eviction cycle reclaims the same excess as the local eviction thread + // (target ratio = high watermark - eviction ratio; clamped to >= 0). + io_pattern_config.report_eviction_high_ratio = + static_cast(eviction_high_watermark_ratio_); + io_pattern_config.report_eviction_target_ratio = static_cast( + std::max(0.0, eviction_high_watermark_ratio_ - eviction_ratio_)); + io_pattern_config.report_driven_observer = + [](const io_pattern::IoPatternRuntime::ReportDrivenCycleReport& rpt) { + auto& metrics = MasterMetricManager::instance(); + metrics.inc_io_pattern_report_cycles(); + if (rpt.degraded) metrics.inc_io_pattern_report_degraded(); + // A cycle "executes" a dimension when a plan reached the storage + // handler. The report-driven cycle only forwards an eviction plan + // with candidate keys (an empty-candidate pressure report is a + // clean no-op, not a failure); prefetch/admission likewise only + // reach their handler when candidates were planned. Failures are + // counted when the storage handler rejected the plan. + if (rpt.eviction_candidates != 0) { + metrics.inc_io_pattern_report_evictions(); + if (rpt.eviction_status != ErrorCode::OK) { + metrics.inc_io_pattern_report_eviction_failures(); + } + } + if (rpt.prefetch_candidates != 0) { + metrics.inc_io_pattern_report_prefetches(); + if (rpt.prefetch_status != ErrorCode::OK) { + metrics.inc_io_pattern_report_prefetch_failures(); + } + } + if (rpt.admissions_admitted != 0) { + metrics.inc_io_pattern_report_admissions(); + if (rpt.admission_status != ErrorCode::OK) { + metrics.inc_io_pattern_report_admission_failures(); + } + } + LOG(INFO) << "[IO-PATTERN-REPORT-CYCLE] cycle=" << rpt.cycle_id + << " keys_analyzed=" << rpt.keys_analyzed + << " analysis_elapsed_us=" << rpt.analysis_elapsed_us + << " degraded=" << rpt.degraded + << " eviction(tier=" + << static_cast(rpt.eviction_tier) + << ", bytes=" << rpt.eviction_target_bytes + << ", candidates=" << rpt.eviction_candidates + << ", status=" << static_cast(rpt.eviction_status) + << ")" + << " prefetch(candidates=" << rpt.prefetch_candidates + << ", status=" << static_cast(rpt.prefetch_status) + << ")" + << " admission(candidates=" << rpt.admission_candidates + << ", admitted=" << rpt.admissions_admitted + << ", status=" << static_cast(rpt.admission_status) + << ")"; + }; io_pattern_runtime_ = std::make_shared( io_pattern::IoPatternRuntime::Handlers{ .eviction = diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index d4b0ca2288..3f381c92c9 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -3,8 +3,12 @@ #include "io_pattern/policy_strategies.h" #include +#include #include #include +#include +#include +#include #include #include #include @@ -1134,6 +1138,124 @@ TEST(IoPatternFrameworkTest, LocalCfmExecutesEvictionOnHighWatermarkSnapshot) { EXPECT_GE(runtime->ObservabilitySnapshot().policy_decisions, 1); } +TEST(IoPatternFrameworkTest, ReportDrivenCycleRunsAfterMergedReport) { + // A merged client report must drive the local analysis -> decision -> + // execution cycle on the receiving runtime (the driver that makes remote + // SubMaster CFM execution observable), not only the local watermark + // thread, the Put admission hook or explicit execute_* RPCs. + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::atomic eviction_handled{0}; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + // Let the detached analyzer finish so pattern selection is deterministic + // in this test (the worker otherwise uses the 500 us bounded budget). + config.analysis_timeout_us = 30'000'000; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [&eviction_handled](const EvictionPlan&) { + ++eviction_handled; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + CfmService service(rt); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "hot-key"}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + batch.storage.push_back( + StorageMetric{.source_id = "reporter", + .observed_at_ns = 1, + .tier = CacheTier::kL1Host, + .used_bytes = 1024ULL * 1024 * 1024, + .capacity_bytes = 1024ULL * 1024 * 1024, + .memory_used_ratio = 0.95F}); + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + + // Wait for the background cycle to drain the merged report. + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(30), [&] { + return last_report.has_value(); + })); + ASSERT_TRUE(last_report.has_value()); + EXPECT_GE(last_report->cycle_id, 1); + EXPECT_GE(last_report->keys_analyzed, 1); + // 0.95 merged host-memory ratio exceeds the default 0.80 high watermark: + // the eviction dimension derived a target and executed through the + // storage handler. + EXPECT_GT(last_report->eviction_target_bytes, 0); + EXPECT_GT(eviction_handled.load(), 0); + EXPECT_EQ(last_report->eviction_status, ErrorCode::OK); +} + +TEST(IoPatternFrameworkTest, ReportDrivenCycleSkipsEvictionWithoutPressure) { + // Below the high watermark the eviction dimension produces no action, but + // the cycle still runs so the report-driven pipeline stays observable. + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::atomic eviction_handled{0}; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [&eviction_handled](const EvictionPlan&) { + ++eviction_handled; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + CfmService service(rt); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "key"}, + .observed_at_ns = 1, + .block_size = 1024, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + batch.storage.push_back( + StorageMetric{.source_id = "reporter", + .observed_at_ns = 1, + .tier = CacheTier::kL1Host, + .used_bytes = 512ULL * 1024 * 1024, + .capacity_bytes = 1024ULL * 1024 * 1024, + .memory_used_ratio = 0.50F}); + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(5), [&] { + return last_report.has_value(); + })); + ASSERT_TRUE(last_report.has_value()); + EXPECT_EQ(last_report->eviction_target_bytes, 0); + EXPECT_EQ(eviction_handled.load(), 0); +} + TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { size_t batches = 0; IoPatternReporter reporter(4, [&](const MetricBatch&) { From 2db825cc267adb86813cb3b85d95bee05a283a5d Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 9 Sep 2026 10:30:25 +0800 Subject: [PATCH 14/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=982?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/source/io_pattern_design.md | 35 +++- .../benchmarks/cfm_client_bench.cpp | 187 ++++++++++++++++++ mooncake-store/include/io_pattern/runtime.h | 25 +++ mooncake-store/include/master_config.h | 25 +++ mooncake-store/src/io_pattern/runtime.cpp | 41 ++++ mooncake-store/src/master.cpp | 39 ++++ mooncake-store/src/master_service.cpp | 17 +- .../tests/io_pattern_framework_test.cpp | 55 ++++++ 8 files changed, 417 insertions(+), 7 deletions(-) diff --git a/docs/source/io_pattern_design.md b/docs/source/io_pattern_design.md index cb7109cad8..a1bb5fbea6 100644 --- a/docs/source/io_pattern_design.md +++ b/docs/source/io_pattern_design.md @@ -1060,8 +1060,14 @@ memory (target bytes = excess over the target ratio × reported capacity, only when the snapshot also contains L1 keys); keys that were recently served as hits and still carry L2/L3 replicas are fed to the prefetch ops, and hot lower-tier keys (not pinned, no L1 replica) are offered to the admission ops. +When no storage watermark is exceeded but the runtime is configured with the +cold-data eviction driver enabled (`report_driven_cold_eviction`), the cycle +instead requests a bounded eviction of the coldest (idle) L1 keys so eviction +is driven by cold/hot analysis and not only by memory pressure; the cycle +report marks such passes with `cold_eviction=true`. Each cycle executes through the same storage-safe handlers, logs one -`[IO-PATTERN-REPORT-CYCLE]` summary line and reports per-dimension outcomes to +`[IO-PATTERN-REPORT-CYCLE]` summary line (including whether the pass was a +cold-eviction driver pass) and reports per-dimension outcomes to `MasterMetricManager` (`io_pattern_report_*` counters, visible on the master `/metrics` endpoint and in the periodic "Master Admin Metrics" log). This is what makes remote-mode policy execution observable: policy is no longer only @@ -1099,11 +1105,28 @@ explicit `execute_*` RPCs. keys to the admission ops; an empty-candidate eviction is a clean no-op, not a failure. Every cycle logs one `[IO-PATTERN-REPORT-CYCLE]` line and reports its per-dimension outcome (eviction/prefetch/admission candidates, handler - statuses, degradation) to the process observer (`MasterMetricManager` - `io_pattern_report_*` counters on the master, visible in `/metrics` and the - "Master Admin Metrics" log). This keeps remote-mode policy execution - observable and data-driven instead of relying only on the local eviction - thread, the Put admission hook or explicit `execute_*` RPCs. + statuses, degradation, cold-driver flag) to the process observer + (`MasterMetricManager` `io_pattern_report_*` counters on the master, visible + in `/metrics` and the "Master Admin Metrics" log). This keeps remote-mode + policy execution observable and data-driven instead of relying only on the + local eviction thread, the Put admission hook or explicit `execute_*` RPCs. +- The cold-data eviction driver (`report_driven_cold_eviction` runtime config + plus `report_driven_cold_eviction_bytes` per-cycle budget and optional + `report_driven_cold_idle_threshold_us` gate) runs on the same report-driven + worker: when a cycle finds no storage-watermark pressure it still plans a + bounded eviction of the coldest non-pinned L1 keys, marks the pass + `cold_eviction=true` and executes through the same storage handlers, so + eviction is not only triggered at the memory high watermark. `MasterService` + surfaces it through `--io_pattern_cold_eviction`, + `--io_pattern_cold_eviction_bytes_per_cycle` and + `--io_pattern_cold_idle_threshold_us`. +- `cfm_client_bench` optionally seeds real KV objects (`--master_server`, + `--num_keys`, `--value_size`, `--replica_num`, `--protocol` and the other + RealClient flags shared with `stress_cluster_bench`) before the simulated + vLLM request stream and reads a subset back, so the IO Pattern handlers run + against replicas that actually exist on the SubMaster (report-only runs leave + eviction/promotion/prefetch counters at zero because the handlers cannot act + on objects the master never stored). - `CfmClientImpl` wraps a single reporting channel for connectors (`ReportSnapshot` / `ReportMetricBatch` / `ExecutePrefetch`); policy runs in the SubMaster's own runtime, so there is no client-side dispatch loop. diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index 42be726ba0..ba514c1afc 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -21,6 +21,15 @@ // like a Store client, then buckets each key to its owning SubMaster. // The receiving side is not observable here, so only client-side latency // is reported. +// +// Remote policy execution is only observable when the SubMaster actually owns +// the reported keys: its eviction/promotion/prefetch handlers operate on real +// replicas, so a report-only run leaves the master-side counters at zero. When +// --master_server and --num_keys are provided, a real-data seeding stage runs +// first ("先种子后仿真"): it writes a batch of real KV objects through +// RealClient (keys share the simulated KvKey naming and tenant) and reads a +// subset back to simulate access heat, then the simulated vLLM request stream +// reports on those same keys. #include #include @@ -43,6 +52,7 @@ #include #include #include +#include #include "gflags/gflags.h" #include "glog/logging.h" @@ -54,6 +64,7 @@ #include "io_pattern/cfm_service.h" #include "io_pattern/rpc_transport.h" #include "io_pattern/runtime.h" +#include "real_client.h" #include "types.h" #ifdef STORE_USE_ETCD #include "etcd_helper.h" @@ -96,6 +107,39 @@ DEFINE_string(cfm_cluster_namespace, "", "etcd leader coordinator). When the cluster was started with a " "non-default cluster_id, pass the same value here"); +// Real Store client parameters used by the optional real-data seeding stage. +// Flag names and defaults mirror stress_cluster_bench.cpp so an existing +// cluster invocation can be reused as-is. Seeding makes the SubMaster hold +// real replicas for the reported keys, which is what the IO Pattern eviction / +// promotion / prefetch handlers operate on (without real objects they remain +// no-ops even when reports trigger policy cycles). +DEFINE_string(master_server, "", + "Master server address (host:port) for RealClient writes; empty " + "disables real-data seeding"); +DEFINE_string(local_hostname, "localhost", + "Local hostname (with optional port, e.g. node1:12345)"); +DEFINE_string(metadata_server, "http://127.0.0.1:8080/metadata", + "Metadata server URL for RealClient setup"); +DEFINE_string(protocol, "tcp", "Transport protocol: tcp, rdma, ub"); +DEFINE_string(device_name, "", "RDMA/UB device name (comma-separated)"); +DEFINE_uint64(global_segment_size, 16ULL * 1024 * 1024 * 1024, + "Global segment size in bytes (per store node)"); +DEFINE_uint64(local_buffer_size, 512ULL * 1024 * 1024, + "Local client buffer size in bytes"); +DEFINE_bool(enable_ssd_offload, false, + "Enable LOCAL_DISK offload when seeding real keys (requires the " + "SubMaster to run with offload enabled)"); +DEFINE_string(ssd_offload_path, "", "SSD offload directory path"); +DEFINE_uint64(num_keys, 0, + "Number of real keys to write during seeding (0 = disabled)"); +DEFINE_uint64(value_size, 4ULL * 1024 * 1024, + "Size in bytes of each seeded real KV object"); +DEFINE_uint64(replica_num, 1, "Number of replicas for each seeded object"); +DEFINE_bool(hard_pin, false, "Pin seeded objects (disable eviction of them)"); +DEFINE_uint64(seed_get_keys, 0, + "How many of the seeded keys to read back with get_into " + "(0 = half of num_keys)"); + uint64_t SteadyNowNs() { return static_cast( std::chrono::duration_cast( @@ -494,6 +538,138 @@ bool ValidateFlags() { FLAGS_memory_used_ratio <= 1.0; } +// Real-data seeding stage (optional). The IO Pattern runtime on the SubMaster +// only executes storage handlers against replicas that actually exist, so a +// report-only benchmark leaves master-side eviction/promotion/prefetch counters +// at zero. When --master_server and --num_keys are provided this stage writes a +// batch of real KV objects through RealClient (same key/tenant naming as the +// simulated requests, so the analysis snapshot and the real metadata overlap) +// and reads a subset back to simulate access heat. +struct SeedStats { + uint64_t written{0}; + uint64_t write_failures{0}; + uint64_t reads{0}; + uint64_t read_failures{0}; +}; + +SeedStats RunRealSeedStage() { + SeedStats stats; + if (FLAGS_master_server.empty() || FLAGS_num_keys == 0 || + FLAGS_value_size == 0) { + return stats; + } + LOG(INFO) << "Real-data seed stage: master_server=" << FLAGS_master_server + << " protocol=" << FLAGS_protocol + << " keys=" << FLAGS_num_keys + << " value_size=" << FLAGS_value_size + << " replica_num=" << FLAGS_replica_num + << " offload=" << (FLAGS_enable_ssd_offload ? "yes" : "no"); + + auto client = mooncake::RealClient::create(); + const size_t block_bytes = std::max(FLAGS_value_size, 4096); + char* buffer = reinterpret_cast(numa_alloc_local(block_bytes)); + if (buffer == nullptr) { + LOG(ERROR) << "numa_alloc_local failed for seed buffer of " + << block_bytes << " bytes"; + return stats; + } + std::memset(buffer, 0xA5, block_bytes); + int ret = client->setup_real( + FLAGS_local_hostname, FLAGS_metadata_server, FLAGS_global_segment_size, + FLAGS_local_buffer_size, FLAGS_protocol, FLAGS_device_name, + FLAGS_master_server, nullptr, "", FLAGS_enable_ssd_offload, + FLAGS_ssd_offload_path, FLAGS_tenant); + if (ret != 0) { + LOG(ERROR) << "setup_real failed: " << ret; + numa_free(buffer, block_bytes); + return stats; + } + ret = client->register_buffer(buffer, block_bytes); + if (ret != 0) { + LOG(ERROR) << "register_buffer failed: " << ret; + numa_free(buffer, block_bytes); + return stats; + } + + // Write keys that share the simulated KvKey naming so later reports and + // the real metadata address the same objects. Enumerate the same + // (session, request, layer, block) space as BuildRequest() and stop after + // --num_keys objects, so the seeded set is exactly the head of the + // reported key universe (real replicas exist for the keys policy will + // select). + mooncake::ReplicateConfig config; + config.replica_num = static_cast(FLAGS_replica_num); + config.with_hard_pin = FLAGS_hard_pin; + const uint64_t total_tokens = FLAGS_prompt_tokens + FLAGS_output_tokens; + const size_t blocks = BlockCount(total_tokens); + const size_t shared_blocks = + std::min(blocks, BlockCount(FLAGS_shared_prefix_tokens)); + uint64_t seeded = 0; + for (size_t request_index = 0; + request_index < FLAGS_requests && seeded < FLAGS_num_keys; + ++request_index) { + const size_t session = request_index % FLAGS_num_sessions; + for (size_t layer = 0; layer < FLAGS_num_layers && seeded < FLAGS_num_keys; + ++layer) { + for (size_t block = 0; block < blocks && seeded < FLAGS_num_keys; + ++block) { + const bool is_shared_prefix = block < shared_blocks; + const std::string key = + KvKey(session, request_index, layer, block, is_shared_prefix); + const int put_ret = + client->put_from(key, buffer, FLAGS_value_size, config); + if (put_ret == 0) { + ++seeded; + } else { + ++stats.write_failures; + LOG(WARNING) << "put_from failed for seed key " << key + << ": " << put_ret; + } + } + } + } + stats.written = seeded; + + // Simulate reads: exercise a hot subset through the real data path so the + // SubMaster records real GET access heat (promotion-on-hit when offloaded). + // Re-enumerate the same key space in the same order and read the first + // read_count keys. + const uint64_t read_count = FLAGS_seed_get_keys == 0 + ? seeded / 2 + : std::min(FLAGS_seed_get_keys, + seeded); + uint64_t read_keys = 0; + for (size_t request_index = 0; + request_index < FLAGS_requests && read_keys < read_count; + ++request_index) { + const size_t session = request_index % FLAGS_num_sessions; + for (size_t layer = 0; + layer < FLAGS_num_layers && read_keys < read_count; ++layer) { + for (size_t block = 0; block < blocks && read_keys < read_count; + ++block) { + const bool is_shared_prefix = block < shared_blocks; + const std::string key = + KvKey(session, request_index, layer, block, is_shared_prefix); + const int64_t got = client->get_into(key, buffer, FLAGS_value_size); + if (got >= 0) { + ++stats.reads; + } else { + ++stats.read_failures; + } + ++read_keys; + } + } + } + + client->unregister_buffer(buffer); + numa_free(buffer, block_bytes); + LOG(INFO) << "Real-data seed stage done: written=" << stats.written + << " write_failures=" << stats.write_failures + << " reads=" << stats.reads + << " read_failures=" << stats.read_failures; + return stats; +} + } // namespace int main(int argc, char* argv[]) { @@ -552,6 +728,13 @@ int main(int argc, char* argv[]) { deployment_description = "remote SubMaster(s) via CFM coro_rpc"; } + // Real-data seeding runs before the simulated request stream ("先种子后仿 + // 真"): the SubMaster must hold real replicas for reported keys before the + // report-driven policy cycle can execute eviction/promotion/prefetch + // against them. Only meaningful with a real SubMaster endpoint + // (--cfm_endpoint) plus RealClient parameters; otherwise it is a no-op. + const SeedStats seed_stats = RunRealSeedStage(); + IoPatternRuntime::Config source_config; source_config.report_capacity = FLAGS_report_capacity; MetricReportStats metric_reports; @@ -654,6 +837,10 @@ int main(int argc, char* argv[]) { << "CFM CLIENT BENCHMARK (vLLM inference request model)\n" << "============================================================\n" << " CFM deployment: " << deployment_description << "\n" + << " Real-data seeding: written=" << seed_stats.written + << " (failures=" << seed_stats.write_failures + << "), reads=" << seed_stats.reads + << " (failures=" << seed_stats.read_failures << ")\n" << " Requests: " << FLAGS_requests << "\n" << " Tokens/request: " << FLAGS_prompt_tokens + FLAGS_output_tokens << " (prompt=" diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 76a745eacc..45e6e51d7d 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -51,6 +51,10 @@ class IoPatternRuntime final { uint64_t eviction_target_bytes{0}; size_t eviction_candidates{0}; ErrorCode eviction_status{ErrorCode::OK}; + // True when the eviction request came from the cold-data driver + // (analysis-selected idle keys) rather than a storage-pressure + // watermark request. + bool cold_eviction{false}; // Prefetch dimension (derived from merged prefix-affinity keys). size_t prefetch_candidates{0}; ErrorCode prefetch_status{ErrorCode::OK}; @@ -91,6 +95,20 @@ class IoPatternRuntime final { // ratio is reached; eviction target bytes are derived as // (peak_ratio - report_eviction_target_ratio) * capacity_bytes. float report_eviction_target_ratio{0.70F}; + // Cold-data eviction driver. When enabled, a report-driven cycle that + // sees no storage-pressure request (merged L1 ratio below the high + // watermark) still runs a bounded eviction of the coldest keys, so + // eviction is driven by cold/hot analysis rather than only by memory + // pressure. Candidate selection and handler execution are identical to + // pressure eviction; the cycle report marks `cold_eviction=true` so + // logs/metrics can distinguish the two drivers. + bool report_driven_cold_eviction{false}; + // Only keys idle (idle_time_us) at least this long are eligible for a + // cold-eviction pass. 0 disables the idle gate. + uint64_t report_driven_cold_idle_threshold_us{0}; + // Max bytes a single cold-eviction pass may request (per drained + // cycle). 0 disables the cold driver regardless of the enable flag. + uint64_t report_driven_cold_eviction_bytes{0}; // Optional per-cycle observer used to surface executions in process // metrics (e.g. MasterMetricManager). Never called from the report // data path; only from the background cycle worker. @@ -177,6 +195,13 @@ class IoPatternRuntime final { float high_ratio, float target_ratio, CacheTier& eviction_tier, uint64_t& eviction_bytes); + // Cold-data eviction driver input: scans the merged snapshot for L1 keys + // that are not pinned and idle at least `idle_threshold_us` (0 = any idle + // gate disabled) and returns the total byte budget bounded by `max_bytes`. + // Returns 0 when there is no idle L1 key to reclaim. + static uint64_t ColdEvictionBudget(const IoPatternSnapshot& snapshot, + uint64_t idle_threshold_us, + uint64_t max_bytes); static TraceHistory DeriveTraceHistory(const IoPatternSnapshot& snapshot); static std::vector DeriveAdmissionCandidates( const IoPatternSnapshot& snapshot); diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index d0ad2c96e7..5eaf5e2fd2 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -48,6 +48,13 @@ struct MasterConfig { double eviction_high_watermark_ratio; double nof_eviction_ratio; double nof_eviction_high_watermark_ratio; + // Report-driven cold-data eviction driver (embedded CFM component). + // When true, merged client reports may drive bounded evictions of the + // coldest keys even when the memory watermark is not exceeded. All three + // default to disabled (watermark-only eviction) unless explicitly set. + bool io_pattern_cold_eviction = false; + uint64_t io_pattern_cold_eviction_bytes_per_cycle = 0; + uint64_t io_pattern_cold_idle_threshold_us = 0; int64_t client_live_ttl_sec; int64_t nof_heartbeat_interval_sec; uint32_t nof_heartbeat_probe_timeout_ms; @@ -524,6 +531,10 @@ class WrappedMasterServiceConfig { double nof_eviction_ratio = DEFAULT_NOF_EVICTION_RATIO; double nof_eviction_high_watermark_ratio = DEFAULT_NOF_EVICTION_HIGH_WATERMARK_RATIO; + // Report-driven cold-data eviction driver (embedded CFM component). + bool io_pattern_cold_eviction = false; + uint64_t io_pattern_cold_eviction_bytes_per_cycle = 0; + uint64_t io_pattern_cold_idle_threshold_us = 0; ViewVersionId view_version = 0; int64_t client_live_ttl_sec = DEFAULT_CLIENT_LIVE_TTL_SEC; int64_t nof_heartbeat_interval_sec = DEFAULT_NOF_HEARTBEAT_INTERVAL_SEC; @@ -630,6 +641,11 @@ class WrappedMasterServiceConfig { nof_eviction_ratio = config.nof_eviction_ratio; nof_eviction_high_watermark_ratio = config.nof_eviction_high_watermark_ratio; + io_pattern_cold_eviction = config.io_pattern_cold_eviction; + io_pattern_cold_eviction_bytes_per_cycle = + config.io_pattern_cold_eviction_bytes_per_cycle; + io_pattern_cold_idle_threshold_us = + config.io_pattern_cold_idle_threshold_us; view_version = view_version_param; client_live_ttl_sec = config.client_live_ttl_sec; nof_heartbeat_interval_sec = config.nof_heartbeat_interval_sec; @@ -1224,6 +1240,10 @@ class MasterServiceConfig { double nof_eviction_ratio = DEFAULT_NOF_EVICTION_RATIO; double nof_eviction_high_watermark_ratio = DEFAULT_NOF_EVICTION_HIGH_WATERMARK_RATIO; + // Report-driven cold-data eviction driver (embedded CFM component). + bool io_pattern_cold_eviction = false; + uint64_t io_pattern_cold_eviction_bytes_per_cycle = 0; + uint64_t io_pattern_cold_idle_threshold_us = 0; ViewVersionId view_version = 0; int64_t client_live_ttl_sec = DEFAULT_CLIENT_LIVE_TTL_SEC; int64_t nof_heartbeat_interval_sec = DEFAULT_NOF_HEARTBEAT_INTERVAL_SEC; @@ -1327,6 +1347,11 @@ class MasterServiceConfig { nof_eviction_ratio = config.nof_eviction_ratio; nof_eviction_high_watermark_ratio = config.nof_eviction_high_watermark_ratio; + io_pattern_cold_eviction = config.io_pattern_cold_eviction; + io_pattern_cold_eviction_bytes_per_cycle = + config.io_pattern_cold_eviction_bytes_per_cycle; + io_pattern_cold_idle_threshold_us = + config.io_pattern_cold_idle_threshold_us; view_version = config.view_version; client_live_ttl_sec = config.client_live_ttl_sec; nof_heartbeat_interval_sec = config.nof_heartbeat_interval_sec; diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index 6b6c0e82cd..2b7af90348 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -2,6 +2,7 @@ #include #include +#include #include namespace mooncake::io_pattern { @@ -396,6 +397,30 @@ void IoPatternRuntime::DeriveEvictionRequest(const IoPatternSnapshot& snapshot, : (capacity_bytes > 0 ? capacity_bytes / 10 : 0); } +uint64_t IoPatternRuntime::ColdEvictionBudget( + const IoPatternSnapshot& snapshot, uint64_t idle_threshold_us, + uint64_t max_bytes) { + if (max_bytes == 0) return 0; + uint64_t budget = 0; + for (const auto& key : snapshot.keys) { + if ((key.replica_tiers & CacheTierBit(CacheTier::kL1Host)) == 0 || + key.pinned) { + continue; + } + if (idle_threshold_us != 0 && + key.idle_time_us < idle_threshold_us) { + continue; + } + // Skip keys with no capacity estimate (block_size unset/unknown). + if (key.block_size == 0) continue; + budget = budget > std::numeric_limits::max() - key.block_size + ? std::numeric_limits::max() + : budget + key.block_size; + if (budget >= max_bytes) return max_bytes; + } + return budget; +} + TraceHistory IoPatternRuntime::DeriveTraceHistory( const IoPatternSnapshot& snapshot) { // Report-driven prefetch input: keys that were recently served as hits and @@ -457,6 +482,22 @@ void IoPatternRuntime::RunReportDrivenCycle() { DeriveEvictionRequest(snapshot, config_.report_eviction_high_ratio, config_.report_eviction_target_ratio, eviction_tier, eviction_bytes); + // Cold-data eviction driver: when the merged storage watermark does not + // trigger a pressure eviction but the analysis-relevant snapshot contains + // idle L1 keys, run a bounded eviction of the coldest keys so eviction is + // driven by cold/hot analysis and not only by memory pressure. Candidate + // selection still goes through the policy engine (ScoreBasedEvictionOps + // ranks the coldest first); this driver only supplies a byte target. + if (eviction_bytes == 0 && config_.report_driven_cold_eviction && + config_.report_driven_cold_eviction_bytes != 0) { + const uint64_t cold_bytes = ColdEvictionBudget( + snapshot, config_.report_driven_cold_idle_threshold_us, + config_.report_driven_cold_eviction_bytes); + if (cold_bytes != 0) { + eviction_bytes = cold_bytes; + report.cold_eviction = true; + } + } report.eviction_tier = eviction_tier; report.eviction_target_bytes = eviction_bytes; const auto trace = DeriveTraceHistory(snapshot); diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index b6176b11e7..223e52e656 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -143,6 +143,16 @@ DEFINE_double(nof_eviction_ratio, mooncake::DEFAULT_NOF_EVICTION_RATIO, DEFINE_double(nof_eviction_high_watermark_ratio, mooncake::DEFAULT_NOF_EVICTION_HIGH_WATERMARK_RATIO, "Ratio of high watermark trigger eviction in NoF SSD"); +DEFINE_bool(io_pattern_cold_eviction, false, + "Enable report-driven cold-data eviction (embedded CFM component): " + "merged client reports may drive bounded evictions of the coldest " + "keys even when the memory high-watermark is not exceeded"); +DEFINE_uint64(io_pattern_cold_eviction_bytes_per_cycle, 0, + "Max bytes one report-driven cold-eviction pass may request " + "(0 disables the cold driver)"); +DEFINE_uint64(io_pattern_cold_idle_threshold_us, 0, + "Minimum idle_time_us for a key to be eligible for a " + "report-driven cold-eviction pass (0 disables the idle gate)"); // RPC server configuration parameters (new, preferred) // TODO: deprecate port and max_threads in the future DEFINE_int32(rpc_thread_num, 0, @@ -533,6 +543,15 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, default_config.GetDouble("nof_eviction_high_watermark_ratio", &master_config.nof_eviction_high_watermark_ratio, FLAGS_nof_eviction_high_watermark_ratio); + default_config.GetBool("io_pattern_cold_eviction", + &master_config.io_pattern_cold_eviction, + FLAGS_io_pattern_cold_eviction); + default_config.GetUInt64("io_pattern_cold_eviction_bytes_per_cycle", + &master_config.io_pattern_cold_eviction_bytes_per_cycle, + FLAGS_io_pattern_cold_eviction_bytes_per_cycle); + default_config.GetUInt64("io_pattern_cold_idle_threshold_us", + &master_config.io_pattern_cold_idle_threshold_us, + FLAGS_io_pattern_cold_idle_threshold_us); default_config.GetInt64("client_live_ttl_sec", &master_config.client_live_ttl_sec, FLAGS_client_ttl); @@ -897,6 +916,26 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, master_config.nof_eviction_high_watermark_ratio = FLAGS_nof_eviction_high_watermark_ratio; } + if ((google::GetCommandLineFlagInfo("io_pattern_cold_eviction", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cold_eviction = + FLAGS_io_pattern_cold_eviction; + } + if ((google::GetCommandLineFlagInfo( + "io_pattern_cold_eviction_bytes_per_cycle", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cold_eviction_bytes_per_cycle = + FLAGS_io_pattern_cold_eviction_bytes_per_cycle; + } + if ((google::GetCommandLineFlagInfo("io_pattern_cold_idle_threshold_us", + &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_cold_idle_threshold_us = + FLAGS_io_pattern_cold_idle_threshold_us; + } if ((google::GetCommandLineFlagInfo("enable_ha", &info) && !info.is_default) || !conf_set) { diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index b49f56818e..67107ea7c0 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -452,6 +452,21 @@ MasterService::MasterService(const MasterServiceConfig& config) static_cast(eviction_high_watermark_ratio_); io_pattern_config.report_eviction_target_ratio = static_cast( std::max(0.0, eviction_high_watermark_ratio_ - eviction_ratio_)); + // Cold-data eviction driver: allow merged reports to reclaim the coldest + // real objects even below the memory watermark (opt-in via master flags). + io_pattern_config.report_driven_cold_eviction = + config.io_pattern_cold_eviction; + io_pattern_config.report_driven_cold_eviction_bytes = + config.io_pattern_cold_eviction_bytes_per_cycle; + io_pattern_config.report_driven_cold_idle_threshold_us = + config.io_pattern_cold_idle_threshold_us; + if (config.io_pattern_cold_eviction) { + LOG(INFO) << "Report-driven cold-data eviction enabled: per-cycle " + "budget=" + << config.io_pattern_cold_eviction_bytes_per_cycle + << " bytes, idle threshold=" + << config.io_pattern_cold_idle_threshold_us << " us"; + } io_pattern_config.report_driven_observer = [](const io_pattern::IoPatternRuntime::ReportDrivenCycleReport& rpt) { auto& metrics = MasterMetricManager::instance(); @@ -490,7 +505,7 @@ MasterService::MasterService(const MasterServiceConfig& config) << ", bytes=" << rpt.eviction_target_bytes << ", candidates=" << rpt.eviction_candidates << ", status=" << static_cast(rpt.eviction_status) - << ")" + << ", cold=" << rpt.cold_eviction << ")" << " prefetch(candidates=" << rpt.prefetch_candidates << ", status=" << static_cast(rpt.prefetch_status) << ")" diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 3f381c92c9..c80a674dad 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1256,6 +1256,61 @@ TEST(IoPatternFrameworkTest, ReportDrivenCycleSkipsEvictionWithoutPressure) { EXPECT_EQ(eviction_handled.load(), 0); } +TEST(IoPatternFrameworkTest, ReportDrivenColdEvictionRunsWithoutPressure) { + // The cold-data eviction driver must run even when no storage watermark + // pressure is present: a report that only contains idle L1 keys still + // yields a bounded eviction request (cold_eviction=true) so eviction is + // driven by cold/hot analysis, not only by memory pressure. + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::atomic eviction_handled{0}; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + config.report_driven_cold_eviction = true; + config.report_driven_cold_eviction_bytes = 128ULL * 1024 * 1024; + config.report_driven_cold_idle_threshold_us = 0; // any idle L1 key counts + config.analysis_timeout_us = 30'000'000; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [&eviction_handled](const EvictionPlan&) { + ++eviction_handled; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + CfmService service(rt); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "cold-key"}, + .observed_at_ns = 1, + .block_size = 1024, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + // No storage metric => no pressure path; only the cold driver can act. + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(30), [&] { + return last_report.has_value(); + })); + ASSERT_TRUE(last_report.has_value()); + EXPECT_TRUE(last_report->cold_eviction); + EXPECT_GT(last_report->eviction_target_bytes, 0); + EXPECT_GT(eviction_handled.load(), 0); + EXPECT_EQ(last_report->eviction_status, ErrorCode::OK); +} + TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { size_t batches = 0; IoPatternReporter reporter(4, [&](const MetricBatch&) { From bf762ed7c988a311e891c1e70dd4f00687173f00 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 9 Sep 2026 14:21:01 +0800 Subject: [PATCH 15/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=983?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../benchmarks/cfm_client_bench.cpp | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index ba514c1afc..a89cdb8c0e 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -139,6 +139,19 @@ DEFINE_bool(hard_pin, false, "Pin seeded objects (disable eviction of them)"); DEFINE_uint64(seed_get_keys, 0, "How many of the seeded keys to read back with get_into " "(0 = half of num_keys)"); +// Report RPC timeout. A merged report can carry hundreds of thousands of +// observations; the old fixed 500 ms budget caused report_metric_batch RPC +// failures on large batches. Also bounds how long the ownership client waits +// per SubMaster delivery. +DEFINE_uint64(cfm_rpc_timeout_ms, 5000, + "Timeout for each CFM report RPC (report_snapshot / " + "report_metric_batch) in milliseconds"); +// Client-side collector/analysis key budget. The default 100k cap drops +// observations once the simulated request stream exceeds it (seen as nonzero +// \"report drops\"); raise it to cover the whole run when reporting many keys. +DEFINE_uint64(max_analysis_keys, 100000, + "Max merged keys kept/analyzed by the client runtime and the " + "embedded SubMaster runtime (raise with the request stream size)"); uint64_t SteadyNowNs() { return static_cast( @@ -700,6 +713,7 @@ int main(int argc, char* argv[]) { // reflect report-triggered policy execution, not only the manual // watermark evaluation at the end of the run. cfm_config.report_driven_execution = true; + cfm_config.max_analysis_keys = FLAGS_max_analysis_keys; cfm_runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [&eviction_commands](const EvictionPlan&) { @@ -720,11 +734,12 @@ int main(int argc, char* argv[]) { std::make_shared(embedded_service); embedded_channel = std::make_shared( std::move(transport), std::make_shared(), - CfmRpcConfig{.timeout = std::chrono::milliseconds(500)}); + CfmRpcConfig{.timeout = + std::chrono::milliseconds(FLAGS_cfm_rpc_timeout_ms)}); } else { const auto resolver = ResolveCfmEndpointOwnership(); - ownership_client = - std::make_shared(resolver, std::chrono::milliseconds(500)); + ownership_client = std::make_shared( + resolver, std::chrono::milliseconds(FLAGS_cfm_rpc_timeout_ms)); deployment_description = "remote SubMaster(s) via CFM coro_rpc"; } @@ -737,6 +752,8 @@ int main(int argc, char* argv[]) { IoPatternRuntime::Config source_config; source_config.report_capacity = FLAGS_report_capacity; + source_config.max_analysis_keys = FLAGS_max_analysis_keys; + source_config.collector.max_total_keys = FLAGS_max_analysis_keys; MetricReportStats metric_reports; const auto report_metric_batch = [&](const MetricBatch& batch) -> bool { const auto started = Clock::now(); From af203faf38dbab8c9ecb7b290653d2cb54a8c08e Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 9 Sep 2026 14:54:04 +0800 Subject: [PATCH 16/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=984?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/src/master_service.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 67107ea7c0..dba35f65bb 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -5483,6 +5483,13 @@ auto MasterService::PutEndInternal( metadata.pending_replaced_quota_charge_bytes = 0; } + // Write-through offload legacy path: every completed MEMORY replica is + // queued to LOCAL_DISK right after PutEnd. This only runs in legacy mode + // (enable_offload=true without offload_on_evict). When offload-on-evict + // is enabled the block below is skipped: the new object stays in L1 and + // demotion happens at eviction time, where the IO Pattern policy selects + // the coldest L1 keys and EvictTenantMemoryForQuota demotes or releases + // them (lower-tier-backed keys are preferred victims). if (enable_offload_ && !offload_on_evict_) { auto& tenant_state = accessor.GetTenantState(); metadata.VisitReplicas( From 92b6485a1533eec426bd3ba6a843d54fcae4823a Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 9 Sep 2026 16:12:54 +0800 Subject: [PATCH 17/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=985?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../benchmarks/cfm_client_bench.cpp | 8 +++++++ .../include/io_pattern/cfm_ownership_client.h | 10 ++++++-- mooncake-store/include/master_config.h | 16 +++++++++++++ .../src/io_pattern/cfm_ownership_client.cpp | 24 ++++++++++++++++--- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index a89cdb8c0e..a755056eb3 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -152,6 +152,13 @@ DEFINE_uint64(cfm_rpc_timeout_ms, 5000, DEFINE_uint64(max_analysis_keys, 100000, "Max merged keys kept/analyzed by the client runtime and the " "embedded SubMaster runtime (raise with the request stream size)"); +// Remote reports normally omit storage watermarks (the owning SubMaster +// reports its own); enabling this attaches the reported storage metrics to +// every owner-addressed snapshot/metric batch so a remote run can drive the +// report-driven eviction dimension from the client side. +DEFINE_bool(report_forward_storage, false, + "Forward StorageMetric observations with remote owner-addressed " + "reports (benchmark/simulation mode)"); uint64_t SteadyNowNs() { return static_cast( @@ -740,6 +747,7 @@ int main(int argc, char* argv[]) { const auto resolver = ResolveCfmEndpointOwnership(); ownership_client = std::make_shared( resolver, std::chrono::milliseconds(FLAGS_cfm_rpc_timeout_ms)); + ownership_client->set_forward_storage(FLAGS_report_forward_storage); deployment_description = "remote SubMaster(s) via CFM coro_rpc"; } diff --git a/mooncake-store/include/io_pattern/cfm_ownership_client.h b/mooncake-store/include/io_pattern/cfm_ownership_client.h index 46b8ca967f..837ed81d1c 100644 --- a/mooncake-store/include/io_pattern/cfm_ownership_client.h +++ b/mooncake-store/include/io_pattern/cfm_ownership_client.h @@ -51,10 +51,15 @@ class CfmOwnershipClient final : public CfmClient { // Sends the metric batch, grouped by the owning SubMaster of each // inference/access object. Storage observations are deliberately not - // routed: the SubMaster that owns the underlying storage already reports - // its own watermark to its local runtime. + // routed by default: the SubMaster that owns the underlying storage + // already reports its own watermark to its local runtime. Benchmark / + // simulation callers can enable forward_storage to have the storage + // metrics ride along with each owner-addressed report so a remote run can + // drive the report-driven eviction dimension. ErrorCode ReportMetricBatch(const MetricBatch& batch) override; + void set_forward_storage(bool forward) { forward_storage_ = forward; } + // Sends an explicit prefetch plan to the SubMaster that owns the first // candidate; that SubMaster executes it through its local storage-safe // handlers. @@ -72,6 +77,7 @@ class CfmOwnershipClient final : public CfmClient { std::unordered_map> channels_; mutable std::mutex channels_mutex_; std::atomic dropped_observations_{0}; + bool forward_storage_{false}; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index 5eaf5e2fd2..7c8cc58dc3 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -292,6 +292,12 @@ class MasterServiceSupervisorConfig { uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; uint32_t promotion_max_per_heartbeat = 1; + // Report-driven cold-data eviction driver (embedded CFM component). + // Mirrors MasterConfig / WrappedMasterServiceConfig; carried through the + // supervisor config used by HA deployments. + bool io_pattern_cold_eviction = false; + uint64_t io_pattern_cold_eviction_bytes_per_cycle = 0; + uint64_t io_pattern_cold_idle_threshold_us = 0; bool enable_kv_events = false; std::string kv_events_bind_endpoint; std::string kv_events_model_name; @@ -332,6 +338,11 @@ class MasterServiceSupervisorConfig { nof_eviction_ratio = config.nof_eviction_ratio; nof_eviction_high_watermark_ratio = config.nof_eviction_high_watermark_ratio; + io_pattern_cold_eviction = config.io_pattern_cold_eviction; + io_pattern_cold_eviction_bytes_per_cycle = + config.io_pattern_cold_eviction_bytes_per_cycle; + io_pattern_cold_idle_threshold_us = + config.io_pattern_cold_idle_threshold_us; client_live_ttl_sec = config.client_live_ttl_sec; nof_heartbeat_interval_sec = config.nof_heartbeat_interval_sec; nof_heartbeat_probe_timeout_ms = config.nof_heartbeat_probe_timeout_ms; @@ -769,6 +780,11 @@ class WrappedMasterServiceConfig { nof_eviction_ratio = config.nof_eviction_ratio; nof_eviction_high_watermark_ratio = config.nof_eviction_high_watermark_ratio; + io_pattern_cold_eviction = config.io_pattern_cold_eviction; + io_pattern_cold_eviction_bytes_per_cycle = + config.io_pattern_cold_eviction_bytes_per_cycle; + io_pattern_cold_idle_threshold_us = + config.io_pattern_cold_idle_threshold_us; view_version = view_version_param; client_live_ttl_sec = config.client_live_ttl_sec; nof_heartbeat_interval_sec = config.nof_heartbeat_interval_sec; diff --git a/mooncake-store/src/io_pattern/cfm_ownership_client.cpp b/mooncake-store/src/io_pattern/cfm_ownership_client.cpp index 943c5f34cb..cfb6e72b4a 100644 --- a/mooncake-store/src/io_pattern/cfm_ownership_client.cpp +++ b/mooncake-store/src/io_pattern/cfm_ownership_client.cpp @@ -33,8 +33,12 @@ std::shared_ptr CfmOwnershipClient::ChannelFor( ErrorCode CfmOwnershipClient::ReportSnapshot(const IoPatternSnapshot& snapshot) { // Bucket keys by their owning SubMaster and report one snapshot per owner. - // Storage observations are not routed here: the SubMaster that owns the - // storage already reports its own watermark into its local runtime. + // Storage observations are not routed by default: the SubMaster that owns + // the storage already reports its own watermark into its local runtime. + // When forward_storage_ is enabled (benchmark / simulation mode) every + // owner-addressed snapshot also carries the reported storage metrics so a + // remote run can drive the report-driven eviction dimension on each + // owning SubMaster. std::unordered_map by_owner; for (const auto& key : snapshot.keys) { const auto endpoint = OwnerEndpoint(key.object); @@ -46,6 +50,12 @@ ErrorCode CfmOwnershipClient::ReportSnapshot(const IoPatternSnapshot& snapshot) owned.generated_at_ns = snapshot.generated_at_ns; owned.keys.push_back(key); } + if (forward_storage_ && !snapshot.storage.empty() && !by_owner.empty()) { + for (auto& [endpoint, owned] : by_owner) { + (void)endpoint; + owned.storage = snapshot.storage; + } + } bool all_ok = true; for (const auto& [endpoint, owned] : by_owner) { auto channel = ChannelFor(endpoint); @@ -72,7 +82,15 @@ ErrorCode CfmOwnershipClient::ReportMetricBatch(const MetricBatch& batch) { } by_owner[endpoint].accesses.push_back(access); } - // batch.storage is intentionally not forwarded (see header/ReportSnapshot). + // batch.storage is intentionally not forwarded by default (see + // ReportSnapshot); forward_storage_ attaches it to every owner-addressed + // batch for benchmark / simulation runs. + if (forward_storage_ && !batch.storage.empty() && !by_owner.empty()) { + for (auto& [endpoint, owned] : by_owner) { + (void)endpoint; + owned.storage = batch.storage; + } + } bool all_ok = true; for (const auto& [endpoint, owned] : by_owner) { auto channel = ChannelFor(endpoint); From d33fd0003a205dd237130d8a41e10734556f63b4 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 9 Sep 2026 17:00:32 +0800 Subject: [PATCH 18/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=986?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../benchmarks/cfm_client_bench.cpp | 135 +++++++++++++++--- 1 file changed, 118 insertions(+), 17 deletions(-) diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index a755056eb3..4b1c21a5bb 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -572,11 +572,21 @@ struct SeedStats { uint64_t read_failures{0}; }; -SeedStats RunRealSeedStage() { +// Captures what the seeding stage actually created so the reporting stage can +// address exactly the real keys (a report stream over synthetic keys whose +// objects were never stored leaves the SubMaster handlers with nothing to act +// on, which shows up as OBJECT_NOT_FOUND / zero master-side evictions). +struct SeedOutcome { SeedStats stats; + std::vector keys; // real keys written, stable order + std::vector hot; // parallel: read back (access heat) +}; + +SeedOutcome RunRealSeedStage() { + SeedOutcome outcome; if (FLAGS_master_server.empty() || FLAGS_num_keys == 0 || FLAGS_value_size == 0) { - return stats; + return outcome; } LOG(INFO) << "Real-data seed stage: master_server=" << FLAGS_master_server << " protocol=" << FLAGS_protocol @@ -591,7 +601,7 @@ SeedStats RunRealSeedStage() { if (buffer == nullptr) { LOG(ERROR) << "numa_alloc_local failed for seed buffer of " << block_bytes << " bytes"; - return stats; + return outcome; } std::memset(buffer, 0xA5, block_bytes); int ret = client->setup_real( @@ -602,13 +612,13 @@ SeedStats RunRealSeedStage() { if (ret != 0) { LOG(ERROR) << "setup_real failed: " << ret; numa_free(buffer, block_bytes); - return stats; + return outcome; } ret = client->register_buffer(buffer, block_bytes); if (ret != 0) { LOG(ERROR) << "register_buffer failed: " << ret; numa_free(buffer, block_bytes); - return stats; + return outcome; } // Write keys that share the simulated KvKey naming so later reports and @@ -640,20 +650,21 @@ SeedStats RunRealSeedStage() { client->put_from(key, buffer, FLAGS_value_size, config); if (put_ret == 0) { ++seeded; + outcome.keys.push_back(key); } else { - ++stats.write_failures; + ++outcome.stats.write_failures; LOG(WARNING) << "put_from failed for seed key " << key << ": " << put_ret; } } } } - stats.written = seeded; + outcome.stats.written = seeded; // Simulate reads: exercise a hot subset through the real data path so the // SubMaster records real GET access heat (promotion-on-hit when offloaded). - // Re-enumerate the same key space in the same order and read the first - // read_count keys. + // Mark the same prefix of the written key list as hot for the report pass. + outcome.hot.assign(outcome.keys.size(), false); const uint64_t read_count = FLAGS_seed_get_keys == 0 ? seeded / 2 : std::min(FLAGS_seed_get_keys, @@ -672,9 +683,12 @@ SeedStats RunRealSeedStage() { KvKey(session, request_index, layer, block, is_shared_prefix); const int64_t got = client->get_into(key, buffer, FLAGS_value_size); if (got >= 0) { - ++stats.reads; + ++outcome.stats.reads; } else { - ++stats.read_failures; + ++outcome.stats.read_failures; + } + if (read_keys < outcome.hot.size()) { + outcome.hot[read_keys] = true; } ++read_keys; } @@ -683,11 +697,11 @@ SeedStats RunRealSeedStage() { client->unregister_buffer(buffer); numa_free(buffer, block_bytes); - LOG(INFO) << "Real-data seed stage done: written=" << stats.written - << " write_failures=" << stats.write_failures - << " reads=" << stats.reads - << " read_failures=" << stats.read_failures; - return stats; + LOG(INFO) << "Real-data seed stage done: written=" << outcome.stats.written + << " write_failures=" << outcome.stats.write_failures + << " reads=" << outcome.stats.reads + << " read_failures=" << outcome.stats.read_failures; + return outcome; } } // namespace @@ -756,7 +770,8 @@ int main(int argc, char* argv[]) { // report-driven policy cycle can execute eviction/promotion/prefetch // against them. Only meaningful with a real SubMaster endpoint // (--cfm_endpoint) plus RealClient parameters; otherwise it is a no-op. - const SeedStats seed_stats = RunRealSeedStage(); + const SeedOutcome seed_outcome = RunRealSeedStage(); + const SeedStats& seed_stats = seed_outcome.stats; IoPatternRuntime::Config source_config; source_config.report_capacity = FLAGS_report_capacity; @@ -790,8 +805,94 @@ int main(int argc, char* argv[]) { uint64_t failed_reports = 0; uint64_t total_blocks = 0; const auto benchmark_start = Clock::now(); + + // When a real seed set was written, report exactly those keys (the ones + // with real replicas) instead of the synthetic request stream. Synthetic + // keys never stored on the SubMaster pollute the merged snapshot: the + // policy engine selects coldest candidates from them, and the storage + // handler then finds no real object to evict (OBJECT_NOT_FOUND / zero + // master-side evictions). The SubMaster's own data path already records + // the real PUT/GET heat for the seeded keys, so reporting the same keys + // gives the report-driven cycle candidates that actually exist. + const bool real_seed_mode = !seed_outcome.keys.empty(); + size_t seed_report_requests = 0; + if (real_seed_mode) { + const auto now_ns = SteadyNowNs(); + const TenantId tenant(FLAGS_tenant); + IoPatternSnapshot real_snapshot; + real_snapshot.generated_at_ns = now_ns; + real_snapshot.keys.reserve(seed_outcome.keys.size()); + uint64_t hot_blocks = 0; + for (size_t i = 0; i < seed_outcome.keys.size(); ++i) { + const auto& key = seed_outcome.keys[i]; + const bool is_hot = seed_outcome.hot[i]; + const ObjectRef object{.tenant_id = tenant, .key = key}; + // Hot keys were read back by the seed stage; cold keys carry an + // older last_access so the analyzer sees a hot/cold split over the + // real set (matching the benchmark's own read pattern). + KeyMetrics key_metrics{ + .object = object, + .session_id = "seed-real-keys", + .last_access_time_ns = + is_hot ? now_ns + : (now_ns > 60'000'000'000ULL + ? now_ns - 60'000'000'000ULL + : 0ULL), + .access_count_window = is_hot ? 4U : 0U, + .block_size = FLAGS_value_size, + .token_count = 16U, + .write_frequency = 1U, + .write_object_size = FLAGS_value_size, + .replica_tiers = CacheTierBit(CacheTier::kL1Host), + .active = is_hot}; + real_snapshot.keys.push_back(std::move(key_metrics)); + if (is_hot) ++hot_blocks; + // Feed the source runtime too so the client-side printout and the + // per-owner report share the same picture. + AccessRecord access{ + .object = object, + .observed_at_ns = now_ns, + .block_size = FLAGS_value_size, + .latency_us = is_hot ? 20U : 200U, + .tier = CacheTier::kL1Host, + .operation = is_hot ? IoOperation::kGet : IoOperation::kPut, + .is_hit = is_hot}; + source_runtime->RecordAccess(key, access); + ++total_blocks; + } + real_snapshot.storage.push_back( + StorageMetric{.source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .read_bandwidth_bytes_per_sec = + 20ULL * 1024 * 1024 * 1024, + .write_bandwidth_bytes_per_sec = + 10ULL * 1024 * 1024 * 1024, + .read_latency_us = 20, + .write_latency_us = 200, + .used_bytes = static_cast( + static_cast(FLAGS_value_size) * + seed_outcome.keys.size()), + .capacity_bytes = + static_cast(FLAGS_num_keys) * + FLAGS_value_size, + .rpc_latency_us = 100, + .memory_used_ratio = + static_cast(FLAGS_memory_used_ratio)}); + const auto report_start = Clock::now(); + const bool sent = send_snapshot(real_snapshot); + report_latency.Record(ToMicroseconds(Clock::now() - report_start)); + if (!sent) ++failed_reports; + seed_report_requests = 1; + LOG(INFO) << "Real-seed report sent: keys=" + << real_snapshot.keys.size() << " hot=" << hot_blocks + << " cold=" << real_snapshot.keys.size() - hot_blocks + << " (synthetic request stream skipped)"; + } + for (size_t request_index = 0; request_index < FLAGS_requests; ++request_index) { + if (real_seed_mode) break; // real keys already reported above auto request = BuildRequest(request_index); total_blocks += request.accesses.size(); for (size_t i = 0; i < request.inference.size(); ++i) { From 18bf2239331154424d7b7ad20f45c37bb70061a9 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 9 Sep 2026 17:28:45 +0800 Subject: [PATCH 19/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=987?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/src/master_service.cpp | 37 +++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index dba35f65bb..d8b371d4c8 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -542,7 +542,19 @@ MasterService::MasterService(const MasterServiceConfig& config) total_freed ? std::numeric_limits::max() : total_freed + result.freed_bytes; + LOG(WARNING) + << "[IO-PATTERN-EVICT-DIAG] io_pattern eviction " + "tenant=" + << tenant.value() << " target=" << target.bytes + << " freed=" << result.freed_bytes + << " evicted_objects=" << result.evicted_objects + << " candidate_keys=" << target.keys.size(); } + LOG(WARNING) + << "[IO-PATTERN-EVICT-DIAG] io_pattern eviction " + "summary plan_target=" + << plan.target_bytes << " total_freed=" << total_freed + << " candidates=" << plan.candidates.size(); return total_freed >= plan.target_bytes ? ErrorCode::OK : ErrorCode::OBJECT_NOT_FOUND; @@ -9742,6 +9754,19 @@ MasterService::EvictTenantMemoryForQuota( OffloadingTask{replica.id(), now, client_id}); } queued = true; + } else { + // Diagnostic: surface why offload-on-evict could not + // enqueue this MEMORY replica (io_pattern quota + // eviction path used by the report-driven cycle). + LOG(WARNING) + << "[IO-PATTERN-EVICT-DIAG] quota offload enqueue " + "failed for key=" + << key << " tenant=" << normalized_tenant.value() + << " error=" + << (result ? toString(result.error()) + : "empty_result") + << " replica_segments=" + << replica.get_segment_names().size(); } }); @@ -10015,6 +10040,18 @@ void MasterService::BatchEvict(double evict_ratio_target, OffloadingTask{replica.id(), now, client_id}); } queued = true; + } else { + // Diagnostic: surface why the offload queue rejected this + // MEMORY replica (empty result = no segment names on the + // replica or no matching LOCAL_DISK holder). + LOG(WARNING) + << "[IO-PATTERN-EVICT-DIAG] BatchEvict offload enqueue " + "failed for key=" + << key << " tenant=" << tenant_id.value() + << " error=" + << (result ? toString(result.error()) : "empty_result") + << " replica_segments=" + << replica.get_segment_names().size(); } }); From 6985a193b7943cce85f6338289baf9d7a8e86d50 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Wed, 9 Sep 2026 17:47:06 +0800 Subject: [PATCH 20/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dcfm=20client=E9=97=AE?= =?UTF-8?q?=E9=A2=988?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/src/master_service.cpp | 42 +++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index d8b371d4c8..475d2688d7 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -549,6 +549,13 @@ MasterService::MasterService(const MasterServiceConfig& config) << " freed=" << result.freed_bytes << " evicted_objects=" << result.evicted_objects << " candidate_keys=" << target.keys.size(); + size_t shown = 0; + for (const auto& key : target.keys) { + if (shown++ >= 3) break; + LOG(WARNING) + << "[IO-PATTERN-EVICT-DIAG] candidate key=" + << key; + } } LOG(WARNING) << "[IO-PATTERN-EVICT-DIAG] io_pattern eviction " @@ -9853,6 +9860,11 @@ MasterService::EvictTenantMemoryForQuota( auto pass = [&](bool allow_soft_pinned) { const size_t start_shard = randomIndex(kNumShards); + size_t diag_candidate_hits = 0; + size_t diag_candidate_skipped_pin = 0; + size_t diag_candidate_skipped_lease = 0; + size_t diag_candidate_skipped_replica = 0; + size_t diag_candidate_evicted = 0; for (size_t scanned = 0; scanned < kNumShards && total.freed_bytes < target_bytes; ++scanned) { @@ -9878,15 +9890,31 @@ MasterService::EvictTenantMemoryForQuota( !metadata.IsLeaseExpired(now) || (!allow_soft_pinned && metadata.IsSoftPinned(now)) || !can_evict_replicas(metadata)) { + if (candidate_keys) { + ++diag_candidate_hits; + if (metadata.IsHardPinned() || + (!allow_soft_pinned && + metadata.IsSoftPinned(now))) { + ++diag_candidate_skipped_pin; + } else if (!metadata.IsLeaseExpired(now)) { + ++diag_candidate_skipped_lease; + } else { + ++diag_candidate_skipped_replica; + } + } ++it; continue; } + if (candidate_keys) ++diag_candidate_hits; auto evict_result = try_evict_group_or_object( it->first, metadata, tenant_state, deferred_replicas, allow_soft_pinned); total.freed_bytes += evict_result.freed_bytes; total.evicted_objects += evict_result.evicted_objects; + if (candidate_keys && evict_result.freed_bytes > 0) { + ++diag_candidate_evicted; + } if (!metadata.IsValid()) { it = EraseMetadata(tenant_state, it, normalized_tenant); } else { @@ -9898,6 +9926,20 @@ MasterService::EvictTenantMemoryForQuota( } } } + if (candidate_keys) { + LOG(WARNING) + << "[IO-PATTERN-EVICT-DIAG] quota pass candidate scan " + "tenant=" + << normalized_tenant.value() + << " allow_soft_pinned=" << allow_soft_pinned + << " hits_in_metadata=" << diag_candidate_hits + << " skipped_pin=" << diag_candidate_skipped_pin + << " skipped_lease=" << diag_candidate_skipped_lease + << " skipped_no_evictable_replica=" + << diag_candidate_skipped_replica + << " evicted=" << diag_candidate_evicted + << " freed_bytes=" << total.freed_bytes; + } }; pass(/*allow_soft_pinned=*/false); From 999b128ae99e1f033ef6bee52cddf3c9ec37fc39 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 10:36:33 +0800 Subject: [PATCH 21/47] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dio=5Fpattern=E9=97=AE?= =?UTF-8?q?=E9=A2=981?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../include/io_pattern/tier_executor.h | 7 ++ mooncake-store/src/io_pattern/runtime.cpp | 17 +++- .../src/io_pattern/tier_executor.cpp | 37 +++++++-- mooncake-store/src/master_service.cpp | 82 ++++++++++++++++--- .../tests/io_pattern_framework_test.cpp | 48 +++++++++++ 5 files changed, 168 insertions(+), 23 deletions(-) diff --git a/mooncake-store/include/io_pattern/tier_executor.h b/mooncake-store/include/io_pattern/tier_executor.h index 30a03bb6fd..2b28a177cc 100644 --- a/mooncake-store/include/io_pattern/tier_executor.h +++ b/mooncake-store/include/io_pattern/tier_executor.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -18,6 +19,12 @@ struct PolicyExecutionStatus { ErrorCode prefetch{ErrorCode::OK}; std::vector admissions; bool degraded{false}; + // Dimensions whose handler was present but declined to act because the + // underlying storage primitive cannot run in the current mode (promotion + // disabled, no lower-tier source replica, or a refusal to move data into an + // inference-runtime-owned tier). A skip is not a policy failure: the policy + // cannot influence that outcome, so it must not drive degradation. + size_t skipped{0}; }; // Bridges policy output to storage/tier mechanisms owned by other modules. diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index 2b7af90348..5b35b0b9ac 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -6,6 +6,17 @@ #include namespace mooncake::io_pattern { +namespace { + +// A dimension that reported UNAVAILABLE_IN_CURRENT_MODE declined to act because +// the storage primitive cannot run in this configuration. The policy cannot +// influence that outcome, so it is not a policy failure. +bool IsPolicyFailure(ErrorCode code) { + return code != ErrorCode::OK && + code != ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; +} + +} // namespace IoPatternRuntime::IoPatternRuntime(Handlers handlers) : IoPatternRuntime(std::move(handlers), Config{}) {} @@ -197,8 +208,8 @@ PolicyExecutionStatus IoPatternRuntime::CommitPolicy(PlannedPolicy& planned) { const auto& result = planned.result; auto status = executor_.Execute(result); status.degraded = status.degraded || result.degraded; - const bool failed = status.eviction != ErrorCode::OK || - status.prefetch != ErrorCode::OK || status.degraded; + const bool failed = IsPolicyFailure(status.eviction) || + IsPolicyFailure(status.prefetch) || status.degraded; if (failed) policy_->RecordFailure(); else @@ -595,7 +606,7 @@ ErrorCode IoPatternRuntime::ExecuteAdmission(const ObjectRef& object, if (analysis_degraded) status.degraded = true; const auto code = status.admissions.empty() ? ErrorCode::OK : status.admissions.front(); - if (code != ErrorCode::OK || status.degraded) + if (IsPolicyFailure(code) || status.degraded) observability_.RecordDegrade(); return code; } diff --git a/mooncake-store/src/io_pattern/tier_executor.cpp b/mooncake-store/src/io_pattern/tier_executor.cpp index 02ba4371de..4bfd0d1b37 100644 --- a/mooncake-store/src/io_pattern/tier_executor.cpp +++ b/mooncake-store/src/io_pattern/tier_executor.cpp @@ -1,33 +1,52 @@ #include "io_pattern/tier_executor.h" namespace mooncake::io_pattern { +namespace { + +// A handler that reports UNAVAILABLE_IN_CURRENT_MODE declined to act because the +// storage primitive cannot run in this configuration. That is not a failure of +// the plan: the policy had no way to influence it, so it must not be counted as +// a policy failure by the caller. +bool IsUnavailable(ErrorCode code) { + return code == ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; +} + +} // namespace PolicyExecutionStatus TierOperationExecutor::Execute( const PolicyResult& result) const { PolicyExecutionStatus status; - if (eviction_ && (!result.eviction.candidates.empty() || - result.eviction.target_bytes != 0)) { + const bool has_eviction_work = !result.eviction.candidates.empty() || + result.eviction.target_bytes != 0; + if (eviction_ && has_eviction_work) { status.eviction = eviction_(result.eviction); - } else if (!result.eviction.candidates.empty() || - result.eviction.target_bytes != 0) { + if (IsUnavailable(status.eviction)) ++status.skipped; + } else if (has_eviction_work) { + // A missing handler is a configuration error rather than a storage + // limitation, so it keeps marking the execution degraded. status.eviction = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; status.degraded = true; } if (prefetch_ && !result.prefetch.candidates.empty()) { status.prefetch = prefetch_(result.prefetch); + if (IsUnavailable(status.prefetch)) ++status.skipped; } else if (!result.prefetch.candidates.empty()) { status.prefetch = ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; status.degraded = true; } for (const auto& admission : result.admissions) { - if (admission_ && admission.decision == AdmissionDecision::kAdmit) { - status.admissions.push_back(admission_(admission)); - } else if (admission.decision == AdmissionDecision::kAdmit) { + if (admission.decision != AdmissionDecision::kAdmit) { + status.admissions.push_back(ErrorCode::OK); + continue; + } + if (!admission_) { status.admissions.push_back(ErrorCode::UNAVAILABLE_IN_CURRENT_MODE); status.degraded = true; - } else { - status.admissions.push_back(ErrorCode::OK); + continue; } + const ErrorCode code = admission_(admission); + if (IsUnavailable(code)) ++status.skipped; + status.admissions.push_back(code); } return status; } diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 475d2688d7..5c1b9d523a 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -514,6 +514,34 @@ MasterService::MasterService(const MasterServiceConfig& config) << ", status=" << static_cast(rpt.admission_status) << ")"; }; + // Promotion is the Store's only cross-tier primitive, and it can decline for + // structural reasons the policy cannot influence: promotion disabled, no + // LOCAL_DISK source replica, or watermark / queue-cap / second-touch + // backpressure. Those map to UNAVAILABLE_IN_CURRENT_MODE so the runtime + // records a skip instead of a policy failure -- otherwise three such cycles + // would permanently replace the workload policy with the legacy fallback. + // Only a vanished object or a genuine enqueue failure stays an error. + // Captured by value: a captureless lambda is an empty, copyable type, so no + // local outlives this constructor. + const auto promotion_outcome_to_error = + [](PromotionQueueResult outcome) -> ErrorCode { + switch (outcome) { + case PromotionQueueResult::kQueued: + case PromotionQueueResult::kAlreadyInFlight: + case PromotionQueueResult::kMemoryReplicaPresent: + return ErrorCode::OK; + case PromotionQueueResult::kDisabled: + case PromotionQueueResult::kFrequencyRejected: + case PromotionQueueResult::kWatermarkRejected: + case PromotionQueueResult::kQueueCapRejected: + case PromotionQueueResult::kNoLocalDiskSource: + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + case PromotionQueueResult::kNotFound: + case PromotionQueueResult::kPushFailed: + return ErrorCode::OBJECT_NOT_FOUND; + } + return ErrorCode::OBJECT_NOT_FOUND; + }; io_pattern_runtime_ = std::make_shared( io_pattern::IoPatternRuntime::Handlers{ .eviction = @@ -567,7 +595,8 @@ MasterService::MasterService(const MasterServiceConfig& config) : ErrorCode::OBJECT_NOT_FOUND; }, .prefetch = - [this](const io_pattern::PrefetchPlan& plan) { + [this, promotion_outcome_to_error]( + const io_pattern::PrefetchPlan& plan) -> ErrorCode { for (const auto& candidate : plan.candidates) { // Store's safe promotion primitive is LOCAL_DISK -> // MEMORY; HBM remains inference-runtime-owned and is @@ -578,26 +607,25 @@ MasterService::MasterService(const MasterServiceConfig& config) } const ObjectIdentity object_id{ candidate.object.tenant_id, candidate.object.key}; - if (TryPushPromotionQueue(object_id, - /*record_candidate=*/false) != - PromotionQueueResult::kQueued) { - return ErrorCode::OBJECT_NOT_FOUND; + const auto code = promotion_outcome_to_error( + TryPushPromotionQueue(object_id, + /*record_candidate=*/false)); + if (code != ErrorCode::OK) { + return code; } } return ErrorCode::OK; }, .admission = - [this](const io_pattern::AdmissionResult& result) { + [this, promotion_outcome_to_error]( + const io_pattern::AdmissionResult& result) -> ErrorCode { if (result.target_tier == io_pattern::CacheTier::kL0Hbm) { return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; } const ObjectIdentity object_id{result.object.tenant_id, result.object.key}; - return TryPushPromotionQueue(object_id, - /*record_candidate=*/false) == - PromotionQueueResult::kQueued - ? ErrorCode::OK - : ErrorCode::OBJECT_NOT_FOUND; + return promotion_outcome_to_error(TryPushPromotionQueue( + object_id, /*record_candidate=*/false)); }}, std::move(io_pattern_config)); io_pattern_cfm_service_ = std::make_shared( @@ -9865,6 +9893,10 @@ MasterService::EvictTenantMemoryForQuota( size_t diag_candidate_skipped_lease = 0; size_t diag_candidate_skipped_replica = 0; size_t diag_candidate_evicted = 0; + size_t diag_no_evictable_has_mem = 0; + size_t diag_no_evictable_completed = 0; + size_t diag_no_evictable_refcnt = 0; + size_t diag_no_evictable_unreadable = 0; for (size_t scanned = 0; scanned < kNumShards && total.freed_bytes < target_bytes; ++scanned) { @@ -9900,6 +9932,30 @@ MasterService::EvictTenantMemoryForQuota( ++diag_candidate_skipped_lease; } else { ++diag_candidate_skipped_replica; + // Replica-level breakdown for why no evictable + // MEMORY replica exists. + bool diag_has_mem = false; + bool diag_all_completed = true; + bool diag_any_refcnt = false; + bool diag_any_unreadable = false; + metadata.VisitReplicas( + [&](const Replica& r) { + if (r.is_memory_replica()) { + diag_has_mem = true; + if (!r.is_completed()) + diag_all_completed = false; + if (r.get_refcnt() != 0) + diag_any_refcnt = true; + if (!IsReplicaReadable(r)) + diag_any_unreadable = true; + } + }); + diag_no_evictable_has_mem += diag_has_mem; + diag_no_evictable_completed += + diag_all_completed; + diag_no_evictable_refcnt += diag_any_refcnt; + diag_no_evictable_unreadable += + diag_any_unreadable; } } ++it; @@ -9937,6 +9993,10 @@ MasterService::EvictTenantMemoryForQuota( << " skipped_lease=" << diag_candidate_skipped_lease << " skipped_no_evictable_replica=" << diag_candidate_skipped_replica + << " (has_mem=" << diag_no_evictable_has_mem + << " completed=" << diag_no_evictable_completed + << " any_refcnt=" << diag_no_evictable_refcnt + << " any_unreadable=" << diag_no_evictable_unreadable << ")" << " evicted=" << diag_candidate_evicted << " freed_bytes=" << total.freed_bytes; } diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index c80a674dad..4f64948cf6 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1743,5 +1743,53 @@ TEST(IoPatternFrameworkTest, OwnershipClientBucketsReportsByResolvedOwner) { server.stop(); } +TEST(IoPatternFrameworkTest, UnavailablePrefetchCapabilityDoesNotDegradePolicy) { + // The prefetch handler's own primitive can be structurally unavailable: + // promotion may be disabled, or the object may have no LOCAL_DISK source + // replica to promote from. The policy cannot influence either outcome, so + // repeating cycles must not be counted as policy failure -- otherwise the + // whole workload policy is permanently replaced by the legacy fallback. + int prefetch_calls = 0; + IoPatternRuntime runtime(IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = + [&prefetch_calls](const PrefetchPlan&) { + ++prefetch_calls; + return ErrorCode::UNAVAILABLE_IN_CURRENT_MODE; + }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + + AccessRecord access{.object = {TenantId("tenant-a"), "cold-key"}, + .block_size = 1024, + .tier = CacheTier::kL3NofSsd, + .operation = IoOperation::kGet, + .is_hit = true}; + // A recommendation-shaped key yields a definitive workload classification + // and a confidence of 1.0, so the prefetch gate is genuinely exercised. + for (int i = 0; i < 21; ++i) { + runtime.RecordAccess(access.object.key, access); + } + runtime.ReportInferenceMetrics( + InferenceMetrics{.object = access.object, .match_length = 300}); + + TraceHistory trace; + trace.events.push_back( + TraceEvent{.object = access.object, .match_length = 300, .is_hit = true}); + + // Three consecutive reported failures is the DegradingPolicyEngine + // threshold used by the runtime. The short pause lets the analyzer thread + // of the previous cycle clear its in-flight flag, which would otherwise + // mark a cycle degraded for reasons unrelated to this test. + for (int cycle = 0; cycle < 3; ++cycle) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + runtime.Execute(CacheTier::kL1Host, 0, trace, {}); + } + + ASSERT_GE(prefetch_calls, 1) << "the prefetch handler must be exercised"; + EXPECT_FALSE(runtime.degraded()) + << "a structurally unavailable prefetch primitive is not a policy " + "failure"; +} + } // namespace } // namespace mooncake::io_pattern From c11bc6dc80c8a9ad28002f4c031ca30463ead639 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 15:04:25 +0800 Subject: [PATCH 22/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../include/io_pattern/collector_impl.h | 6 + mooncake-store/include/io_pattern/runtime.h | 8 ++ mooncake-store/include/master_service.h | 21 ++++ .../src/io_pattern/collector_impl.cpp | 35 ++++++ mooncake-store/src/io_pattern/runtime.cpp | 5 + mooncake-store/src/master_service.cpp | 106 +++++++++++++++--- .../tests/io_pattern_framework_test.cpp | 51 +++++++++ .../tests/offload_on_evict_test.cpp | 32 ++++++ 8 files changed, 248 insertions(+), 16 deletions(-) diff --git a/mooncake-store/include/io_pattern/collector_impl.h b/mooncake-store/include/io_pattern/collector_impl.h index f3de308dc9..a25e833fd9 100644 --- a/mooncake-store/include/io_pattern/collector_impl.h +++ b/mooncake-store/include/io_pattern/collector_impl.h @@ -37,6 +37,12 @@ class IoPatternCollectorImpl final : public IoPatternCollector { // Ingests a CFM snapshot without replaying it through the asynchronous // reporter. The sender is already the reporting side of that pipeline. void MergeSnapshot(const IoPatternSnapshot& snapshot); + // Applies an authoritative tier transition reported by the owner of the + // replica metadata (insert, removal, or move between tiers). This is how a + // tier bit stops being claimed once the replica is actually gone. Presence + // must not be inferred from access recency: a cold key that has simply not + // been read has to stay eligible for eviction. + void RecordTierEvent(const CacheEvent& event); IoPatternSnapshot GetSnapshot() const override; uint64_t dropped() const; bool degraded() const; diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 45e6e51d7d..4ade433d12 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -62,6 +62,11 @@ class IoPatternRuntime final { size_t admission_candidates{0}; size_t admissions_admitted{0}; ErrorCode admission_status{ErrorCode::OK}; + // Dimensions whose storage handler declined to act because the + // primitive cannot run in the current mode (promotion disabled, no + // lower-tier source replica, HBM refusal). Such a skip is expected and + // is not counted as a policy failure. + size_t skipped_dimensions{0}; }; using ReportDrivenObserver = std::function; @@ -123,6 +128,9 @@ class IoPatternRuntime final { void RecordAccess(const std::string& key, const AccessRecord& record); void RecordStorageMetric(const StorageMetric& metric); void MergeSnapshot(const IoPatternSnapshot& snapshot); + // Applies an authoritative tier transition reported by the owner of the + // replica metadata. See IoPatternCollectorImpl::RecordTierEvent. + void RecordTierEvent(const CacheEvent& event); bool FlushReports(); void StopReports(); diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 19f62d4067..a1df8ca9f0 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -124,6 +124,7 @@ class MasterService { // members friend class ha::MasterSnapshotCodecTest; // codec round-trip unit test friend class test::MasterServiceHATest; + friend class test::OffloadOnEvictTest; public: using NoFProbeFn = @@ -1024,6 +1025,22 @@ class MasterService { const TenantId& tenant_id, uint64_t target_bytes, const std::unordered_set* candidate_keys = nullptr); + // After an eviction pass, reports a kRemoved tier event for every candidate + // key whose MEMORY replica is actually gone, so the IO Pattern collector + // stops claiming an L1 replica for it. Presence is read from authoritative + // metadata rather than inferred from access recency: an evicted key must + // become promotable again, while a cold key that has simply not been read + // must stay eligible for eviction. + void ReportEvictedKeysAsTierRemovals( + const TenantId& tenant, + const std::unordered_set& candidate_keys); + + // Runs the legacy lease-ordered eviction for an IO Pattern plan that could + // not free its byte target (stale candidates, active leases, pins, or an + // empty candidate set), so a watermark request still makes progress. + // Returns true when the fallback ran. + bool RunLegacyEvictionFallback(uint64_t shortfall_bytes); + // Helper to get a snapshot of alive clients (under client_mutex_ shared // lock) std::unordered_set> getAliveClientsSnapshot() const; @@ -1819,6 +1836,10 @@ class MasterService { // Eviction thread related members std::thread eviction_thread_; std::atomic eviction_running_{false}; + // Serializes legacy eviction fallbacks: the watermark thread and the + // report-driven worker can both observe a shortfall, and BatchEvict has no + // re-entrancy protection. + std::mutex legacy_eviction_mutex_; static constexpr uint64_t kEvictionThreadSleepMs = 10; // 10 ms sleep between eviction checks diff --git a/mooncake-store/src/io_pattern/collector_impl.cpp b/mooncake-store/src/io_pattern/collector_impl.cpp index 416ef811fa..7adcb97aec 100644 --- a/mooncake-store/src/io_pattern/collector_impl.cpp +++ b/mooncake-store/src/io_pattern/collector_impl.cpp @@ -218,6 +218,41 @@ void IoPatternCollectorImpl::MergeSnapshot(const IoPatternSnapshot& snapshot) { } } +void IoPatternCollectorImpl::RecordTierEvent(const CacheEvent& event) { + if (event.type == CacheEventType::kUnknown) { + return; + } + std::lock_guard lock(mutex_); + const auto it = key_metrics_.find(event.object); + if (it == key_metrics_.end()) { + return; + } + const auto with_tier = [](CacheTierMask mask, CacheTier tier, + bool present) -> CacheTierMask { + const CacheTierMask bit = CacheTierBit(tier); + return present ? static_cast(mask | bit) + : static_cast(mask & ~bit); + }; + switch (event.type) { + case CacheEventType::kInserted: + it->second.replica_tiers = + with_tier(it->second.replica_tiers, event.target_tier, true); + break; + case CacheEventType::kRemoved: + it->second.replica_tiers = + with_tier(it->second.replica_tiers, event.source_tier, false); + break; + case CacheEventType::kTierChanged: + it->second.replica_tiers = + with_tier(it->second.replica_tiers, event.source_tier, false); + it->second.replica_tiers = + with_tier(it->second.replica_tiers, event.target_tier, true); + break; + case CacheEventType::kUnknown: + return; + } +} + IoPatternSnapshot IoPatternCollectorImpl::GetSnapshot() const { std::lock_guard lock(mutex_); IoPatternSnapshot snapshot; diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index 5b35b0b9ac..ce40099c01 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -160,6 +160,10 @@ void IoPatternRuntime::MergeSnapshot(const IoPatternSnapshot& snapshot) { } } +void IoPatternRuntime::RecordTierEvent(const CacheEvent& event) { + collector_->RecordTierEvent(event); +} + bool IoPatternRuntime::FlushReports() { return collector_->FlushReports(); } void IoPatternRuntime::StopReports() { collector_->StopReports(); } @@ -539,6 +543,7 @@ void IoPatternRuntime::RunReportDrivenCycle() { report.degraded = status.degraded || planned.result.degraded; report.eviction_status = status.eviction; report.prefetch_status = status.prefetch; + report.skipped_dimensions = status.skipped; if (!status.admissions.empty()) { report.admission_status = status.admissions.front(); } diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 5c1b9d523a..1d6f6ce629 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -500,6 +500,7 @@ MasterService::MasterService(const MasterServiceConfig& config) << " keys_analyzed=" << rpt.keys_analyzed << " analysis_elapsed_us=" << rpt.analysis_elapsed_us << " degraded=" << rpt.degraded + << " skipped=" << rpt.skipped_dimensions << " eviction(tier=" << static_cast(rpt.eviction_tier) << ", bytes=" << rpt.eviction_target_bytes @@ -551,8 +552,6 @@ MasterService::MasterService(const MasterServiceConfig& config) std::unordered_set keys; }; if (plan.target_bytes == 0) return ErrorCode::OK; - if (plan.candidates.empty()) - return ErrorCode::OBJECT_NOT_FOUND; uint64_t total_freed = 0; std::unordered_map targets; @@ -584,13 +583,29 @@ MasterService::MasterService(const MasterServiceConfig& config) << "[IO-PATTERN-EVICT-DIAG] candidate key=" << key; } + ReportEvictedKeysAsTierRemovals(tenant, target.keys); } LOG(WARNING) << "[IO-PATTERN-EVICT-DIAG] io_pattern eviction " "summary plan_target=" << plan.target_bytes << " total_freed=" << total_freed << " candidates=" << plan.candidates.size(); - return total_freed >= plan.target_bytes + if (total_freed >= plan.target_bytes) { + return ErrorCode::OK; + } + // The plan under-delivered: its candidates may be stale, may + // still hold leases or pins, or may be empty. Fall back to + // the legacy lease-ordered eviction so the watermark request + // still makes progress. This covers both the local watermark + // thread and the report-driven worker, which is why the + // thread no longer runs its own fallback. + const uint64_t shortfall = plan.target_bytes - total_freed; + LOG(WARNING) + << "[IO-PATTERN-EVICT-FALLBACK] policy plan under-" + "delivered plan_target=" + << plan.target_bytes << " freed=" << total_freed + << " shortfall=" << shortfall; + return RunLegacyEvictionFallback(shortfall) ? ErrorCode::OK : ErrorCode::OBJECT_NOT_FOUND; }, @@ -8193,6 +8208,18 @@ auto MasterService::NotifyOffloadSuccess( local_disk_segment->ssd_used_bytes.fetch_add( metadata.data_size, std::memory_order_relaxed); } + if (added_new_local_disk_replica && io_pattern_runtime_) { + // The object now holds a durable lower-tier replica, so record it: + // once its MEMORY replica is reclaimed the key must be promotable + // again instead of looking like it has no replica at all. LOCAL_DISK + // is reported under kL3NofSsd until CacheTier grows a dedicated + // member for it. + io_pattern_runtime_->RecordTierEvent( + io_pattern::CacheEvent{ + .type = io_pattern::CacheEventType::kInserted, + .object = {object_id.tenant_id, object_id.user_key}, + .target_tier = io_pattern::CacheTier::kL3NofSsd}); + } } return {}; @@ -9102,9 +9129,11 @@ void MasterService::EvictionThreadFunc() { const auto status = io_pattern_runtime_->Execute( io_pattern::CacheTier::kL1Host, static_cast(evict_ratio_target * capacity), {}); - if (status.eviction != ErrorCode::OK) { - BatchEvict(evict_ratio_target, evict_ratio_lowerbound); - } + // A shortfall is already handled inside the runtime's eviction + // handler (RunLegacyEvictionFallback), which also covers the + // report-driven worker; running BatchEvict again here would + // evict twice for one watermark breach. + (void)status; } else { BatchEvict(evict_ratio_target, evict_ratio_lowerbound); } @@ -9705,6 +9734,52 @@ tl::expected MasterService::ApplySnapshotState( return {}; } +void MasterService::ReportEvictedKeysAsTierRemovals( + const TenantId& tenant, + const std::unordered_set& candidate_keys) { + if (!io_pattern_runtime_ || candidate_keys.empty()) { + return; + } + for (const auto& key : candidate_keys) { + const ObjectIdentity object_id{tenant, key}; + MetadataAccessorRO accessor(this, object_id); + if (accessor.Exists() && accessor.Get().HasMemReplica()) { + continue; + } + // The MEMORY replica is gone, so the policy must stop treating this key + // as already resident in the head tier; otherwise it is never offered + // to admission and never promoted back. + io_pattern_runtime_->RecordTierEvent( + io_pattern::CacheEvent{ + .type = io_pattern::CacheEventType::kRemoved, + .object = {tenant, key}, + .source_tier = io_pattern::CacheTier::kL1Host}); + } +} + +bool MasterService::RunLegacyEvictionFallback(uint64_t shortfall_bytes) { + if (shortfall_bytes == 0) { + return true; + } + const auto capacity = std::max( + 0, MasterMetricManager::instance().get_total_mem_capacity()); + if (capacity == 0) { + return false; + } + // Only one fallback at a time: the watermark thread and the report-driven + // worker can both observe a shortfall, and BatchEvict has no re-entrancy + // protection. + std::unique_lock lock(legacy_eviction_mutex_, std::try_to_lock); + if (!lock.owns_lock()) { + return false; + } + const double ratio = + std::min(1.0, static_cast(shortfall_bytes) / + static_cast(capacity)); + BatchEvict(ratio, ratio * 0.5); + return true; +} + MasterService::TenantQuotaEvictionResult MasterService::EvictTenantMemoryForQuota( const TenantId& tenant_id, uint64_t target_bytes, @@ -9939,16 +10014,15 @@ MasterService::EvictTenantMemoryForQuota( bool diag_any_refcnt = false; bool diag_any_unreadable = false; metadata.VisitReplicas( - [&](const Replica& r) { - if (r.is_memory_replica()) { - diag_has_mem = true; - if (!r.is_completed()) - diag_all_completed = false; - if (r.get_refcnt() != 0) - diag_any_refcnt = true; - if (!IsReplicaReadable(r)) - diag_any_unreadable = true; - } + &Replica::fn_is_memory_replica, + [&](Replica& r) { + diag_has_mem = true; + if (!r.is_completed()) + diag_all_completed = false; + if (r.get_refcnt() != 0) + diag_any_refcnt = true; + if (!IsReplicaReadable(r)) + diag_any_unreadable = true; }); diag_no_evictable_has_mem += diag_has_mem; diag_no_evictable_completed += diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 4f64948cf6..75f5437571 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1791,5 +1791,56 @@ TEST(IoPatternFrameworkTest, UnavailablePrefetchCapabilityDoesNotDegradePolicy) "failure"; } +TEST(IoPatternFrameworkTest, TierRemovalEventRetiresTheReplicaBit) { + // An object evicted from host memory must stop claiming an L1 replica, so + // it becomes promotable again instead of being treated as already resident + // in the head tier. + IoPatternRuntime runtime( + {.eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}); + AccessRecord access{.object = {TenantId("tenant-a"), "tiered-key"}, + .block_size = 64, + .tier = CacheTier::kL1Host, + .is_hit = true}; + runtime.RecordAccess(access.object.key, access); + ASSERT_EQ(runtime.Snapshot().keys.size(), 1U); + EXPECT_EQ(runtime.Snapshot().keys.front().replica_tiers, + CacheTierBit(CacheTier::kL1Host)); + + runtime.RecordTierEvent(CacheEvent{.type = CacheEventType::kRemoved, + .object = access.object, + .source_tier = CacheTier::kL1Host}); + + EXPECT_EQ(runtime.Snapshot().keys.front().replica_tiers, + static_cast(0)); +} + +TEST(IoPatternFrameworkTest, TierChangeEventMovesTheReplicaBit) { + // Demotion keeps the lower-tier claim without keeping the upper one; a + // fresh insert adds a claim. + IoPatternCollectorImpl collector; + AccessRecord access{.object = {TenantId("tenant-a"), "moved-key"}, + .block_size = 64, + .tier = CacheTier::kL1Host, + .is_hit = true}; + collector.RecordAccess(access.object.key, access); + + collector.RecordTierEvent( + CacheEvent{.type = CacheEventType::kTierChanged, + .object = access.object, + .source_tier = CacheTier::kL1Host, + .target_tier = CacheTier::kL3NofSsd}); + EXPECT_EQ(collector.GetSnapshot().keys.front().replica_tiers, + CacheTierBit(CacheTier::kL3NofSsd)); + + collector.RecordTierEvent(CacheEvent{.type = CacheEventType::kInserted, + .object = access.object, + .target_tier = CacheTier::kL2Segment}); + EXPECT_EQ(collector.GetSnapshot().keys.front().replica_tiers, + static_cast(CacheTierBit(CacheTier::kL3NofSsd) | + CacheTierBit(CacheTier::kL2Segment))); +} + } // namespace } // namespace mooncake::io_pattern diff --git a/mooncake-store/tests/offload_on_evict_test.cpp b/mooncake-store/tests/offload_on_evict_test.cpp index 08810e6fdf..54f0e35032 100644 --- a/mooncake-store/tests/offload_on_evict_test.cpp +++ b/mooncake-store/tests/offload_on_evict_test.cpp @@ -23,6 +23,15 @@ class OffloadOnEvictTest : public ::testing::Test { void TearDown() override { google::ShutdownGoogleLogging(); } + // Friend access to the private legacy-eviction fallback, which the runtime's + // eviction handler runs when an IO Pattern plan cannot free its byte target. + // OffloadOnEvictTest is friended; TEST_F-generated subclasses are not, hence + // this static funnel. + static bool RunLegacyEvictionFallbackForTesting(MasterService* service, + uint64_t shortfall_bytes) { + return service->RunLegacyEvictionFallback(shortfall_bytes); + } + static constexpr size_t kDefaultSegmentBase = 0x300000000; Segment MakeSegment(std::string name, size_t base, size_t size) const { @@ -424,6 +433,29 @@ TEST_F(OffloadOnEvictTest, BatchRemoveDropsOffloadingObjectsMirror) { "offloading_objects."; } +// The runtime's eviction handler falls back to the legacy lease-ordered +// eviction when an IO Pattern plan cannot free its byte target. That path is +// what keeps a watermark request making progress when the plan's candidates are +// stale, leased, pinned, or empty -- the report-driven worker relies on it +// because it has no eviction thread of its own. +TEST_F(OffloadOnEvictTest, LegacyEvictionFallbackRunsForUnderDeliveringPlan) { + MasterServiceConfig config; + config.default_kv_lease_ttl = 100; + auto service = std::make_unique(config); + auto ctx = PrepareSegment(*service, "fallback-segment", kDefaultSegmentBase, + 8 * 1024 * 1024); + for (int i = 0; i < 8; ++i) { + PutObject(*service, ctx.client_id, "fb-" + std::to_string(i), 4096); + } + + // No shortfall is a no-op and reports success. + EXPECT_TRUE(RunLegacyEvictionFallbackForTesting(service.get(), 0)); + + // A real shortfall runs the legacy path: the segment is mounted, so a + // capacity is known and the guard mutex is free. + EXPECT_TRUE(RunLegacyEvictionFallbackForTesting(service.get(), 1)); +} + } // namespace mooncake::test int main(int argc, char** argv) { From 8b55fae86d71a5a1ee130136fedae8288cd9605f Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 15:17:22 +0800 Subject: [PATCH 23/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/master_service.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index a1df8ca9f0..4d310e5a0c 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -89,6 +89,9 @@ class SnapshotChildProcessTest; class PromotionOnHitTest; class MasterServiceTenantQuotaTest; class MasterServiceHATest; +// Friended so the offload-on-evict tests can drive the legacy eviction fallback +// that the IO Pattern eviction handler runs when a plan under-delivers. +class OffloadOnEvictTest; } // namespace test namespace benchmarks { class BatchEvictBench; From 00dd4fa0f511c5ff71d2891058ef1d070c60455f Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 15:26:48 +0800 Subject: [PATCH 24/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/src/master_service.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 1d6f6ce629..366055c012 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -8213,11 +8213,14 @@ auto MasterService::NotifyOffloadSuccess( // once its MEMORY replica is reclaimed the key must be promotable // again instead of looking like it has no replica at all. LOCAL_DISK // is reported under kL3NofSsd until CacheTier grows a dedicated - // member for it. + // member for it. The per-object identity resolved inside the add + // branch is scoped to that branch, so use the loop-level request + // identity here. io_pattern_runtime_->RecordTierEvent( io_pattern::CacheEvent{ .type = io_pattern::CacheEventType::kInserted, - .object = {object_id.tenant_id, object_id.user_key}, + .object = {request_object_id.tenant_id, + request_object_id.user_key}, .target_tier = io_pattern::CacheTier::kL3NofSsd}); } } From 818560ae5a3d509bbe53cf9506b48bafb2fb6e19 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 15:38:53 +0800 Subject: [PATCH 25/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/io_pattern_framework_test.cpp | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 75f5437571..6af52555e2 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1403,19 +1403,19 @@ TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { IoPatternSnapshot first; first.generated_at_ns = 10; first.keys.push_back(KeyMetrics{.object = {TenantId("tenant"), "first"}, + .access_count_window = 1, + .block_size = 100, .token_count = 20 * 1024, .prefix_fanout = 20, - .match_length = 512, - .block_size = 100, - .access_count_window = 1}); + .match_length = 512}); IoPatternSnapshot second; second.generated_at_ns = 50; second.keys.push_back(KeyMetrics{.object = {TenantId("tenant"), "second"}, + .access_count_window = 5, + .block_size = 300, .token_count = 30, .prefix_fanout = 20, - .match_length = 300, - .block_size = 300, - .access_count_window = 5}); + .match_length = 300}); EXPECT_EQ(analyzer.DetectWorkloadType(second), WorkloadType::kMixed); const auto stats = analyzer.FeatureStats(); EXPECT_EQ(stats.samples, 2); @@ -1431,9 +1431,9 @@ TEST(IoPatternFrameworkTest, SlidingWindowDeduplicatesObjectsAndBoundsHistory) { IoPatternSnapshot snapshot; snapshot.generated_at_ns = timestamp; snapshot.keys.push_back(KeyMetrics{.object = object, + .access_count_window = timestamp, .token_count = - static_cast(timestamp), - .access_count_window = timestamp}); + static_cast(timestamp)}); analyzer.Analyze(snapshot); } @@ -1469,8 +1469,8 @@ TEST(IoPatternFrameworkTest, KMeansFallbackLabelsIndependentSessions) { .match_length = 512}, KeyMetrics{.object = {TenantId("tenant-b"), "small"}, .session_id = "recommendation-session", - .block_size = 64 * 1024, - .access_count_window = 30}, + .access_count_window = 30, + .block_size = 64 * 1024}, }; const auto result = analyzer.Analyze(snapshot); From 68390b69fbcbd2cae6dffc684260e7f5070712b5 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 15:49:38 +0800 Subject: [PATCH 26/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/tests/io_pattern_framework_test.cpp | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 6af52555e2..6416118d61 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -546,6 +546,10 @@ TEST(IoPatternFrameworkTest, TracePrefetchPlansOnlyLongPrefixMatches) { key.block_size = 4096; key.replica_tiers = CacheTierBit(CacheTier::kL3NofSsd); context.snapshot.keys.push_back(key); + // The prefetch gate also requires analyzer confidence, so supply the key + // pattern the production pipeline would derive for this object. + context.analysis.keys = { + KeyPattern{.object = key.object, .confidence = 1.0F}}; TraceHistory trace; trace.events.push_back( @@ -568,6 +572,10 @@ TEST(IoPatternFrameworkTest, TracePrefetchDeduplicatesObjects) { key.block_size = 128; key.replica_tiers = CacheTierBit(CacheTier::kL2Segment); context.snapshot.keys.push_back(key); + // The prefetch gate also requires analyzer confidence, so supply the key + // pattern the production pipeline would derive for this object. + context.analysis.keys = { + KeyPattern{.object = key.object, .confidence = 1.0F}}; TraceHistory trace; trace.events.push_back( @@ -1632,8 +1640,11 @@ TEST(IoPatternFrameworkTest, RuntimeConnectsCollectionAnalysisPolicyAndHandlers) }, }); + // observed_at_ns is deliberately left unset so the collector stamps this + // access with its own clock: the frequency window is a true rolling window, + // so a synthetic epoch timestamp would be pruned and the admission would be + // rejected for frequency instead of exercising the handler. AccessRecord access{.object = {TenantId("tenant-a"), "runtime-key"}, - .observed_at_ns = 1, .block_size = 64, .tier = CacheTier::kL2Segment, .is_hit = true}; From 2dc7f4b20558d3f3adfd76f192379e34e8834322 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 15:59:09 +0800 Subject: [PATCH 27/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/tests/io_pattern_framework_test.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 6416118d61..0b86b435b8 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -941,10 +941,17 @@ TEST(IoPatternFrameworkTest, InProcessTransportDoesNotHoldLockAcrossHandler) { std::promise handler_entered; std::promise release_handler; auto release = release_handler.get_future().share(); + // Only the first invocation blocks and signals entry: the test deliberately + // issues a second concurrent Send, and re-satisfying the promise from that + // invocation would throw std::future_error inside the handler and abort the + // process instead of proving that the transport lock is not held across it. + std::atomic handler_calls{0}; auto transport = std::make_shared( [&](std::string_view, std::string_view) { - handler_entered.set_value(); - release.wait(); + if (handler_calls.fetch_add(1) == 0) { + handler_entered.set_value(); + release.wait(); + } return true; }); From 33ed27d01f4512a212d5b508993ba492efc4bdd3 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 16:09:20 +0800 Subject: [PATCH 28/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/io_pattern/threshold_analyzer.cpp | 60 +++++++++++++------ .../tests/io_pattern_framework_test.cpp | 23 +++++-- 2 files changed, 59 insertions(+), 24 deletions(-) diff --git a/mooncake-store/src/io_pattern/threshold_analyzer.cpp b/mooncake-store/src/io_pattern/threshold_analyzer.cpp index f84d8a2c1b..7bcfe4028f 100644 --- a/mooncake-store/src/io_pattern/threshold_analyzer.cpp +++ b/mooncake-store/src/io_pattern/threshold_analyzer.cpp @@ -17,12 +17,29 @@ bool IsRecommendation(const KeyMetrics& key, key.access_count_window > config.recommendation_frequency; } +// A code-agent key already satisfies the conversation dimensions (fanout and +// match length), so conversation is scoped to the non-huge-context case. +// Without that bound every code-agent key also matched this rule, matched > 1 +// forced kMixed, and kCodeAgent became unreachable under the default config. bool IsConversation(const KeyMetrics& key, const ThresholdAnalyzerConfig& config) { - return key.prefix_fanout > config.conversation_prefix_fanout && + return key.token_count <= config.code_agent_token_count && + key.prefix_fanout > config.conversation_prefix_fanout && key.match_length > config.conversation_match_length; } +// Similarity of one reported dimension to a rule threshold. A dimension the key +// does not report at all (zero) contributes nothing instead of acting as a +// zero-similarity term: combining dimensions with std::min forced every partial +// match to zero as soon as a single metric was absent, which is the common case +// for a key observed through only one data path. +float ReportedRatio(float value, float threshold) { + if (value <= 0.0F) { + return 0.0F; + } + return std::min(1.0F, value / std::max(1.0F, threshold)); +} + float RuleConfidence(const KeyMetrics& key, const ThresholdAnalyzerConfig& config) { float score = 0.0F; @@ -32,23 +49,30 @@ float RuleConfidence(const KeyMetrics& key, // A partial match is useful to policies, but must not look like a // definitive workload classification. if (score == 0.0F) { - const float code = std::min( - {static_cast(key.token_count) / - std::max(1.0F, static_cast(config.code_agent_token_count)), - static_cast(key.prefix_fanout) / - std::max(1.0F, static_cast(config.code_agent_prefix_fanout)), - static_cast(key.match_length) / - std::max(1.0F, static_cast(config.code_agent_match_length))}); - const float recommendation = std::min( - static_cast(config.recommendation_block_size) / - std::max(1.0F, static_cast(key.block_size)), - static_cast(key.access_count_window) / - std::max(1.0F, static_cast(config.recommendation_frequency))); - const float conversation = std::min( - static_cast(key.prefix_fanout) / - std::max(1.0F, static_cast(config.conversation_prefix_fanout)), - static_cast(key.match_length) / - std::max(1.0F, static_cast(config.conversation_match_length))); + const float code = std::max( + {ReportedRatio(static_cast(key.token_count), + static_cast(config.code_agent_token_count)), + ReportedRatio(static_cast(key.prefix_fanout), + static_cast(config.code_agent_prefix_fanout)), + ReportedRatio(static_cast(key.match_length), + static_cast(config.code_agent_match_length))}); + // The recommendation rule rewards a *small* block, so its block term is + // inverted; an unset block_size counts as unreported. + const float block_ratio = + key.block_size == 0 + ? 0.0F + : std::min(1.0F, + static_cast(config.recommendation_block_size) / + static_cast(key.block_size)); + const float recommendation = std::max( + block_ratio, + ReportedRatio(static_cast(key.access_count_window), + static_cast(config.recommendation_frequency))); + const float conversation = std::max( + ReportedRatio(static_cast(key.prefix_fanout), + static_cast(config.conversation_prefix_fanout)), + ReportedRatio(static_cast(key.match_length), + static_cast(config.conversation_match_length))); score = std::clamp(std::max({code, recommendation, conversation}), 0.0F, 1.0F); } diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 0b86b435b8..b050ebd5f2 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1363,19 +1363,28 @@ TEST(IoPatternFrameworkTest, FeedbackWindowAggregatesBoundedSamples) { window.Record({.hit_rate_delta = -0.4F, .prefetch_accuracy = 0.3F}); const auto stats = window.Snapshot(); EXPECT_EQ(stats.samples, 2); - EXPECT_FLOAT_EQ(stats.hit_rate_delta, (-0.2F - 0.4F) / 2.0F); + // Capacity 2 keeps the two most recent samples, i.e. the second and third + // records: (-0.4 + 0.1) / 2. The previous expectation averaged the first and + // third records, which no bounded window can produce. + EXPECT_FLOAT_EQ(stats.hit_rate_delta, (0.1F - 0.4F) / 2.0F); EXPECT_FLOAT_EQ(stats.prefetch_accuracy, (0.9F + 0.3F) / 2.0F); } TEST(IoPatternFrameworkTest, AdaptiveTunerChangesWeightsAfterNegativeStreak) { AdaptivePolicyTuner tuner(3); ScoreBasedEvictionConfig config; - EXPECT_FALSE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); - EXPECT_FALSE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); - EXPECT_TRUE(tuner.Tune({.hit_rate_delta = -0.1F}, config)); + // prefetch_accuracy is stated explicitly: leaving it at its 0.0F default + // would trip the tuner's conservative branch (prefetch_accuracy < 0.2F) on + // the very first sample and the frequency/idle streak path would never run. + const PolicyFeedbackStats negative{.hit_rate_delta = -0.1F, + .prefetch_accuracy = 1.0F}; + EXPECT_FALSE(tuner.Tune(negative, config)); + EXPECT_FALSE(tuner.Tune(negative, config)); + EXPECT_TRUE(tuner.Tune(negative, config)); EXPECT_FLOAT_EQ(config.frequency_weight, 0.8F); EXPECT_FLOAT_EQ(config.idle_weight, 1.1F); - EXPECT_FALSE(tuner.Tune({.hit_rate_delta = 0.0F}, config)); + EXPECT_FALSE(tuner.Tune({.hit_rate_delta = 0.0F, .prefetch_accuracy = 1.0F}, + config)); } TEST(IoPatternFrameworkTest, AdaptiveTunerHandlesChurnAndPersistsChanges) { @@ -1434,7 +1443,9 @@ TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { EXPECT_EQ(analyzer.DetectWorkloadType(second), WorkloadType::kMixed); const auto stats = analyzer.FeatureStats(); EXPECT_EQ(stats.samples, 2); - EXPECT_EQ(stats.token_median, 30); + // Percentile() ranks with size/2, so two samples select the upper-middle + // element: sorted {30, 20480} at index 1. + EXPECT_EQ(stats.token_median, 20480); EXPECT_EQ(stats.fanout_p90, 20); EXPECT_EQ(stats.block_p90, 300); } From 96253744d62ba5ea56b2f07d1d5c334f5451f0ae Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 16:17:23 +0800 Subject: [PATCH 29/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/tests/io_pattern_framework_test.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index b050ebd5f2..25c804a710 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1440,6 +1440,12 @@ TEST(IoPatternFrameworkTest, SlidingWindowAnalyzerComputesPercentiles) { .token_count = 30, .prefix_fanout = 20, .match_length = 300}); + // Both snapshots must reach the analyzer: the aggregate has to hold two + // samples for kMixed (one code-agent shaped key, one conversation shaped + // key), and the percentile expectations below are computed over both. The + // first snapshot was previously built and then never fed to the analyzer, + // which left a single sample and made the kMixed expectation unreachable. + analyzer.Analyze(first); EXPECT_EQ(analyzer.DetectWorkloadType(second), WorkloadType::kMixed); const auto stats = analyzer.FeatureStats(); EXPECT_EQ(stats.samples, 2); From f139a038cb401963e7bc303f5984697e1b843d96 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 16:24:17 +0800 Subject: [PATCH 30/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/io_pattern/policy_engine.h | 13 ++++++++++--- mooncake-store/src/io_pattern/runtime.cpp | 4 ++-- mooncake-store/tests/io_pattern_framework_test.cpp | 10 ++++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h index afa8865903..31a36f3c96 100644 --- a/mooncake-store/include/io_pattern/policy_engine.h +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -31,9 +31,15 @@ class PolicyEngine { const PolicyContext& context) const = 0; // Executes the three policy dimensions through one uniform result seam. + // Eviction and admission target different tiers: an eviction plan reclaims + // from `eviction_tier`, while each admission decides whether an object may + // enter `admission_tier`. The two are passed separately because using the + // eviction tier as the admission target silently applied the eviction tier's + // watermark and prefix gates to the promotion decision. virtual PolicyResult ExecutePolicy(const PolicyContext& context, CacheTier eviction_tier, uint64_t eviction_bytes, + CacheTier admission_tier, const TraceHistory& trace, const std::vector& admissions = {}) const { PolicyResult result; @@ -41,7 +47,7 @@ class PolicyEngine { result.prefetch = PlanPrefetch(context, trace); for (const auto& object : admissions) { result.admissions.push_back( - DecideAdmission(object, eviction_tier, context)); + DecideAdmission(object, admission_tier, context)); } return result; } @@ -120,10 +126,11 @@ class RegistryPolicyEngine final : public PolicyEngine { PolicyResult ExecutePolicy(const PolicyContext& context, CacheTier tier, uint64_t bytes, + CacheTier admission_tier, const TraceHistory& trace, const std::vector& admissions = {}) const override { - auto result = PolicyEngine::ExecutePolicy(context, tier, bytes, trace, - admissions); + auto result = PolicyEngine::ExecutePolicy( + context, tier, bytes, admission_tier, trace, admissions); std::shared_lock lock(mutex_); result.degraded = !registries_ || !registries_->eviction.Create(eviction_name_) || diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index ce40099c01..4f03fd4313 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -274,8 +274,8 @@ IoPatternRuntime::PlannedPolicy IoPatternRuntime::BuildPolicy( workload_policy_->AdvanceTransitionWindow(); planned.result = policy_->ExecutePolicy( PolicyContext{.snapshot = planned.snapshot, .analysis = analysis, - .session_id = session_id}, eviction_tier, - eviction_bytes, trace, admissions); + .session_id = session_id}, + eviction_tier, eviction_bytes, CacheTier::kL1Host, trace, admissions); planned.result.degraded = planned.result.degraded || collector_->degraded() || planned.analysis_degraded || diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 25c804a710..9f7bf05d6e 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -707,7 +707,7 @@ TEST(IoPatternFrameworkTest, UnifiedPolicyResultSeamDelegates) { PolicyContext context; context.snapshot = snapshot; const auto result = engine.ExecutePolicy( - context, CacheTier::kL1Host, 1024, {}, {key.object}); + context, CacheTier::kL1Host, 1024, CacheTier::kL1Host, {}, {key.object}); EXPECT_EQ(result.admissions.size(), 1); EXPECT_EQ(result.admissions.front().object, key.object); } @@ -722,13 +722,15 @@ TEST(IoPatternFrameworkTest, RegistryPolicyEngineResolvesNamedOps) { "prefix", [] { return std::make_shared(); })); RegistryPolicyEngine engine(registries, "score", "trace", "prefix"); const ObjectRef object{TenantId("tenant-a"), "key"}; - const auto result = engine.ExecutePolicy({}, CacheTier::kL1Host, 1024, {}, - {object}); + const auto result = engine.ExecutePolicy( + {}, CacheTier::kL1Host, 1024, CacheTier::kL1Host, {}, {object}); ASSERT_EQ(result.admissions.size(), 1); EXPECT_EQ(result.admissions.front().object, object); RegistryPolicyEngine missing(registries, "missing", "trace", "prefix"); - EXPECT_TRUE(missing.ExecutePolicy({}, CacheTier::kL1Host, 0, {}).degraded); + EXPECT_TRUE( + missing.ExecutePolicy({}, CacheTier::kL1Host, 0, CacheTier::kL1Host, {}) + .degraded); } TEST(IoPatternFrameworkTest, ReporterBatchesBoundsAndCountsDrops) { From 8a501893cf58230c21934e547c292ca98925552c Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 16:36:42 +0800 Subject: [PATCH 31/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/io_pattern/types.h | 31 +++++++++++++++++++ .../src/io_pattern/policy_strategies.cpp | 22 ++++++++++--- .../tests/io_pattern_framework_test.cpp | 25 +++++++++++++++ 3 files changed, 73 insertions(+), 5 deletions(-) diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h index 503766e0f3..f9f2f29c42 100644 --- a/mooncake-store/include/io_pattern/types.h +++ b/mooncake-store/include/io_pattern/types.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -14,6 +15,10 @@ enum class CacheTier : uint8_t { kL1Host = 1, kL2Segment = 2, kL3NofSsd = 3, + // Client-local SSD, which is the store's only promotable lower tier + // (LOCAL_DISK -> MEMORY). Appended last so the numeric values of the + // existing tiers stay stable for the CFM wire format. + kLocalDisk = 4, }; using CacheTierMask = uint8_t; @@ -22,6 +27,32 @@ constexpr CacheTierMask CacheTierBit(CacheTier tier) { return static_cast(1U << static_cast(tier)); } +// Every tier, so callers do not have to assume that declaration order is also +// storage order. +inline constexpr std::array kAllCacheTiers{ + CacheTier::kL0Hbm, CacheTier::kL1Host, CacheTier::kLocalDisk, + CacheTier::kL2Segment, CacheTier::kL3NofSsd}; + +// Distance from the compute along the storage ladder, for decisions that need a +// "closer to the head tier" comparison. Declaration order is deliberately not +// this order: kLocalDisk is appended last for wire compatibility but sits +// directly below host memory, above the pooled segment and NoF tiers. +constexpr int TierDepth(CacheTier tier) { + switch (tier) { + case CacheTier::kL0Hbm: + return 0; + case CacheTier::kL1Host: + return 1; + case CacheTier::kLocalDisk: + return 2; + case CacheTier::kL2Segment: + return 3; + case CacheTier::kL3NofSsd: + return 4; + } + return 4; +} + enum class IoOperation : uint8_t { kGet, kPut, diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp index d11a542bfc..fb848a828a 100644 --- a/mooncake-store/src/io_pattern/policy_strategies.cpp +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -23,11 +23,17 @@ const KeyMetrics* FindMetrics(const ObjectRef& object, return it == snapshot.keys.end() ? nullptr : &*it; } +// Whether the object already has a replica deeper in the storage ladder, which +// is what makes evicting this copy safe. Compared by TierDepth rather than by +// enum value: kLocalDisk is declared last for wire compatibility but sits +// directly below host memory, so index order is not storage order. bool HasLowerTierReplica(const KeyMetrics& key, CacheTier tier) { - const auto tier_index = static_cast(tier); - for (uint8_t index = tier_index + 1; - index <= static_cast(CacheTier::kL3NofSsd); ++index) { - if (key.replica_tiers & static_cast(1U << index)) { + const int depth = TierDepth(tier); + for (const CacheTier candidate : kAllCacheTiers) { + if (TierDepth(candidate) <= depth) { + continue; + } + if (key.replica_tiers & CacheTierBit(candidate)) { return true; } } @@ -40,7 +46,10 @@ CacheTier TierDownTarget(CacheTier source, TierDownMode mode) { return CacheTier::kL2Segment; } // Prefix-affinity keeps the immediate next tier as the placement target; - // callers may co-locate grouped prefixes within that tier. + // callers may co-locate grouped prefixes within that tier. kLocalDisk is not + // reachable from here: it is appended last for wire compatibility, and the + // MEMORY -> LOCAL_DISK demotion is selected explicitly by the tier-down + // driver rather than derived from this ladder. return static_cast(static_cast(source) + 1); } @@ -86,6 +95,9 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, config_.recompute_weight * pattern->recompute_score; break; case CacheTier::kL1Host: + // Host-local SSD is scored like host memory: idle time dominates, + // and an existing deeper replica makes reclaiming it safe. + case CacheTier::kLocalDisk: case CacheTier::kL2Segment: score = config_.idle_weight * pattern->idle_score - config_.frequency_weight * pattern->frequency_score + diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 9f7bf05d6e..58760612db 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1879,5 +1879,30 @@ TEST(IoPatternFrameworkTest, TierChangeEventMovesTheReplicaBit) { CacheTierBit(CacheTier::kL2Segment))); } +TEST(IoPatternFrameworkTest, LocalDiskCountsAsALowerTierThanHostMemory) { + // TierDepth, not declaration order, defines the storage ladder: kLocalDisk is + // declared last so the existing tier values stay stable on the CFI wire. + EXPECT_LT(TierDepth(CacheTier::kL1Host), TierDepth(CacheTier::kLocalDisk)); + EXPECT_LT(TierDepth(CacheTier::kLocalDisk), TierDepth(CacheTier::kL2Segment)); + EXPECT_LT(TierDepth(CacheTier::kL2Segment), TierDepth(CacheTier::kL3NofSsd)); + + // A local-disk copy is what makes reclaiming the host copy safe, which is + // exactly what the eviction score's lower-replica term rewards. + ScoreBasedEvictionOps eviction; + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "offloaded"}; + key.block_size = 64; + key.replica_tiers = CacheTierBit(CacheTier::kL1Host) | + CacheTierBit(CacheTier::kLocalDisk); + context.snapshot.keys.push_back(key); + context.analysis.keys = {KeyPattern{.object = key.object, .idle_score = 1.0F}}; + + const auto plan = eviction.Evaluate(context, CacheTier::kL1Host, 64); + ASSERT_EQ(plan.candidates.size(), 1); + // idle_weight(1.0) * idle_score(1.0) + lower_replica_weight(1.0) * 1.0. + EXPECT_FLOAT_EQ(plan.candidates.front().score, 2.0F); +} + } // namespace } // namespace mooncake::io_pattern From 813c038d0991f115ce09393723c6090637282126 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 16:46:49 +0800 Subject: [PATCH 32/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/io_pattern/policy_strategies.cpp | 12 +++++++----- mooncake-store/src/io_pattern/runtime.cpp | 11 +++++++++-- mooncake-store/src/master_service.cpp | 17 ++++++++++++++--- .../tests/io_pattern_framework_test.cpp | 8 ++++---- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp index fb848a828a..6b6863574e 100644 --- a/mooncake-store/src/io_pattern/policy_strategies.cpp +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -207,11 +207,13 @@ PrefetchPlan TraceBasedPrefetchOps::Evaluate( if (candidate.confidence < config_.minimum_confidence) { continue; } - if (key->replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) { - candidate.source_tier = CacheTier::kL3NofSsd; - candidate.target_tier = CacheTier::kL2Segment; - } else if (key->replica_tiers & CacheTierBit(CacheTier::kL2Segment)) { - candidate.source_tier = CacheTier::kL2Segment; + // Only one hop is real: the store promotes a LOCAL_DISK replica into + // host memory. There is no L3 -> L2 or L2 -> L1 mover, so reporting a + // fabricated kL2Segment target made every candidate look executable + // while the handler could only ever promote a local-disk source -- a + // NoF-only object therefore produced a plan the executor had to reject. + if (key->replica_tiers & CacheTierBit(CacheTier::kLocalDisk)) { + candidate.source_tier = CacheTier::kLocalDisk; candidate.target_tier = CacheTier::kL1Host; } else { continue; diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index 4f03fd4313..dbe674d38f 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -449,8 +449,14 @@ TraceHistory IoPatternRuntime::DeriveTraceHistory( .count()); for (const auto& key : snapshot.keys) { if (!key.active || key.access_count_window == 0) continue; - if ((key.replica_tiers & CacheTierBit(CacheTier::kL2Segment)) == 0 && - (key.replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) == 0) { + // Any replica deeper than host memory is a promotion candidate; local + // disk is the only one the store can actually move up, but the others + // still belong in the trace so the ops layer decides. + const bool has_lower_replica = + (key.replica_tiers & CacheTierBit(CacheTier::kLocalDisk)) != 0 || + (key.replica_tiers & CacheTierBit(CacheTier::kL2Segment)) != 0 || + (key.replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) != 0; + if (!has_lower_replica) { continue; } trace.events.push_back( @@ -474,6 +480,7 @@ std::vector IoPatternRuntime::DeriveAdmissionCandidates( const bool in_head = (key.replica_tiers & CacheTierBit(CacheTier::kL1Host)) != 0; const bool lower_tier = + (key.replica_tiers & CacheTierBit(CacheTier::kLocalDisk)) != 0 || (key.replica_tiers & CacheTierBit(CacheTier::kL2Segment)) != 0 || (key.replica_tiers & CacheTierBit(CacheTier::kL3NofSsd)) != 0; if (!in_head && lower_tier) candidates.push_back(key.object); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 366055c012..5be42c7a1b 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -4612,9 +4612,14 @@ auto MasterService::GetReplicaListLocal(const ObjectIdentity& object_id) std::chrono::steady_clock::now().time_since_epoch()) .count()); io_access.block_size = metadata.size; + // LOCAL_DISK is reported as its own tier: it is the only tier the store + // can promote from, and collapsing it into kL3NofSsd made local-disk and + // NoF-only objects indistinguishable to the policy. io_access.tier = resp.replicas[0].is_memory_replica() ? io_pattern::CacheTier::kL1Host - : io_pattern::CacheTier::kL3NofSsd; + : (resp.replicas[0].is_local_disk_replica() + ? io_pattern::CacheTier::kLocalDisk + : io_pattern::CacheTier::kL3NofSsd); io_access.operation = io_pattern::IoOperation::kGet; io_access.is_hit = true; record_io_access = true; @@ -4867,7 +4872,11 @@ MasterService::BatchGetReplicaListLocal(const std::vector& keys, .tier = results[original_idx]->replicas[0].is_memory_replica() ? io_pattern::CacheTier::kL1Host - : io_pattern::CacheTier::kL3NofSsd, + : (results[original_idx] + ->replicas[0] + .is_local_disk_replica() + ? io_pattern::CacheTier::kLocalDisk + : io_pattern::CacheTier::kL3NofSsd), .operation = io_pattern::IoOperation::kGet, .is_hit = true}); } @@ -5597,7 +5606,9 @@ auto MasterService::PutEndInternal( .block_size = metadata.size, .tier = replica_type == ReplicaType::MEMORY ? io_pattern::CacheTier::kL1Host - : io_pattern::CacheTier::kL3NofSsd, + : (replica_type == ReplicaType::LOCAL_DISK + ? io_pattern::CacheTier::kLocalDisk + : io_pattern::CacheTier::kL3NofSsd), .operation = io_pattern::IoOperation::kPut, .is_hit = true, .write_batch_size = write_batch_size, diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 58760612db..ec0c3010ed 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -544,7 +544,7 @@ TEST(IoPatternFrameworkTest, TracePrefetchPlansOnlyLongPrefixMatches) { KeyMetrics key; key.object = {TenantId("tenant-a"), "block"}; key.block_size = 4096; - key.replica_tiers = CacheTierBit(CacheTier::kL3NofSsd); + key.replica_tiers = CacheTierBit(CacheTier::kLocalDisk); context.snapshot.keys.push_back(key); // The prefetch gate also requires analyzer confidence, so supply the key // pattern the production pipeline would derive for this object. @@ -559,8 +559,8 @@ TEST(IoPatternFrameworkTest, TracePrefetchPlansOnlyLongPrefixMatches) { const auto plan = prefetch.Evaluate(context, trace); ASSERT_EQ(plan.candidates.size(), 1); - EXPECT_EQ(plan.candidates.front().source_tier, CacheTier::kL3NofSsd); - EXPECT_EQ(plan.candidates.front().target_tier, CacheTier::kL2Segment); + EXPECT_EQ(plan.candidates.front().source_tier, CacheTier::kLocalDisk); + EXPECT_EQ(plan.candidates.front().target_tier, CacheTier::kL1Host); EXPECT_EQ(plan.candidates.front().bytes, 4096); } @@ -570,7 +570,7 @@ TEST(IoPatternFrameworkTest, TracePrefetchDeduplicatesObjects) { KeyMetrics key; key.object = {TenantId("tenant-a"), "block"}; key.block_size = 128; - key.replica_tiers = CacheTierBit(CacheTier::kL2Segment); + key.replica_tiers = CacheTierBit(CacheTier::kLocalDisk); context.snapshot.keys.push_back(key); // The prefetch gate also requires analyzer confidence, so supply the key // pattern the production pipeline would derive for this object. From 87c759ca6f8e689e2f6c9fe87e349ea969a4580b Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 16:54:41 +0800 Subject: [PATCH 33/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/tests/io_pattern_framework_test.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index ec0c3010ed..699814b4e0 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1798,7 +1798,9 @@ TEST(IoPatternFrameworkTest, UnavailablePrefetchCapabilityDoesNotDegradePolicy) AccessRecord access{.object = {TenantId("tenant-a"), "cold-key"}, .block_size = 1024, - .tier = CacheTier::kL3NofSsd, + // Local disk is the only tier the store can promote from, + // and the only one that yields a prefetch candidate. + .tier = CacheTier::kLocalDisk, .operation = IoOperation::kGet, .is_hit = true}; // A recommendation-shaped key yields a definitive workload classification From 95c4fd01b39cb91bb3e433c718ebc575267cd8d9 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 17:03:51 +0800 Subject: [PATCH 34/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/io_pattern/runtime.h | 10 +--- mooncake-store/include/io_pattern/types.h | 5 ++ .../src/io_pattern/policy_strategies.cpp | 8 +++ mooncake-store/src/io_pattern/runtime.cpp | 50 ++++++------------- .../tests/io_pattern_framework_test.cpp | 30 +++++++++++ 5 files changed, 60 insertions(+), 43 deletions(-) diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 4ade433d12..d58d24014b 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -186,7 +186,8 @@ class IoPatternRuntime final { uint64_t eviction_bytes, const TraceHistory& trace, const std::vector& admissions, - const std::string& session_id); + const std::string& session_id, + uint64_t min_idle_time_us = 0); void AdmissionWorker(); ErrorCode ExecuteAdmission(const ObjectRef& object, CacheTier target_tier, const std::string& session_id); @@ -203,13 +204,6 @@ class IoPatternRuntime final { float high_ratio, float target_ratio, CacheTier& eviction_tier, uint64_t& eviction_bytes); - // Cold-data eviction driver input: scans the merged snapshot for L1 keys - // that are not pinned and idle at least `idle_threshold_us` (0 = any idle - // gate disabled) and returns the total byte budget bounded by `max_bytes`. - // Returns 0 when there is no idle L1 key to reclaim. - static uint64_t ColdEvictionBudget(const IoPatternSnapshot& snapshot, - uint64_t idle_threshold_us, - uint64_t max_bytes); static TraceHistory DeriveTraceHistory(const IoPatternSnapshot& snapshot); static std::vector DeriveAdmissionCandidates( const IoPatternSnapshot& snapshot); diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h index f9f2f29c42..ef0930c43f 100644 --- a/mooncake-store/include/io_pattern/types.h +++ b/mooncake-store/include/io_pattern/types.h @@ -204,6 +204,11 @@ struct PolicyContext { IoPatternSnapshot snapshot; PatternResult analysis; std::string session_id; + // Cold-data eviction gate: when non-zero, only objects idle at least this + // long are eligible. Applied here rather than by pre-summing idle bytes into + // the byte target, so the budget and the selected victims describe the same + // keys. 0 disables the gate. + uint64_t min_idle_time_us{0}; }; struct TraceEvent { diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp index 6b6863574e..ab97b8c793 100644 --- a/mooncake-store/src/io_pattern/policy_strategies.cpp +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -65,6 +65,10 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, if ((key.replica_tiers & CacheTierBit(tier)) == 0 || key.pinned) { continue; } + if (context.min_idle_time_us != 0 && + key.idle_time_us < context.min_idle_time_us) { + continue; + } max_block_size = std::max(max_block_size, key.block_size); max_other_replicas = std::max(max_other_replicas, key.other_replica_count); @@ -73,6 +77,10 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, if ((key.replica_tiers & CacheTierBit(tier)) == 0 || key.pinned) { continue; } + if (context.min_idle_time_us != 0 && + key.idle_time_us < context.min_idle_time_us) { + continue; + } const auto* pattern = FindPattern(key.object, context.analysis); if (pattern == nullptr) { continue; diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index dbe674d38f..231056d3e2 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -257,7 +257,8 @@ PolicyResult IoPatternRuntime::Plan( IoPatternRuntime::PlannedPolicy IoPatternRuntime::BuildPolicy( CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, - const std::vector& admissions, const std::string& session_id) { + const std::vector& admissions, const std::string& session_id, + uint64_t min_idle_time_us) { PlannedPolicy planned; planned.snapshot = collector_->GetSnapshot(); const auto start = std::chrono::steady_clock::now(); @@ -273,8 +274,10 @@ IoPatternRuntime::PlannedPolicy IoPatternRuntime::BuildPolicy( workload_policy_->SetSessionWorkloads(analysis.sessions); workload_policy_->AdvanceTransitionWindow(); planned.result = policy_->ExecutePolicy( - PolicyContext{.snapshot = planned.snapshot, .analysis = analysis, - .session_id = session_id}, + PolicyContext{.snapshot = planned.snapshot, + .analysis = analysis, + .session_id = session_id, + .min_idle_time_us = min_idle_time_us}, eviction_tier, eviction_bytes, CacheTier::kL1Host, trace, admissions); planned.result.degraded = planned.result.degraded || collector_->degraded() || @@ -412,30 +415,6 @@ void IoPatternRuntime::DeriveEvictionRequest(const IoPatternSnapshot& snapshot, : (capacity_bytes > 0 ? capacity_bytes / 10 : 0); } -uint64_t IoPatternRuntime::ColdEvictionBudget( - const IoPatternSnapshot& snapshot, uint64_t idle_threshold_us, - uint64_t max_bytes) { - if (max_bytes == 0) return 0; - uint64_t budget = 0; - for (const auto& key : snapshot.keys) { - if ((key.replica_tiers & CacheTierBit(CacheTier::kL1Host)) == 0 || - key.pinned) { - continue; - } - if (idle_threshold_us != 0 && - key.idle_time_us < idle_threshold_us) { - continue; - } - // Skip keys with no capacity estimate (block_size unset/unknown). - if (key.block_size == 0) continue; - budget = budget > std::numeric_limits::max() - key.block_size - ? std::numeric_limits::max() - : budget + key.block_size; - if (budget >= max_bytes) return max_bytes; - } - return budget; -} - TraceHistory IoPatternRuntime::DeriveTraceHistory( const IoPatternSnapshot& snapshot) { // Report-driven prefetch input: keys that were recently served as hits and @@ -510,15 +489,15 @@ void IoPatternRuntime::RunReportDrivenCycle() { // driven by cold/hot analysis and not only by memory pressure. Candidate // selection still goes through the policy engine (ScoreBasedEvictionOps // ranks the coldest first); this driver only supplies a byte target. + uint64_t cold_idle_threshold_us = 0; if (eviction_bytes == 0 && config_.report_driven_cold_eviction && config_.report_driven_cold_eviction_bytes != 0) { - const uint64_t cold_bytes = ColdEvictionBudget( - snapshot, config_.report_driven_cold_idle_threshold_us, - config_.report_driven_cold_eviction_bytes); - if (cold_bytes != 0) { - eviction_bytes = cold_bytes; - report.cold_eviction = true; - } + // The idle gate is applied where the policy selects victims, so the byte + // budget and the victim set describe the same keys. Pre-summing the idle + // keys' bytes here produced a target the selector was free to ignore. + eviction_bytes = config_.report_driven_cold_eviction_bytes; + cold_idle_threshold_us = config_.report_driven_cold_idle_threshold_us; + report.cold_eviction = true; } report.eviction_tier = eviction_tier; report.eviction_target_bytes = eviction_bytes; @@ -527,7 +506,8 @@ void IoPatternRuntime::RunReportDrivenCycle() { report.admission_candidates = admissions.size(); auto planned = BuildPolicy(eviction_tier, eviction_bytes, trace, - admissions, "report-driven"); + admissions, "report-driven", + cold_idle_threshold_us); report.analysis_elapsed_us = planned.analysis_elapsed_us; const auto& result = planned.result; report.eviction_candidates = result.eviction.candidates.size(); diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 699814b4e0..a3de4eefd1 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1906,5 +1906,35 @@ TEST(IoPatternFrameworkTest, LocalDiskCountsAsALowerTierThanHostMemory) { EXPECT_FLOAT_EQ(plan.candidates.front().score, 2.0F); } +TEST(IoPatternFrameworkTest, ScoreEvictionHonoursTheColdIdleGate) { + // The cold-eviction driver gates victims by idle time. Applying that gate + // where candidates are selected -- rather than pre-summing idle bytes into + // the byte target -- keeps the budget and the victim set describing the same + // keys, so a pass cannot reclaim objects the driver never considered cold. + PolicyContext context; + context.min_idle_time_us = 1'000'000; // 1 s + KeyMetrics fresh; + fresh.object = {TenantId("tenant-a"), "fresh"}; + fresh.idle_time_us = 500'000; // below the gate + fresh.block_size = 64; + fresh.replica_tiers = CacheTierBit(CacheTier::kL1Host); + KeyMetrics cold; + cold.object = {TenantId("tenant-a"), "cold"}; + cold.idle_time_us = 5'000'000; // above the gate + cold.block_size = 64; + cold.replica_tiers = CacheTierBit(CacheTier::kL1Host); + context.snapshot.keys = {fresh, cold}; + context.analysis.keys = { + KeyPattern{.object = fresh.object, .idle_score = 1.0F}, + KeyPattern{.object = cold.object, .idle_score = 0.1F}}; + + ScoreBasedEvictionOps eviction; + const auto plan = eviction.Evaluate(context, CacheTier::kL1Host, 64); + + ASSERT_EQ(plan.candidates.size(), 1U); + // Only the genuinely idle key is eligible, even though "fresh" scores higher. + EXPECT_EQ(plan.candidates.front().object.key, "cold"); +} + } // namespace } // namespace mooncake::io_pattern From 2ae5da46a670c15c56caebebe8de510ded35a4a6 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 17:14:09 +0800 Subject: [PATCH 35/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/io_pattern/policy_strategies.h | 6 +++++- mooncake-store/tests/io_pattern_framework_test.cpp | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/mooncake-store/include/io_pattern/policy_strategies.h b/mooncake-store/include/io_pattern/policy_strategies.h index 6ef274405d..9ce2bbf981 100644 --- a/mooncake-store/include/io_pattern/policy_strategies.h +++ b/mooncake-store/include/io_pattern/policy_strategies.h @@ -35,7 +35,11 @@ class ScoreBasedEvictionOps final : public EvictionOps { struct PrefixMatchAdmissionConfig { uint32_t hbm_match_length{64}; - uint64_t frequency_threshold{1}; + // Minimum accesses in the rolling window before an object may be admitted to + // a non-HBM tier. Defaults to 2, matching the master's own + // promotion_admission_threshold second-touch gate; a threshold of 1 admitted + // on first sight and made this gate a no-op. + uint64_t frequency_threshold{2}; float max_memory_used_ratio{0.90F}; }; diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index a3de4eefd1..bce32306ae 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1674,6 +1674,10 @@ TEST(IoPatternFrameworkTest, RuntimeConnectsCollectionAnalysisPolicyAndHandlers) .block_size = 64, .tier = CacheTier::kL2Segment, .is_hit = true}; + // Two accesses: the default admission frequency gate is 2, aligned with the + // master's promotion_admission_threshold. A single observation would be + // rejected for frequency instead of exercising the handler. + runtime.RecordAccess(access.object.key, access); runtime.RecordAccess(access.object.key, access); runtime.ReportInferenceMetrics( InferenceMetrics{.object = access.object, .match_length = 512}); @@ -1720,6 +1724,9 @@ TEST(IoPatternFrameworkTest, RuntimeSchedulesAdmissionOffTheProducerPath) { AccessRecord access{.object = {TenantId("tenant"), "disk-key"}, .tier = CacheTier::kL3NofSsd, .operation = IoOperation::kPut}; + // The default admission frequency gate is 2, so two observations are needed + // before the admission handler is reached. + runtime.RecordAccess(access.object.key, access); runtime.RecordAccess(access.object.key, access); EXPECT_TRUE(runtime.ScheduleAdmission(access.object, CacheTier::kL1Host)); From 7569327a5f562ce2e8c1fc08be05497feef0f270 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 17:32:33 +0800 Subject: [PATCH 36/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../include/io_pattern/policy_engine.h | 12 +++++++++-- mooncake-store/include/io_pattern/runtime.h | 4 ++++ mooncake-store/src/io_pattern/runtime.cpp | 3 ++- mooncake-store/src/master_service.cpp | 5 +++++ .../tests/io_pattern_framework_test.cpp | 20 +++++++++++++++++++ 5 files changed, 41 insertions(+), 3 deletions(-) diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h index 31a36f3c96..28c603b5ec 100644 --- a/mooncake-store/include/io_pattern/policy_engine.h +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -159,9 +159,11 @@ class RegistryPolicyEngine final : public PolicyEngine { class WorkloadPolicyEngine final : public PolicyEngine { public: explicit WorkloadPolicyEngine(WorkloadType type = WorkloadType::kMixed, - uint32_t transition_windows = 3) + uint32_t transition_windows = 3, + float admission_watermark_ratio = 0.90F) : transition_windows_(transition_windows), workload_type_(type), - previous_type_(type) { + previous_type_(type), + admission_watermark_ratio_(admission_watermark_ratio) { Configure(type); } @@ -327,6 +329,11 @@ class WorkloadPolicyEngine final : public PolicyEngine { break; } if (tuned_eviction_ && !eviction_override) eviction = *tuned_eviction_; + // Derived from the store's eviction high watermark rather than a fixed + // 0.90: admission must stop before eviction starts, otherwise the window + // between the two watermarks keeps admitting objects the store is + // already reclaiming. + admission.max_memory_used_ratio = admission_watermark_ratio_; return std::make_shared( std::make_shared(eviction), std::make_shared(prefetch), @@ -354,6 +361,7 @@ class WorkloadPolicyEngine final : public PolicyEngine { std::optional tuned_eviction_; ScoreBasedEvictionConfig active_eviction_; ScoreBasedEvictionConfig previous_eviction_; + float admission_watermark_ratio_{0.90F}; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index d58d24014b..5c319d592e 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -96,6 +96,10 @@ class IoPatternRuntime final { // snapshot is considered under pressure and an eviction cycle is // executed. Mirrors the master's own high-watermark trigger. float report_eviction_high_ratio{0.80F}; + // Storage ratio at or above which admission into the head tier is + // refused. MasterService derives it from the same eviction high + // watermark so admission stops before eviction starts. + float admission_watermark_ratio{0.90F}; // After an eviction cycle the tier is considered relieved once this // ratio is reached; eviction target bytes are derived as // (peak_ratio - report_eviction_target_ratio) * capacity_bytes. diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index 231056d3e2..a09abdc0c1 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -42,7 +42,8 @@ IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) auto sliding = std::make_shared( config.analysis_window_ns); analyzer_ = std::make_shared(std::move(sliding)); - workload_policy_ = std::make_shared(); + workload_policy_ = std::make_shared( + WorkloadType::kMixed, 3, config_.admission_watermark_ratio); std::shared_ptr legacy_strategy; if (config_.legacy_fallback == LegacyFallback::kFifo) { legacy_strategy = std::make_shared(); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 5be42c7a1b..5dac96241e 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -452,6 +452,11 @@ MasterService::MasterService(const MasterServiceConfig& config) static_cast(eviction_high_watermark_ratio_); io_pattern_config.report_eviction_target_ratio = static_cast( std::max(0.0, eviction_high_watermark_ratio_ - eviction_ratio_)); + // Admission must stop before eviction starts. Deriving the admission + // watermark from the same high watermark removes the window in which the + // store evicts while still admitting new objects. + io_pattern_config.admission_watermark_ratio = + static_cast(eviction_high_watermark_ratio_); // Cold-data eviction driver: allow merged reports to reclaim the coldest // real objects even below the memory watermark (opt-in via master flags). io_pattern_config.report_driven_cold_eviction = diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index bce32306ae..17addf87e0 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1943,5 +1943,25 @@ TEST(IoPatternFrameworkTest, ScoreEvictionHonoursTheColdIdleGate) { EXPECT_EQ(plan.candidates.front().object.key, "cold"); } +TEST(IoPatternFrameworkTest, AdmissionWatermarkFollowsTheConfiguredHighWatermark) { + // Admission must stop before eviction starts, so the watermark is derived + // from the store's eviction high watermark instead of a fixed 0.90: a store + // configured to evict at 0.75 also refuses admission at 0.75 rather than + // continuing to admit through the 0.75..0.90 window. + WorkloadPolicyEngine engine(WorkloadType::kMixed, 3, 0.75F); + PolicyContext context; + KeyMetrics key; + key.object = {TenantId("tenant-a"), "hot"}; + key.access_count_window = 32; + context.snapshot.keys.push_back(key); + context.snapshot.storage = {StorageMetric{.source_id = "host", + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.80F}}; + + EXPECT_EQ(engine.DecideAdmission(key.object, CacheTier::kL1Host, context) + .decision, + AdmissionDecision::kRejectWatermark); +} + } // namespace } // namespace mooncake::io_pattern From 324c31cdbf09a0065dcfca3cea90b8aab8c5aa43 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 17:47:27 +0800 Subject: [PATCH 37/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/master_config.h | 72 ++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index 7c8cc58dc3..a6362d6e93 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -169,6 +169,17 @@ struct MasterConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; + // Policy-driven tier down: copy a selected object down to local disk while + // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by + // a per-cycle byte budget. A zero budget disables the driver. + bool io_pattern_tier_down = false; + uint64_t io_pattern_tier_down_bytes_per_cycle = 0; + // Admission gates. The frequency threshold defaults to 2, matching the + // master's own second-touch promotion gate; 1 restores the previous + // admit-on-first-sight behaviour. A negative watermark means "derive it from + // eviction_high_watermark_ratio", so admission stops before eviction starts. + uint32_t io_pattern_admission_frequency_threshold = 2; + double io_pattern_admission_watermark_ratio = -1.0; // Max promotion tasks PromotionObjectHeartbeat returns to a single // client per call. Each task is a synchronous SSD-read + RDMA-write // on the client; serializing them avoids blocking past the client- @@ -291,6 +302,17 @@ class MasterServiceSupervisorConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; + // Policy-driven tier down: copy a selected object down to local disk while + // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by + // a per-cycle byte budget. A zero budget disables the driver. + bool io_pattern_tier_down = false; + uint64_t io_pattern_tier_down_bytes_per_cycle = 0; + // Admission gates. The frequency threshold defaults to 2, matching the + // master's own second-touch promotion gate; 1 restores the previous + // admit-on-first-sight behaviour. A negative watermark means "derive it from + // eviction_high_watermark_ratio", so admission stops before eviction starts. + uint32_t io_pattern_admission_frequency_threshold = 2; + double io_pattern_admission_watermark_ratio = -1.0; uint32_t promotion_max_per_heartbeat = 1; // Report-driven cold-data eviction driver (embedded CFM component). // Mirrors MasterConfig / WrappedMasterServiceConfig; carried through the @@ -357,6 +379,13 @@ class MasterServiceSupervisorConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; + io_pattern_tier_down = config.io_pattern_tier_down; + io_pattern_tier_down_bytes_per_cycle = + config.io_pattern_tier_down_bytes_per_cycle; + io_pattern_admission_frequency_threshold = + config.io_pattern_admission_frequency_threshold; + io_pattern_admission_watermark_ratio = + config.io_pattern_admission_watermark_ratio; promotion_max_per_heartbeat = config.promotion_max_per_heartbeat; enable_kv_events = config.enable_kv_events; kv_events_bind_endpoint = config.kv_events_bind_endpoint; @@ -563,6 +592,17 @@ class WrappedMasterServiceConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; + // Policy-driven tier down: copy a selected object down to local disk while + // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by + // a per-cycle byte budget. A zero budget disables the driver. + bool io_pattern_tier_down = false; + uint64_t io_pattern_tier_down_bytes_per_cycle = 0; + // Admission gates. The frequency threshold defaults to 2, matching the + // master's own second-touch promotion gate; 1 restores the previous + // admit-on-first-sight behaviour. A negative watermark means "derive it from + // eviction_high_watermark_ratio", so admission stops before eviction starts. + uint32_t io_pattern_admission_frequency_threshold = 2; + double io_pattern_admission_watermark_ratio = -1.0; uint32_t promotion_max_per_heartbeat = 1; bool enable_kv_events = false; std::string kv_events_bind_endpoint; @@ -673,6 +713,13 @@ class WrappedMasterServiceConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; + io_pattern_tier_down = config.io_pattern_tier_down; + io_pattern_tier_down_bytes_per_cycle = + config.io_pattern_tier_down_bytes_per_cycle; + io_pattern_admission_frequency_threshold = + config.io_pattern_admission_frequency_threshold; + io_pattern_admission_watermark_ratio = + config.io_pattern_admission_watermark_ratio; promotion_max_per_heartbeat = config.promotion_max_per_heartbeat; enable_kv_events = config.enable_kv_events; kv_events_bind_endpoint = config.kv_events_bind_endpoint; @@ -802,6 +849,13 @@ class WrappedMasterServiceConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; + io_pattern_tier_down = config.io_pattern_tier_down; + io_pattern_tier_down_bytes_per_cycle = + config.io_pattern_tier_down_bytes_per_cycle; + io_pattern_admission_frequency_threshold = + config.io_pattern_admission_frequency_threshold; + io_pattern_admission_watermark_ratio = + config.io_pattern_admission_watermark_ratio; promotion_max_per_heartbeat = config.promotion_max_per_heartbeat; enable_kv_events = config.enable_kv_events; kv_events_bind_endpoint = config.kv_events_bind_endpoint; @@ -1277,6 +1331,17 @@ class MasterServiceConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; + // Policy-driven tier down: copy a selected object down to local disk while + // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by + // a per-cycle byte budget. A zero budget disables the driver. + bool io_pattern_tier_down = false; + uint64_t io_pattern_tier_down_bytes_per_cycle = 0; + // Admission gates. The frequency threshold defaults to 2, matching the + // master's own second-touch promotion gate; 1 restores the previous + // admit-on-first-sight behaviour. A negative watermark means "derive it from + // eviction_high_watermark_ratio", so admission stops before eviction starts. + uint32_t io_pattern_admission_frequency_threshold = 2; + double io_pattern_admission_watermark_ratio = -1.0; uint32_t promotion_max_per_heartbeat = 1; bool enable_kv_events = false; std::string kv_events_bind_endpoint; @@ -1384,6 +1449,13 @@ class MasterServiceConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; + io_pattern_tier_down = config.io_pattern_tier_down; + io_pattern_tier_down_bytes_per_cycle = + config.io_pattern_tier_down_bytes_per_cycle; + io_pattern_admission_frequency_threshold = + config.io_pattern_admission_frequency_threshold; + io_pattern_admission_watermark_ratio = + config.io_pattern_admission_watermark_ratio; promotion_max_per_heartbeat = config.promotion_max_per_heartbeat; enable_kv_events = config.enable_kv_events; kv_events_bind_endpoint = config.kv_events_bind_endpoint; From 4264679a89978cece9aad13f5e5b5e3b3fbd5e35 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 18:00:27 +0800 Subject: [PATCH 38/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/io_pattern/policy_engine.h | 10 ++++++++-- mooncake-store/include/io_pattern/runtime.h | 4 ++++ mooncake-store/src/io_pattern/runtime.cpp | 3 ++- mooncake-store/src/master_service.cpp | 14 ++++++++++---- 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/mooncake-store/include/io_pattern/policy_engine.h b/mooncake-store/include/io_pattern/policy_engine.h index 28c603b5ec..b089d3a569 100644 --- a/mooncake-store/include/io_pattern/policy_engine.h +++ b/mooncake-store/include/io_pattern/policy_engine.h @@ -160,10 +160,12 @@ class WorkloadPolicyEngine final : public PolicyEngine { public: explicit WorkloadPolicyEngine(WorkloadType type = WorkloadType::kMixed, uint32_t transition_windows = 3, - float admission_watermark_ratio = 0.90F) + float admission_watermark_ratio = 0.90F, + uint32_t admission_frequency_threshold = 2) : transition_windows_(transition_windows), workload_type_(type), previous_type_(type), - admission_watermark_ratio_(admission_watermark_ratio) { + admission_watermark_ratio_(admission_watermark_ratio), + admission_frequency_threshold_(admission_frequency_threshold) { Configure(type); } @@ -309,6 +311,9 @@ class WorkloadPolicyEngine final : public PolicyEngine { ScoreBasedEvictionConfig eviction = eviction_override.value_or(EvictionConfigFor(type)); PrefixMatchAdmissionConfig admission; + // Applied before the per-workload switch, so a template that states its + // own frequency gate (generative recommendation) still wins. + admission.frequency_threshold = admission_frequency_threshold_; TraceBasedPrefetchConfig prefetch; switch (type) { case WorkloadType::kCodeAgent: @@ -362,6 +367,7 @@ class WorkloadPolicyEngine final : public PolicyEngine { ScoreBasedEvictionConfig active_eviction_; ScoreBasedEvictionConfig previous_eviction_; float admission_watermark_ratio_{0.90F}; + uint32_t admission_frequency_threshold_{2}; }; } // namespace mooncake::io_pattern diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 5c319d592e..2a52220612 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -100,6 +100,10 @@ class IoPatternRuntime final { // refused. MasterService derives it from the same eviction high // watermark so admission stops before eviction starts. float admission_watermark_ratio{0.90F}; + // Minimum accesses in the rolling window before admission into a + // non-HBM tier. Defaults to the ops default (2); 1 restores the previous + // admit-on-first-sight behaviour. + uint32_t admission_frequency_threshold{2}; // After an eviction cycle the tier is considered relieved once this // ratio is reached; eviction target bytes are derived as // (peak_ratio - report_eviction_target_ratio) * capacity_bytes. diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index a09abdc0c1..bb49c5824b 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -43,7 +43,8 @@ IoPatternRuntime::IoPatternRuntime(Handlers handlers, Config config) config.analysis_window_ns); analyzer_ = std::make_shared(std::move(sliding)); workload_policy_ = std::make_shared( - WorkloadType::kMixed, 3, config_.admission_watermark_ratio); + WorkloadType::kMixed, 3, config_.admission_watermark_ratio, + config_.admission_frequency_threshold); std::shared_ptr legacy_strategy; if (config_.legacy_fallback == LegacyFallback::kFifo) { legacy_strategy = std::make_shared(); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 5dac96241e..bcc5f19791 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -452,11 +452,17 @@ MasterService::MasterService(const MasterServiceConfig& config) static_cast(eviction_high_watermark_ratio_); io_pattern_config.report_eviction_target_ratio = static_cast( std::max(0.0, eviction_high_watermark_ratio_ - eviction_ratio_)); - // Admission must stop before eviction starts. Deriving the admission - // watermark from the same high watermark removes the window in which the - // store evicts while still admitting new objects. + // Admission must stop before eviction starts: a negative configured watermark + // means "derive it from the eviction high watermark", so the store does not + // evict while still admitting new objects. A non-negative value pins it. io_pattern_config.admission_watermark_ratio = - static_cast(eviction_high_watermark_ratio_); + config.io_pattern_admission_watermark_ratio >= 0.0 + ? static_cast(config.io_pattern_admission_watermark_ratio) + : static_cast(eviction_high_watermark_ratio_); + // Configurable so a deployment can restore the previous admit-on-first-sight + // behaviour (threshold 1) without recompiling. + io_pattern_config.admission_frequency_threshold = + config.io_pattern_admission_frequency_threshold; // Cold-data eviction driver: allow merged reports to reclaim the coldest // real objects even below the memory watermark (opt-in via master flags). io_pattern_config.report_driven_cold_eviction = From f67aeaeec5b3838f774bc0b97376116776ef3def Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Thu, 10 Sep 2026 20:39:21 +0800 Subject: [PATCH 39/47] =?UTF-8?q?io=5Fpattern=E4=BF=AE=E5=A4=8D8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mooncake-store/include/io_pattern/runtime.h | 18 +- mooncake-store/include/io_pattern/types.h | 28 ++ mooncake-store/include/master_config.h | 44 +-- mooncake-store/include/master_service.h | 39 +++ .../src/io_pattern/cfm_protocol.cpp | 26 ++ .../src/io_pattern/policy_strategies.cpp | 13 +- mooncake-store/src/io_pattern/runtime.cpp | 26 +- mooncake-store/src/master.cpp | 46 +++ mooncake-store/src/master_service.cpp | 146 ++++++++- .../tests/io_pattern_framework_test.cpp | 287 ++++++++++++++++++ .../tests/offload_on_evict_test.cpp | 208 +++++++++++++ 11 files changed, 844 insertions(+), 37 deletions(-) diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 2a52220612..4344e430b2 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -55,6 +55,11 @@ class IoPatternRuntime final { // (analysis-selected idle keys) rather than a storage-pressure // watermark request. bool cold_eviction{false}; + // True when the eviction dimension was a demotion pass instead: the + // candidates were copied down to LOCAL_DISK and kept their MEMORY + // replica, so eviction_target_bytes is a demotion budget and + // eviction_status is the demotion outcome. Nothing was reclaimed. + bool tier_down{false}; // Prefetch dimension (derived from merged prefix-affinity keys). size_t prefetch_candidates{0}; ErrorCode prefetch_status{ErrorCode::OK}; @@ -122,6 +127,16 @@ class IoPatternRuntime final { // Max bytes a single cold-eviction pass may request (per drained // cycle). 0 disables the cold driver regardless of the enable flag. uint64_t report_driven_cold_eviction_bytes{0}; + // Policy-driven tier down: the below-watermark placement action. When a + // report-driven cycle finds no reclaim request (neither storage pressure + // nor the cold-eviction driver) it spends this budget copying the coldest + // in-memory keys down to LOCAL_DISK while keeping their MEMORY replica, + // so a later reclaim of those keys can discard them safely instead of + // paying for the copy then. A demotion frees nothing, so a reclaim always + // wins the cycle, and the cycle report marks `tier_down=true` so a + // demotion is never counted as an eviction. There is no separate enable + // flag: 0 keeps the driver off, so this budget is the whole control. + uint64_t tier_down_bytes_per_cycle{0}; // Optional per-cycle observer used to surface executions in process // metrics (e.g. MasterMetricManager). Never called from the report // data path; only from the background cycle worker. @@ -195,7 +210,8 @@ class IoPatternRuntime final { const TraceHistory& trace, const std::vector& admissions, const std::string& session_id, - uint64_t min_idle_time_us = 0); + uint64_t min_idle_time_us = 0, + bool tier_down = false); void AdmissionWorker(); ErrorCode ExecuteAdmission(const ObjectRef& object, CacheTier target_tier, const std::string& session_id); diff --git a/mooncake-store/include/io_pattern/types.h b/mooncake-store/include/io_pattern/types.h index ef0930c43f..a4f9190f4b 100644 --- a/mooncake-store/include/io_pattern/types.h +++ b/mooncake-store/include/io_pattern/types.h @@ -209,6 +209,14 @@ struct PolicyContext { // the byte target, so the budget and the selected victims describe the same // keys. 0 disables the gate. uint64_t min_idle_time_us{0}; + // Tier-down driver gate. When set, the strategy labels the candidates it + // selects as kTierDown instead of kEvict: the same victims are chosen, but + // the executor copies them down instead of reclaiming them. The action is + // decided by the driver here and can never be derived from a candidate's + // target_tier: TierDownTarget() returns source+1 for every tier it is given, + // so deriving it would label every candidate a demotion and disable eviction + // entirely. + bool tier_down{false}; }; struct TraceEvent { @@ -243,17 +251,37 @@ struct PrefetchPlan { std::vector candidates; }; +// What the executor must do with one selected candidate. Chosen by the driver +// that built the plan (PolicyContext::tier_down), never inferred from +// target_tier: see PolicyContext::tier_down. +enum class EvictionAction : uint8_t { + // Reclaim the MEMORY replica (the original behaviour). + kEvict, + // Copy the object down to LOCAL_DISK and keep the MEMORY replica, so a later + // Get still hits memory and nothing is freed by this action. + kTierDown, +}; + struct EvictionCandidate { ObjectRef object; uint64_t bytes{0}; float score{0.0F}; CacheTier target_tier{CacheTier::kL3NofSsd}; + // Appended last so existing positional and designated initializers keep + // their meaning; the default preserves the previous reclaim semantics for + // every producer that does not set an action. + EvictionAction action{EvictionAction::kEvict}; }; struct EvictionPlan { CacheTier source_tier{CacheTier::kL0Hbm}; uint64_t target_bytes{0}; std::vector candidates; + // How many of target_bytes are meant to be demoted rather than reclaimed. + // 0 for a pure eviction plan. This rides the CFM wire so a remote executor + // can tell a tier-down budget from a reclaim budget instead of treating the + // whole plan as memory to free. + uint64_t tier_down_target_bytes{0}; }; enum class AdmissionDecision : uint8_t { diff --git a/mooncake-store/include/master_config.h b/mooncake-store/include/master_config.h index a6362d6e93..85435520ef 100644 --- a/mooncake-store/include/master_config.h +++ b/mooncake-store/include/master_config.h @@ -169,10 +169,12 @@ struct MasterConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; - // Policy-driven tier down: copy a selected object down to local disk while - // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by - // a per-cycle byte budget. A zero budget disables the driver. - bool io_pattern_tier_down = false; + // Policy-driven tier down: below the memory watermark, copy the coldest + // objects down to local disk while keeping their MEMORY replica (tier down is + // a copy, not a reclaim), so a later reclaim can discard them safely. There + // is no separate enable switch: a non-zero per-cycle budget is the control, + // and 0 keeps the driver off. A reclaim always wins, so a cycle that reaches + // the watermark evicts instead of demoting. uint64_t io_pattern_tier_down_bytes_per_cycle = 0; // Admission gates. The frequency threshold defaults to 2, matching the // master's own second-touch promotion gate; 1 restores the previous @@ -302,10 +304,12 @@ class MasterServiceSupervisorConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; - // Policy-driven tier down: copy a selected object down to local disk while - // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by - // a per-cycle byte budget. A zero budget disables the driver. - bool io_pattern_tier_down = false; + // Policy-driven tier down: below the memory watermark, copy the coldest + // objects down to local disk while keeping their MEMORY replica (tier down is + // a copy, not a reclaim), so a later reclaim can discard them safely. There + // is no separate enable switch: a non-zero per-cycle budget is the control, + // and 0 keeps the driver off. A reclaim always wins, so a cycle that reaches + // the watermark evicts instead of demoting. uint64_t io_pattern_tier_down_bytes_per_cycle = 0; // Admission gates. The frequency threshold defaults to 2, matching the // master's own second-touch promotion gate; 1 restores the previous @@ -379,7 +383,6 @@ class MasterServiceSupervisorConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; - io_pattern_tier_down = config.io_pattern_tier_down; io_pattern_tier_down_bytes_per_cycle = config.io_pattern_tier_down_bytes_per_cycle; io_pattern_admission_frequency_threshold = @@ -592,10 +595,12 @@ class WrappedMasterServiceConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; - // Policy-driven tier down: copy a selected object down to local disk while - // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by - // a per-cycle byte budget. A zero budget disables the driver. - bool io_pattern_tier_down = false; + // Policy-driven tier down: below the memory watermark, copy the coldest + // objects down to local disk while keeping their MEMORY replica (tier down is + // a copy, not a reclaim), so a later reclaim can discard them safely. There + // is no separate enable switch: a non-zero per-cycle budget is the control, + // and 0 keeps the driver off. A reclaim always wins, so a cycle that reaches + // the watermark evicts instead of demoting. uint64_t io_pattern_tier_down_bytes_per_cycle = 0; // Admission gates. The frequency threshold defaults to 2, matching the // master's own second-touch promotion gate; 1 restores the previous @@ -713,7 +718,6 @@ class WrappedMasterServiceConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; - io_pattern_tier_down = config.io_pattern_tier_down; io_pattern_tier_down_bytes_per_cycle = config.io_pattern_tier_down_bytes_per_cycle; io_pattern_admission_frequency_threshold = @@ -849,7 +853,6 @@ class WrappedMasterServiceConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; - io_pattern_tier_down = config.io_pattern_tier_down; io_pattern_tier_down_bytes_per_cycle = config.io_pattern_tier_down_bytes_per_cycle; io_pattern_admission_frequency_threshold = @@ -1331,10 +1334,12 @@ class MasterServiceConfig { bool promotion_on_hit = false; uint32_t promotion_admission_threshold = 2; uint32_t promotion_queue_limit = 50000; - // Policy-driven tier down: copy a selected object down to local disk while - // keeping its MEMORY replica (tier down is a copy, not a reclaim), bounded by - // a per-cycle byte budget. A zero budget disables the driver. - bool io_pattern_tier_down = false; + // Policy-driven tier down: below the memory watermark, copy the coldest + // objects down to local disk while keeping their MEMORY replica (tier down is + // a copy, not a reclaim), so a later reclaim can discard them safely. There + // is no separate enable switch: a non-zero per-cycle budget is the control, + // and 0 keeps the driver off. A reclaim always wins, so a cycle that reaches + // the watermark evicts instead of demoting. uint64_t io_pattern_tier_down_bytes_per_cycle = 0; // Admission gates. The frequency threshold defaults to 2, matching the // master's own second-touch promotion gate; 1 restores the previous @@ -1449,7 +1454,6 @@ class MasterServiceConfig { promotion_on_hit = config.promotion_on_hit; promotion_admission_threshold = config.promotion_admission_threshold; promotion_queue_limit = config.promotion_queue_limit; - io_pattern_tier_down = config.io_pattern_tier_down; io_pattern_tier_down_bytes_per_cycle = config.io_pattern_tier_down_bytes_per_cycle; io_pattern_admission_frequency_threshold = diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index 4d310e5a0c..ae2e3a3dfc 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -993,6 +993,22 @@ class MasterService { */ void setHttpMetadataRemoteUrl(const std::string& metadata_connstring); + /** + * @brief Policy-driven tier-down dispatch counters. Independent of the + * eviction counters because a demotion copies an object down and keeps its + * MEMORY replica: it frees no bytes, so folding it into freed memory would + * make a tier-down plan look like a plan that under-delivered. + */ + uint64_t tier_down_attempt_count() const { + return tier_down_attempts_.load(std::memory_order_relaxed); + } + uint64_t tier_down_success_count() const { + return tier_down_successes_.load(std::memory_order_relaxed); + } + uint64_t tier_down_failure_count() const { + return tier_down_failures_.load(std::memory_order_relaxed); + } + private: std::unique_ptr CreateSnapshotCatalogStore(); @@ -1786,6 +1802,20 @@ class MasterService { */ PromotionQueueResult TryPushPromotionQueue(const ObjectIdentity& object_id, bool record_candidate = true); + + /** + * @brief Queue one MEMORY replica of `object_id` for a LOCAL_DISK copy + * (tier down) and keep the MEMORY replica in place. + * + * This is the demotion counterpart of an eviction: it never removes a + * replica and never frees bytes, so the caller must not count its result as + * reclaimed memory. Acquires its own RW shard accessor; safe to call from + * the io_pattern eviction handler, which does not hold one while this runs. + * Returns false when the key vanished, has no completed MEMORY replica, or + * the holder client cannot accept the offload. A key that is already queued + * counts as success: it is already on its way down. + */ + bool TryQueueTierDown(const ObjectIdentity& object_id); void RecordOrUpdateCandidate(TenantState& tenant_state, const std::string& key, uint8_t sketch_score, PromotionCandidateReason reason, @@ -2253,6 +2283,15 @@ class MasterService { std::shared_ptr io_pattern_runtime_; std::shared_ptr io_pattern_cfm_service_; + // Policy-driven tier down. Kept separate from the eviction counters because a + // demotion queues a disk copy and frees no memory: folding it into freed + // bytes would make a tier-down plan look like a plan that under-delivered. + // Written only from the io_pattern eviction handler; relaxed order is enough + // because these are advisory observability counters. + std::atomic tier_down_attempts_{0}; + std::atomic tier_down_successes_{0}; + std::atomic tier_down_failures_{0}; + const std::string ha_backend_type_; const std::string ha_backend_connstring_; diff --git a/mooncake-store/src/io_pattern/cfm_protocol.cpp b/mooncake-store/src/io_pattern/cfm_protocol.cpp index 229025a017..8b5886b09b 100644 --- a/mooncake-store/src/io_pattern/cfm_protocol.cpp +++ b/mooncake-store/src/io_pattern/cfm_protocol.cpp @@ -252,6 +252,14 @@ std::string CfmBinaryCodec::EncodePolicy(const PolicyCommand& command) const { Append(output, candidate.score); AppendEnum(output, candidate.target_tier); } + // Tier-down trailer, appended after the original layout so a decoder + // that predates it still finds the candidate list it expects. Without + // this a plan sent to a remote SubMaster would arrive as pure eviction + // and the executor would reclaim the keys the driver meant to keep. + Append(output, eviction->tier_down_target_bytes); + for (const auto& candidate : eviction->candidates) { + AppendEnum(output, candidate.action); + } } else if (const auto* prefetch = std::get_if(&command)) { AppendHeader(output, 'P'); AppendPrefetchPlan(output, *prefetch); @@ -296,6 +304,24 @@ std::optional CfmBinaryCodec::DecodePolicy( } plan.candidates.push_back(std::move(candidate)); } + // Optional tier-down trailer. A payload encoded before tier down existed + // ends here, and both fields keep their defaults (kEvict / 0), which is + // exactly the pre-tier-down meaning of the plan. A partly written trailer + // is a decode error rather than a silent default. + if (offset < payload.size()) { + if (!Read(payload, offset, plan.tier_down_target_bytes)) { + return std::nullopt; + } + for (auto& candidate : plan.candidates) { + uint8_t raw_action = 0; + if (!Read(payload, offset, raw_action) || + raw_action > + static_cast(EvictionAction::kTierDown)) { + return std::nullopt; + } + candidate.action = static_cast(raw_action); + } + } return offset == payload.size() ? std::optional(std::move(plan)) : std::nullopt; diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp index ab97b8c793..c9169becb1 100644 --- a/mooncake-store/src/io_pattern/policy_strategies.cpp +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -59,6 +59,16 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, CacheTier tier, uint64_t target_bytes) const { EvictionPlan plan{.source_tier = tier, .target_bytes = target_bytes}; + // The driver owns the action: a tier-down pass selects the same victims an + // eviction pass would, and only the executor's action differs. Deriving it + // from target_tier is not possible -- TierDownTarget() returns source+1 for + // L0/L1/L2 alike, so every candidate would look like a demotion. + const EvictionAction action = context.tier_down + ? EvictionAction::kTierDown + : EvictionAction::kEvict; + if (context.tier_down) { + plan.tier_down_target_bytes = target_bytes; + } uint64_t max_block_size = 0; uint32_t max_other_replicas = 0; for (const auto& key : context.snapshot.keys) { @@ -127,7 +137,8 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, .bytes = key.block_size, .score = score, .target_tier = TierDownTarget( - tier, config_.tier_down_mode)}); + tier, config_.tier_down_mode), + .action = action}); } std::sort(plan.candidates.begin(), plan.candidates.end(), [](const EvictionCandidate& lhs, const EvictionCandidate& rhs) { diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index bb49c5824b..bf1c1f118f 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -260,7 +260,7 @@ PolicyResult IoPatternRuntime::Plan( IoPatternRuntime::PlannedPolicy IoPatternRuntime::BuildPolicy( CacheTier eviction_tier, uint64_t eviction_bytes, const TraceHistory& trace, const std::vector& admissions, const std::string& session_id, - uint64_t min_idle_time_us) { + uint64_t min_idle_time_us, bool tier_down) { PlannedPolicy planned; planned.snapshot = collector_->GetSnapshot(); const auto start = std::chrono::steady_clock::now(); @@ -279,7 +279,8 @@ IoPatternRuntime::PlannedPolicy IoPatternRuntime::BuildPolicy( PolicyContext{.snapshot = planned.snapshot, .analysis = analysis, .session_id = session_id, - .min_idle_time_us = min_idle_time_us}, + .min_idle_time_us = min_idle_time_us, + .tier_down = tier_down}, eviction_tier, eviction_bytes, CacheTier::kL1Host, trace, admissions); planned.result.degraded = planned.result.degraded || collector_->degraded() || @@ -485,12 +486,31 @@ void IoPatternRuntime::RunReportDrivenCycle() { DeriveEvictionRequest(snapshot, config_.report_eviction_high_ratio, config_.report_eviction_target_ratio, eviction_tier, eviction_bytes); + // Tier-down driver: the below-watermark placement action. Copy the coldest + // in-memory keys down to LOCAL_DISK while keeping their MEMORY replica, so a + // later reclaim of those keys can discard them safely instead of copying then. + // A demotion frees nothing, so it must never win over a reclaim: the pressure + // request above takes the cycle whenever the watermark is reached, and this + // driver only fills the cycle that would otherwise do nothing. The budget is + // the whole control (no enable flag); the driver decides the action here, and + // the policy must never derive it from target_tier. + bool tier_down = false; + if (eviction_bytes == 0 && config_.tier_down_bytes_per_cycle != 0) { + eviction_bytes = config_.tier_down_bytes_per_cycle; + tier_down = true; + report.tier_down = true; + } // Cold-data eviction driver: when the merged storage watermark does not // trigger a pressure eviction but the analysis-relevant snapshot contains // idle L1 keys, run a bounded eviction of the coldest keys so eviction is // driven by cold/hot analysis and not only by memory pressure. Candidate // selection still goes through the policy engine (ScoreBasedEvictionOps // ranks the coldest first); this driver only supplies a byte target. + // + // It also acts below the watermark, but it reclaims instead of placing, so it + // only takes the slot when no tier-down budget is configured: demoting a key + // the same cycle would have discarded defeats the point of paving cold data + // down first. uint64_t cold_idle_threshold_us = 0; if (eviction_bytes == 0 && config_.report_driven_cold_eviction && config_.report_driven_cold_eviction_bytes != 0) { @@ -509,7 +529,7 @@ void IoPatternRuntime::RunReportDrivenCycle() { auto planned = BuildPolicy(eviction_tier, eviction_bytes, trace, admissions, "report-driven", - cold_idle_threshold_us); + cold_idle_threshold_us, tier_down); report.analysis_elapsed_us = planned.analysis_elapsed_us; const auto& result = planned.result; report.eviction_candidates = result.eviction.candidates.size(); diff --git a/mooncake-store/src/master.cpp b/mooncake-store/src/master.cpp index 223e52e656..00cb663194 100644 --- a/mooncake-store/src/master.cpp +++ b/mooncake-store/src/master.cpp @@ -153,6 +153,21 @@ DEFINE_uint64(io_pattern_cold_eviction_bytes_per_cycle, 0, DEFINE_uint64(io_pattern_cold_idle_threshold_us, 0, "Minimum idle_time_us for a key to be eligible for a " "report-driven cold-eviction pass (0 disables the idle gate)"); +DEFINE_uint64(io_pattern_tier_down_bytes_per_cycle, 0, + "Max bytes one report-driven tier-down pass may copy per cycle. " + "The pass runs below the memory watermark and copies the coldest " + "objects down while keeping their MEMORY replica, so a later " + "reclaim can discard them safely; a cycle that reaches the " + "watermark evicts instead. 0 keeps the driver off, so this budget " + "is the only switch the driver has"); +DEFINE_uint32(io_pattern_admission_frequency_threshold, 2, + "Min access count for a reported key before the io_pattern " + "policy admits it (set 1 to admit on first sight)"); +DEFINE_double(io_pattern_admission_watermark_ratio, -1.0, + "Capacity watermark for io_pattern admission, as a ratio of " + "capacity; a negative value derives it from " + "eviction_high_watermark_ratio so admission stops before " + "eviction starts"); // RPC server configuration parameters (new, preferred) // TODO: deprecate port and max_threads in the future DEFINE_int32(rpc_thread_num, 0, @@ -552,6 +567,16 @@ void InitMasterConf(const mooncake::DefaultConfig& default_config, default_config.GetUInt64("io_pattern_cold_idle_threshold_us", &master_config.io_pattern_cold_idle_threshold_us, FLAGS_io_pattern_cold_idle_threshold_us); + default_config.GetUInt64("io_pattern_tier_down_bytes_per_cycle", + &master_config.io_pattern_tier_down_bytes_per_cycle, + FLAGS_io_pattern_tier_down_bytes_per_cycle); + default_config.GetUInt32( + "io_pattern_admission_frequency_threshold", + &master_config.io_pattern_admission_frequency_threshold, + FLAGS_io_pattern_admission_frequency_threshold); + default_config.GetDouble("io_pattern_admission_watermark_ratio", + &master_config.io_pattern_admission_watermark_ratio, + FLAGS_io_pattern_admission_watermark_ratio); default_config.GetInt64("client_live_ttl_sec", &master_config.client_live_ttl_sec, FLAGS_client_ttl); @@ -936,6 +961,27 @@ void LoadConfigFromCmdline(mooncake::MasterConfig& master_config, master_config.io_pattern_cold_idle_threshold_us = FLAGS_io_pattern_cold_idle_threshold_us; } + if ((google::GetCommandLineFlagInfo("io_pattern_tier_down_bytes_per_cycle", + &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_tier_down_bytes_per_cycle = + FLAGS_io_pattern_tier_down_bytes_per_cycle; + } + if ((google::GetCommandLineFlagInfo( + "io_pattern_admission_frequency_threshold", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_admission_frequency_threshold = + FLAGS_io_pattern_admission_frequency_threshold; + } + if ((google::GetCommandLineFlagInfo( + "io_pattern_admission_watermark_ratio", &info) && + !info.is_default) || + !conf_set) { + master_config.io_pattern_admission_watermark_ratio = + FLAGS_io_pattern_admission_watermark_ratio; + } if ((google::GetCommandLineFlagInfo("enable_ha", &info) && !info.is_default) || !conf_set) { diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index bcc5f19791..4448afeabf 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -478,6 +478,16 @@ MasterService::MasterService(const MasterServiceConfig& config) << " bytes, idle threshold=" << config.io_pattern_cold_idle_threshold_us << " us"; } + // Policy-driven tier-down driver: below the memory watermark, copy the + // coldest in-memory keys down to LOCAL_DISK while keeping their MEMORY + // replica, so a later reclaim can discard them safely. The per-cycle budget + // is the whole control: 0 (the default) keeps the driver off. + io_pattern_config.tier_down_bytes_per_cycle = + config.io_pattern_tier_down_bytes_per_cycle; + if (config.io_pattern_tier_down_bytes_per_cycle != 0) { + LOG(INFO) << "Policy-driven tier down enabled: per-cycle budget=" + << config.io_pattern_tier_down_bytes_per_cycle << " bytes"; + } io_pattern_config.report_driven_observer = [](const io_pattern::IoPatternRuntime::ReportDrivenCycleReport& rpt) { auto& metrics = MasterMetricManager::instance(); @@ -489,7 +499,20 @@ MasterService::MasterService(const MasterServiceConfig& config) // clean no-op, not a failure); prefetch/admission likewise only // reach their handler when candidates were planned. Failures are // counted when the storage handler rejected the plan. - if (rpt.eviction_candidates != 0) { + // + // A tier-down cycle is excluded from the eviction counters: it copies + // keys down and frees nothing, so counting it as a report-driven + // eviction would overstate reclaim. Its outcome is visible through + // MasterService's own tier_down_attempt/success/failure counters. + if (rpt.tier_down) { + LOG(INFO) << "[IO-PATTERN-TIER-DOWN] report-driven tier-down " + "cycle=" + << rpt.cycle_id + << " budget=" << rpt.eviction_target_bytes + << " candidates=" << rpt.eviction_candidates + << " status=" << static_cast(rpt.eviction_status) + << " skipped=" << rpt.skipped_dimensions; + } else if (rpt.eviction_candidates != 0) { metrics.inc_io_pattern_report_evictions(); if (rpt.eviction_status != ErrorCode::OK) { metrics.inc_io_pattern_report_eviction_failures(); @@ -517,7 +540,8 @@ MasterService::MasterService(const MasterServiceConfig& config) << ", bytes=" << rpt.eviction_target_bytes << ", candidates=" << rpt.eviction_candidates << ", status=" << static_cast(rpt.eviction_status) - << ", cold=" << rpt.cold_eviction << ")" + << ", cold=" << rpt.cold_eviction + << ", tier_down=" << rpt.tier_down << ")" << " prefetch(candidates=" << rpt.prefetch_candidates << ", status=" << static_cast(rpt.prefetch_status) << ")" @@ -563,14 +587,59 @@ MasterService::MasterService(const MasterServiceConfig& config) std::unordered_set keys; }; if (plan.target_bytes == 0) return ErrorCode::OK; + // A tier-down plan copies objects down and keeps their + // MEMORY replica, so it frees nothing. Only the reclaim share + // of the plan may drive the legacy fallback below: counting + // the demotion budget as a shortfall would make the fallback + // evict exactly the keys the driver just chose to keep. + const uint64_t tier_down_target = + std::min(plan.tier_down_target_bytes, + plan.target_bytes); + const uint64_t reclaim_target = + plan.target_bytes - tier_down_target; uint64_t total_freed = 0; + uint64_t tier_down_attempts = 0; + uint64_t tier_down_queued = 0; + uint64_t tier_down_failed = 0; std::unordered_map targets; for (const auto& candidate : plan.candidates) { + // The action is per candidate and comes from the plan + // (which got it from the driver), never from + // target_tier. A kTierDown candidate is copied down and + // keeps its MEMORY replica, so it is excluded from the + // quota eviction below and from freed accounting. + if (candidate.action == + io_pattern::EvictionAction::kTierDown) { + ++tier_down_attempts; + if (TryQueueTierDown(ObjectIdentity{ + candidate.object.tenant_id, + candidate.object.key})) { + ++tier_down_queued; + } else { + ++tier_down_failed; + } + continue; + } auto& target = targets[candidate.object.tenant_id]; target.bytes += candidate.bytes; target.keys.insert(candidate.object.key); } + if (tier_down_attempts != 0) { + tier_down_attempts_.fetch_add( + tier_down_attempts, std::memory_order_relaxed); + tier_down_successes_.fetch_add( + tier_down_queued, std::memory_order_relaxed); + tier_down_failures_.fetch_add( + tier_down_failed, std::memory_order_relaxed); + LOG(WARNING) + << "[IO-PATTERN-TIER-DOWN] io_pattern tier down " + "plan_target=" + << tier_down_target + << " attempts=" << tier_down_attempts + << " queued=" << tier_down_queued + << " failed=" << tier_down_failed; + } for (const auto& [tenant, target] : targets) { const auto result = EvictTenantMemoryForQuota( tenant, target.bytes, &target.keys); @@ -599,22 +668,27 @@ MasterService::MasterService(const MasterServiceConfig& config) LOG(WARNING) << "[IO-PATTERN-EVICT-DIAG] io_pattern eviction " "summary plan_target=" - << plan.target_bytes << " total_freed=" << total_freed + << plan.target_bytes + << " reclaim_target=" << reclaim_target + << " total_freed=" << total_freed << " candidates=" << plan.candidates.size(); - if (total_freed >= plan.target_bytes) { + if (total_freed >= reclaim_target) { return ErrorCode::OK; } - // The plan under-delivered: its candidates may be stale, may - // still hold leases or pins, or may be empty. Fall back to - // the legacy lease-ordered eviction so the watermark request - // still makes progress. This covers both the local watermark + // The plan under-delivered its *reclaim* target: its + // candidates may be stale, may still hold leases or pins, or + // may be empty. Fall back to the legacy lease-ordered + // eviction so the watermark request still makes progress. + // This covers both the local watermark // thread and the report-driven worker, which is why the - // thread no longer runs its own fallback. - const uint64_t shortfall = plan.target_bytes - total_freed; + // thread no longer runs its own fallback. The tier-down share + // of the plan is deliberately excluded from the shortfall: a + // demotion is not a failed reclaim. + const uint64_t shortfall = reclaim_target - total_freed; LOG(WARNING) << "[IO-PATTERN-EVICT-FALLBACK] policy plan under-" - "delivered plan_target=" - << plan.target_bytes << " freed=" << total_freed + "delivered reclaim_target=" + << reclaim_target << " freed=" << total_freed << " shortfall=" << shortfall; return RunLegacyEvictionFallback(shortfall) ? ErrorCode::OK @@ -8301,6 +8375,54 @@ tl::expected, ErrorCode> MasterService::PushOffloadingQueue( return queued_clients; } +// Policy-driven tier down: queue one MEMORY replica of `object_id` for a +// LOCAL_DISK copy, keeping the MEMORY replica in place. Shares the offload +// bookkeeping used by the offload-on-evict path (refcnt pin plus an +// offloading_tasks entry, both released when the client reports the copy back), +// and deliberately frees nothing: demotion is a copy, not a reclaim. +bool MasterService::TryQueueTierDown(const ObjectIdentity& object_id) { + MetadataAccessorRW accessor(this, object_id); + if (!accessor.Exists()) { + return false; + } + auto& metadata = accessor.Get(); + auto& tenant_state = accessor.GetTenantState(); + + // One offload per key. An in-flight task already pins the MEMORY replica this + // demotion would pin again, and the holder's queue is keyed by the object, so + // re-pushing would only fail with OBJECT_ALREADY_EXISTS. The key is already + // on its way down, which is what the caller asked for. + if (tenant_state.offloading_tasks.count(object_id.user_key) > 0) { + return true; + } + + const auto now = std::chrono::system_clock::now(); + bool queued = false; + metadata.VisitReplicas( + [](const Replica& replica) { + return replica.is_completed() && replica.is_memory_replica(); + }, + [this, &object_id, &tenant_state, &now, &queued](Replica& replica) { + if (queued) return; // only one replica needs to be copied down + auto result = PushOffloadingQueue(object_id, replica); + if (!result || result.value().empty()) { + VLOG(1) << "tier_down_push_failed key=" << object_id.user_key + << " error=" + << (result ? "empty_result" : toString(result.error())) + << " replica_segments=" + << replica.get_segment_names().size(); + return; + } + auto& tasks = tenant_state.offloading_tasks[object_id.user_key]; + for (const auto& client_id : result.value()) { + replica.inc_refcnt(); + tasks.push_back(OffloadingTask{replica.id(), now, client_id}); + } + queued = true; + }); + return queued; +} + // Promotion-on-hit // Push a key onto the holder client's promotion_objects map. Resolves the diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 17addf87e0..95d358eaad 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -920,6 +920,55 @@ TEST(IoPatternFrameworkTest, BinaryCfmCodecRoundTripsAllPolicyCommands) { EXPECT_TRUE(decoded_batch->accesses.front().is_hit); } +// An EvictionPlan crosses the CFM wire to a remote SubMaster, so the +// per-candidate action and the tier-down budget have to survive the codec: +// otherwise the remote executor receives a plan that looks like pure eviction +// and reclaims the very keys the driver chose to demote. +TEST(IoPatternFrameworkTest, BinaryCfmCodecCarriesTierDownActionAndBudget) { + CfmBinaryCodec codec; + EvictionPlan plan{.source_tier = CacheTier::kL1Host, + .target_bytes = 4096, + .candidates = {EvictionCandidate{ + .object = {TenantId("tenant"), "demote"}, + .bytes = 4096, + .score = 0.5F, + .target_tier = CacheTier::kLocalDisk, + .action = EvictionAction::kTierDown}, + EvictionCandidate{ + .object = {TenantId("tenant"), "reclaim"}, + .bytes = 4096, + .score = 0.25F, + .target_tier = CacheTier::kL3NofSsd, + .action = EvictionAction::kEvict}}, + .tier_down_target_bytes = 4096}; + + const auto payload = codec.EncodePolicy(plan); + const auto decoded = codec.DecodePolicy(payload); + ASSERT_TRUE(decoded.has_value()); + ASSERT_TRUE(std::holds_alternative(*decoded)); + const auto& out = std::get(*decoded); + ASSERT_EQ(out.candidates.size(), 2); + EXPECT_EQ(out.candidates[0].action, EvictionAction::kTierDown); + EXPECT_EQ(out.candidates[1].action, EvictionAction::kEvict); + EXPECT_EQ(out.tier_down_target_bytes, 4096u); + EXPECT_EQ(out.target_bytes, 4096u); + + // A payload written before tier down existed stops after the candidate list + // and still decodes. Both new fields then keep their pre-tier-down meaning + // (kEvict / 0) instead of silently turning the plan into a demotion. + const size_t trailer_bytes = sizeof(uint64_t) + out.candidates.size(); + ASSERT_GT(payload.size(), trailer_bytes); + const auto legacy = + codec.DecodePolicy(payload.substr(0, payload.size() - trailer_bytes)); + ASSERT_TRUE(legacy.has_value()); + ASSERT_TRUE(std::holds_alternative(*legacy)); + const auto& old = std::get(*legacy); + ASSERT_EQ(old.candidates.size(), 2); + EXPECT_EQ(old.candidates[0].action, EvictionAction::kEvict); + EXPECT_EQ(old.candidates[1].action, EvictionAction::kEvict); + EXPECT_EQ(old.tier_down_target_bytes, 0u); +} + TEST(IoPatternFrameworkTest, InProcessCfmTransportDispatchesReports) { CfmBinaryCodec codec; bool received_snapshot = false; @@ -1328,6 +1377,212 @@ TEST(IoPatternFrameworkTest, ReportDrivenColdEvictionRunsWithoutPressure) { EXPECT_EQ(last_report->eviction_status, ErrorCode::OK); } +TEST(IoPatternFrameworkTest, ReportDrivenTierDownLabelsCandidatesWithoutPressure) { + // The tier-down driver must make the policy label its candidates as + // demotions: the plan it produces copies the selected keys down to + // LOCAL_DISK and keeps their MEMORY replica, so nothing is reclaimed. It + // fires from a report with no storage pressure at all. + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::mutex plan_mutex; + std::optional handled_plan; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + // The budget alone enables the driver: there is no enable flag. + config.tier_down_bytes_per_cycle = 96ULL * 1024 * 1024; + // Let the detached analyzer finish so candidate selection is deterministic. + config.analysis_timeout_us = 30'000'000; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = + [&](const EvictionPlan& plan) { + std::lock_guard lock(plan_mutex); + handled_plan = plan; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + CfmService service(rt); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "demote-key"}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + // No storage metric, no cold-eviction driver: only tier down can act. + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(30), [&] { + return last_report.has_value(); + })); + ASSERT_TRUE(last_report.has_value()); + EXPECT_TRUE(last_report->tier_down); + EXPECT_FALSE(last_report->cold_eviction); + EXPECT_EQ(last_report->eviction_target_bytes, 96ULL * 1024 * 1024); + EXPECT_GT(last_report->eviction_candidates, 0u); + EXPECT_EQ(last_report->eviction_status, ErrorCode::OK); + + std::lock_guard plan_lock(plan_mutex); + ASSERT_TRUE(handled_plan.has_value()); + // The budget is carried as a demotion budget, and every candidate is + // labelled a demotion -- the action comes from the driver, not target_tier. + EXPECT_EQ(handled_plan->tier_down_target_bytes, 96ULL * 1024 * 1024); + ASSERT_FALSE(handled_plan->candidates.empty()); + for (const auto& candidate : handled_plan->candidates) { + EXPECT_EQ(candidate.action, EvictionAction::kTierDown); + } +} + +TEST(IoPatternFrameworkTest, ReportDrivenTierDownYieldsToPressureEviction) { + // A demotion frees nothing, so a reclaim always wins the cycle: with the + // tier-down driver enabled and host memory above the high watermark, the + // same configuration must produce an eviction plan instead of a demotion. + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::mutex plan_mutex; + std::optional handled_plan; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + // Same budget-only enablement, with host memory above the high watermark. + config.tier_down_bytes_per_cycle = 96ULL * 1024 * 1024; + config.analysis_timeout_us = 30'000'000; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = + [&](const EvictionPlan& plan) { + std::lock_guard lock(plan_mutex); + handled_plan = plan; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + CfmService service(rt); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "hot-key"}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + batch.storage.push_back( + StorageMetric{.source_id = "reporter", + .observed_at_ns = 1, + .tier = CacheTier::kL1Host, + .used_bytes = 1024ULL * 1024 * 1024, + .capacity_bytes = 1024ULL * 1024 * 1024, + .memory_used_ratio = 0.95F}); + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(30), [&] { + return last_report.has_value(); + })); + ASSERT_TRUE(last_report.has_value()); + EXPECT_FALSE(last_report->tier_down); + EXPECT_FALSE(last_report->cold_eviction); + EXPECT_GT(last_report->eviction_target_bytes, 0); + EXPECT_EQ(last_report->eviction_status, ErrorCode::OK); + + std::lock_guard plan_lock(plan_mutex); + ASSERT_TRUE(handled_plan.has_value()); + EXPECT_EQ(handled_plan->tier_down_target_bytes, 0u); + ASSERT_FALSE(handled_plan->candidates.empty()); + for (const auto& candidate : handled_plan->candidates) { + EXPECT_EQ(candidate.action, EvictionAction::kEvict); + } +} + +TEST(IoPatternFrameworkTest, ReportDrivenTierDownPavesColdDataBeforeColdEviction) { + // Both drivers act below the watermark, but only one can own the cycle. The + // tier-down budget takes it: demoting a key that the same cycle would have + // discarded defeats the point of paving cold data down first, so the + // cold-eviction driver only keeps the slot when no tier-down budget is + // configured (see ReportDrivenColdEvictionRunsWithoutPressure). + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::mutex plan_mutex; + std::optional handled_plan; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + config.tier_down_bytes_per_cycle = 96ULL * 1024 * 1024; + config.report_driven_cold_eviction = true; + config.report_driven_cold_eviction_bytes = 128ULL * 1024 * 1024; + config.report_driven_cold_idle_threshold_us = 0; + config.analysis_timeout_us = 30'000'000; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = + [&](const EvictionPlan& plan) { + std::lock_guard lock(plan_mutex); + handled_plan = plan; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + CfmService service(rt); + CfmBinaryCodec codec; + MetricBatch batch; + batch.accesses.push_back( + AccessRecord{.object = {TenantId("tenant"), "cold-key"}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + ASSERT_TRUE(service.Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(30), [&] { + return last_report.has_value(); + })); + ASSERT_TRUE(last_report.has_value()); + EXPECT_TRUE(last_report->tier_down); + EXPECT_FALSE(last_report->cold_eviction); + // The tier-down budget, not the cold-eviction budget, sized this cycle. + EXPECT_EQ(last_report->eviction_target_bytes, 96ULL * 1024 * 1024); + + std::lock_guard plan_lock(plan_mutex); + ASSERT_TRUE(handled_plan.has_value()); + EXPECT_EQ(handled_plan->tier_down_target_bytes, 96ULL * 1024 * 1024); + ASSERT_FALSE(handled_plan->candidates.empty()); + for (const auto& candidate : handled_plan->candidates) { + EXPECT_EQ(candidate.action, EvictionAction::kTierDown); + } +} + TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { size_t batches = 0; IoPatternReporter reporter(4, [&](const MetricBatch&) { @@ -1617,6 +1872,38 @@ TEST(IoPatternFrameworkTest, TierDownTemplatesChooseDocumentedTargets) { CacheTier::kL1Host); } +// The candidate action is decided by the driver (PolicyContext::tier_down) and +// must never be derived from target_tier: TierDownTarget() returns source+1 for +// L0/L1/L2 alike, so a derived action would label every candidate a demotion and +// eviction would stop working entirely. +TEST(IoPatternFrameworkTest, TierDownContextLabelsCandidatesAndBudget) { + PolicyContext context; + context.snapshot.keys = {KeyMetrics{ + .object = {TenantId("tenant"), "key"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}}; + context.analysis.keys = { + KeyPattern{.object = context.snapshot.keys.front().object}}; + + ScoreBasedEvictionOps ops; + + const auto reclaimed = ops.Evaluate(context, CacheTier::kL1Host, 64); + ASSERT_EQ(reclaimed.candidates.size(), 1); + EXPECT_EQ(reclaimed.candidates.front().object.key, "key"); + EXPECT_EQ(reclaimed.candidates.front().action, EvictionAction::kEvict); + EXPECT_EQ(reclaimed.tier_down_target_bytes, 0u); + + context.tier_down = true; + const auto demoted = ops.Evaluate(context, CacheTier::kL1Host, 64); + ASSERT_EQ(demoted.candidates.size(), 1); + EXPECT_EQ(demoted.candidates.front().object.key, "key"); + EXPECT_EQ(demoted.candidates.front().action, EvictionAction::kTierDown); + EXPECT_EQ(demoted.tier_down_target_bytes, 64u); + // Same victims either way: only the action differs. + EXPECT_EQ(demoted.candidates.front().target_tier, + reclaimed.candidates.front().target_tier); +} + TEST(IoPatternFrameworkTest, PrefetchRequiresConfidenceAndNeverPromotesToHbm) { PolicyContext context; context.snapshot.keys = { diff --git a/mooncake-store/tests/offload_on_evict_test.cpp b/mooncake-store/tests/offload_on_evict_test.cpp index 54f0e35032..85d49e835a 100644 --- a/mooncake-store/tests/offload_on_evict_test.cpp +++ b/mooncake-store/tests/offload_on_evict_test.cpp @@ -11,6 +11,10 @@ #include #include "types.h" +#include "io_pattern/cfm_protocol.h" +#include "io_pattern/cfm_service.h" +#include "io_pattern/runtime.h" +#include "io_pattern/types.h" namespace mooncake::test { @@ -32,6 +36,43 @@ class OffloadOnEvictTest : public ::testing::Test { return service->RunLegacyEvictionFallback(shortfall_bytes); } + // Friend access to the embedded io_pattern runtime, so a test can hand the + // eviction handler a plan directly and observe how each candidate action is + // dispatched. OffloadOnEvictTest is friended; TEST_F-generated subclasses are + // not, hence this static funnel. + static ErrorCode ExecuteEvictionPlan(MasterService* service, + const io_pattern::EvictionPlan& plan) { + return service->io_pattern_runtime_->ExecuteCommand(plan); + } + + // Friend access to the embedded CFM endpoint, so a test can drive a real + // report-driven cycle through the MasterService wiring (flags -> config -> + // runtime driver -> policy -> handler) instead of calling the runtime + // directly. Returns false when the report was rejected. + static bool SendIoPatternReport(MasterService* service, + const io_pattern::MetricBatch& batch) { + io_pattern::CfmBinaryCodec codec; + return service->io_pattern_cfm_service_->Send( + "report_metric_batch", codec.EncodeMetricBatch(batch)); + } + + // A demotion must leave a readable MEMORY replica behind; an eviction must + // not. Reads the same client-facing view a Get would. + bool HasCompleteMemoryReplica(MasterService& service, + const std::string& key) const { + auto replica_list = service.GetReplicaList(key, TenantId::Default()); + if (!replica_list.has_value()) { + return false; + } + for (const auto& descriptor : replica_list.value().replicas) { + if (descriptor.is_memory_replica() && + descriptor.status == ReplicaStatus::COMPLETE) { + return true; + } + } + return false; + } + static constexpr size_t kDefaultSegmentBase = 0x300000000; Segment MakeSegment(std::string name, size_t base, size_t size) const { @@ -456,6 +497,173 @@ TEST_F(OffloadOnEvictTest, LegacyEvictionFallbackRunsForUnderDeliveringPlan) { EXPECT_TRUE(RunLegacyEvictionFallbackForTesting(service.get(), 1)); } +// ============================================================================= +// Policy-driven tier down: the io_pattern eviction handler must copy the key +// down to LOCAL_DISK while keeping its MEMORY replica. A demotion frees no +// bytes, so it may not be counted as freed memory and may not turn the plan into +// a reclaim shortfall -- otherwise the legacy fallback evicts exactly the keys +// the driver chose to keep. +// ============================================================================= + +TEST_F(OffloadOnEvictTest, TierDownPlanQueuesOffloadAndKeepsMemoryReplica) { + MasterServiceConfig config; + config.enable_offload = true; + config.offload_on_evict = true; + // No lease protection: if the handler wrongly treated the demotion budget as + // a reclaim shortfall, the legacy fallback would be free to reclaim one of + // these keys immediately, so a surviving MEMORY replica is real evidence the + // fallback never ran. + config.default_kv_lease_ttl = 0; + auto service = std::make_unique(config); + + constexpr size_t seg_size = 1024 * 1024 * 16; + auto ctx = PrepareSegment(*service, "tier_down_segment", kDefaultSegmentBase, + seg_size); + auto mount_ld = service->MountLocalDiskSegment(ctx.client_id, true); + ASSERT_TRUE(mount_ld.has_value()); + + PutObject(*service, ctx.client_id, "td_demote"); + PutObject(*service, ctx.client_id, "td_keep"); + + const io_pattern::EvictionPlan plan{ + .source_tier = io_pattern::CacheTier::kL1Host, + .target_bytes = 1024, + .candidates = {io_pattern::EvictionCandidate{ + .object = {TenantId::Default(), "td_demote"}, + .bytes = 1024, + .score = 1.0F, + .target_tier = io_pattern::CacheTier::kLocalDisk, + .action = io_pattern::EvictionAction::kTierDown}}, + .tier_down_target_bytes = 1024}; + + EXPECT_EQ(ExecuteEvictionPlan(service.get(), plan), ErrorCode::OK); + + // The chosen key was queued for a LOCAL_DISK copy ... + auto queued = DrainOffloadQueue(*service, ctx.client_id); + ASSERT_EQ(queued.size(), 1u); + EXPECT_TRUE(queued.count("td_demote") > 0); + + // ... and both keys still serve from MEMORY: demotion is a copy, not a + // reclaim, so nothing was freed and the fallback was never asked to free it. + EXPECT_TRUE(HasCompleteMemoryReplica(*service, "td_demote")); + EXPECT_TRUE(HasCompleteMemoryReplica(*service, "td_keep")); + + // Tier down is counted on its own counters, independently of freed bytes. + EXPECT_EQ(service->tier_down_attempt_count(), 1u); + EXPECT_EQ(service->tier_down_success_count(), 1u); + EXPECT_EQ(service->tier_down_failure_count(), 0u); + + service->RemoveAll(); +} + +// The contrast case: the same plan shape labelled kEvict must go to the quota +// eviction path and never to the demotion path. +// +// The configuration matters here. With offload_on_evict enabled, ordinary +// eviction *also* defers its victim to the offload queue ("No memory freed ... +// deferred for disk offload"), so an empty queue would prove nothing. This test +// therefore runs the legacy write-through mode (enable_offload=true, +// offload_on_evict=false), where only a tier-down dispatch can queue an offload: +// PutEnd pushes one entry, which is drained before the plan runs, and eviction +// reclaims without offloading. +TEST_F(OffloadOnEvictTest, EvictActionPlanNeverQueuesOffload) { + MasterServiceConfig config; + config.enable_offload = true; + config.offload_on_evict = false; + config.default_kv_lease_ttl = 0; + auto service = std::make_unique(config); + + constexpr size_t seg_size = 1024 * 1024 * 16; + auto ctx = PrepareSegment(*service, "evict_action_segment", + kDefaultSegmentBase, seg_size); + auto mount_ld = service->MountLocalDiskSegment(ctx.client_id, true); + ASSERT_TRUE(mount_ld.has_value()); + + PutObject(*service, ctx.client_id, "ev_reclaim"); + + // Legacy write-through: one entry from PutEnd, drained so that anything left + // in the queue afterwards can only have come from the plan below. + auto pre_plan = DrainOffloadQueue(*service, ctx.client_id); + ASSERT_EQ(pre_plan.size(), 1u); + + const io_pattern::EvictionPlan plan{ + .source_tier = io_pattern::CacheTier::kL1Host, + .target_bytes = 1024, + .candidates = {io_pattern::EvictionCandidate{ + .object = {TenantId::Default(), "ev_reclaim"}, + .bytes = 1024, + .score = 1.0F, + .target_tier = io_pattern::CacheTier::kL3NofSsd, + .action = io_pattern::EvictionAction::kEvict}}, + .tier_down_target_bytes = 0}; + + ExecuteEvictionPlan(service.get(), plan); + + auto queued = DrainOffloadQueue(*service, ctx.client_id); + EXPECT_TRUE(queued.empty()) + << "a kEvict candidate must not be queued for a LOCAL_DISK copy"; + EXPECT_EQ(service->tier_down_attempt_count(), 0u); + EXPECT_EQ(service->tier_down_success_count(), 0u); + EXPECT_EQ(service->tier_down_failure_count(), 0u); + + service->RemoveAll(); +} + +// End to end through the MasterService wiring: the config flag must reach the +// report-driven driver, the driver must label the plan, and the handler must +// copy the key down while keeping it in memory. +TEST_F(OffloadOnEvictTest, TierDownDriverDemotesReportedKeyWithoutReclaiming) { + MasterServiceConfig config; + config.enable_offload = true; + config.offload_on_evict = true; + config.default_kv_lease_ttl = 0; + // The wiring under test: the budget alone enables the driver, there is no + // enable flag. + config.io_pattern_tier_down_bytes_per_cycle = 4096; + auto service = std::make_unique(config); + + constexpr size_t seg_size = 1024 * 1024 * 16; + auto ctx = PrepareSegment(*service, "tier_down_report_segment", + kDefaultSegmentBase, seg_size); + auto mount_ld = service->MountLocalDiskSegment(ctx.client_id, true); + ASSERT_TRUE(mount_ld.has_value()); + + PutObject(*service, ctx.client_id, "td_report", 4096); + + // No storage metric: the merged report carries no pressure, so only the + // tier-down driver can act on this cycle. The access record is what puts the + // key into the analyzed snapshot with an L1 replica bit. + io_pattern::MetricBatch batch; + batch.accesses.push_back(io_pattern::AccessRecord{ + .object = {TenantId::Default(), "td_report"}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = io_pattern::CacheTier::kL1Host, + .operation = io_pattern::IoOperation::kGet, + .is_hit = true}); + ASSERT_TRUE(SendIoPatternReport(service.get(), batch)); + + // The cycle runs on the runtime's background worker, so wait for the + // demotion to reach the offload queue. + std::unordered_map queued; + WaitUntil([&] { + queued = DrainOffloadQueue(*service, ctx.client_id); + return !queued.empty(); + }); + ASSERT_EQ(queued.size(), 1u) << "tier-down driver queued " + << queued.size() << " object(s)"; + EXPECT_TRUE(queued.count("td_report") > 0); + + // The demotion copied the key down and kept serving it from MEMORY: nothing + // was reclaimed, so the reclaim path was never involved. + EXPECT_TRUE(HasCompleteMemoryReplica(*service, "td_report")); + EXPECT_GE(service->tier_down_attempt_count(), 1u); + EXPECT_GE(service->tier_down_success_count(), 1u); + EXPECT_EQ(service->tier_down_failure_count(), 0u); + + service->RemoveAll(); +} + } // namespace mooncake::test int main(int argc, char** argv) { From 4bc5fe3f4c8b8c4001c0809ab40b192fb61ee5df Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 09:44:41 +0800 Subject: [PATCH 40/47] add io_pattern metric --- mooncake-store/include/io_pattern/runtime.h | 4 + .../include/master_metric_manager.h | 14 +++ mooncake-store/src/io_pattern/runtime.cpp | 9 ++ mooncake-store/src/master_metric_manager.cpp | 93 +++++++++++++++++++ mooncake-store/src/master_service.cpp | 3 + .../tests/io_pattern_framework_test.cpp | 80 ++++++++++++++++ mooncake-store/tests/master_metrics_test.cpp | 33 +++++++ 7 files changed, 236 insertions(+) diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 4344e430b2..65fd8a09b1 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -191,7 +191,9 @@ class IoPatternRuntime final { std::string session_id = {}); void RecordFeedback(PolicyFeedbackSample sample); + PolicyFeedbackStats FeedbackSnapshot() const; IoPatternSnapshot Snapshot() const; + // With no explicit window, QPS is averaged over this runtime's lifetime. IoPatternObservabilitySnapshot ObservabilitySnapshot( double window_seconds = 0.0) const; bool degraded() const; @@ -248,6 +250,8 @@ class IoPatternRuntime final { PolicyFeedbackWindow feedback_; AdaptivePolicyTuner tuner_; IoPatternObservability observability_; + const std::chrono::steady_clock::time_point started_at_{ + std::chrono::steady_clock::now()}; mutable std::mutex feedback_state_mutex_; std::unordered_set pending_prefetches_; uint64_t feedback_accesses_{0}; diff --git a/mooncake-store/include/master_metric_manager.h b/mooncake-store/include/master_metric_manager.h index 83d2f77385..bc26c246d0 100644 --- a/mooncake-store/include/master_metric_manager.h +++ b/mooncake-store/include/master_metric_manager.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,10 @@ namespace mooncake { +namespace io_pattern { +class IoPatternRuntime; +} + class MasterMetricManager { public: // --- Singleton Access --- @@ -316,6 +321,12 @@ class MasterMetricManager { int64_t get_io_pattern_report_admission_failures(); int64_t get_io_pattern_report_degraded(); + // Scrape the local runtime directly, including access-only traffic. A weak + // reference keeps the metrics singleton from extending MasterService life. + void set_io_pattern_runtime( + std::weak_ptr runtime); + void clear_io_pattern_runtime(const io_pattern::IoPatternRuntime* runtime); + // PutStart Discard Metrics void inc_put_start_discard_cnt(int64_t count, int64_t size); void inc_put_start_release_cnt(int64_t count, int64_t size); @@ -447,6 +458,7 @@ class MasterMetricManager { // Update all metrics once to ensure zero values are serialized void update_metrics_for_zero_output(); + std::string serialize_io_pattern_metrics(); std::string get_summary_string(bool update_summary_snapshot); struct SummaryCounters { @@ -549,6 +561,8 @@ class MasterMetricManager { // --- Metric Members --- std::mutex summary_snapshot_mutex_; + std::mutex io_pattern_runtime_mutex_; + std::weak_ptr io_pattern_runtime_; SummarySnapshot summary_snapshot_; // Memory Storage Metrics diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index bf1c1f118f..e85b7beb7e 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -638,12 +638,21 @@ void IoPatternRuntime::RecordFeedback(PolicyFeedbackSample sample) { } } +PolicyFeedbackStats IoPatternRuntime::FeedbackSnapshot() const { + return feedback_.Snapshot(); +} + IoPatternSnapshot IoPatternRuntime::Snapshot() const { return collector_->GetSnapshot(); } IoPatternObservabilitySnapshot IoPatternRuntime::ObservabilitySnapshot( double window_seconds) const { + if (window_seconds <= 0.0) { + window_seconds = std::chrono::duration( + std::chrono::steady_clock::now() - started_at_) + .count(); + } return observability_.Snapshot(window_seconds); } diff --git a/mooncake-store/src/master_metric_manager.cpp b/mooncake-store/src/master_metric_manager.cpp index ca45ae5b22..9b40715e14 100644 --- a/mooncake-store/src/master_metric_manager.cpp +++ b/mooncake-store/src/master_metric_manager.cpp @@ -8,6 +8,7 @@ #include #include "utils.h" +#include "io_pattern/runtime.h" namespace mooncake { @@ -1880,6 +1881,97 @@ int64_t MasterMetricManager::get_update_task_failures() { return mark_task_to_complete_failures_.value(); } +void MasterMetricManager::set_io_pattern_runtime( + std::weak_ptr runtime) { + std::lock_guard lock(io_pattern_runtime_mutex_); + io_pattern_runtime_ = std::move(runtime); +} + +void MasterMetricManager::clear_io_pattern_runtime( + const io_pattern::IoPatternRuntime* runtime) { + std::lock_guard lock(io_pattern_runtime_mutex_); + if (io_pattern_runtime_.lock().get() == runtime) + io_pattern_runtime_.reset(); +} + +std::string MasterMetricManager::serialize_io_pattern_metrics() { + io_pattern::IoPatternObservabilitySnapshot observation; + io_pattern::PolicyFeedbackStats feedback; + { + // Unregister waits for snapshots and their temporary strong reference + // to finish, so a scrape cannot defer runtime worker shutdown beyond + // the lifetime of the MasterService captured by its handlers. + std::lock_guard lock(io_pattern_runtime_mutex_); + auto runtime = io_pattern_runtime_.lock(); + if (!runtime) return {}; + observation = runtime->ObservabilitySnapshot(); + feedback = runtime->FeedbackSnapshot(); + } + + // Serialize snapshots rather than incrementing counters by cumulative + // values. Local metric objects keep concurrent scrapes independent and + // cannot retain stale values from a previous runtime. + std::string result; + const auto gauge = [&result](const char* name, const char* help, + double value) { + ylt::metric::gauge_d metric(name, help); + metric.update(value); + metric.serialize(result); + }; + const auto counter = [&result](const char* name, const char* help, + uint64_t value) { + ylt::metric::counter_t metric(name, help); + metric.inc(value); + metric.serialize(result); + }; + gauge("master_io_pattern_collect_latency_us", + "Maximum collection latency in microseconds since runtime startup", + observation.collect_latency_us); + gauge("master_io_pattern_analyze_latency_us", + "Maximum analysis latency in microseconds since runtime startup", + observation.analyze_latency_us); + gauge("master_io_pattern_policy_decision_qps", + "Average policy decisions per second since runtime startup; use rate " + "of master_io_pattern_policy_decisions_total for a rolling rate", + observation.policy_decision_qps); + counter("master_io_pattern_policy_decisions_total", + "Total policy decisions since runtime startup", + observation.policy_decisions); + gauge("master_io_pattern_strategy_hit_rate", + "Fraction of policy decisions with eviction or prefetch candidates", + observation.strategy_hit_rate); + gauge("master_io_pattern_false_positive_rate", + "Observed prefetch misses divided by total policy decisions", + observation.false_positive_rate); + counter("master_io_pattern_degrade_count", + "Total recorded degradation events since runtime startup", + observation.degrade_count); + counter("master_io_pattern_report_drop_count", + "Total collector drops recorded since runtime startup", + observation.report_drop_count); + gauge( + "master_io_pattern_hit_rate_delta", + "Mean hit rate delta in the bounded feedback sample window; automatic " + "samples compare consecutive groups of 64 accesses", + feedback.hit_rate_delta); + gauge("master_io_pattern_eviction_churn", + "Mean eviction feedback in the bounded sample window; automatic " + "samples are eviction candidate count divided by snapshot key count", + feedback.eviction_churn); + gauge("master_io_pattern_ttft_delta", + "Mean externally supplied TTFT delta in the bounded feedback sample " + "window; zero unless supplied via RecordFeedback", + feedback.ttft_delta); + gauge("master_io_pattern_prefetch_accuracy", + "Mean prefetch accuracy in the bounded feedback sample window; " + "samples without prefetch feedback currently contribute zero", + feedback.prefetch_accuracy); + gauge("master_io_pattern_feedback_samples", + "Number of samples currently retained in the feedback window", + feedback.samples); + return result; +} + // --- Serialization --- std::string MasterMetricManager::serialize_metrics() { // Note: Following Prometheus style, metrics with value 0 that haven't @@ -2053,6 +2145,7 @@ std::string MasterMetricManager::serialize_metrics() { serialize_metric(io_pattern_report_admissions_); serialize_metric(io_pattern_report_admission_failures_); serialize_metric(io_pattern_report_degraded_); + ss << serialize_io_pattern_metrics(); // Serialize PutStart Discard Metrics serialize_metric(put_start_discard_cnt_); diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 4448afeabf..1f6ad30fd0 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -728,6 +728,7 @@ MasterService::MasterService(const MasterServiceConfig& config) object_id, /*record_candidate=*/false)); }}, std::move(io_pattern_config)); + MasterMetricManager::instance().set_io_pattern_runtime(io_pattern_runtime_); io_pattern_cfm_service_ = std::make_shared( io_pattern_runtime_); @@ -1930,6 +1931,8 @@ MasterService::~MasterService() { // Its admission worker executes handlers that capture this service. Stop // and join it while all handler dependencies are still alive. + MasterMetricManager::instance().clear_io_pattern_runtime( + io_pattern_runtime_.get()); io_pattern_cfm_service_.reset(); io_pattern_runtime_.reset(); diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 95d358eaad..1c7de16211 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1,4 +1,5 @@ #include "io_pattern/io_pattern.h" +#include "master_metric_manager.h" #include "io_pattern/threshold_analyzer.h" #include "io_pattern/policy_strategies.h" @@ -2250,5 +2251,84 @@ TEST(IoPatternFrameworkTest, AdmissionWatermarkFollowsTheConfiguredHighWatermark AdmissionDecision::kRejectWatermark); } +TEST(IoPatternFrameworkTest, RuntimeMetricsExport) { + auto& metrics = MasterMetricManager::instance(); + IoPatternRuntime::Config config; + config.collector.max_total_keys = 1; + auto runtime = std::make_shared( + IoPatternRuntime::Handlers{}, config); + metrics.set_io_pattern_runtime(runtime); + + const auto value = [](const std::string& text, const std::string& name) { + const auto offset = text.find("\n" + name + " "); + EXPECT_NE(offset, std::string::npos) << name; + return offset == std::string::npos + ? -1.0 + : std::stod(text.substr(offset + name.size() + 2)); + }; + const auto initial = metrics.serialize_metrics(); + for (const auto* name : + {"policy_decisions_total", "feedback_samples", "collect_latency_us", + "analyze_latency_us", "strategy_hit_rate", "false_positive_rate", + "degrade_count", "report_drop_count"}) { + EXPECT_DOUBLE_EQ( + value(initial, std::string("master_io_pattern_") + name), 0.0); + } + runtime->RecordFeedback({.hit_rate_delta = -0.25F, + .eviction_churn = 0.5F, + .ttft_delta = -0.125F, + .prefetch_accuracy = 0.75F}); + runtime->Plan(CacheTier::kL1Host, 0, {}); + const auto first = metrics.serialize_metrics(); + EXPECT_DOUBLE_EQ(value(first, "master_io_pattern_hit_rate_delta"), -0.25); + EXPECT_DOUBLE_EQ(value(first, "master_io_pattern_eviction_churn"), 0.5); + EXPECT_DOUBLE_EQ(value(first, "master_io_pattern_ttft_delta"), -0.125); + EXPECT_DOUBLE_EQ(value(first, "master_io_pattern_prefetch_accuracy"), 0.75); + EXPECT_DOUBLE_EQ(value(first, "master_io_pattern_feedback_samples"), 1.0); + EXPECT_DOUBLE_EQ(value(first, "master_io_pattern_policy_decisions_total"), + 1.0); + EXPECT_GT(value(first, "master_io_pattern_policy_decision_qps"), 0.0); + // Scrapes must not increment cumulative counters or consume feedback. + const auto second = metrics.serialize_metrics(); + EXPECT_DOUBLE_EQ(value(second, "master_io_pattern_policy_decisions_total"), + 1.0); + EXPECT_DOUBLE_EQ(value(second, "master_io_pattern_feedback_samples"), 1.0); + EXPECT_NE( + second.find("# TYPE master_io_pattern_policy_decisions_total counter"), + std::string::npos); + EXPECT_NE(second.find("# TYPE master_io_pattern_hit_rate_delta gauge"), + std::string::npos); + + runtime->RecordAccess( + "one", {.object = {TenantId::Default(), "one"}, .is_hit = true}); + runtime->RecordAccess( + "two", {.object = {TenantId::Default(), "two"}, .is_hit = true}); + runtime->Execute(CacheTier::kL1Host, 0, {}); + const auto degraded = metrics.serialize_metrics(); + EXPECT_DOUBLE_EQ(value(degraded, "master_io_pattern_report_drop_count"), + 1.0); + EXPECT_GE(value(degraded, "master_io_pattern_degrade_count"), 1.0); + + // The singleton must not retain a runtime (or its MasterService handlers). + std::weak_ptr weak = runtime; + runtime.reset(); + EXPECT_TRUE(weak.expired()); + EXPECT_EQ(metrics.serialize_metrics().find( + "# TYPE master_io_pattern_hit_rate_delta "), + std::string::npos); + + auto replacement = + std::make_shared(IoPatternRuntime::Handlers{}); + metrics.set_io_pattern_runtime(replacement); + metrics.clear_io_pattern_runtime(nullptr); + EXPECT_DOUBLE_EQ(value(metrics.serialize_metrics(), + "master_io_pattern_policy_decisions_total"), + 0.0); + metrics.clear_io_pattern_runtime(replacement.get()); + EXPECT_EQ(metrics.serialize_metrics().find( + "# TYPE master_io_pattern_hit_rate_delta "), + std::string::npos); +} + } // namespace } // namespace mooncake::io_pattern diff --git a/mooncake-store/tests/master_metrics_test.cpp b/mooncake-store/tests/master_metrics_test.cpp index 6f6889d59e..62174b7e13 100644 --- a/mooncake-store/tests/master_metrics_test.cpp +++ b/mooncake-store/tests/master_metrics_test.cpp @@ -18,6 +18,7 @@ #include "types.h" #include "master_config.h" #include "master_metric_manager.h" +#include "io_pattern/runtime.h" namespace mooncake::test { @@ -1021,6 +1022,38 @@ TEST_F(MasterMetricsTest, SsdOffloadCacheHitAndTotalConsistent) { service_.Remove(ssd_only_key, "default"); } +TEST_F(MasterMetricsTest, AdminMetricsExposeIoPatternFeedbackFromAccesses) { + const int http_port = getFreeTcpPort(); + WrappedMasterServiceConfig config; + config.enable_metric_reporting = false; + WrappedMasterService service(config); + MasterAdminServer admin_server(static_cast(http_port), + /*enable_metric_reporting=*/true); + ASSERT_TRUE(admin_server.Start()); + + // Exercise the production CFM ingress and MasterService registration. + io_pattern::MetricBatch batch; + for (int i = 0; i < 64; ++i) { + batch.accesses.push_back( + {.object = {TenantId::Default(), "missing"}, .is_hit = false}); + } + io_pattern::CfmBinaryCodec codec; + ASSERT_TRUE(service.CfmRpcEndpoint().Send("report_metric_batch", + codec.EncodeMetricBatch(batch))); + const auto response = FetchUrl(http_port, "/metrics"); + ASSERT_EQ(response.http_status, 200); + EXPECT_NE( + response.body.find("\nmaster_io_pattern_feedback_samples 1.000000\n"), + std::string::npos); + EXPECT_NE( + response.body.find("# TYPE master_io_pattern_collect_latency_us gauge"), + std::string::npos); + EXPECT_NE(response.body.find( + "# TYPE master_io_pattern_policy_decisions_total counter"), + std::string::npos); + admin_server.Stop(); +} + } // namespace mooncake::test int main(int argc, char** argv) { From 090cc12cfbf45b9122b0b0943df2c1676c04d943 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 14:44:08 +0800 Subject: [PATCH 41/47] add io_pattern metric --- .../ha/standby/hot_standby_service_test.cpp | 65 ++++++++++--------- mooncake-store/tests/master_metrics_test.cpp | 4 ++ 2 files changed, 38 insertions(+), 31 deletions(-) diff --git a/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp b/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp index d6d1784285..be3612e774 100644 --- a/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp +++ b/mooncake-store/tests/ha/standby/hot_standby_service_test.cpp @@ -26,6 +26,10 @@ namespace mooncake::test { namespace { +// Existing batch fixtures use the unscoped (empty master_id) OpLog namespace. +const ha::MasterSources kTestSources{ + {.master_id = "", .address = "primary_unused"}}; + class FakeSnapshotProvider final : public SnapshotProvider { public: explicit FakeSnapshotProvider( @@ -121,7 +125,7 @@ TEST_F(HotStandbyServiceTest, TestStart) { << "Requires real etcd connection, run in integration environment."; #else ErrorCode err = - service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); EXPECT_EQ(StandbyState::FAILED, service_->GetState()); #endif @@ -135,10 +139,10 @@ TEST_F(HotStandbyServiceTest, TestStart_AlreadyRunning) { // After the first Start fails and state becomes FAILED, the second Start // should still return INTERNAL_ERROR ErrorCode err1 = - service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err1); ErrorCode err2 = - service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err2); #endif } @@ -149,7 +153,7 @@ TEST_F(HotStandbyServiceTest, TestStart_InvalidEtcdEndpoints) { #else std::string invalid_endpoints = "invalid_endpoint"; ErrorCode err = - service_->Start("primary_unused", invalid_endpoints, cluster_id_); + service_->Start(kTestSources, invalid_endpoints, cluster_id_); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); #endif } @@ -178,7 +182,7 @@ TEST_F(HotStandbyServiceTest, TestStateTransition_StartToWatching) { // to FAILED EXPECT_EQ(StandbyState::STOPPED, service_->GetState()); ErrorCode err = - service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); EXPECT_EQ(StandbyState::FAILED, service_->GetState()); #endif @@ -191,8 +195,7 @@ TEST_F(HotStandbyServiceTest, TestStateTransition_ConnectionFailed) { #else // In non-etcd mode we cannot distinguish detailed connection errors; only // verify it doesn't crash - ErrorCode err = - service_->Start("primary_unused", "bad_endpoint", cluster_id_); + ErrorCode err = service_->Start(kTestSources, "bad_endpoint", cluster_id_); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); #endif } @@ -205,7 +208,7 @@ TEST_F(HotStandbyServiceTest, TestStateTransition_SyncFailed) { // In non-etcd mode, the sync phase is not actually executed; just ensure // the call is safe ErrorCode err = - service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_); EXPECT_EQ(ErrorCode::INTERNAL_ERROR, err); #endif } @@ -229,7 +232,7 @@ TEST_F(HotStandbyServiceTest, TestGetSyncStatus_AfterSync) { #else // In non-etcd mode, calling Start will not change applied/primary, but the // state machine enters FAILED - (void)service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + (void)service_->Start(kTestSources, oplog_endpoints_, cluster_id_); StandbySyncStatus status = service_->GetSyncStatus(); EXPECT_EQ(StandbyState::FAILED, status.state); #endif @@ -310,7 +313,7 @@ TEST_F(HotStandbyServiceTest, TestWarmStart_WithLocalState) { "test warm start."; #else // In non-etcd mode, only verify that Start is safe to call - (void)service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + (void)service_->Start(kTestSources, oplog_endpoints_, cluster_id_); SUCCEED(); #endif } @@ -319,7 +322,7 @@ TEST_F(HotStandbyServiceTest, TestWarmStart_WithoutLocalState) { #ifdef STORE_USE_ETCD GTEST_SKIP() << "Requires real etcd and snapshot provider configuration."; #else - (void)service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + (void)service_->Start(kTestSources, oplog_endpoints_, cluster_id_); SUCCEED(); #endif } @@ -332,7 +335,7 @@ TEST_F(HotStandbyServiceTest, TestWarmStart_WithSnapshot) { config_.enable_snapshot_bootstrap = true; // Recreate service to apply the new configuration service_.reset(new HotStandbyService(config_)); - (void)service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + (void)service_->Start(kTestSources, oplog_endpoints_, cluster_id_); SUCCEED(); #endif } @@ -501,7 +504,7 @@ TEST_F(HotStandbyServiceTest, TestVerificationLoop_WhenEnabled) { #else config_.enable_verification = true; service_.reset(new HotStandbyService(config_)); - (void)service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + (void)service_->Start(kTestSources, oplog_endpoints_, cluster_id_); service_->Stop(); SUCCEED(); #endif @@ -513,7 +516,7 @@ TEST_F(HotStandbyServiceTest, TestVerificationLoop_WhenDisabled) { #ifdef STORE_USE_ETCD GTEST_SKIP() << "Requires real etcd connection to start service."; #else - (void)service_->Start("primary_unused", oplog_endpoints_, cluster_id_); + (void)service_->Start(kTestSources, oplog_endpoints_, cluster_id_); service_->Stop(); SUCCEED(); #endif @@ -640,7 +643,7 @@ TEST_F(HotStandbyServiceTest, auto service = std::make_unique(config); service->SetCatchUpBatchKvBackendForTesting(batch_backend); ASSERT_EQ(ErrorCode::OK, - service->Start("primary_unused", "unused", cluster_id)); + service->Start(kTestSources, "unused", cluster_id)); for (int i = 0; i < 100 && service->GetLatestAppliedSequenceId() < 1; ++i) { std::this_thread::sleep_for(std::chrono::milliseconds(2)); @@ -671,7 +674,7 @@ TEST_F(HotStandbyServiceTest, BatchRecordRetriesTransientBackendFailure) { auto service = std::make_unique(config); service->SetCatchUpBatchKvBackendForTesting(batch_backend); ASSERT_EQ(ErrorCode::OK, - service->Start("primary_unused", "unused", cluster_id)); + service->Start(kTestSources, "unused", cluster_id)); for (int i = 0; i < 100 && (service->GetLatestAppliedSequenceId() < 1 || @@ -697,8 +700,8 @@ TEST_F(HotStandbyServiceTest, BatchRecordRetryTimeoutTransitionsToFailed) { auto service = std::make_unique(config); service->SetCatchUpBatchKvBackendForTesting(batch_backend); - ASSERT_EQ(ErrorCode::OK, service->Start("primary_unused", "unused", - "batch-standby-timeout")); + ASSERT_EQ(ErrorCode::OK, + service->Start(kTestSources, "unused", "batch-standby-timeout")); for (int i = 0; i < 100 && service->GetState() != StandbyState::FAILED; ++i) { @@ -722,8 +725,8 @@ TEST_F(HotStandbyServiceTest, BatchRecordCanRestartAfterRetryTimeout) { auto service = std::make_unique(config); service->SetCatchUpBatchKvBackendForTesting(batch_backend); - ASSERT_EQ(ErrorCode::OK, service->Start("primary_unused", "unused", - "batch-standby-restart")); + ASSERT_EQ(ErrorCode::OK, + service->Start(kTestSources, "unused", "batch-standby-restart")); for (int i = 0; i < 100 && service->GetState() != StandbyState::FAILED; ++i) { std::this_thread::sleep_for(std::chrono::milliseconds(2)); @@ -731,8 +734,8 @@ TEST_F(HotStandbyServiceTest, BatchRecordCanRestartAfterRetryTimeout) { ASSERT_EQ(StandbyState::FAILED, service->GetState()); batch_backend->SetGetError(ErrorCode::OK); - ASSERT_EQ(ErrorCode::OK, service->Start("primary_unused", "unused", - "batch-standby-restart")); + ASSERT_EQ(ErrorCode::OK, + service->Start(kTestSources, "unused", "batch-standby-restart")); EXPECT_EQ(StandbyState::WATCHING, service->GetState()); EXPECT_EQ(ErrorCode::OK, service->GetSyncStatus().last_error); @@ -787,7 +790,7 @@ TEST_F(HotStandbyServiceTest, auto service = std::make_unique(config); ASSERT_EQ(ErrorCode::OK, - service->Start("primary_unused", etcd_endpoints, cluster_id)); + service->Start(kTestSources, etcd_endpoints, cluster_id)); const uint64_t expected_seq = prefix.last_seq + 1; constexpr int kMaxAttempts = 100; @@ -811,7 +814,7 @@ TEST_F(PromotionCatchUpTest, UsesDurablePrefixLastSeqAsCatchUpTarget) { batch_backend_->Put(BuildBatchRecordKey(cluster_id_, 1), EncodeOpLogBatchRecord(MakeBatch(1, 1, 2)))); - auto err = service_->Start({}, oplog_endpoints_, cluster_id_); + auto err = service_->Start(kTestSources, oplog_endpoints_, cluster_id_); if (err != ErrorCode::OK) { GTEST_SKIP() << "Service could not reach WATCHING state; " "skipping promotion test"; @@ -836,7 +839,7 @@ TEST_F(PromotionCatchUpTest, RetriesTransientDurablePrefixReadFailure) { service_->SetCatchUpBatchKvBackendForTesting(batch_backend); ASSERT_EQ(ErrorCode::OK, - service_->Start({}, oplog_endpoints_, cluster_id_)); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_)); batch_backend->FailNextGet(ErrorCode::ETCD_OPERATION_ERROR); StandbySnapshot out; @@ -846,7 +849,7 @@ TEST_F(PromotionCatchUpTest, RetriesTransientDurablePrefixReadFailure) { TEST_F(PromotionCatchUpTest, MissingDurablePrefixPromotesAtSequenceZero) { ASSERT_EQ(ErrorCode::OK, - service_->Start({}, oplog_endpoints_, cluster_id_)); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_)); ASSERT_EQ(StandbyState::WATCHING, service_->GetState()); StandbySnapshot out; ASSERT_EQ(ErrorCode::OK, service_->PromoteAndExportSnapshot(out)); @@ -861,7 +864,7 @@ TEST_F(PromotionCatchUpTest, MissingDurablePrefixRejectsNonzeroSequence) { std::optional(MakeSnapshot("baseline", 1, "key", 1)))); ASSERT_EQ(ErrorCode::OK, - service_->Start({}, oplog_endpoints_, cluster_id_)); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_)); ASSERT_EQ(StandbyState::WATCHING, service_->GetState()); StandbySnapshot out; EXPECT_EQ(ErrorCode::INCOMPLETE_OPLOG_CATCH_UP, @@ -871,7 +874,7 @@ TEST_F(PromotionCatchUpTest, MissingDurablePrefixRejectsNonzeroSequence) { TEST_F(PromotionCatchUpTest, CatchesUpPrefixThatAppearsBeforePromotion) { ASSERT_EQ(ErrorCode::OK, - service_->Start({}, oplog_endpoints_, cluster_id_)); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_)); ASSERT_EQ(StandbyState::WATCHING, service_->GetState()); ASSERT_EQ(ErrorCode::OK, batch_backend_->Put( @@ -902,7 +905,7 @@ TEST_F(PromotionCatchUpTest, PaginatesBatchRecords) { } service_->SetCatchUpBatchKvBackendForTesting(batch_backend); - auto err = service_->Start({}, oplog_endpoints_, cluster_id_); + auto err = service_->Start(kTestSources, oplog_endpoints_, cluster_id_); if (err != ErrorCode::OK) { GTEST_SKIP() << "Service could not reach WATCHING state; " "skipping promotion test"; @@ -915,7 +918,7 @@ TEST_F(PromotionCatchUpTest, PaginatesBatchRecords) { TEST_F(PromotionCatchUpTest, FailsPromotionWhenDurablePrefixUnreadable) { ASSERT_EQ(ErrorCode::OK, - service_->Start({}, oplog_endpoints_, cluster_id_)); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_)); batch_backend_->SetGetError(ErrorCode::PERSISTENT_FAIL); StandbySnapshot out; @@ -926,7 +929,7 @@ TEST_F(PromotionCatchUpTest, FailsPromotionWhenDurablePrefixUnreadable) { TEST_F(PromotionCatchUpTest, FailsPromotionWhenTargetBatchUnreadable) { ASSERT_EQ(ErrorCode::OK, - service_->Start({}, oplog_endpoints_, cluster_id_)); + service_->Start(kTestSources, oplog_endpoints_, cluster_id_)); ASSERT_EQ(ErrorCode::OK, batch_backend_->Put( BuildDurablePrefixKey(cluster_id_), diff --git a/mooncake-store/tests/master_metrics_test.cpp b/mooncake-store/tests/master_metrics_test.cpp index 62174b7e13..8e2d351940 100644 --- a/mooncake-store/tests/master_metrics_test.cpp +++ b/mooncake-store/tests/master_metrics_test.cpp @@ -921,6 +921,9 @@ TEST_F(MasterMetricsTest, SsdOffloadCacheHitAndTotalConsistent) { const int64_t base_file_cache_nums = metrics.get_file_cache_nums(); // Step 1: Mount segment and create a completed MEMORY replica. + // MountSegment registers the owner in client_host_id_ only with a host ID. + // LOCAL_DISK reads filter out replicas whose owner is not registered. + segment.host_id = "ssd-metrics-host"; auto mount_result = service_.MountSegment(segment, client_id); ASSERT_TRUE(mount_result.has_value()); auto put_start_result = @@ -1025,6 +1028,7 @@ TEST_F(MasterMetricsTest, SsdOffloadCacheHitAndTotalConsistent) { TEST_F(MasterMetricsTest, AdminMetricsExposeIoPatternFeedbackFromAccesses) { const int http_port = getFreeTcpPort(); WrappedMasterServiceConfig config; + config.default_kv_lease_ttl = 100; config.enable_metric_reporting = false; WrappedMasterService service(config); MasterAdminServer admin_server(static_cast(http_port), From cfc531928aac9953644acdfcea3c8620bf0bfabd Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 15:18:07 +0800 Subject: [PATCH 42/47] add io_pattern metric --- .../benchmarks/cfm_client_bench.cpp | 1052 ++++++++++++----- mooncake-store/src/master_metric_manager.cpp | 36 + .../tests/io_pattern_framework_test.cpp | 25 + mooncake-store/tests/master_metrics_test.cpp | 5 + 4 files changed, 801 insertions(+), 317 deletions(-) diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index 4b1c21a5bb..c52c12ab9a 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -30,30 +30,28 @@ // RealClient (keys share the simulated KvKey naming and tenant) and reads a // subset back to simulate access heat, then the simulated vLLM request stream // reports on those same keys. - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include - #include "gflags/gflags.h" #include "glog/logging.h" #include "cvm/cvm_types.h" @@ -69,15 +67,12 @@ #ifdef STORE_USE_ETCD #include "etcd_helper.h" #endif - namespace { - using Clock = std::chrono::steady_clock; using mooncake::ErrorCode; using mooncake::TenantId; using mooncake::toString; using namespace mooncake::io_pattern; - DEFINE_uint64(requests, 20, "Number of vLLM-style inference requests"); DEFINE_uint64(prompt_tokens, 1024, "Input tokens in each inference request"); DEFINE_uint64(output_tokens, 128, "Decode tokens in each inference request"); @@ -106,7 +101,6 @@ DEFINE_string(cfm_cluster_namespace, "", "MC_STORE_CLUSTER_ID or mooncake_cluster (same rule as the " "etcd leader coordinator). When the cluster was started with a " "non-default cluster_id, pass the same value here"); - // Real Store client parameters used by the optional real-data seeding stage. // Flag names and defaults mirror stress_cluster_bench.cpp so an existing // cluster invocation can be reused as-is. Seeding makes the SubMaster hold @@ -118,8 +112,10 @@ DEFINE_string(master_server, "", "disables real-data seeding"); DEFINE_string(local_hostname, "localhost", "Local hostname (with optional port, e.g. node1:12345)"); -DEFINE_string(metadata_server, "http://127.0.0.1:8080/metadata", - "Metadata server URL for RealClient setup"); +DEFINE_string( + metadata_server, + "[http://127.0.0.1:8080/metadata](http://127.0.0.1:8080/metadata)", + "Metadata server URL for RealClient setup"); DEFINE_string(protocol, "tcp", "Transport protocol: tcp, rdma, ub"); DEFINE_string(device_name, "", "RDMA/UB device name (comma-separated)"); DEFINE_uint64(global_segment_size, 16ULL * 1024 * 1024 * 1024, @@ -148,10 +144,11 @@ DEFINE_uint64(cfm_rpc_timeout_ms, 5000, "report_metric_batch) in milliseconds"); // Client-side collector/analysis key budget. The default 100k cap drops // observations once the simulated request stream exceeds it (seen as nonzero -// \"report drops\"); raise it to cover the whole run when reporting many keys. -DEFINE_uint64(max_analysis_keys, 100000, - "Max merged keys kept/analyzed by the client runtime and the " - "embedded SubMaster runtime (raise with the request stream size)"); +// "report drops"); raise it to cover the whole run when reporting many keys. +DEFINE_uint64( + max_analysis_keys, 100000, + "Max merged keys kept/analyzed by the client runtime and the " + "embedded SubMaster runtime (raise with the request stream size)"); // Remote reports normally omit storage watermarks (the owning SubMaster // reports its own); enabling this attaches the reported storage metrics to // every owner-addressed snapshot/metric batch so a remote run can drive the @@ -159,50 +156,138 @@ DEFINE_uint64(max_analysis_keys, 100000, DEFINE_bool(report_forward_storage, false, "Forward StorageMetric observations with remote owner-addressed " "reports (benchmark/simulation mode)"); - +DEFINE_uint64(promotion_test_wait_sec, 0, + "When >0: seed keys, wait N seconds for IO Pattern cold eviction " + "to offload+cull MEMORY replicas, then re-read the seeded keys " + "with get_into to trigger promotion-on-hit. Requires real-data " + "seeding (--master-server + --num-keys)."); +DEFINE_uint64(post_promo_evict_wait_sec, 0, + "When >0 (and promotion_test_wait_sec>0): after the promotion " + "re-read, report ALL seeded keys as cold for N seconds so the " + "IO Pattern cold eviction re-evicts the freshly promoted MEMORY " + "replicas (S7.5: promoted replicas must be evictable again)."); +DEFINE_bool(admission_test_mode, false, + "S8.1: during the promotion-test wait phase report ALL seeded keys " + "as cold with LOCAL_DISK replica_tiers so the report-driven cold " + "eviction offloads all of them; then send one hot+LOCAL_DISK " + "report so the report-driven admission path promotes them " + "(master_io_pattern_report_admissions_total increments). Requires " + "promotion_test_wait_sec>0, real-data seeding, and a SubMaster " + "running with --enable_offload and " + "--io_pattern_admission_frequency_threshold=1."); +DEFINE_string(force_workload_type, "", + "Force reported key metrics to match a specific workload type " + "for S4 analysis layer testing. Options: code_agent, " + "recommendation, conversation. Empty = use original metrics."); +DEFINE_string(force_session_workload_types, "", + "S4.2: comma-separated per-session workload type overrides " + "applied to sessions in order (e.g. " + "'code_agent,recommendation' makes session 0 report code-agent " + "metrics and session 1 report recommendation metrics, forcing " + "kMixed -> K-means on the SubMaster). A session index past the " + "list falls back to --force_workload_type. Empty = use " + "--force_workload_type / original metrics."); +DEFINE_bool(prefetch_test_mode, false, + "S11: report-driven prefetch test. During the promotion-test wait " + "phase report ALL seeded keys as cold + LOCAL_DISK so the " + "report-driven cold eviction offloads them (same offload as " + "admission_test_mode); then send --prefetch_repeat_reports " + "hot+LOCAL_DISK trigger reports carrying match_length=" + "--prefetch_match_length and code-agent metrics, so " + "DeriveTraceHistory -> TraceBasedPrefetchOps::Evaluate emits " + "LOCAL_DISK -> kL1Host candidates and the master prefetch handler " + "pushes them into the promotion queue. Requires " + "promotion_test_wait_sec>0, real-data seeding, and a SubMaster " + "with --enable_offload and --promotion_on_hit=true."); +DEFINE_uint64(prefetch_match_length, 1024, + "S11: match_length reported by the prefetch trigger. Must exceed " + "the workload's prefetch.match_length_threshold (code_agent=512, " + "Mixed default 256) for TraceBasedPrefetchOps::Evaluate to emit " + "candidates; a value at/below the threshold leaves candidates=0 " + "(S11.2)."); +DEFINE_string(prefetch_replica_variant, "local_disk_l1", + "S11: replica_tiers reported by the prefetch trigger. " + "local_disk_l1 = LOCAL_DISK|L1Host (prefetch fires, admission " + "suppressed via in_head), local_disk = LOCAL_DISK only (prefetch " + "and admission both fire), l1_only = L1Host only (no lower " + "replica, trace empty, candidates=0), l2_only / l3_only = lower " + "tier without LOCAL_DISK (in trace but Evaluate skips, " + "candidates=0) -- S11.3."); +DEFINE_uint64(prefetch_repeat_reports, 1, + "S11.5: number of identical prefetch trigger reports to send; " + "the promotion queue must not grow on the repeats (dedup)."); +DEFINE_bool(prefetch_fake_local_report, false, + "S11.6: after the synthetic request stream (no real seeding) send " + "one report claiming LOCAL_DISK replicas on keys with no real " + "object so the prefetch handler fails " + "(UNAVAILABLE_IN_CURRENT_MODE / OBJECT_NOT_FOUND) and " + "io_pattern_report_prefetch_failures increments."); +DEFINE_int32(prefetch_ready_poll_sec, 120, + "S11: max seconds to poll the master for ALL seeded keys to hold " + "a real LOCAL_DISK replica before sending the prefetch trigger " + "reports. The offload driven by the wait phase is asynchronous; " + "triggering before it finishes lets the same-cycle fallback " + "eviction delete a MEMORY-only key (OpType::REMOVE) so the " + "prefetch handler hits OBJECT_NOT_FOUND(-704) and aborts the " + "whole plan. Polling get_replica_desc both confirms readiness and " + "holds the keys' leases so eviction cannot delete them. 0 " + "disables the poll."); uint64_t SteadyNowNs() { - return static_cast( - std::chrono::duration_cast( - Clock::now().time_since_epoch()) - .count()); + return static_cast(std::chrono::duration_caststd::chrono::nanoseconds( + Clock::now().time_since_epoch()) + .count()); } - double ToMicroseconds(Clock::duration duration) { return std::chrono::duration(duration).count(); } - size_t BlockCount(uint64_t tokens) { - return static_cast((tokens + FLAGS_tokens_per_block - 1) / - FLAGS_tokens_per_block); + return static_cast((tokens + FLAGS_tokens_per_block - 1) / + FLAGS_tokens_per_block); } - std::string KvKey(size_t session, size_t request, size_t layer, size_t block, bool is_shared_prefix) { - const auto owner = is_shared_prefix ? std::string("prefix") - : std::string("request-") + - std::to_string(request); - return "vllm/" + FLAGS_node_id + "/session-" + - std::to_string(session) + "/" + owner + "/layer-" + - std::to_string(layer) + "/block-" + std::to_string(block); + const auto owner = is_shared_prefix + ? std::string("prefix") + : std::string("request-") + std::to_string(request); + return "vllm/" + FLAGS_node_id + "/session-" + std::to_string(session) + + "/" + owner + "/layer-" + std::to_string(layer) + "/block-" + + std::to_string(block); +} +// Per-session workload type for S4.2: index into the comma-separated +// --force_session_workload_types list by session id; fall back to the global +// --force_workload_type when the list is empty or the session is out of range. +std::string SessionWorkloadType(size_t session) { + const std::string& spec = FLAGS_force_session_workload_types; + if (spec.empty()) return FLAGS_force_workload_type; + size_t start = 0; + size_t index = 0; + while (start <= spec.size()) { + const size_t comma = spec.find(',', start); + const std::string token = + spec.substr(start, comma == std::string::npos ? std::string::npos + : comma - start); + if (index == session) return token; + if (comma == std::string::npos) break; + start = comma + 1; + ++index; + } + return FLAGS_force_workload_type; } - // Sends reports straight into an embedded SubMaster's CFM component. This is // the ownership-addressed path collapsed to the single owning SubMaster of a // benchmark run, exercised without network. class EmbeddedCfmTransport final : public CfmRpcTransport { public: - explicit EmbeddedCfmTransport(std::shared_ptr service) + explicit EmbeddedCfmTransport(std::shared_ptr service) : service_(std::move(service)) {} - bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds) override { return service_ && service_->Send(method, payload, FLAGS_node_id); } private: - std::shared_ptr service_; + std::shared_ptr service_; }; - #ifdef STORE_USE_ETCD // Resolves the CVM cluster namespace used by --cfm_endpoint when it carries an // etcd:// backend. Mirrors EtcdLeaderCoordinator::ResolveClusterNamespace: @@ -217,7 +302,6 @@ std::string ResolveCvmNamespace() { } return mooncake::DEFAULT_CLUSTER_ID; } - // Key that stores the leader address for single-leader HA. // Mirrors EtcdLeaderCoordinator::BuildMasterViewKey. std::string BuildMasterViewKey(const std::string& cluster_namespace) { @@ -228,12 +312,11 @@ std::string BuildMasterViewKey(const std::string& cluster_namespace) { return "mooncake-store/" + normalized + "/master_view"; } #endif // STORE_USE_ETCD - // If --cfm_endpoint names a single SubMaster directly ("host:port") this // returns an ownership resolver that routes every key to it. If it is an // etcd:// entry, it resolves the cluster like a Store client: a present // leader master_view yields a single target; otherwise the CVM -// /cvm//masters registry plus slot ownership is used to bucket keys to +// /cvm//masters registry plus slot ownership is used to bucket keys to // their owning SubMaster. Returns an empty resolver on any resolution failure // (the caller aborts instead of hanging). SubmasterEndpointResolver ResolveCfmEndpointOwnership() { @@ -243,8 +326,10 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { // Plain host:port -> every observed key belongs to this single // SubMaster (the equivalent of the old single-endpoint remote mode). const std::string endpoint = entry; - return [endpoint](const TenantId&, const std::string&) - -> std::optional { return endpoint; }; + return [endpoint](const TenantId&, + const std::string&) -> std::optionalstd::string { + return endpoint; + }; } #ifndef STORE_USE_ETCD LOG(FATAL) << "cfm_endpoint entry '" << entry @@ -260,14 +345,12 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { } const std::string connstring = entry.substr(scheme_pos + 3); const std::string cluster_namespace = ResolveCvmNamespace(); - ErrorCode err = mooncake::EtcdHelper::ConnectToEtcdStoreClient(connstring); if (err != ErrorCode::OK) { LOG(FATAL) << "cfm_endpoint: failed to connect etcd '" << connstring << "': " << toString(err); return {}; } - // Single-leader HA: leader master_view holds the master address. const std::string view_key = BuildMasterViewKey(cluster_namespace); std::string leader_address; @@ -275,20 +358,21 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { err = mooncake::EtcdHelper::Get(view_key.data(), view_key.size(), leader_address, revision); if (err == ErrorCode::OK && !leader_address.empty()) { - LOG(INFO) << "cfm_endpoint: single-leader HA via " << view_key - << " -> " << leader_address; + LOG(INFO) << "cfm_endpoint: single-leader HA via " << view_key << " -> " + << leader_address; const std::string endpoint = std::move(leader_address); - return [endpoint](const TenantId&, const std::string&) - -> std::optional { return endpoint; }; + return [endpoint](const TenantId&, + const std::string&) -> std::optionalstd::string { + return endpoint; + }; } if (err != ErrorCode::OK && err != ErrorCode::ETCD_KEY_NOT_EXIST) { LOG(FATAL) << "cfm_endpoint: failed to read " << view_key << ": " << toString(err); return {}; } - // CVM multi-submaster: masters registry + slot ownership. - std::vector masters; + std::vectormooncake::cvm::MasterRegistration masters; mooncake::ViewVersionId version = 0; err = mooncake::cvm::EtcdViewStore::LoadAllMasters(cluster_namespace, masters, version); @@ -297,12 +381,10 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { << cluster_namespace << "': " << toString(err); return {}; } - std::map address_by_master; // id -> host:port - std::vector primary_ids; + std::vectorstd::string primary_ids; for (const auto& reg : masters) { - if (reg.role == - static_cast(mooncake::cvm::MasterRole::kPrimary) && + if (reg.role == static_cast(mooncake::cvm::MasterRole::kPrimary) && !reg.address.empty()) { address_by_master[reg.master_id] = reg.address; primary_ids.push_back(reg.master_id); @@ -315,17 +397,15 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { return {}; } std::sort(primary_ids.begin(), primary_ids.end()); - // Prefer the authoritative slot owner table published by CvmController; // fall back to the consistent-hash ring used by the masters themselves. std::unordered_map owner_by_slot; - std::vector slot_owners; + std::vectormooncake::cvm::SlotOwner slot_owners; const ErrorCode slot_err = mooncake::cvm::EtcdViewStore::LoadAllSlotOwners( cluster_namespace, slot_owners, version); if (slot_err == ErrorCode::OK) { for (const auto& owner : slot_owners) { - if (owner.state == - static_cast(mooncake::cvm::SlotState::kStable) && + if (owner.state == static_cast(mooncake::cvm::SlotState::kStable) && !owner.primary_master_id.empty()) { owner_by_slot[owner.slot] = owner.primary_master_id; } @@ -334,15 +414,13 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { const bool has_owner_table = !owner_by_slot.empty(); LOG(INFO) << "cfm_endpoint: CVM namespace '" << cluster_namespace << "' has " << primary_ids.size() << " primary submaster(s), " - << (has_owner_table ? owner_by_slot.size() : 0) - << " slot owners" + << (has_owner_table ? owner_by_slot.size() : 0) << " slot owners" << (has_owner_table ? "" : " (falling back to hash ring)"); - return [address_by_master = std::move(address_by_master), primary_ids = std::move(primary_ids), owner_by_slot = std::move(owner_by_slot), has_owner_table]( const TenantId& tenant, - const std::string& key) -> std::optional { + const std::string& key) -> std::optionalstd::string { const uint16_t slot = mooncake::cvm::KeySlot(tenant, key); std::string owner; if (has_owner_table) { @@ -358,23 +436,19 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { }; #endif } - class LatencyStats final { public: void Record(double value_us) { values_us_.push_back(value_us); } - double Percentile(double percentile) const { if (values_us_.empty()) return 0.0; const double rank = percentile / 100.0 * (values_us_.size() - 1); - const auto lower = static_cast(rank); + const auto lower = static_cast(rank); const auto upper = std::min(lower + 1, values_us_.size() - 1); const double fraction = rank - lower; return values_us_[lower] * (1.0 - fraction) + values_us_[upper] * fraction; } - void Finalize() { std::sort(values_us_.begin(), values_us_.end()); } - double Mean() const { if (values_us_.empty()) return 0.0; return std::accumulate(values_us_.begin(), values_us_.end(), 0.0) / @@ -382,27 +456,24 @@ class LatencyStats final { } private: - std::vector values_us_; + std::vector values_us_; }; - struct MetricReportSnapshot { uint64_t calls{0}; uint64_t failures{0}; uint64_t observations{0}; LatencyStats latency; }; - class MetricReportStats final { public: void Record(const MetricBatch& batch, double latency_us, bool success) { std::lock_guard lock(mutex_); ++calls; if (!success) ++failures; - observations += - batch.inference.size() + batch.accesses.size() + batch.storage.size(); + observations += batch.inference.size() + batch.accesses.size() + + batch.storage.size(); latency.Record(latency_us); } - MetricReportSnapshot Finalize() { std::lock_guard lock(mutex_); latency.Finalize(); @@ -419,13 +490,11 @@ class MetricReportStats final { uint64_t observations{0}; LatencyStats latency; }; - struct RequestData { IoPatternSnapshot snapshot; - std::vector inference; - std::vector accesses; + std::vector inference; + std::vector accesses; }; - RequestData BuildRequest(size_t request_index) { const size_t session = request_index % FLAGS_num_sessions; const uint64_t total_tokens = FLAGS_prompt_tokens + FLAGS_output_tokens; @@ -434,7 +503,6 @@ RequestData BuildRequest(size_t request_index) { std::min(blocks, BlockCount(FLAGS_shared_prefix_tokens)); const bool prefix_is_cached = request_index >= FLAGS_num_sessions; const uint64_t now_ns = SteadyNowNs(); - RequestData request; request.snapshot.generated_at_ns = now_ns; request.inference.reserve(blocks * FLAGS_num_layers); @@ -442,37 +510,31 @@ RequestData BuildRequest(size_t request_index) { request.snapshot.keys.reserve(blocks * FLAGS_num_layers); const auto tenant = TenantId(FLAGS_tenant); const auto session_id = "vllm-session-" + std::to_string(session); - + const std::string workload_type = SessionWorkloadType(session); for (size_t layer = 0; layer < FLAGS_num_layers; ++layer) { for (size_t block = 0; block < blocks; ++block) { const bool is_shared_prefix = block < shared_blocks; const bool is_hit = is_shared_prefix && prefix_is_cached; - const ObjectRef object{ - .tenant_id = tenant, - .key = KvKey(session, request_index, layer, block, - is_shared_prefix)}; - const auto block_end = std::min( - total_tokens, (block + 1) * FLAGS_tokens_per_block); - const auto block_tokens = static_cast( - block_end - block * FLAGS_tokens_per_block); - + const ObjectRef object{.tenant_id = tenant, + .key = KvKey(session, request_index, layer, + block, is_shared_prefix)}; + const auto block_end = + std::min(total_tokens, (block + 1) * FLAGS_tokens_per_block); + const auto block_tokens = + static_cast(block_end - block * FLAGS_tokens_per_block); InferenceMetrics inference{ .object = object, .session_id = session_id, .layout = CacheLayout::kLayerFirst, - .layout_group = static_cast(layer), - .prefix_depth = static_cast(shared_blocks), - .prefix_fanout = static_cast(FLAGS_num_sessions), - .match_length = is_hit - ? static_cast( - FLAGS_shared_prefix_tokens) - : 0U, - .continuous_prefix_length = is_hit - ? static_cast( - FLAGS_shared_prefix_tokens) - : 0U, + .layout_group = static_cast(layer), + .prefix_depth = static_cast(shared_blocks), + .prefix_fanout = static_cast(FLAGS_num_sessions), + .match_length = + is_hit ? static_cast(FLAGS_shared_prefix_tokens) : 0U, + .continuous_prefix_length = + is_hit ? static_cast(FLAGS_shared_prefix_tokens) : 0U, .token_count = block_tokens, - .recompute_cost = is_hit ? 0.0F : static_cast(block_tokens), + .recompute_cost = is_hit ? 0.0F : static_cast(block_tokens), .request_priority = 1}; AccessRecord access{ .object = object, @@ -482,55 +544,80 @@ RequestData BuildRequest(size_t request_index) { .tier = CacheTier::kL1Host, .operation = is_hit ? IoOperation::kGet : IoOperation::kPut, .is_hit = is_hit, - .write_batch_size = is_hit - ? 0U - : static_cast(FLAGS_num_layers), + .write_batch_size = is_hit ? 0U : static_cast(FLAGS_num_layers), .overwrite = !is_hit && is_shared_prefix}; + // Override reported metrics for S4 workload type testing + if (!workload_type.empty()) { + if (workload_type == "code_agent") { + inference.token_count = 16385; + inference.prefix_fanout = 32; + inference.match_length = 512; + access.block_size = 512 * 1024; + } else if (workload_type == "recommendation") { + access.block_size = 65536; + access.is_hit = true; + } else if (workload_type == "conversation") { + inference.token_count = 8000; + inference.prefix_fanout = 32; + inference.match_length = 512; + } + } request.inference.push_back(inference); request.accesses.push_back(access); - request.snapshot.keys.push_back( - KeyMetrics{.object = object, - .session_id = session_id, - .last_access_time_ns = now_ns, - .access_count_window = 1, - .block_size = FLAGS_kv_block_bytes, - .token_count = block_tokens, - .prefix_depth = static_cast(shared_blocks), - .prefix_fanout = static_cast(FLAGS_num_sessions), - .match_length = inference.match_length, - .continuous_prefix_length = - inference.continuous_prefix_length, - .write_batch_size = access.write_batch_size, - .write_frequency = - access.operation == IoOperation::kPut ? 1U : 0U, - .write_object_size = FLAGS_kv_block_bytes, - .recompute_cost = inference.recompute_cost, - .overwrite_ratio = access.overwrite ? 1.0F : 0.0F, - .replica_tiers = CacheTierBit(CacheTier::kL1Host), - .layout = CacheLayout::kLayerFirst, - .layout_group = static_cast(layer), - .request_priority = 1, - .active = is_hit, - .write_burst = !is_hit}); + KeyMetrics key_metrics{ + .object = object, + .session_id = session_id, + .last_access_time_ns = now_ns, + .access_count_window = 3, + .block_size = FLAGS_kv_block_bytes, + .token_count = block_tokens, + .prefix_depth = static_cast(shared_blocks), + .prefix_fanout = static_cast(FLAGS_num_sessions), + .match_length = inference.match_length, + .continuous_prefix_length = inference.continuous_prefix_length, + .write_batch_size = access.write_batch_size, + .write_frequency = + access.operation == IoOperation::kPut ? 1U : 0U, + .write_object_size = FLAGS_kv_block_bytes, + .recompute_cost = inference.recompute_cost, + .overwrite_ratio = access.overwrite ? 1.0F : 0.0F, + .replica_tiers = CacheTierBit(CacheTier::kL1Host), + .layout = CacheLayout::kLayerFirst, + .layout_group = static_cast(layer), + .request_priority = 1, + .active = is_hit, + .write_burst = !is_hit}; + if (workload_type == "code_agent") { + key_metrics.token_count = 16385; + key_metrics.prefix_fanout = 32; + key_metrics.match_length = 512; + key_metrics.block_size = 512 * 1024; + } else if (workload_type == "recommendation") { + key_metrics.block_size = 65536; + key_metrics.access_count_window = 30; + key_metrics.recompute_cost = 0.0F; + } else if (workload_type == "conversation") { + key_metrics.token_count = 8000; + key_metrics.prefix_fanout = 32; + key_metrics.match_length = 512; + } + request.snapshot.keys.push_back(key_metrics); } } - request.snapshot.storage.push_back( - StorageMetric{.source_id = FLAGS_node_id, - .observed_at_ns = now_ns, - .tier = CacheTier::kL1Host, - .read_bandwidth_bytes_per_sec = 20ULL * 1024 * 1024 * 1024, - .write_bandwidth_bytes_per_sec = 10ULL * 1024 * 1024 * 1024, - .read_latency_us = 20, - .write_latency_us = 200, - .used_bytes = static_cast( - FLAGS_memory_used_ratio * 1024 * 1024 * 1024), - .capacity_bytes = 1024ULL * 1024 * 1024, - .rpc_latency_us = 100, - .memory_used_ratio = - static_cast(FLAGS_memory_used_ratio)}); + request.snapshot.storage.push_back(StorageMetric{ + .source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .read_bandwidth_bytes_per_sec = 20ULL * 1024 * 1024 * 1024, + .write_bandwidth_bytes_per_sec = 10ULL * 1024 * 1024 * 1024, + .read_latency_us = 20, + .write_latency_us = 200, + .used_bytes = static_cast(FLAGS_memory_used_ratio * 1024 * 1024 * 1024), + .capacity_bytes = 1024ULL * 1024 * 1024, + .rpc_latency_us = 100, + .memory_used_ratio = static_cast(FLAGS_memory_used_ratio)}); return request; } - void PrintObservability(std::string_view name, const IoPatternObservabilitySnapshot& metrics) { std::cout << "\n " << name << " IO Pattern metrics\n" @@ -549,15 +636,14 @@ void PrintObservability(std::string_view name, << " report drops: " << metrics.report_drop_count << "\n"; } - bool ValidateFlags() { - return FLAGS_requests != 0 && FLAGS_prompt_tokens + FLAGS_output_tokens != 0 && + return FLAGS_requests != 0 && + FLAGS_prompt_tokens + FLAGS_output_tokens != 0 && FLAGS_tokens_per_block != 0 && FLAGS_num_layers != 0 && FLAGS_kv_block_bytes != 0 && FLAGS_num_sessions != 0 && FLAGS_report_capacity != 0 && FLAGS_memory_used_ratio >= 0.0 && FLAGS_memory_used_ratio <= 1.0; } - // Real-data seeding stage (optional). The IO Pattern runtime on the SubMaster // only executes storage handlers against replicas that actually exist, so a // report-only benchmark leaves master-side eviction/promotion/prefetch counters @@ -571,17 +657,15 @@ struct SeedStats { uint64_t reads{0}; uint64_t read_failures{0}; }; - // Captures what the seeding stage actually created so the reporting stage can // address exactly the real keys (a report stream over synthetic keys whose // objects were never stored leaves the SubMaster handlers with nothing to act // on, which shows up as OBJECT_NOT_FOUND / zero master-side evictions). struct SeedOutcome { SeedStats stats; - std::vector keys; // real keys written, stable order - std::vector hot; // parallel: read back (access heat) + std::vectorstd::string keys; // real keys written, stable order + std::vector hot; // parallel: read back (access heat) }; - SeedOutcome RunRealSeedStage() { SeedOutcome outcome; if (FLAGS_master_server.empty() || FLAGS_num_keys == 0 || @@ -589,14 +673,12 @@ SeedOutcome RunRealSeedStage() { return outcome; } LOG(INFO) << "Real-data seed stage: master_server=" << FLAGS_master_server - << " protocol=" << FLAGS_protocol - << " keys=" << FLAGS_num_keys + << " protocol=" << FLAGS_protocol << " keys=" << FLAGS_num_keys << " value_size=" << FLAGS_value_size << " replica_num=" << FLAGS_replica_num << " offload=" << (FLAGS_enable_ssd_offload ? "yes" : "no"); - auto client = mooncake::RealClient::create(); - const size_t block_bytes = std::max(FLAGS_value_size, 4096); + const size_t block_bytes = std::max(FLAGS_value_size, 4096); char* buffer = reinterpret_cast(numa_alloc_local(block_bytes)); if (buffer == nullptr) { LOG(ERROR) << "numa_alloc_local failed for seed buffer of " @@ -620,7 +702,6 @@ SeedOutcome RunRealSeedStage() { numa_free(buffer, block_bytes); return outcome; } - // Write keys that share the simulated KvKey naming so later reports and // the real metadata address the same objects. Enumerate the same // (session, request, layer, block) space as BuildRequest() and stop after @@ -628,7 +709,7 @@ SeedOutcome RunRealSeedStage() { // reported key universe (real replicas exist for the keys policy will // select). mooncake::ReplicateConfig config; - config.replica_num = static_cast(FLAGS_replica_num); + config.replica_num = static_cast(FLAGS_replica_num); config.with_hard_pin = FLAGS_hard_pin; const uint64_t total_tokens = FLAGS_prompt_tokens + FLAGS_output_tokens; const size_t blocks = BlockCount(total_tokens); @@ -639,13 +720,13 @@ SeedOutcome RunRealSeedStage() { request_index < FLAGS_requests && seeded < FLAGS_num_keys; ++request_index) { const size_t session = request_index % FLAGS_num_sessions; - for (size_t layer = 0; layer < FLAGS_num_layers && seeded < FLAGS_num_keys; - ++layer) { + for (size_t layer = 0; + layer < FLAGS_num_layers && seeded < FLAGS_num_keys; ++layer) { for (size_t block = 0; block < blocks && seeded < FLAGS_num_keys; ++block) { const bool is_shared_prefix = block < shared_blocks; - const std::string key = - KvKey(session, request_index, layer, block, is_shared_prefix); + const std::string key = KvKey(session, request_index, layer, + block, is_shared_prefix); const int put_ret = client->put_from(key, buffer, FLAGS_value_size, config); if (put_ret == 0) { @@ -660,15 +741,13 @@ SeedOutcome RunRealSeedStage() { } } outcome.stats.written = seeded; - // Simulate reads: exercise a hot subset through the real data path so the // SubMaster records real GET access heat (promotion-on-hit when offloaded). // Mark the same prefix of the written key list as hot for the report pass. outcome.hot.assign(outcome.keys.size(), false); const uint64_t read_count = FLAGS_seed_get_keys == 0 ? seeded / 2 - : std::min(FLAGS_seed_get_keys, - seeded); + : std::min(FLAGS_seed_get_keys, seeded); uint64_t read_keys = 0; for (size_t request_index = 0; request_index < FLAGS_requests && read_keys < read_count; @@ -679,11 +758,20 @@ SeedOutcome RunRealSeedStage() { for (size_t block = 0; block < blocks && read_keys < read_count; ++block) { const bool is_shared_prefix = block < shared_blocks; - const std::string key = - KvKey(session, request_index, layer, block, is_shared_prefix); - const int64_t got = client->get_into(key, buffer, FLAGS_value_size); + const std::string key = KvKey(session, request_index, layer, + block, is_shared_prefix); + // First get_into: may hit LOCAL_DISK-only replica, triggering + // promotion-on-hit if admission gate passes. + int64_t got = client->get_into(key, buffer, FLAGS_value_size); if (got >= 0) { ++outcome.stats.reads; + // Two more get_into calls to raise CountMinSketch frequency + // above promotion_admission_threshold (default 2), which + // makes the promotion admission gate pass on the first hit. + got = client->get_into(key, buffer, FLAGS_value_size); + if (got >= 0) ++outcome.stats.reads; + got = client->get_into(key, buffer, FLAGS_value_size); + if (got >= 0) ++outcome.stats.reads; } else { ++outcome.stats.read_failures; } @@ -694,7 +782,6 @@ SeedOutcome RunRealSeedStage() { } } } - client->unregister_buffer(buffer); numa_free(buffer, block_bytes); LOG(INFO) << "Real-data seed stage done: written=" << outcome.stats.written @@ -703,9 +790,7 @@ SeedOutcome RunRealSeedStage() { << " read_failures=" << outcome.stats.read_failures; return outcome; } - } // namespace - int main(int argc, char* argv[]) { google::InitGoogleLogging(argv[0]); gflags::ParseCommandLineFlags(&argc, &argv, true); @@ -714,17 +799,15 @@ int main(int argc, char* argv[]) { "--memory_used_ratio must be within [0, 1]"; return 1; } - - std::atomic eviction_commands{0}; - std::atomic prefetch_commands{0}; - std::atomic admission_commands{0}; - + std::atomic eviction_commands{0}; + std::atomic prefetch_commands{0}; + std::atomic admission_commands{0}; // The SubMaster-side CFM component (embedded mode) or the ownership // resolver used by the remote reporter. - std::shared_ptr embedded_service; - std::shared_ptr cfm_runtime; - std::shared_ptr ownership_client; - std::shared_ptr embedded_channel; + std::shared_ptr embedded_service; + std::shared_ptr cfm_runtime; + std::shared_ptr ownership_client; + std::shared_ptr embedded_channel; std::string deployment_description; if (FLAGS_cfm_endpoint.empty()) { deployment_description = "embedded SubMaster (local CFM)"; @@ -735,36 +818,37 @@ int main(int argc, char* argv[]) { // watermark evaluation at the end of the run. cfm_config.report_driven_execution = true; cfm_config.max_analysis_keys = FLAGS_max_analysis_keys; - cfm_runtime = std::make_shared( + cfm_runtime = std::make_shared( IoPatternRuntime::Handlers{ - .eviction = [&eviction_commands](const EvictionPlan&) { - ++eviction_commands; - return ErrorCode::OK; - }, - .prefetch = [&prefetch_commands](const PrefetchPlan&) { - ++prefetch_commands; - return ErrorCode::OK; - }, - .admission = [&admission_commands](const AdmissionResult&) { - ++admission_commands; - return ErrorCode::OK; - }}, + .eviction = + [&eviction_commands](const EvictionPlan&) { + ++eviction_commands; + return ErrorCode::OK; + }, + .prefetch = + [&prefetch_commands](const PrefetchPlan&) { + ++prefetch_commands; + return ErrorCode::OK; + }, + .admission = + [&admission_commands](const AdmissionResult&) { + ++admission_commands; + return ErrorCode::OK; + }}, std::move(cfm_config)); - embedded_service = std::make_shared(cfm_runtime); - auto transport = - std::make_shared(embedded_service); - embedded_channel = std::make_shared( - std::move(transport), std::make_shared(), - CfmRpcConfig{.timeout = - std::chrono::milliseconds(FLAGS_cfm_rpc_timeout_ms)}); + embedded_service = std::make_shared(cfm_runtime); + auto transport = std::make_shared(embedded_service); + embedded_channel = + std::make_shared(std::move(transport), std::make_shared(), + CfmRpcConfig{.timeout = std::chrono::milliseconds( + FLAGS_cfm_rpc_timeout_ms)}); } else { const auto resolver = ResolveCfmEndpointOwnership(); - ownership_client = std::make_shared( + ownership_client = std::make_shared( resolver, std::chrono::milliseconds(FLAGS_cfm_rpc_timeout_ms)); ownership_client->set_forward_storage(FLAGS_report_forward_storage); deployment_description = "remote SubMaster(s) via CFM coro_rpc"; } - // Real-data seeding runs before the simulated request stream ("先种子后仿 // 真"): the SubMaster must hold real replicas for reported keys before the // report-driven policy cycle can execute eviction/promotion/prefetch @@ -772,7 +856,6 @@ int main(int argc, char* argv[]) { // (--cfm_endpoint) plus RealClient parameters; otherwise it is a no-op. const SeedOutcome seed_outcome = RunRealSeedStage(); const SeedStats& seed_stats = seed_outcome.stats; - IoPatternRuntime::Config source_config; source_config.report_capacity = FLAGS_report_capacity; source_config.max_analysis_keys = FLAGS_max_analysis_keys; @@ -781,31 +864,31 @@ int main(int argc, char* argv[]) { const auto report_metric_batch = [&](const MetricBatch& batch) -> bool { const auto started = Clock::now(); const bool success = - ownership_client ? ownership_client->ReportMetricBatch(batch) == ErrorCode::OK - : (embedded_channel && embedded_channel->SendMetricBatch(batch)); + ownership_client + ? ownership_client->ReportMetricBatch(batch) == ErrorCode::OK + : (embedded_channel && + embedded_channel->SendMetricBatch(batch)); metric_reports.Record(batch, ToMicroseconds(Clock::now() - started), success); return success; }; source_config.report_sink = report_metric_batch; - auto source_runtime = std::make_shared( + auto source_runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, source_config); - const auto send_snapshot = [&](const IoPatternSnapshot& snapshot) -> bool { return ownership_client ? ownership_client->ReportSnapshot(snapshot) == ErrorCode::OK - : (embedded_channel && embedded_channel->SendSnapshot(snapshot)); + : (embedded_channel && + embedded_channel->SendSnapshot(snapshot)); }; - LatencyStats report_latency; uint64_t failed_reports = 0; uint64_t total_blocks = 0; const auto benchmark_start = Clock::now(); - // When a real seed set was written, report exactly those keys (the ones // with real replicas) instead of the synthetic request stream. Synthetic // keys never stored on the SubMaster pollute the merged snapshot: the @@ -833,11 +916,11 @@ int main(int argc, char* argv[]) { KeyMetrics key_metrics{ .object = object, .session_id = "seed-real-keys", - .last_access_time_ns = - is_hot ? now_ns - : (now_ns > 60'000'000'000ULL - ? now_ns - 60'000'000'000ULL - : 0ULL), + .last_access_time_ns = is_hot + ? now_ns + : (now_ns > 60'000'000'000ULL + ? now_ns - 60'000'000'000ULL + : 0ULL), .access_count_window = is_hot ? 4U : 0U, .block_size = FLAGS_value_size, .token_count = 16U, @@ -860,36 +943,334 @@ int main(int argc, char* argv[]) { source_runtime->RecordAccess(key, access); ++total_blocks; } - real_snapshot.storage.push_back( - StorageMetric{.source_id = FLAGS_node_id, - .observed_at_ns = now_ns, - .tier = CacheTier::kL1Host, - .read_bandwidth_bytes_per_sec = - 20ULL * 1024 * 1024 * 1024, - .write_bandwidth_bytes_per_sec = - 10ULL * 1024 * 1024 * 1024, - .read_latency_us = 20, - .write_latency_us = 200, - .used_bytes = static_cast( - static_cast(FLAGS_value_size) * - seed_outcome.keys.size()), - .capacity_bytes = - static_cast(FLAGS_num_keys) * - FLAGS_value_size, - .rpc_latency_us = 100, - .memory_used_ratio = - static_cast(FLAGS_memory_used_ratio)}); + real_snapshot.storage.push_back(StorageMetric{ + .source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .read_bandwidth_bytes_per_sec = 20ULL * 1024 * 1024 * 1024, + .write_bandwidth_bytes_per_sec = 10ULL * 1024 * 1024 * 1024, + .read_latency_us = 20, + .write_latency_us = 200, + .used_bytes = static_cast(static_cast(FLAGS_value_size) * + seed_outcome.keys.size()), + .capacity_bytes = static_cast(FLAGS_num_keys) * FLAGS_value_size, + .rpc_latency_us = 100, + .memory_used_ratio = static_cast(FLAGS_memory_used_ratio)}); const auto report_start = Clock::now(); const bool sent = send_snapshot(real_snapshot); report_latency.Record(ToMicroseconds(Clock::now() - report_start)); if (!sent) ++failed_reports; seed_report_requests = 1; - LOG(INFO) << "Real-seed report sent: keys=" - << real_snapshot.keys.size() << " hot=" << hot_blocks + LOG(INFO) << "Real-seed report sent: keys=" << real_snapshot.keys.size() + << " hot=" << hot_blocks << " cold=" << real_snapshot.keys.size() - hot_blocks << " (synthetic request stream skipped)"; } - + // Promotion test: wait for IO Pattern cold eviction to cull MEMORY + // replicas, then re-read seeded keys to trigger promotion-on-hit. + if (FLAGS_promotion_test_wait_sec > 0 && !seed_outcome.keys.empty()) { + LOG(INFO) << "[PROMO-TEST] waiting " << FLAGS_promotion_test_wait_sec + << "s for cold eviction, sending reports every 5s..."; + const auto wait_end = + Clock::now() + std::chrono::seconds(FLAGS_promotion_test_wait_sec); + while (Clock::now() < wait_end) { + const uint64_t now_ns = SteadyNowNs(); + const TenantId tenant(FLAGS_tenant); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = now_ns; + for (size_t i = 0; i < seed_outcome.keys.size(); ++i) { + // Keep the cold half cold during the wait so the report-driven + // cold eviction keeps selecting them (once leases expire) and + // offloads them to LOCAL_DISK; the final re-read then hits + // LOCAL_DISK-only replicas and triggers promotion-on-hit. In + // admission_test_mode / prefetch_test_mode ALL keys are + // reported cold + LOCAL_DISK so every seeded key gets + // offloaded and becomes a report-driven admission / prefetch + // candidate. + const bool offload_all = + FLAGS_admission_test_mode || FLAGS_prefetch_test_mode; + const bool is_hot = offload_all ? false : seed_outcome.hot[i]; + snapshot.keys.push_back(KeyMetrics{ + .object = {.tenant_id = tenant, + .key = seed_outcome.keys[i]}, + .session_id = "promo-test", + .last_access_time_ns = + is_hot ? now_ns + : (now_ns > 60'000'000'000ULL + ? now_ns - 60'000'000'000ULL + : 0ULL), + .access_count_window = is_hot ? 3U : 0U, + .block_size = FLAGS_kv_block_bytes, + .token_count = 16, + .prefix_depth = 0, + .prefix_fanout = 1, + .match_length = 0, + .recompute_cost = 0.0F, + .replica_tiers = offload_all + ? CacheTierBit(CacheTier::kLocalDisk) + : CacheTierBit(CacheTier::kL1Host), + .request_priority = 1, + .active = is_hot}); + } + snapshot.storage.push_back( + StorageMetric{.source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.5F}); + send_snapshot(snapshot); + std::this_thread::sleep_for(std::chrono::seconds(5)); + } + // S8.1: report-driven admission trigger. After the wait all seeded + // keys have been offloaded to LOCAL_DISK; report them hot + LOCAL_DISK + // so DeriveAdmissionCandidates selects them and the report-driven + // admission path promotes them (the promotion counter increments and + // the admission observer raises master_io_pattern_report_admissions). + // Skipped in prefetch_test_mode so the prefetch trigger below is the + // only report-driven dimension observed for S11. + if (FLAGS_admission_test_mode && !FLAGS_prefetch_test_mode) { + const uint64_t now_ns = SteadyNowNs(); + const TenantId tenant(FLAGS_tenant); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = now_ns; + for (size_t i = 0; i < seed_outcome.keys.size(); ++i) { + snapshot.keys.push_back(KeyMetrics{ + .object = {.tenant_id = tenant, + .key = seed_outcome.keys[i]}, + .session_id = "admission-test", + .last_access_time_ns = now_ns, + .access_count_window = 3U, + .block_size = FLAGS_kv_block_bytes, + .token_count = 16, + .prefix_depth = 0, + .prefix_fanout = 1, + .match_length = 0, + .recompute_cost = 0.0F, + .replica_tiers = CacheTierBit(CacheTier::kLocalDisk), + .request_priority = 1, + .active = true}); + } + snapshot.storage.push_back( + StorageMetric{.source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.5F}); + send_snapshot(snapshot); + LOG(INFO) << "[ADMISSION-TEST] sent hot+LOCAL_DISK report keys=" + << snapshot.keys.size(); + std::this_thread::sleep_for(std::chrono::seconds(5)); + } + // S11: report-driven prefetch trigger. After the wait all seeded keys + // have real LOCAL_DISK replicas (offloaded by the cold-eviction + // driver). Report them as active hits carrying a high match_length and + // code-agent metrics (RuleConfidence=1.0 >= minimum_confidence=0.6) so + // DeriveTraceHistory emits trace events, TraceBasedPrefetchOps:: + // Evaluate emits LOCAL_DISK -> kL1Host candidates (match-length gate + + // confidence gate + LOCAL_DISK source gate), and the master prefetch + // handler pushes them into the promotion queue. The replica variant + // selects which gates S11.2/S11.3 exercise. + if (FLAGS_prefetch_test_mode) { + const uint64_t now_ns = SteadyNowNs(); + const TenantId tenant(FLAGS_tenant); + const std::string& variant = FLAGS_prefetch_replica_variant; + CacheTierMask tiers = 0; + if (variant == "local_disk") { + tiers = CacheTierBit(CacheTier::kLocalDisk); + } else if (variant == "l1_only") { + tiers = CacheTierBit(CacheTier::kL1Host); + } else if (variant == "l2_only") { + tiers = CacheTierBit(CacheTier::kL2Segment); + } else if (variant == "l3_only") { + tiers = CacheTierBit(CacheTier::kL3NofSsd); + } else { + tiers = CacheTierBit(CacheTier::kLocalDisk) | + CacheTierBit(CacheTier::kL1Host); + } + // S11 readiness gate: the wait phase queues async offloads, so a + // trigger sent before they finish lets the same-cycle fallback + // eviction delete a MEMORY-only key (OpType::REMOVE). That key is + // then still listed as a prefetch candidate and the handler aborts + // with OBJECT_NOT_FOUND(-704), which also degrades the policy + // engine. Poll get_replica_desc (Query also renews the keys' + // leases, so eviction skips them while they drain) until every + // seeded key holds a real LOCAL_DISK replica. Keys that are never + // ready (already deleted by a fallback eviction) are filtered out + // of the trigger report so the prefetch handler only sees objects + // that actually exist with a LOCAL_DISK source. + std::vectorstd::string prefetch_keys = seed_outcome.keys; + if (FLAGS_prefetch_ready_poll_sec > 0 && + !seed_outcome.keys.empty() && + (tiers & CacheTierBit(CacheTier::kLocalDisk)) != 0) { + auto wait_client = mooncake::RealClient::create(); + const int setup_ret = wait_client->setup_real( + FLAGS_local_hostname, FLAGS_metadata_server, + FLAGS_global_segment_size, FLAGS_local_buffer_size, + FLAGS_protocol, FLAGS_device_name, FLAGS_master_server, + nullptr, "", false, "", FLAGS_tenant); + if (setup_ret != 0) { + LOG(ERROR) << "[PREFETCH-TEST] setup_real failed for " + "LOCAL_DISK readiness poll: " + << setup_ret; + } else { + const auto poll_start = Clock::now(); + const auto poll_end = + poll_start + + std::chrono::seconds(FLAGS_prefetch_ready_poll_sec); + size_t ready = 0; + while (Clock::now() < poll_end) { + ready = 0; + prefetch_keys.clear(); + for (const auto& key : seed_outcome.keys) { + const auto descs = + wait_client->get_replica_desc(key); + bool has_local_disk = false; + for (const auto& d : descs) { + if (d.is_local_disk_replica()) { + has_local_disk = true; + break; + } + } + if (has_local_disk) { + ++ready; + prefetch_keys.push_back(key); + } + } + const auto elapsed_s = + std::chrono::duration_cast( + Clock::now() - poll_start) + .count(); + LOG(INFO) + << "[PREFETCH-TEST] local_disk-ready keys=" << ready + << "/" << seed_outcome.keys.size() + << " elapsed_sec=" << elapsed_s; + if (ready == seed_outcome.keys.size()) break; + std::this_thread::sleep_for(std::chrono::seconds(2)); + } + if (ready < seed_outcome.keys.size()) { + LOG(WARNING) + << "[PREFETCH-TEST] LOCAL_DISK readiness " + "TIMEOUT: ready=" + << ready << "/" << seed_outcome.keys.size() + << " after " << FLAGS_prefetch_ready_poll_sec + << "s; trigger will use only the " + "confirmed-ready keys"; + } + } + } + for (uint64_t rep = 0; rep < FLAGS_prefetch_repeat_reports; ++rep) { + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = now_ns + rep; + snapshot.keys.reserve(prefetch_keys.size()); + for (size_t i = 0; i < prefetch_keys.size(); ++i) { + snapshot.keys.push_back( + KeyMetrics{.object = {.tenant_id = tenant, + .key = prefetch_keys[i]}, + .session_id = "prefetch-test", + .last_access_time_ns = now_ns, + .access_count_window = 3U, + .block_size = 512U * 1024U, + .token_count = 16385U, + .prefix_depth = 0, + .prefix_fanout = 32U, + .match_length = + static_cast(FLAGS_prefetch_match_length), + .continuous_prefix_length = + static_cast(FLAGS_prefetch_match_length), + .recompute_cost = 0.0F, + .replica_tiers = tiers, + .request_priority = 1, + .active = true}); + } + snapshot.storage.push_back( + StorageMetric{.source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.5F}); + send_snapshot(snapshot); + LOG(INFO) << "[PREFETCH-TEST] trigger rep=" << rep + << " keys=" << snapshot.keys.size() + << " match_length=" << FLAGS_prefetch_match_length + << " variant=" << variant; + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + LOG(INFO) << "[PREFETCH-TEST] triggers done"; + } + LOG(INFO) << "[PROMO-TEST] wait done, re-reading seeded keys..."; + auto client = mooncake::RealClient::create(); + const size_t block_bytes = std::max(FLAGS_value_size, 4096); + char* buffer = reinterpret_cast(numa_alloc_local(block_bytes)); + if (buffer) { + std::memset(buffer, 0xA5, block_bytes); + int ret = client->setup_real( + FLAGS_local_hostname, FLAGS_metadata_server, + FLAGS_global_segment_size, FLAGS_local_buffer_size, + FLAGS_protocol, FLAGS_device_name, FLAGS_master_server, nullptr, + "", false, "", FLAGS_tenant); + if (ret == 0) ret = client->register_buffer(buffer, block_bytes); + if (ret == 0) { + uint64_t promo_hits = 0, promo_misses = 0; + for (const auto& key : seed_outcome.keys) { + for (int a = 0; a < 5; ++a) { + int64_t got = + client->get_into(key, buffer, FLAGS_value_size); + if (got >= 0) + ++promo_hits; + else + ++promo_misses; + } + } + LOG(INFO) << "[PROMO-TEST] get_into hits=" << promo_hits + << " misses=" << promo_misses; + client->unregister_buffer(buffer); + } + numa_free(buffer, block_bytes); + } + } + // S7.5: after promotion completes, re-mark ALL seeded keys cold so the + // report-driven cold eviction must re-evict the freshly promoted MEMORY + // replicas (they hold LOCAL_DISK copies, so eviction deletes the MEMORY + // replica directly). If promotion leaked pins/refcnts the eviction would + // stall on these keys. + if (FLAGS_post_promo_evict_wait_sec > 0 && !seed_outcome.keys.empty()) { + LOG(INFO) << "[PROMO-RECYCLE] waiting " + << FLAGS_post_promo_evict_wait_sec + << "s reporting all seeded keys cold for re-eviction..."; + const auto recycle_end = + Clock::now() + + std::chrono::seconds(FLAGS_post_promo_evict_wait_sec); + while (Clock::now() < recycle_end) { + const uint64_t now_ns = SteadyNowNs(); + const TenantId tenant(FLAGS_tenant); + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = now_ns; + for (size_t i = 0; i < seed_outcome.keys.size(); ++i) { + snapshot.keys.push_back(KeyMetrics{ + .object = {.tenant_id = tenant, + .key = seed_outcome.keys[i]}, + .session_id = "promo-recycle", + .last_access_time_ns = now_ns > 60'000'000'000ULL + ? now_ns - 60'000'000'000ULL + : 0ULL, + .access_count_window = 0U, + .block_size = FLAGS_kv_block_bytes, + .token_count = 16, + .prefix_depth = 0, + .prefix_fanout = 1, + .match_length = 0, + .recompute_cost = 0.0F, + .replica_tiers = CacheTierBit(CacheTier::kL1Host), + .request_priority = 1, + .active = false}); + } + snapshot.storage.push_back( + StorageMetric{.source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.5F}); + send_snapshot(snapshot); + std::this_thread::sleep_for(std::chrono::seconds(5)); + } + LOG(INFO) << "[PROMO-RECYCLE] recycle wait done"; + } for (size_t request_index = 0; request_index < FLAGS_requests; ++request_index) { if (real_seed_mode) break; // real keys already reported above @@ -901,28 +1282,67 @@ int main(int argc, char* argv[]) { request.accesses[i]); } source_runtime->RecordStorageMetric(request.snapshot.storage.front()); - const auto report_start = Clock::now(); const bool sent = send_snapshot(request.snapshot); report_latency.Record(ToMicroseconds(Clock::now() - report_start)); if (!sent) ++failed_reports; } + // S11.6: prefetch failure path. Synthetic keys claim a LOCAL_DISK replica + // that does not exist, so the master prefetch handler finds no real object + // (kNotFound -> OBJECT_NOT_FOUND) or no LOCAL_DISK source and returns a + // non-OK status, which increments io_pattern_report_prefetch_failures. + // Real-data seeding must be off for the objects to be absent. + if (FLAGS_prefetch_fake_local_report && !real_seed_mode) { + const uint64_t now_ns = SteadyNowNs(); + const TenantId tenant(FLAGS_tenant); + for (uint64_t rep = 0; rep < FLAGS_prefetch_repeat_reports; ++rep) { + IoPatternSnapshot snapshot; + snapshot.generated_at_ns = now_ns + rep; + const auto source_keys = source_runtime->Snapshot().keys; + snapshot.keys.reserve(source_keys.size()); + for (const auto& key : source_keys) { + snapshot.keys.push_back(KeyMetrics{ + .object = key.object, + .session_id = "prefetch-fake", + .last_access_time_ns = now_ns, + .access_count_window = 3U, + .block_size = 512U * 1024U, + .token_count = 16385U, + .prefix_depth = 0, + .prefix_fanout = 32U, + .match_length = static_cast(FLAGS_prefetch_match_length), + .continuous_prefix_length = + static_cast(FLAGS_prefetch_match_length), + .recompute_cost = 0.0F, + .replica_tiers = CacheTierBit(CacheTier::kLocalDisk), + .request_priority = 1, + .active = true}); + } + snapshot.storage.push_back( + StorageMetric{.source_id = FLAGS_node_id, + .observed_at_ns = now_ns, + .tier = CacheTier::kL1Host, + .memory_used_ratio = 0.1F}); + send_snapshot(snapshot); + LOG(INFO) << "[PREFETCH-TEST] fake LOCAL_DISK report rep=" << rep + << " keys=" << snapshot.keys.size() + << " (no real objects -> handler failure expected)"; + std::this_thread::sleep_for(std::chrono::seconds(3)); + } + } const auto submission_seconds = - std::chrono::duration(Clock::now() - benchmark_start).count(); - + std::chrono::duration(Clock::now() - benchmark_start).count(); // Stop joins the reporter worker and performs its final flush. No new // metric batch can reach the SubMaster after this returns. source_runtime->StopReports(); std::this_thread::sleep_for( std::chrono::milliseconds(FLAGS_report_flush_wait_ms)); - // The report-driven worker executes one cycle per merged report. Wait for // it to drain before reading handler counters / snapshots so the printed // numbers are deterministic. if (cfm_runtime && cfm_runtime->report_driven_execution()) { cfm_runtime->WaitForReportDrivenIdle(); } - // Embedded mode: when the report-driven worker is disabled, evaluate and // execute policy once locally (the pre-worker high-watermark trigger that // the production EvictionThreadFunc runs). With report_driven_execution @@ -932,21 +1352,18 @@ int main(int argc, char* argv[]) { if (cfm_runtime && !cfm_runtime->report_driven_execution() && !cfm_runtime->Snapshot().keys.empty()) { const auto capacity = 1024ULL * 1024 * 1024; - const auto target = - static_cast((FLAGS_memory_used_ratio - 0.80F) * - static_cast(capacity)); + const auto target = static_cast((FLAGS_memory_used_ratio - 0.80F) * + static_cast(capacity)); const auto status = cfm_runtime->Execute( - CacheTier::kL1Host, - target > 0 ? target : capacity / 10, TraceHistory{}); + CacheTier::kL1Host, target > 0 ? target : capacity / 10, + TraceHistory{}); if (status.eviction != ErrorCode::OK && status.prefetch != ErrorCode::OK && status.degraded) { LOG(WARNING) << "Local CFM evaluation degraded"; } } - const auto end_to_end_seconds = - std::chrono::duration(Clock::now() - benchmark_start).count(); - + std::chrono::duration(Clock::now() - benchmark_start).count(); const auto source_snapshot = source_runtime->Snapshot(); const auto source_metrics = source_runtime->ObservabilitySnapshot(end_to_end_seconds); @@ -958,55 +1375,54 @@ int main(int argc, char* argv[]) { const auto cfm_metrics = embedded_service ? embedded_service->Observability(end_to_end_seconds) : IoPatternObservabilitySnapshot{}; - - std::cout << "\n============================================================\n" - << "CFM CLIENT BENCHMARK (vLLM inference request model)\n" - << "============================================================\n" - << " CFM deployment: " << deployment_description << "\n" - << " Real-data seeding: written=" << seed_stats.written - << " (failures=" << seed_stats.write_failures - << "), reads=" << seed_stats.reads - << " (failures=" << seed_stats.read_failures << ")\n" - << " Requests: " << FLAGS_requests << "\n" - << " Tokens/request: " - << FLAGS_prompt_tokens + FLAGS_output_tokens << " (prompt=" - << FLAGS_prompt_tokens << ", decode=" << FLAGS_output_tokens - << ")\n" - << " KV blocks/request: " << BlockCount( - FLAGS_prompt_tokens + FLAGS_output_tokens) * FLAGS_num_layers - << " (layers=" << FLAGS_num_layers << ")\n" - << " Total KV blocks: " << total_blocks << "\n" - << " Request submission time: " << std::fixed - << std::setprecision(2) << submission_seconds << " s\n" - << " Submission requests/sec: " - << FLAGS_requests / submission_seconds << "\n" - << " End-to-end time: " << end_to_end_seconds << " s\n" - << "\n CFM SendSnapshot latency\n" - << " failed reports: " << failed_reports << "\n" - << " mean: " << report_latency.Mean() << " us\n" - << " p50 / p90 / p99: " << report_latency.Percentile(50) - << " / " << report_latency.Percentile(90) << " / " - << report_latency.Percentile(99) << " us\n" - << "\n CFM report_metric_batch latency\n" - << " calls / failures: " << metric_report_snapshot.calls - << " / " << metric_report_snapshot.failures << "\n" - << " observations: " - << metric_report_snapshot.observations - << "\n" - << " mean: " - << metric_report_snapshot.latency.Mean() - << " us\n" - << " p50 / p90 / p99: " - << metric_report_snapshot.latency.Percentile(50) << " / " - << metric_report_snapshot.latency.Percentile(90) << " / " - << metric_report_snapshot.latency.Percentile(99) << " us\n" - << "\n Local policy handlers executed\n" - << " evictions: " << eviction_commands << "\n" - << " prefetches: " << prefetch_commands << "\n" - << " admissions: " << admission_commands << "\n" - << "\n IO Pattern snapshots\n" - << " client keys / storage: " << source_snapshot.keys.size() << " / " - << source_snapshot.storage.size() << "\n"; + std::cout + << "\n============================================================\n" + << "CFM CLIENT BENCHMARK (vLLM inference request model)\n" + << "============================================================\n" + << " CFM deployment: " << deployment_description << "\n" + << " Real-data seeding: written=" << seed_stats.written + << " (failures=" << seed_stats.write_failures + << "), reads=" << seed_stats.reads + << " (failures=" << seed_stats.read_failures << ")\n" + << " Requests: " << FLAGS_requests << "\n" + << " Tokens/request: " + << FLAGS_prompt_tokens + FLAGS_output_tokens + << " (prompt=" << FLAGS_prompt_tokens + << ", decode=" << FLAGS_output_tokens << ")\n" + << " KV blocks/request: " + << BlockCount(FLAGS_prompt_tokens + FLAGS_output_tokens) * + FLAGS_num_layers + << " (layers=" << FLAGS_num_layers << ")\n" + << " Total KV blocks: " << total_blocks << "\n" + << " Request submission time: " << std::fixed << std::setprecision(2) + << submission_seconds << " s\n" + << " Submission requests/sec: " << FLAGS_requests / submission_seconds + << "\n" + << " End-to-end time: " << end_to_end_seconds << " s\n" + << "\n CFM SendSnapshot latency\n" + << " failed reports: " << failed_reports << "\n" + << " mean: " << report_latency.Mean() << " us\n" + << " p50 / p90 / p99: " << report_latency.Percentile(50) + << " / " << report_latency.Percentile(90) << " / " + << report_latency.Percentile(99) << " us\n" + << "\n CFM report_metric_batch latency\n" + << " calls / failures: " << metric_report_snapshot.calls + << " / " << metric_report_snapshot.failures << "\n" + << " observations: " << metric_report_snapshot.observations + << "\n" + << " mean: " + << metric_report_snapshot.latency.Mean() << " us\n" + << " p50 / p90 / p99: " + << metric_report_snapshot.latency.Percentile(50) << " / " + << metric_report_snapshot.latency.Percentile(90) << " / " + << metric_report_snapshot.latency.Percentile(99) << " us\n" + << "\n Local policy handlers executed\n" + << " evictions: " << eviction_commands << "\n" + << " prefetches: " << prefetch_commands << "\n" + << " admissions: " << admission_commands << "\n" + << "\n IO Pattern snapshots\n" + << " client keys / storage: " << source_snapshot.keys.size() << " / " + << source_snapshot.storage.size() << "\n"; if (embedded_service) { std::cout << " CFM keys / storage: " << cfm_snapshot.keys.size() << " / " << cfm_snapshot.storage.size() << "\n"; @@ -1015,11 +1431,13 @@ int main(int argc, char* argv[]) { // receiving SubMaster's own master admin metrics (`io_pattern_report_*` // in `/metrics` and the periodic "Master Admin Metrics" log) and its // [IO-PATTERN-REPORT-CYCLE] log lines. - std::cout << " CFM keys / storage: remote endpoint (see SubMaster " - "master admin metrics)\n"; + std::cout + << " CFM keys / storage: remote endpoint (see SubMaster " + "master admin metrics)\n"; } PrintObservability("Client", source_metrics); if (embedded_service) PrintObservability("CFM", cfm_metrics); - std::cout << "============================================================\n"; + std::cout + << "============================================================\n"; return failed_reports == 0 ? 0 : 2; -} +} \ No newline at end of file diff --git a/mooncake-store/src/master_metric_manager.cpp b/mooncake-store/src/master_metric_manager.cpp index 9b40715e14..ed85529ee5 100644 --- a/mooncake-store/src/master_metric_manager.cpp +++ b/mooncake-store/src/master_metric_manager.cpp @@ -6,6 +6,7 @@ #include // For string building during serialization #include // Required by histogram serialization #include +#include #include "utils.h" #include "io_pattern/runtime.h" @@ -2812,6 +2813,41 @@ std::string MasterMetricManager::get_summary_string( << io_pattern_report_admission_failures_.value() << ", " << "degraded=" << io_pattern_report_degraded_.value(); + std::optional observation; + io_pattern::PolicyFeedbackStats feedback; + { + // As with /metrics, release the temporary strong reference under the + // registration lock so service shutdown cannot race a summary read. + std::lock_guard lock(io_pattern_runtime_mutex_); + auto runtime = io_pattern_runtime_.lock(); + if (runtime) { + observation = runtime->ObservabilitySnapshot(); + feedback = runtime->FeedbackSnapshot(); + } + } + if (observation) { + // Keep precision local: small deltas must remain visible without + // changing the formatting of the rest of the admin summary. + std::ostringstream io_summary; + io_summary + << std::setprecision(6) << " | IO Pattern (runtime, lifetime): " + << "collect_latency_max_us=" << observation->collect_latency_us + << ", analyze_latency_max_us=" << observation->analyze_latency_us + << ", policy_decision_qps=" << observation->policy_decision_qps + << ", policy_decisions=" << observation->policy_decisions + << ", strategy_hit_rate=" << observation->strategy_hit_rate + << ", false_positive_rate=" << observation->false_positive_rate + << ", degrade_count=" << observation->degrade_count + << ", report_drop_count=" << observation->report_drop_count + << " | IO Pattern (feedback, sample window): " + << "hit_rate_delta=" << feedback.hit_rate_delta + << ", eviction_churn=" << feedback.eviction_churn + << ", ttft_delta=" << feedback.ttft_delta + << ", prefetch_accuracy=" << feedback.prefetch_accuracy + << ", feedback_samples=" << feedback.samples; + ss << io_summary.str(); + } + // Discard summary ss << " | Discard: " << "Released/Total=" << put_start_release_cnt << "/" diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 1c7de16211..e536f9842b 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -2288,6 +2288,23 @@ TEST(IoPatternFrameworkTest, RuntimeMetricsExport) { EXPECT_DOUBLE_EQ(value(first, "master_io_pattern_policy_decisions_total"), 1.0); EXPECT_GT(value(first, "master_io_pattern_policy_decision_qps"), 0.0); + // Both the HTTP summary and periodic Master Admin Metrics log must expose + // the same runtime and feedback values without consuming the counters. + for (const auto& summary : + {metrics.get_summary_string(), + metrics.get_summary_string_and_update_snapshot()}) { + EXPECT_NE(summary.find("IO Pattern (runtime, lifetime):"), + std::string::npos); + for (const auto* field : + {"collect_latency_max_us=", "analyze_latency_max_us=", + "policy_decision_qps=", "policy_decisions=1", + "strategy_hit_rate=0", "false_positive_rate=0", "degrade_count=0", + "report_drop_count=0", "hit_rate_delta=-0.25", + "eviction_churn=0.5", "ttft_delta=-0.125", + "prefetch_accuracy=0.75", "feedback_samples=1"}) { + EXPECT_NE(summary.find(field), std::string::npos) << field; + } + } // Scrapes must not increment cumulative counters or consume feedback. const auto second = metrics.serialize_metrics(); EXPECT_DOUBLE_EQ(value(second, "master_io_pattern_policy_decisions_total"), @@ -2308,11 +2325,16 @@ TEST(IoPatternFrameworkTest, RuntimeMetricsExport) { EXPECT_DOUBLE_EQ(value(degraded, "master_io_pattern_report_drop_count"), 1.0); EXPECT_GE(value(degraded, "master_io_pattern_degrade_count"), 1.0); + EXPECT_NE(metrics.get_summary_string().find("report_drop_count=1"), + std::string::npos); // The singleton must not retain a runtime (or its MasterService handlers). std::weak_ptr weak = runtime; runtime.reset(); EXPECT_TRUE(weak.expired()); + EXPECT_EQ( + metrics.get_summary_string().find("IO Pattern (runtime, lifetime):"), + std::string::npos); EXPECT_EQ(metrics.serialize_metrics().find( "# TYPE master_io_pattern_hit_rate_delta "), std::string::npos); @@ -2325,6 +2347,9 @@ TEST(IoPatternFrameworkTest, RuntimeMetricsExport) { "master_io_pattern_policy_decisions_total"), 0.0); metrics.clear_io_pattern_runtime(replacement.get()); + EXPECT_EQ(metrics.get_summary_string_and_update_snapshot().find( + "IO Pattern (feedback, sample window):"), + std::string::npos); EXPECT_EQ(metrics.serialize_metrics().find( "# TYPE master_io_pattern_hit_rate_delta "), std::string::npos); diff --git a/mooncake-store/tests/master_metrics_test.cpp b/mooncake-store/tests/master_metrics_test.cpp index 8e2d351940..36d47760c1 100644 --- a/mooncake-store/tests/master_metrics_test.cpp +++ b/mooncake-store/tests/master_metrics_test.cpp @@ -1055,6 +1055,11 @@ TEST_F(MasterMetricsTest, AdminMetricsExposeIoPatternFeedbackFromAccesses) { EXPECT_NE(response.body.find( "# TYPE master_io_pattern_policy_decisions_total counter"), std::string::npos); + const auto summary = FetchUrl(http_port, "/metrics/summary"); + ASSERT_EQ(summary.http_status, 200); + EXPECT_NE(summary.body.find("IO Pattern (runtime, lifetime):"), + std::string::npos); + EXPECT_NE(summary.body.find("feedback_samples=1"), std::string::npos); admin_server.Stop(); } From 987b07e08dc5f1c53b5684ce2ff5eeffcc03f29a Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 15:36:06 +0800 Subject: [PATCH 43/47] add io_pattern metric --- .../benchmarks/cfm_client_bench.cpp | 225 +++++++++--------- 1 file changed, 119 insertions(+), 106 deletions(-) diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index c52c12ab9a..efc693c4c4 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -25,32 +25,32 @@ // Remote policy execution is only observable when the SubMaster actually owns // the reported keys: its eviction/promotion/prefetch handlers operate on real // replicas, so a report-only run leaves the master-side counters at zero. When -// --master_server and --num_keys are provided, a real-data seeding stage runs +//--master_server and --num_keys are provided, a real-data seeding stage runs // first ("先种子后仿真"): it writes a batch of real KV objects through // RealClient (keys share the simulated KvKey naming and tenant) and reads a // subset back to simulate access heat, then the simulated vLLM request stream // reports on those same keys. -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include "gflags/gflags.h" #include "glog/logging.h" @@ -110,7 +110,7 @@ DEFINE_string(cfm_cluster_namespace, "", DEFINE_string(master_server, "", "Master server address (host:port) for RealClient writes; empty " "disables real-data seeding"); -DEFINE_string(local_hostname, "localhost", +DEFINE_string(local_hostname, "[localhost](https://localhost)", "Local hostname (with optional port, e.g. node1:12345)"); DEFINE_string( metadata_server, @@ -233,16 +233,17 @@ DEFINE_int32(prefetch_ready_poll_sec, 120, "holds the keys' leases so eviction cannot delete them. 0 " "disables the poll."); uint64_t SteadyNowNs() { - return static_cast(std::chrono::duration_caststd::chrono::nanoseconds( - Clock::now().time_since_epoch()) - .count()); + return static_cast( + std::chrono::duration_caststd::chrono::nanoseconds( + Clock::now().time_since_epoch()) + .count()); } double ToMicroseconds(Clock::duration duration) { return std::chrono::duration(duration).count(); } size_t BlockCount(uint64_t tokens) { - return static_cast((tokens + FLAGS_tokens_per_block - 1) / - FLAGS_tokens_per_block); + return static_cast((tokens + FLAGS_tokens_per_block - 1) / + FLAGS_tokens_per_block); } std::string KvKey(size_t session, size_t request, size_t layer, size_t block, bool is_shared_prefix) { @@ -278,7 +279,7 @@ std::string SessionWorkloadType(size_t session) { // benchmark run, exercised without network. class EmbeddedCfmTransport final : public CfmRpcTransport { public: - explicit EmbeddedCfmTransport(std::shared_ptr service) + explicit EmbeddedCfmTransport(std::shared_ptr service) : service_(std::move(service)) {} bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds) override { @@ -286,7 +287,7 @@ class EmbeddedCfmTransport final : public CfmRpcTransport { } private: - std::shared_ptr service_; + std::shared_ptr service_; }; #ifdef STORE_USE_ETCD // Resolves the CVM cluster namespace used by --cfm_endpoint when it carries an @@ -316,7 +317,7 @@ std::string BuildMasterViewKey(const std::string& cluster_namespace) { // returns an ownership resolver that routes every key to it. If it is an // etcd:// entry, it resolves the cluster like a Store client: a present // leader master_view yields a single target; otherwise the CVM -// /cvm//masters registry plus slot ownership is used to bucket keys to +// /cvm//masters registry plus slot ownership is used to bucket keys to // their owning SubMaster. Returns an empty resolver on any resolution failure // (the caller aborts instead of hanging). SubmasterEndpointResolver ResolveCfmEndpointOwnership() { @@ -384,7 +385,8 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { std::map address_by_master; // id -> host:port std::vectorstd::string primary_ids; for (const auto& reg : masters) { - if (reg.role == static_cast(mooncake::cvm::MasterRole::kPrimary) && + if (reg.role == + static_cast(mooncake::cvm::MasterRole::kPrimary) && !reg.address.empty()) { address_by_master[reg.master_id] = reg.address; primary_ids.push_back(reg.master_id); @@ -405,7 +407,8 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { cluster_namespace, slot_owners, version); if (slot_err == ErrorCode::OK) { for (const auto& owner : slot_owners) { - if (owner.state == static_cast(mooncake::cvm::SlotState::kStable) && + if (owner.state == + static_cast(mooncake::cvm::SlotState::kStable) && !owner.primary_master_id.empty()) { owner_by_slot[owner.slot] = owner.primary_master_id; } @@ -442,7 +445,7 @@ class LatencyStats final { double Percentile(double percentile) const { if (values_us_.empty()) return 0.0; const double rank = percentile / 100.0 * (values_us_.size() - 1); - const auto lower = static_cast(rank); + const auto lower = static_cast(rank); const auto upper = std::min(lower + 1, values_us_.size() - 1); const double fraction = rank - lower; return values_us_[lower] * (1.0 - fraction) + @@ -456,7 +459,7 @@ class LatencyStats final { } private: - std::vector values_us_; + std::vector values_us_; }; struct MetricReportSnapshot { uint64_t calls{0}; @@ -492,8 +495,8 @@ class MetricReportStats final { }; struct RequestData { IoPatternSnapshot snapshot; - std::vector inference; - std::vector accesses; + std::vector inference; + std::vector accesses; }; RequestData BuildRequest(size_t request_index) { const size_t session = request_index % FLAGS_num_sessions; @@ -518,23 +521,26 @@ RequestData BuildRequest(size_t request_index) { const ObjectRef object{.tenant_id = tenant, .key = KvKey(session, request_index, layer, block, is_shared_prefix)}; - const auto block_end = - std::min(total_tokens, (block + 1) * FLAGS_tokens_per_block); - const auto block_tokens = - static_cast(block_end - block * FLAGS_tokens_per_block); + const auto block_end = std::min( + total_tokens, (block + 1) * FLAGS_tokens_per_block); + const auto block_tokens = static_cast( + block_end - block * FLAGS_tokens_per_block); InferenceMetrics inference{ .object = object, .session_id = session_id, .layout = CacheLayout::kLayerFirst, - .layout_group = static_cast(layer), - .prefix_depth = static_cast(shared_blocks), - .prefix_fanout = static_cast(FLAGS_num_sessions), + .layout_group = static_cast(layer), + .prefix_depth = static_cast(shared_blocks), + .prefix_fanout = static_cast(FLAGS_num_sessions), .match_length = - is_hit ? static_cast(FLAGS_shared_prefix_tokens) : 0U, + is_hit ? static_cast(FLAGS_shared_prefix_tokens) + : 0U, .continuous_prefix_length = - is_hit ? static_cast(FLAGS_shared_prefix_tokens) : 0U, + is_hit ? static_cast(FLAGS_shared_prefix_tokens) + : 0U, .token_count = block_tokens, - .recompute_cost = is_hit ? 0.0F : static_cast(block_tokens), + .recompute_cost = + is_hit ? 0.0F : static_cast(block_tokens), .request_priority = 1}; AccessRecord access{ .object = object, @@ -544,7 +550,8 @@ RequestData BuildRequest(size_t request_index) { .tier = CacheTier::kL1Host, .operation = is_hit ? IoOperation::kGet : IoOperation::kPut, .is_hit = is_hit, - .write_batch_size = is_hit ? 0U : static_cast(FLAGS_num_layers), + .write_batch_size = + is_hit ? 0U : static_cast(FLAGS_num_layers), .overwrite = !is_hit && is_shared_prefix}; // Override reported metrics for S4 workload type testing if (!workload_type.empty()) { @@ -571,8 +578,8 @@ RequestData BuildRequest(size_t request_index) { .access_count_window = 3, .block_size = FLAGS_kv_block_bytes, .token_count = block_tokens, - .prefix_depth = static_cast(shared_blocks), - .prefix_fanout = static_cast(FLAGS_num_sessions), + .prefix_depth = static_cast(shared_blocks), + .prefix_fanout = static_cast(FLAGS_num_sessions), .match_length = inference.match_length, .continuous_prefix_length = inference.continuous_prefix_length, .write_batch_size = access.write_batch_size, @@ -583,7 +590,7 @@ RequestData BuildRequest(size_t request_index) { .overwrite_ratio = access.overwrite ? 1.0F : 0.0F, .replica_tiers = CacheTierBit(CacheTier::kL1Host), .layout = CacheLayout::kLayerFirst, - .layout_group = static_cast(layer), + .layout_group = static_cast(layer), .request_priority = 1, .active = is_hit, .write_burst = !is_hit}; @@ -612,10 +619,11 @@ RequestData BuildRequest(size_t request_index) { .write_bandwidth_bytes_per_sec = 10ULL * 1024 * 1024 * 1024, .read_latency_us = 20, .write_latency_us = 200, - .used_bytes = static_cast(FLAGS_memory_used_ratio * 1024 * 1024 * 1024), + .used_bytes = + static_cast(FLAGS_memory_used_ratio * 1024 * 1024 * 1024), .capacity_bytes = 1024ULL * 1024 * 1024, .rpc_latency_us = 100, - .memory_used_ratio = static_cast(FLAGS_memory_used_ratio)}); + .memory_used_ratio = static_cast(FLAGS_memory_used_ratio)}); return request; } void PrintObservability(std::string_view name, @@ -664,7 +672,7 @@ struct SeedStats { struct SeedOutcome { SeedStats stats; std::vectorstd::string keys; // real keys written, stable order - std::vector hot; // parallel: read back (access heat) + std::vector hot; // parallel: read back (access heat) }; SeedOutcome RunRealSeedStage() { SeedOutcome outcome; @@ -678,7 +686,7 @@ SeedOutcome RunRealSeedStage() { << " replica_num=" << FLAGS_replica_num << " offload=" << (FLAGS_enable_ssd_offload ? "yes" : "no"); auto client = mooncake::RealClient::create(); - const size_t block_bytes = std::max(FLAGS_value_size, 4096); + const size_t block_bytes = std::max(FLAGS_value_size, 4096); char* buffer = reinterpret_cast(numa_alloc_local(block_bytes)); if (buffer == nullptr) { LOG(ERROR) << "numa_alloc_local failed for seed buffer of " @@ -709,7 +717,7 @@ SeedOutcome RunRealSeedStage() { // reported key universe (real replicas exist for the keys policy will // select). mooncake::ReplicateConfig config; - config.replica_num = static_cast(FLAGS_replica_num); + config.replica_num = static_cast(FLAGS_replica_num); config.with_hard_pin = FLAGS_hard_pin; const uint64_t total_tokens = FLAGS_prompt_tokens + FLAGS_output_tokens; const size_t blocks = BlockCount(total_tokens); @@ -745,9 +753,10 @@ SeedOutcome RunRealSeedStage() { // SubMaster records real GET access heat (promotion-on-hit when offloaded). // Mark the same prefix of the written key list as hot for the report pass. outcome.hot.assign(outcome.keys.size(), false); - const uint64_t read_count = FLAGS_seed_get_keys == 0 - ? seeded / 2 - : std::min(FLAGS_seed_get_keys, seeded); + const uint64_t read_count = + FLAGS_seed_get_keys == 0 + ? seeded / 2 + : std::min(FLAGS_seed_get_keys, seeded); uint64_t read_keys = 0; for (size_t request_index = 0; request_index < FLAGS_requests && read_keys < read_count; @@ -799,15 +808,15 @@ int main(int argc, char* argv[]) { "--memory_used_ratio must be within [0, 1]"; return 1; } - std::atomic eviction_commands{0}; - std::atomic prefetch_commands{0}; - std::atomic admission_commands{0}; + std::atomic eviction_commands{0}; + std::atomic prefetch_commands{0}; + std::atomic admission_commands{0}; // The SubMaster-side CFM component (embedded mode) or the ownership // resolver used by the remote reporter. - std::shared_ptr embedded_service; - std::shared_ptr cfm_runtime; - std::shared_ptr ownership_client; - std::shared_ptr embedded_channel; + std::shared_ptr embedded_service; + std::shared_ptr cfm_runtime; + std::shared_ptr ownership_client; + std::shared_ptr embedded_channel; std::string deployment_description; if (FLAGS_cfm_endpoint.empty()) { deployment_description = "embedded SubMaster (local CFM)"; @@ -818,7 +827,7 @@ int main(int argc, char* argv[]) { // watermark evaluation at the end of the run. cfm_config.report_driven_execution = true; cfm_config.max_analysis_keys = FLAGS_max_analysis_keys; - cfm_runtime = std::make_shared( + cfm_runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [&eviction_commands](const EvictionPlan&) { @@ -836,21 +845,22 @@ int main(int argc, char* argv[]) { return ErrorCode::OK; }}, std::move(cfm_config)); - embedded_service = std::make_shared(cfm_runtime); - auto transport = std::make_shared(embedded_service); - embedded_channel = - std::make_shared(std::move(transport), std::make_shared(), - CfmRpcConfig{.timeout = std::chrono::milliseconds( - FLAGS_cfm_rpc_timeout_ms)}); + embedded_service = std::make_shared(cfm_runtime); + auto transport = + std::make_shared(embedded_service); + embedded_channel = std::make_shared( + std::move(transport), std::make_shared(), + CfmRpcConfig{.timeout = std::chrono::milliseconds( + FLAGS_cfm_rpc_timeout_ms)}); } else { const auto resolver = ResolveCfmEndpointOwnership(); - ownership_client = std::make_shared( + ownership_client = std::make_shared( resolver, std::chrono::milliseconds(FLAGS_cfm_rpc_timeout_ms)); ownership_client->set_forward_storage(FLAGS_report_forward_storage); - deployment_description = "remote SubMaster(s) via CFM coro_rpc"; + deployment_description = "remote SubMaster (s) via CFM coro_rpc"; } - // Real-data seeding runs before the simulated request stream ("先种子后仿 - // 真"): the SubMaster must hold real replicas for reported keys before the + // Real-data seeding runs before the simulated request stream (" 先种子后仿 + // 真 "): the SubMaster must hold real replicas for reported keys before the // report-driven policy cycle can execute eviction/promotion/prefetch // against them. Only meaningful with a real SubMaster endpoint // (--cfm_endpoint) plus RealClient parameters; otherwise it is a no-op. @@ -873,7 +883,7 @@ int main(int argc, char* argv[]) { return success; }; source_config.report_sink = report_metric_batch; - auto source_runtime = std::make_shared( + auto source_runtime = std::make_shared( IoPatternRuntime::Handlers{ .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, @@ -951,11 +961,13 @@ int main(int argc, char* argv[]) { .write_bandwidth_bytes_per_sec = 10ULL * 1024 * 1024 * 1024, .read_latency_us = 20, .write_latency_us = 200, - .used_bytes = static_cast(static_cast(FLAGS_value_size) * + .used_bytes = + static_cast(static_cast(FLAGS_value_size) * seed_outcome.keys.size()), - .capacity_bytes = static_cast(FLAGS_num_keys) * FLAGS_value_size, + .capacity_bytes = + static_cast(FLAGS_num_keys) * FLAGS_value_size, .rpc_latency_us = 100, - .memory_used_ratio = static_cast(FLAGS_memory_used_ratio)}); + .memory_used_ratio = static_cast(FLAGS_memory_used_ratio)}); const auto report_start = Clock::now(); const bool sent = send_snapshot(real_snapshot); report_latency.Record(ToMicroseconds(Clock::now() - report_start)); @@ -1161,24 +1173,24 @@ int main(int argc, char* argv[]) { snapshot.generated_at_ns = now_ns + rep; snapshot.keys.reserve(prefetch_keys.size()); for (size_t i = 0; i < prefetch_keys.size(); ++i) { - snapshot.keys.push_back( - KeyMetrics{.object = {.tenant_id = tenant, - .key = prefetch_keys[i]}, - .session_id = "prefetch-test", - .last_access_time_ns = now_ns, - .access_count_window = 3U, - .block_size = 512U * 1024U, - .token_count = 16385U, - .prefix_depth = 0, - .prefix_fanout = 32U, - .match_length = - static_cast(FLAGS_prefetch_match_length), - .continuous_prefix_length = - static_cast(FLAGS_prefetch_match_length), - .recompute_cost = 0.0F, - .replica_tiers = tiers, - .request_priority = 1, - .active = true}); + snapshot.keys.push_back(KeyMetrics{ + .object = {.tenant_id = tenant, + .key = prefetch_keys[i]}, + .session_id = "prefetch-test", + .last_access_time_ns = now_ns, + .access_count_window = 3U, + .block_size = 512U * 1024U, + .token_count = 16385U, + .prefix_depth = 0, + .prefix_fanout = 32U, + .match_length = + static_cast(FLAGS_prefetch_match_length), + .continuous_prefix_length = + static_cast(FLAGS_prefetch_match_length), + .recompute_cost = 0.0F, + .replica_tiers = tiers, + .request_priority = 1, + .active = true}); } snapshot.storage.push_back( StorageMetric{.source_id = FLAGS_node_id, @@ -1196,7 +1208,7 @@ int main(int argc, char* argv[]) { } LOG(INFO) << "[PROMO-TEST] wait done, re-reading seeded keys..."; auto client = mooncake::RealClient::create(); - const size_t block_bytes = std::max(FLAGS_value_size, 4096); + const size_t block_bytes = std::max(FLAGS_value_size, 4096); char* buffer = reinterpret_cast(numa_alloc_local(block_bytes)); if (buffer) { std::memset(buffer, 0xA5, block_bytes); @@ -1310,9 +1322,10 @@ int main(int argc, char* argv[]) { .token_count = 16385U, .prefix_depth = 0, .prefix_fanout = 32U, - .match_length = static_cast(FLAGS_prefetch_match_length), + .match_length = + static_cast(FLAGS_prefetch_match_length), .continuous_prefix_length = - static_cast(FLAGS_prefetch_match_length), + static_cast(FLAGS_prefetch_match_length), .recompute_cost = 0.0F, .replica_tiers = CacheTierBit(CacheTier::kLocalDisk), .request_priority = 1, @@ -1331,7 +1344,7 @@ int main(int argc, char* argv[]) { } } const auto submission_seconds = - std::chrono::duration(Clock::now() - benchmark_start).count(); + std::chrono::duration(Clock::now() - benchmark_start).count(); // Stop joins the reporter worker and performs its final flush. No new // metric batch can reach the SubMaster after this returns. source_runtime->StopReports(); @@ -1352,8 +1365,8 @@ int main(int argc, char* argv[]) { if (cfm_runtime && !cfm_runtime->report_driven_execution() && !cfm_runtime->Snapshot().keys.empty()) { const auto capacity = 1024ULL * 1024 * 1024; - const auto target = static_cast((FLAGS_memory_used_ratio - 0.80F) * - static_cast(capacity)); + const auto target = static_cast( + (FLAGS_memory_used_ratio - 0.80F) * static_cast(capacity)); const auto status = cfm_runtime->Execute( CacheTier::kL1Host, target > 0 ? target : capacity / 10, TraceHistory{}); @@ -1363,7 +1376,7 @@ int main(int argc, char* argv[]) { } } const auto end_to_end_seconds = - std::chrono::duration(Clock::now() - benchmark_start).count(); + std::chrono::duration(Clock::now() - benchmark_start).count(); const auto source_snapshot = source_runtime->Snapshot(); const auto source_metrics = source_runtime->ObservabilitySnapshot(end_to_end_seconds); From 3ab266a7aa5987d4d2c87520d913fba47c621305 Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 15:48:43 +0800 Subject: [PATCH 44/47] add io_pattern metric --- .../benchmarks/cfm_client_bench.cpp | 101 ++++++++++++++---- 1 file changed, 81 insertions(+), 20 deletions(-) diff --git a/mooncake-store/benchmarks/cfm_client_bench.cpp b/mooncake-store/benchmarks/cfm_client_bench.cpp index efc693c4c4..810c035ef4 100644 --- a/mooncake-store/benchmarks/cfm_client_bench.cpp +++ b/mooncake-store/benchmarks/cfm_client_bench.cpp @@ -25,11 +25,12 @@ // Remote policy execution is only observable when the SubMaster actually owns // the reported keys: its eviction/promotion/prefetch handlers operate on real // replicas, so a report-only run leaves the master-side counters at zero. When -//--master_server and --num_keys are provided, a real-data seeding stage runs +// --master_server and --num_keys are provided, a real-data seeding stage runs // first ("先种子后仿真"): it writes a batch of real KV objects through // RealClient (keys share the simulated KvKey naming and tenant) and reads a // subset back to simulate access heat, then the simulated vLLM request stream // reports on those same keys. + #include #include #include @@ -52,6 +53,7 @@ #include #include #include + #include "gflags/gflags.h" #include "glog/logging.h" #include "cvm/cvm_types.h" @@ -67,12 +69,15 @@ #ifdef STORE_USE_ETCD #include "etcd_helper.h" #endif + namespace { + using Clock = std::chrono::steady_clock; using mooncake::ErrorCode; using mooncake::TenantId; using mooncake::toString; using namespace mooncake::io_pattern; + DEFINE_uint64(requests, 20, "Number of vLLM-style inference requests"); DEFINE_uint64(prompt_tokens, 1024, "Input tokens in each inference request"); DEFINE_uint64(output_tokens, 128, "Decode tokens in each inference request"); @@ -101,6 +106,7 @@ DEFINE_string(cfm_cluster_namespace, "", "MC_STORE_CLUSTER_ID or mooncake_cluster (same rule as the " "etcd leader coordinator). When the cluster was started with a " "non-default cluster_id, pass the same value here"); + // Real Store client parameters used by the optional real-data seeding stage. // Flag names and defaults mirror stress_cluster_bench.cpp so an existing // cluster invocation can be reused as-is. Seeding makes the SubMaster hold @@ -110,12 +116,10 @@ DEFINE_string(cfm_cluster_namespace, "", DEFINE_string(master_server, "", "Master server address (host:port) for RealClient writes; empty " "disables real-data seeding"); -DEFINE_string(local_hostname, "[localhost](https://localhost)", +DEFINE_string(local_hostname, "localhost", "Local hostname (with optional port, e.g. node1:12345)"); -DEFINE_string( - metadata_server, - "[http://127.0.0.1:8080/metadata](http://127.0.0.1:8080/metadata)", - "Metadata server URL for RealClient setup"); +DEFINE_string(metadata_server, "http://127.0.0.1:8080/metadata", + "Metadata server URL for RealClient setup"); DEFINE_string(protocol, "tcp", "Transport protocol: tcp, rdma, ub"); DEFINE_string(device_name, "", "RDMA/UB device name (comma-separated)"); DEFINE_uint64(global_segment_size, 16ULL * 1024 * 1024 * 1024, @@ -144,7 +148,7 @@ DEFINE_uint64(cfm_rpc_timeout_ms, 5000, "report_metric_batch) in milliseconds"); // Client-side collector/analysis key budget. The default 100k cap drops // observations once the simulated request stream exceeds it (seen as nonzero -// "report drops"); raise it to cover the whole run when reporting many keys. +// \"report drops\"); raise it to cover the whole run when reporting many keys. DEFINE_uint64( max_analysis_keys, 100000, "Max merged keys kept/analyzed by the client runtime and the " @@ -232,19 +236,23 @@ DEFINE_int32(prefetch_ready_poll_sec, 120, "whole plan. Polling get_replica_desc both confirms readiness and " "holds the keys' leases so eviction cannot delete them. 0 " "disables the poll."); + uint64_t SteadyNowNs() { return static_cast( - std::chrono::duration_caststd::chrono::nanoseconds( + std::chrono::duration_cast( Clock::now().time_since_epoch()) .count()); } + double ToMicroseconds(Clock::duration duration) { return std::chrono::duration(duration).count(); } + size_t BlockCount(uint64_t tokens) { return static_cast((tokens + FLAGS_tokens_per_block - 1) / FLAGS_tokens_per_block); } + std::string KvKey(size_t session, size_t request, size_t layer, size_t block, bool is_shared_prefix) { const auto owner = is_shared_prefix @@ -254,6 +262,7 @@ std::string KvKey(size_t session, size_t request, size_t layer, size_t block, "/" + owner + "/layer-" + std::to_string(layer) + "/block-" + std::to_string(block); } + // Per-session workload type for S4.2: index into the comma-separated // --force_session_workload_types list by session id; fall back to the global // --force_workload_type when the list is empty or the session is out of range. @@ -274,6 +283,7 @@ std::string SessionWorkloadType(size_t session) { } return FLAGS_force_workload_type; } + // Sends reports straight into an embedded SubMaster's CFM component. This is // the ownership-addressed path collapsed to the single owning SubMaster of a // benchmark run, exercised without network. @@ -281,6 +291,7 @@ class EmbeddedCfmTransport final : public CfmRpcTransport { public: explicit EmbeddedCfmTransport(std::shared_ptr service) : service_(std::move(service)) {} + bool Send(std::string_view method, std::string_view payload, std::chrono::milliseconds) override { return service_ && service_->Send(method, payload, FLAGS_node_id); @@ -289,6 +300,7 @@ class EmbeddedCfmTransport final : public CfmRpcTransport { private: std::shared_ptr service_; }; + #ifdef STORE_USE_ETCD // Resolves the CVM cluster namespace used by --cfm_endpoint when it carries an // etcd:// backend. Mirrors EtcdLeaderCoordinator::ResolveClusterNamespace: @@ -303,6 +315,7 @@ std::string ResolveCvmNamespace() { } return mooncake::DEFAULT_CLUSTER_ID; } + // Key that stores the leader address for single-leader HA. // Mirrors EtcdLeaderCoordinator::BuildMasterViewKey. std::string BuildMasterViewKey(const std::string& cluster_namespace) { @@ -313,6 +326,7 @@ std::string BuildMasterViewKey(const std::string& cluster_namespace) { return "mooncake-store/" + normalized + "/master_view"; } #endif // STORE_USE_ETCD + // If --cfm_endpoint names a single SubMaster directly ("host:port") this // returns an ownership resolver that routes every key to it. If it is an // etcd:// entry, it resolves the cluster like a Store client: a present @@ -328,7 +342,7 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { // SubMaster (the equivalent of the old single-endpoint remote mode). const std::string endpoint = entry; return [endpoint](const TenantId&, - const std::string&) -> std::optionalstd::string { + const std::string&) -> std::optional { return endpoint; }; } @@ -346,12 +360,14 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { } const std::string connstring = entry.substr(scheme_pos + 3); const std::string cluster_namespace = ResolveCvmNamespace(); + ErrorCode err = mooncake::EtcdHelper::ConnectToEtcdStoreClient(connstring); if (err != ErrorCode::OK) { LOG(FATAL) << "cfm_endpoint: failed to connect etcd '" << connstring << "': " << toString(err); return {}; } + // Single-leader HA: leader master_view holds the master address. const std::string view_key = BuildMasterViewKey(cluster_namespace); std::string leader_address; @@ -363,7 +379,7 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { << leader_address; const std::string endpoint = std::move(leader_address); return [endpoint](const TenantId&, - const std::string&) -> std::optionalstd::string { + const std::string&) -> std::optional { return endpoint; }; } @@ -372,8 +388,9 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { << toString(err); return {}; } + // CVM multi-submaster: masters registry + slot ownership. - std::vectormooncake::cvm::MasterRegistration masters; + std::vector masters; mooncake::ViewVersionId version = 0; err = mooncake::cvm::EtcdViewStore::LoadAllMasters(cluster_namespace, masters, version); @@ -382,8 +399,9 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { << cluster_namespace << "': " << toString(err); return {}; } + std::map address_by_master; // id -> host:port - std::vectorstd::string primary_ids; + std::vector primary_ids; for (const auto& reg : masters) { if (reg.role == static_cast(mooncake::cvm::MasterRole::kPrimary) && @@ -399,10 +417,11 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { return {}; } std::sort(primary_ids.begin(), primary_ids.end()); + // Prefer the authoritative slot owner table published by CvmController; // fall back to the consistent-hash ring used by the masters themselves. std::unordered_map owner_by_slot; - std::vectormooncake::cvm::SlotOwner slot_owners; + std::vector slot_owners; const ErrorCode slot_err = mooncake::cvm::EtcdViewStore::LoadAllSlotOwners( cluster_namespace, slot_owners, version); if (slot_err == ErrorCode::OK) { @@ -419,11 +438,12 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { << "' has " << primary_ids.size() << " primary submaster(s), " << (has_owner_table ? owner_by_slot.size() : 0) << " slot owners" << (has_owner_table ? "" : " (falling back to hash ring)"); + return [address_by_master = std::move(address_by_master), primary_ids = std::move(primary_ids), owner_by_slot = std::move(owner_by_slot), has_owner_table]( const TenantId& tenant, - const std::string& key) -> std::optionalstd::string { + const std::string& key) -> std::optional { const uint16_t slot = mooncake::cvm::KeySlot(tenant, key); std::string owner; if (has_owner_table) { @@ -439,9 +459,11 @@ SubmasterEndpointResolver ResolveCfmEndpointOwnership() { }; #endif } + class LatencyStats final { public: void Record(double value_us) { values_us_.push_back(value_us); } + double Percentile(double percentile) const { if (values_us_.empty()) return 0.0; const double rank = percentile / 100.0 * (values_us_.size() - 1); @@ -451,7 +473,9 @@ class LatencyStats final { return values_us_[lower] * (1.0 - fraction) + values_us_[upper] * fraction; } + void Finalize() { std::sort(values_us_.begin(), values_us_.end()); } + double Mean() const { if (values_us_.empty()) return 0.0; return std::accumulate(values_us_.begin(), values_us_.end(), 0.0) / @@ -461,12 +485,14 @@ class LatencyStats final { private: std::vector values_us_; }; + struct MetricReportSnapshot { uint64_t calls{0}; uint64_t failures{0}; uint64_t observations{0}; LatencyStats latency; }; + class MetricReportStats final { public: void Record(const MetricBatch& batch, double latency_us, bool success) { @@ -477,6 +503,7 @@ class MetricReportStats final { batch.storage.size(); latency.Record(latency_us); } + MetricReportSnapshot Finalize() { std::lock_guard lock(mutex_); latency.Finalize(); @@ -493,11 +520,13 @@ class MetricReportStats final { uint64_t observations{0}; LatencyStats latency; }; + struct RequestData { IoPatternSnapshot snapshot; std::vector inference; std::vector accesses; }; + RequestData BuildRequest(size_t request_index) { const size_t session = request_index % FLAGS_num_sessions; const uint64_t total_tokens = FLAGS_prompt_tokens + FLAGS_output_tokens; @@ -506,6 +535,7 @@ RequestData BuildRequest(size_t request_index) { std::min(blocks, BlockCount(FLAGS_shared_prefix_tokens)); const bool prefix_is_cached = request_index >= FLAGS_num_sessions; const uint64_t now_ns = SteadyNowNs(); + RequestData request; request.snapshot.generated_at_ns = now_ns; request.inference.reserve(blocks * FLAGS_num_layers); @@ -514,6 +544,7 @@ RequestData BuildRequest(size_t request_index) { const auto tenant = TenantId(FLAGS_tenant); const auto session_id = "vllm-session-" + std::to_string(session); const std::string workload_type = SessionWorkloadType(session); + for (size_t layer = 0; layer < FLAGS_num_layers; ++layer) { for (size_t block = 0; block < blocks; ++block) { const bool is_shared_prefix = block < shared_blocks; @@ -525,6 +556,7 @@ RequestData BuildRequest(size_t request_index) { total_tokens, (block + 1) * FLAGS_tokens_per_block); const auto block_tokens = static_cast( block_end - block * FLAGS_tokens_per_block); + InferenceMetrics inference{ .object = object, .session_id = session_id, @@ -626,6 +658,7 @@ RequestData BuildRequest(size_t request_index) { .memory_used_ratio = static_cast(FLAGS_memory_used_ratio)}); return request; } + void PrintObservability(std::string_view name, const IoPatternObservabilitySnapshot& metrics) { std::cout << "\n " << name << " IO Pattern metrics\n" @@ -644,6 +677,7 @@ void PrintObservability(std::string_view name, << " report drops: " << metrics.report_drop_count << "\n"; } + bool ValidateFlags() { return FLAGS_requests != 0 && FLAGS_prompt_tokens + FLAGS_output_tokens != 0 && @@ -652,6 +686,7 @@ bool ValidateFlags() { FLAGS_report_capacity != 0 && FLAGS_memory_used_ratio >= 0.0 && FLAGS_memory_used_ratio <= 1.0; } + // Real-data seeding stage (optional). The IO Pattern runtime on the SubMaster // only executes storage handlers against replicas that actually exist, so a // report-only benchmark leaves master-side eviction/promotion/prefetch counters @@ -665,15 +700,17 @@ struct SeedStats { uint64_t reads{0}; uint64_t read_failures{0}; }; + // Captures what the seeding stage actually created so the reporting stage can // address exactly the real keys (a report stream over synthetic keys whose // objects were never stored leaves the SubMaster handlers with nothing to act // on, which shows up as OBJECT_NOT_FOUND / zero master-side evictions). struct SeedOutcome { SeedStats stats; - std::vectorstd::string keys; // real keys written, stable order - std::vector hot; // parallel: read back (access heat) + std::vector keys; // real keys written, stable order + std::vector hot; // parallel: read back (access heat) }; + SeedOutcome RunRealSeedStage() { SeedOutcome outcome; if (FLAGS_master_server.empty() || FLAGS_num_keys == 0 || @@ -685,6 +722,7 @@ SeedOutcome RunRealSeedStage() { << " value_size=" << FLAGS_value_size << " replica_num=" << FLAGS_replica_num << " offload=" << (FLAGS_enable_ssd_offload ? "yes" : "no"); + auto client = mooncake::RealClient::create(); const size_t block_bytes = std::max(FLAGS_value_size, 4096); char* buffer = reinterpret_cast(numa_alloc_local(block_bytes)); @@ -710,6 +748,7 @@ SeedOutcome RunRealSeedStage() { numa_free(buffer, block_bytes); return outcome; } + // Write keys that share the simulated KvKey naming so later reports and // the real metadata address the same objects. Enumerate the same // (session, request, layer, block) space as BuildRequest() and stop after @@ -749,6 +788,7 @@ SeedOutcome RunRealSeedStage() { } } outcome.stats.written = seeded; + // Simulate reads: exercise a hot subset through the real data path so the // SubMaster records real GET access heat (promotion-on-hit when offloaded). // Mark the same prefix of the written key list as hot for the report pass. @@ -791,6 +831,7 @@ SeedOutcome RunRealSeedStage() { } } } + client->unregister_buffer(buffer); numa_free(buffer, block_bytes); LOG(INFO) << "Real-data seed stage done: written=" << outcome.stats.written @@ -799,7 +840,9 @@ SeedOutcome RunRealSeedStage() { << " read_failures=" << outcome.stats.read_failures; return outcome; } + } // namespace + int main(int argc, char* argv[]) { google::InitGoogleLogging(argv[0]); gflags::ParseCommandLineFlags(&argc, &argv, true); @@ -808,9 +851,11 @@ int main(int argc, char* argv[]) { "--memory_used_ratio must be within [0, 1]"; return 1; } + std::atomic eviction_commands{0}; std::atomic prefetch_commands{0}; std::atomic admission_commands{0}; + // The SubMaster-side CFM component (embedded mode) or the ownership // resolver used by the remote reporter. std::shared_ptr embedded_service; @@ -857,15 +902,17 @@ int main(int argc, char* argv[]) { ownership_client = std::make_shared( resolver, std::chrono::milliseconds(FLAGS_cfm_rpc_timeout_ms)); ownership_client->set_forward_storage(FLAGS_report_forward_storage); - deployment_description = "remote SubMaster (s) via CFM coro_rpc"; + deployment_description = "remote SubMaster(s) via CFM coro_rpc"; } - // Real-data seeding runs before the simulated request stream (" 先种子后仿 - // 真 "): the SubMaster must hold real replicas for reported keys before the + + // Real-data seeding runs before the simulated request stream ("先种子后仿 + // 真"): the SubMaster must hold real replicas for reported keys before the // report-driven policy cycle can execute eviction/promotion/prefetch // against them. Only meaningful with a real SubMaster endpoint // (--cfm_endpoint) plus RealClient parameters; otherwise it is a no-op. const SeedOutcome seed_outcome = RunRealSeedStage(); const SeedStats& seed_stats = seed_outcome.stats; + IoPatternRuntime::Config source_config; source_config.report_capacity = FLAGS_report_capacity; source_config.max_analysis_keys = FLAGS_max_analysis_keys; @@ -889,16 +936,19 @@ int main(int argc, char* argv[]) { .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, source_config); + const auto send_snapshot = [&](const IoPatternSnapshot& snapshot) -> bool { return ownership_client ? ownership_client->ReportSnapshot(snapshot) == ErrorCode::OK : (embedded_channel && embedded_channel->SendSnapshot(snapshot)); }; + LatencyStats report_latency; uint64_t failed_reports = 0; uint64_t total_blocks = 0; const auto benchmark_start = Clock::now(); + // When a real seed set was written, report exactly those keys (the ones // with real replicas) instead of the synthetic request stream. Synthetic // keys never stored on the SubMaster pollute the merged snapshot: the @@ -978,6 +1028,7 @@ int main(int argc, char* argv[]) { << " cold=" << real_snapshot.keys.size() - hot_blocks << " (synthetic request stream skipped)"; } + // Promotion test: wait for IO Pattern cold eviction to cull MEMORY // replicas, then re-read seeded keys to trigger promotion-on-hit. if (FLAGS_promotion_test_wait_sec > 0 && !seed_outcome.keys.empty()) { @@ -1108,7 +1159,7 @@ int main(int argc, char* argv[]) { // ready (already deleted by a fallback eviction) are filtered out // of the trigger report so the prefetch handler only sees objects // that actually exist with a LOCAL_DISK source. - std::vectorstd::string prefetch_keys = seed_outcome.keys; + std::vector prefetch_keys = seed_outcome.keys; if (FLAGS_prefetch_ready_poll_sec > 0 && !seed_outcome.keys.empty() && (tiers & CacheTierBit(CacheTier::kLocalDisk)) != 0) { @@ -1237,6 +1288,7 @@ int main(int argc, char* argv[]) { numa_free(buffer, block_bytes); } } + // S7.5: after promotion completes, re-mark ALL seeded keys cold so the // report-driven cold eviction must re-evict the freshly promoted MEMORY // replicas (they hold LOCAL_DISK copies, so eviction deletes the MEMORY @@ -1283,6 +1335,7 @@ int main(int argc, char* argv[]) { } LOG(INFO) << "[PROMO-RECYCLE] recycle wait done"; } + for (size_t request_index = 0; request_index < FLAGS_requests; ++request_index) { if (real_seed_mode) break; // real keys already reported above @@ -1294,11 +1347,13 @@ int main(int argc, char* argv[]) { request.accesses[i]); } source_runtime->RecordStorageMetric(request.snapshot.storage.front()); + const auto report_start = Clock::now(); const bool sent = send_snapshot(request.snapshot); report_latency.Record(ToMicroseconds(Clock::now() - report_start)); if (!sent) ++failed_reports; } + // S11.6: prefetch failure path. Synthetic keys claim a LOCAL_DISK replica // that does not exist, so the master prefetch handler finds no real object // (kNotFound -> OBJECT_NOT_FOUND) or no LOCAL_DISK source and returns a @@ -1345,17 +1400,20 @@ int main(int argc, char* argv[]) { } const auto submission_seconds = std::chrono::duration(Clock::now() - benchmark_start).count(); + // Stop joins the reporter worker and performs its final flush. No new // metric batch can reach the SubMaster after this returns. source_runtime->StopReports(); std::this_thread::sleep_for( std::chrono::milliseconds(FLAGS_report_flush_wait_ms)); + // The report-driven worker executes one cycle per merged report. Wait for // it to drain before reading handler counters / snapshots so the printed // numbers are deterministic. if (cfm_runtime && cfm_runtime->report_driven_execution()) { cfm_runtime->WaitForReportDrivenIdle(); } + // Embedded mode: when the report-driven worker is disabled, evaluate and // execute policy once locally (the pre-worker high-watermark trigger that // the production EvictionThreadFunc runs). With report_driven_execution @@ -1375,8 +1433,10 @@ int main(int argc, char* argv[]) { LOG(WARNING) << "Local CFM evaluation degraded"; } } + const auto end_to_end_seconds = std::chrono::duration(Clock::now() - benchmark_start).count(); + const auto source_snapshot = source_runtime->Snapshot(); const auto source_metrics = source_runtime->ObservabilitySnapshot(end_to_end_seconds); @@ -1388,6 +1448,7 @@ int main(int argc, char* argv[]) { const auto cfm_metrics = embedded_service ? embedded_service->Observability(end_to_end_seconds) : IoPatternObservabilitySnapshot{}; + std::cout << "\n============================================================\n" << "CFM CLIENT BENCHMARK (vLLM inference request model)\n" From 3aa69a807f8fe57aeeb8486be2bae5b84f4b9aee Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 17:50:06 +0800 Subject: [PATCH 45/47] add io_pattern metric --- mooncake-store/include/master_service.h | 33 +++++++- .../src/io_pattern/policy_strategies.cpp | 17 ++++ mooncake-store/src/master_service.cpp | 78 +++++++++++++++---- .../tests/io_pattern_framework_test.cpp | 39 ++++++++++ .../tests/offload_on_evict_test.cpp | 65 ++++++++++++++++ 5 files changed, 215 insertions(+), 17 deletions(-) diff --git a/mooncake-store/include/master_service.h b/mooncake-store/include/master_service.h index ae2e3a3dfc..5408b1b2a0 100644 --- a/mooncake-store/include/master_service.h +++ b/mooncake-store/include/master_service.h @@ -1008,6 +1008,18 @@ class MasterService { uint64_t tier_down_failure_count() const { return tier_down_failures_.load(std::memory_order_relaxed); } + /// Candidates skipped because their disk copy is still in flight: a + /// non-zero, persistent value means the tier-down driver is waiting on the + /// client's offload queue rather than making progress. + uint64_t tier_down_skipped_in_flight_count() const { + return tier_down_skipped_in_flight_.load(std::memory_order_relaxed); + } + /// Candidates skipped because they already hold a LOCAL_DISK replica. A + /// persistent value means selection is re-picking keys that are already + /// paved down (each cycle's budget is being spent on nothing). + uint64_t tier_down_skipped_paved_count() const { + return tier_down_skipped_paved_.load(std::memory_order_relaxed); + } private: std::unique_ptr CreateSnapshotCatalogStore(); @@ -1811,11 +1823,20 @@ class MasterService { * replica and never frees bytes, so the caller must not count its result as * reclaimed memory. Acquires its own RW shard accessor; safe to call from * the io_pattern eviction handler, which does not hold one while this runs. - * Returns false when the key vanished, has no completed MEMORY replica, or - * the holder client cannot accept the offload. A key that is already queued - * counts as success: it is already on its way down. + * + * The outcome is reported instead of a bare bool: "nothing left to do" and + * "queued" are operationally different, and collapsing a key that already + * holds a LOCAL_DISK replica -- or whose copy is still in flight -- into + * success is what hides a spinning tier-down driver. */ - bool TryQueueTierDown(const ObjectIdentity& object_id); + enum class TierDownOutcome { + kQueued, + kAlreadyInFlight, + kAlreadyPaved, + kNotFound, + kPushFailed, + }; + TierDownOutcome TryQueueTierDown(const ObjectIdentity& object_id); void RecordOrUpdateCandidate(TenantState& tenant_state, const std::string& key, uint8_t sketch_score, PromotionCandidateReason reason, @@ -2291,6 +2312,10 @@ class MasterService { std::atomic tier_down_attempts_{0}; std::atomic tier_down_successes_{0}; std::atomic tier_down_failures_{0}; + // Outcomes that are neither progress nor failure. Kept separate so a stalled + // driver is visible instead of being counted as successful demotions. + std::atomic tier_down_skipped_in_flight_{0}; + std::atomic tier_down_skipped_paved_{0}; const std::string ha_backend_type_; diff --git a/mooncake-store/src/io_pattern/policy_strategies.cpp b/mooncake-store/src/io_pattern/policy_strategies.cpp index c9169becb1..e71dd71ebf 100644 --- a/mooncake-store/src/io_pattern/policy_strategies.cpp +++ b/mooncake-store/src/io_pattern/policy_strategies.cpp @@ -69,6 +69,17 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, if (context.tier_down) { plan.tier_down_target_bytes = target_bytes; } + // A demotion copies the key down and keeps its MEMORY replica, so a key that + // already has a replica below this tier has nothing left to copy. It must be + // excluded or the driver spins forever on the same victims: the key stays an + // L1 candidate, the scorer prefers lower-replica-backed victims for eviction + // (see lower_replica_weight below), and every cycle would spend its whole + // budget re-copying the same keys while the rest of the cold set is never + // paved. Eviction has no such problem -- it removes the replica, so the next + // cycle picks fresh victims -- which is why this exclusion is tier-down only. + const auto is_already_paved = [&context, tier](const KeyMetrics& key) { + return context.tier_down && HasLowerTierReplica(key, tier); + }; uint64_t max_block_size = 0; uint32_t max_other_replicas = 0; for (const auto& key : context.snapshot.keys) { @@ -79,6 +90,9 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, key.idle_time_us < context.min_idle_time_us) { continue; } + if (is_already_paved(key)) { + continue; + } max_block_size = std::max(max_block_size, key.block_size); max_other_replicas = std::max(max_other_replicas, key.other_replica_count); @@ -91,6 +105,9 @@ EvictionPlan ScoreBasedEvictionOps::Evaluate(const PolicyContext& context, key.idle_time_us < context.min_idle_time_us) { continue; } + if (is_already_paved(key)) { + continue; + } const auto* pattern = FindPattern(key.object, context.analysis); if (pattern == nullptr) { continue; diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 1f6ad30fd0..791f1da96b 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -600,7 +600,26 @@ MasterService::MasterService(const MasterServiceConfig& config) uint64_t total_freed = 0; uint64_t tier_down_attempts = 0; uint64_t tier_down_queued = 0; + uint64_t tier_down_in_flight = 0; + uint64_t tier_down_paved = 0; uint64_t tier_down_failed = 0; + std::vector tier_down_diag; + const auto tier_down_outcome_name = + [](TierDownOutcome outcome) -> const char* { + switch (outcome) { + case TierDownOutcome::kQueued: + return "queued"; + case TierDownOutcome::kAlreadyInFlight: + return "already_in_flight"; + case TierDownOutcome::kAlreadyPaved: + return "already_paved"; + case TierDownOutcome::kNotFound: + return "not_found"; + case TierDownOutcome::kPushFailed: + return "push_failed"; + } + return "unknown"; + }; std::unordered_map targets; for (const auto& candidate : plan.candidates) { @@ -612,12 +631,29 @@ MasterService::MasterService(const MasterServiceConfig& config) if (candidate.action == io_pattern::EvictionAction::kTierDown) { ++tier_down_attempts; - if (TryQueueTierDown(ObjectIdentity{ + const TierDownOutcome outcome = + TryQueueTierDown(ObjectIdentity{ candidate.object.tenant_id, - candidate.object.key})) { - ++tier_down_queued; - } else { - ++tier_down_failed; + candidate.object.key}); + switch (outcome) { + case TierDownOutcome::kQueued: + ++tier_down_queued; + break; + case TierDownOutcome::kAlreadyInFlight: + ++tier_down_in_flight; + break; + case TierDownOutcome::kAlreadyPaved: + ++tier_down_paved; + break; + case TierDownOutcome::kNotFound: + case TierDownOutcome::kPushFailed: + ++tier_down_failed; + break; + } + if (tier_down_diag.size() < 3) { + tier_down_diag.push_back( + candidate.object.key + "=" + + tier_down_outcome_name(outcome)); } continue; } @@ -632,13 +668,24 @@ MasterService::MasterService(const MasterServiceConfig& config) tier_down_queued, std::memory_order_relaxed); tier_down_failures_.fetch_add( tier_down_failed, std::memory_order_relaxed); + tier_down_skipped_in_flight_.fetch_add( + tier_down_in_flight, std::memory_order_relaxed); + tier_down_skipped_paved_.fetch_add( + tier_down_paved, std::memory_order_relaxed); LOG(WARNING) << "[IO-PATTERN-TIER-DOWN] io_pattern tier down " "plan_target=" << tier_down_target << " attempts=" << tier_down_attempts << " queued=" << tier_down_queued + << " already_in_flight=" << tier_down_in_flight + << " already_paved=" << tier_down_paved << " failed=" << tier_down_failed; + for (const auto& diag : tier_down_diag) { + LOG(WARNING) + << "[IO-PATTERN-TIER-DOWN] candidate " + << diag; + } } for (const auto& [tenant, target] : targets) { const auto result = EvictTenantMemoryForQuota( @@ -8383,20 +8430,25 @@ tl::expected, ErrorCode> MasterService::PushOffloadingQueue( // bookkeeping used by the offload-on-evict path (refcnt pin plus an // offloading_tasks entry, both released when the client reports the copy back), // and deliberately frees nothing: demotion is a copy, not a reclaim. -bool MasterService::TryQueueTierDown(const ObjectIdentity& object_id) { +MasterService::TierDownOutcome MasterService::TryQueueTierDown( + const ObjectIdentity& object_id) { MetadataAccessorRW accessor(this, object_id); if (!accessor.Exists()) { - return false; + return TierDownOutcome::kNotFound; } auto& metadata = accessor.Get(); auto& tenant_state = accessor.GetTenantState(); - // One offload per key. An in-flight task already pins the MEMORY replica this - // demotion would pin again, and the holder's queue is keyed by the object, so - // re-pushing would only fail with OBJECT_ALREADY_EXISTS. The key is already - // on its way down, which is what the caller asked for. + // Already on disk: a demotion is a copy down, so there is nothing left to + // copy. Reported separately from kQueued because a driver that keeps + // selecting paved keys is not making progress. + if (metadata.HasReplica(&Replica::fn_is_local_disk_replica)) { + return TierDownOutcome::kAlreadyPaved; + } + // A copy is already in flight. Re-pushing would only fail with + // OBJECT_ALREADY_EXISTS, and like kAlreadyPaved this is not progress. if (tenant_state.offloading_tasks.count(object_id.user_key) > 0) { - return true; + return TierDownOutcome::kAlreadyInFlight; } const auto now = std::chrono::system_clock::now(); @@ -8423,7 +8475,7 @@ bool MasterService::TryQueueTierDown(const ObjectIdentity& object_id) { } queued = true; }); - return queued; + return queued ? TierDownOutcome::kQueued : TierDownOutcome::kPushFailed; } // Promotion-on-hit diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index e536f9842b..2d79438025 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1905,6 +1905,45 @@ TEST(IoPatternFrameworkTest, TierDownContextLabelsCandidatesAndBudget) { reclaimed.candidates.front().target_tier); } +// A demotion keeps the MEMORY replica, so a key that already has a replica below +// it has nothing left to copy down. Without that exclusion the driver spins on +// the same keys every cycle -- they stay L1 candidates and the eviction scorer +// actively prefers lower-replica-backed victims -- so the rest of the cold set is +// never paved and the SSD never grows past one budget. +TEST(IoPatternFrameworkTest, TierDownSkipsKeysThatAlreadyHaveALowerReplica) { + PolicyContext context; + context.snapshot.keys = { + KeyMetrics{.object = {TenantId("tenant"), "already-paved"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL1Host) | + CacheTierBit(CacheTier::kL3NofSsd)}, + KeyMetrics{.object = {TenantId("tenant"), "not-paved"}, + .block_size = 64, + .replica_tiers = CacheTierBit(CacheTier::kL1Host)}, + }; + context.analysis.keys = { + KeyPattern{.object = context.snapshot.keys[0].object}, + KeyPattern{.object = context.snapshot.keys[1].object}, + }; + + ScoreBasedEvictionOps ops; + + // Eviction keeps both keys eligible: the exclusion below is tier-down only, + // and the scorer still ranks the lower-replica-backed key first as the safe + // victim to reclaim. + const auto reclaimed = ops.Evaluate(context, CacheTier::kL1Host, 128); + ASSERT_EQ(reclaimed.candidates.size(), 2); + EXPECT_EQ(reclaimed.candidates.front().object.key, "already-paved"); + EXPECT_EQ(reclaimed.candidates.front().action, EvictionAction::kEvict); + + // Tier down must only consider the key that still needs a disk copy. + context.tier_down = true; + const auto demoted = ops.Evaluate(context, CacheTier::kL1Host, 128); + ASSERT_EQ(demoted.candidates.size(), 1); + EXPECT_EQ(demoted.candidates.front().object.key, "not-paved"); + EXPECT_EQ(demoted.candidates.front().action, EvictionAction::kTierDown); +} + TEST(IoPatternFrameworkTest, PrefetchRequiresConfidenceAndNeverPromotesToHbm) { PolicyContext context; context.snapshot.keys = { diff --git a/mooncake-store/tests/offload_on_evict_test.cpp b/mooncake-store/tests/offload_on_evict_test.cpp index 85d49e835a..1ad21176c4 100644 --- a/mooncake-store/tests/offload_on_evict_test.cpp +++ b/mooncake-store/tests/offload_on_evict_test.cpp @@ -56,6 +56,18 @@ class OffloadOnEvictTest : public ::testing::Test { "report_metric_batch", codec.EncodeMetricBatch(batch)); } + // Friend access to the tier-event hook the master uses when an offload + // completes, so a test can make a key look already-paved without having to + // drive a whole offload round trip. + static void MarkLowerTierReplicaForTesting(MasterService* service, + const std::string& key, + io_pattern::CacheTier tier) { + service->io_pattern_runtime_->RecordTierEvent(io_pattern::CacheEvent{ + .type = io_pattern::CacheEventType::kInserted, + .object = {TenantId::Default(), key}, + .target_tier = tier}); + } + // A demotion must leave a readable MEMORY replica behind; an eviction must // not. Reads the same client-facing view a Get would. bool HasCompleteMemoryReplica(MasterService& service, @@ -664,6 +676,59 @@ TEST_F(OffloadOnEvictTest, TierDownDriverDemotesReportedKeyWithoutReclaiming) { service->RemoveAll(); } +// End to end through the driver: a key that already holds a lower-tier replica +// must not consume the tier-down budget, otherwise the same keys are re-copied +// every cycle and the SSD never grows past one budget's worth. +TEST_F(OffloadOnEvictTest, TierDownDriverPavesFreshKeysBeforePavedOnes) { + MasterServiceConfig config; + config.enable_offload = true; + config.offload_on_evict = true; + config.default_kv_lease_ttl = 0; + // Exactly one object per cycle, so the chosen candidate is unambiguous. + config.io_pattern_tier_down_bytes_per_cycle = 4096; + auto service = std::make_unique(config); + + constexpr size_t seg_size = 1024 * 1024 * 16; + auto ctx = PrepareSegment(*service, "tier_down_progress_segment", + kDefaultSegmentBase, seg_size); + auto mount_ld = service->MountLocalDiskSegment(ctx.client_id, true); + ASSERT_TRUE(mount_ld.has_value()); + + PutObject(*service, ctx.client_id, "td_paved", 4096); + PutObject(*service, ctx.client_id, "td_fresh", 4096); + // "td_paved" was put first, so it is the colder key and would normally win + // the budget. Marking it as already holding a lower-tier replica is what an + // offload completion does, and it must hand the budget to "td_fresh". + MarkLowerTierReplicaForTesting(service.get(), "td_paved", + io_pattern::CacheTier::kL3NofSsd); + + io_pattern::MetricBatch batch; + for (const char* key : {"td_paved", "td_fresh"}) { + batch.accesses.push_back(io_pattern::AccessRecord{ + .object = {TenantId::Default(), key}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = io_pattern::CacheTier::kL1Host, + .operation = io_pattern::IoOperation::kGet, + .is_hit = true}); + } + ASSERT_TRUE(SendIoPatternReport(service.get(), batch)); + + std::unordered_map queued; + WaitUntil([&] { + queued = DrainOffloadQueue(*service, ctx.client_id); + return !queued.empty(); + }); + ASSERT_EQ(queued.size(), 1u); + EXPECT_TRUE(queued.count("td_fresh") > 0) + << "tier down re-picked a key that already has a disk replica"; + // Both keys keep serving from MEMORY: demotion is a copy, not a reclaim. + EXPECT_TRUE(HasCompleteMemoryReplica(*service, "td_paved")); + EXPECT_TRUE(HasCompleteMemoryReplica(*service, "td_fresh")); + + service->RemoveAll(); +} + } // namespace mooncake::test int main(int argc, char** argv) { From 85c1f1472fe37cdb47b0c9a51831fd1b5c64d3af Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 18:42:43 +0800 Subject: [PATCH 46/47] add io_pattern metric --- mooncake-store/include/io_pattern/runtime.h | 13 +- mooncake-store/src/io_pattern/runtime.cpp | 45 ++++- .../tests/io_pattern_framework_test.cpp | 154 ++++++++++++++++++ 3 files changed, 203 insertions(+), 9 deletions(-) diff --git a/mooncake-store/include/io_pattern/runtime.h b/mooncake-store/include/io_pattern/runtime.h index 65fd8a09b1..49e382c1d3 100644 --- a/mooncake-store/include/io_pattern/runtime.h +++ b/mooncake-store/include/io_pattern/runtime.h @@ -137,6 +137,15 @@ class IoPatternRuntime final { // demotion is never counted as an eviction. There is no separate enable // flag: 0 keeps the driver off, so this budget is the whole control. uint64_t tier_down_bytes_per_cycle{0}; + // Periodic driver tick, in milliseconds. Reports only flow while the + // workload does, but tier down and cold eviction are exactly the drivers + // that must act on an idle cluster, so when either is configured the + // report-driven worker also wakes on this interval. A tick cycle + // deliberately ignores recorded storage pressure: the master's own + // watermark thread owns pressure reclaims (and the ratio it records + // lingers in the collector until the next breach), so a tick that also + // reclaimed would evict twice for one breach. 0 disables the tick. + uint64_t tick_interval_ms{10'000}; // Optional per-cycle observer used to surface executions in process // metrics (e.g. MasterMetricManager). Never called from the report // data path; only from the background cycle worker. @@ -220,7 +229,9 @@ class IoPatternRuntime final { // Report-driven cycle internals (single background worker). void ReportDrivenWorker(); - void RunReportDrivenCycle(); + // allow_pressure=false is the periodic-tick path: the pressure request is + // skipped so only the below-watermark drivers act. + void RunReportDrivenCycle(bool allow_pressure); // Runs the executor over an already planned policy and records the shared // outcome bookkeeping (policy failure/success, degradation, pending // prefetch set and feedback). Used by both Execute() and the diff --git a/mooncake-store/src/io_pattern/runtime.cpp b/mooncake-store/src/io_pattern/runtime.cpp index e85b7beb7e..14c13e7d93 100644 --- a/mooncake-store/src/io_pattern/runtime.cpp +++ b/mooncake-store/src/io_pattern/runtime.cpp @@ -336,18 +336,42 @@ void IoPatternRuntime::WaitForReportDrivenIdle() { } void IoPatternRuntime::ReportDrivenWorker() { + // Below-watermark drivers (tier down, cold eviction) must not depend on client + // reports: reports only flow while there is traffic, so an idle cluster would + // never pave cold data down or reclaim it. When either driver is configured + // the worker wakes on a timer as well; clusters that configure neither pay no + // periodic analysis, and a tick can never fire with a zero interval. + const bool tick_enabled = config_.report_driven_execution && + config_.tick_interval_ms != 0 && + (config_.tier_down_bytes_per_cycle != 0 || + config_.report_driven_cold_eviction); + const auto tick_interval = + std::chrono::milliseconds(static_cast(config_.tick_interval_ms)); while (true) { + bool tick_triggered = false; { std::unique_lock lock(report_mutex_); - report_condition_.wait(lock, [this] { - return report_stopping_ || report_pending_; - }); + if (tick_enabled) { + if (!report_condition_.wait_for(lock, tick_interval, [this] { + return report_stopping_ || report_pending_; + })) { + tick_triggered = true; + } + } else { + report_condition_.wait(lock, [this] { + return report_stopping_ || report_pending_; + }); + } if (report_stopping_) return; report_pending_ = false; report_worker_busy_ = true; } try { - RunReportDrivenCycle(); + // A tick cycle ignores any recorded storage pressure on purpose: the + // master's own watermark thread owns pressure reclaims, and the ratio + // it records lingers in the collector until the next breach, so + // honouring it here would reclaim the same excess a second time. + RunReportDrivenCycle(/*allow_pressure=*/!tick_triggered); } catch (...) { policy_->RecordFailure(); observability_.RecordDegrade(); @@ -470,7 +494,7 @@ std::vector IoPatternRuntime::DeriveAdmissionCandidates( return candidates; } -void IoPatternRuntime::RunReportDrivenCycle() { +void IoPatternRuntime::RunReportDrivenCycle(bool allow_pressure) { IoPatternSnapshot snapshot = collector_->GetSnapshot(); ReportDrivenCycleReport report; report.cycle_id = ++report_cycle_id_; @@ -483,9 +507,14 @@ void IoPatternRuntime::RunReportDrivenCycle() { } CacheTier eviction_tier = CacheTier::kL1Host; uint64_t eviction_bytes = 0; - DeriveEvictionRequest(snapshot, config_.report_eviction_high_ratio, - config_.report_eviction_target_ratio, eviction_tier, - eviction_bytes); + // A tick-driven cycle skips the pressure request entirely, so the + // below-watermark drivers below decide (tier down, then cold eviction). The + // master's watermark thread already handled any real breach. + if (allow_pressure) { + DeriveEvictionRequest(snapshot, config_.report_eviction_high_ratio, + config_.report_eviction_target_ratio, + eviction_tier, eviction_bytes); + } // Tier-down driver: the below-watermark placement action. Copy the coldest // in-memory keys down to LOCAL_DISK while keeping their MEMORY replica, so a // later reclaim of those keys can discard them safely instead of copying then. diff --git a/mooncake-store/tests/io_pattern_framework_test.cpp b/mooncake-store/tests/io_pattern_framework_test.cpp index 2d79438025..5604818152 100644 --- a/mooncake-store/tests/io_pattern_framework_test.cpp +++ b/mooncake-store/tests/io_pattern_framework_test.cpp @@ -1584,6 +1584,160 @@ TEST(IoPatternFrameworkTest, ReportDrivenTierDownPavesColdDataBeforeColdEviction } } +// The below-watermark drivers must not depend on client reports: reports only +// flow while the workload does, so an idle cluster would never pave cold data +// down. The tick lets them run on their own, with no report at all. +TEST(IoPatternFrameworkTest, ReportDrivenTickRunsTierDownWithoutAnyReport) { + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::mutex plan_mutex; + std::optional handled_plan; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + config.tier_down_bytes_per_cycle = 64ULL * 1024 * 1024; + config.tick_interval_ms = 50; + config.analysis_timeout_us = 30'000'000; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = + [&](const EvictionPlan& plan) { + std::lock_guard lock(plan_mutex); + handled_plan = plan; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + + // Recorded locally on the owning SubMaster, exactly as PutEnd/Get do; no + // report is ever sent, so the tick is the only possible wakeup. + rt->RecordAccess("local-key", + AccessRecord{.object = {TenantId("tenant"), "local-key"}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(10), [&] { + return last_report.has_value(); + })) << "the periodic tick never ran a cycle"; + ASSERT_TRUE(last_report.has_value()); + EXPECT_TRUE(last_report->tier_down); + EXPECT_EQ(last_report->eviction_target_bytes, 64ULL * 1024 * 1024); + + std::lock_guard plan_lock(plan_mutex); + ASSERT_TRUE(handled_plan.has_value()); + EXPECT_EQ(handled_plan->tier_down_target_bytes, 64ULL * 1024 * 1024); + for (const auto& candidate : handled_plan->candidates) { + EXPECT_EQ(candidate.action, EvictionAction::kTierDown); + } +} + +// A tick cycle must not reclaim: the master's watermark thread owns pressure, and +// the ratio it records stays in the collector until the next breach, so honouring +// it here would reclaim the same excess twice. +TEST(IoPatternFrameworkTest, ReportDrivenTickIgnoresRecordedPressure) { + std::mutex observer_mutex; + std::condition_variable observer_condition; + std::optional last_report; + std::mutex plan_mutex; + std::optional handled_plan; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + config.tier_down_bytes_per_cycle = 64ULL * 1024 * 1024; + config.tick_interval_ms = 50; + config.analysis_timeout_us = 30'000'000; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport& report) { + std::lock_guard lock(observer_mutex); + last_report = report; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = + [&](const EvictionPlan& plan) { + std::lock_guard lock(plan_mutex); + handled_plan = plan; + return ErrorCode::OK; + }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + + rt->RecordAccess("local-key", + AccessRecord{.object = {TenantId("tenant"), "local-key"}, + .observed_at_ns = 1, + .block_size = 4096, + .tier = CacheTier::kL1Host, + .operation = IoOperation::kGet, + .is_hit = true}); + // Host memory far above the high watermark: a report-triggered cycle would + // derive a reclaim request from this, a tick-triggered one must not. + rt->RecordStorageMetric(StorageMetric{.source_id = "master-memory", + .observed_at_ns = 1, + .tier = CacheTier::kL1Host, + .used_bytes = 1024ULL * 1024 * 1024, + .capacity_bytes = + 1024ULL * 1024 * 1024, + .memory_used_ratio = 0.95F}); + + std::unique_lock lock(observer_mutex); + ASSERT_TRUE(observer_condition.wait_for(lock, std::chrono::seconds(10), [&] { + return last_report.has_value(); + })); + ASSERT_TRUE(last_report.has_value()); + EXPECT_TRUE(last_report->tier_down); + EXPECT_FALSE(last_report->cold_eviction); + EXPECT_EQ(last_report->eviction_target_bytes, 64ULL * 1024 * 1024); + + std::lock_guard plan_lock(plan_mutex); + ASSERT_TRUE(handled_plan.has_value()); + EXPECT_EQ(handled_plan->tier_down_target_bytes, 64ULL * 1024 * 1024); + ASSERT_FALSE(handled_plan->candidates.empty()); + for (const auto& candidate : handled_plan->candidates) { + EXPECT_EQ(candidate.action, EvictionAction::kTierDown); + } +} + +// The tick is gated on a below-watermark driver being configured, so a cluster +// that only uses the watermark path pays no periodic analysis (and gets no +// surprise cycles while idle). +TEST(IoPatternFrameworkTest, ReportDrivenTickIsOffWithoutABelowWatermarkDriver) { + std::mutex observer_mutex; + std::condition_variable observer_condition; + bool observed = false; + IoPatternRuntime::Config config; + config.report_driven_execution = true; + config.tick_interval_ms = 50; + config.report_driven_observer = + [&](const IoPatternRuntime::ReportDrivenCycleReport&) { + std::lock_guard lock(observer_mutex); + observed = true; + observer_condition.notify_all(); + }; + auto rt = std::make_shared( + IoPatternRuntime::Handlers{ + .eviction = [](const EvictionPlan&) { return ErrorCode::OK; }, + .prefetch = [](const PrefetchPlan&) { return ErrorCode::OK; }, + .admission = [](const AdmissionResult&) { return ErrorCode::OK; }}, + std::move(config)); + + std::unique_lock lock(observer_mutex); + EXPECT_FALSE(observer_condition.wait_for(lock, std::chrono::milliseconds(500), + [&] { return observed; })) + << "no driver is configured, so nothing should be running cycles"; +} + TEST(IoPatternFrameworkTest, ReporterBackgroundLifecycleFlushesOnStop) { size_t batches = 0; IoPatternReporter reporter(4, [&](const MetricBatch&) { From d4b3c53b0d94cd225c670d7357314b83cf92025b Mon Sep 17 00:00:00 2001 From: Gzure <740684863@qq.com> Date: Mon, 14 Sep 2026 18:51:08 +0800 Subject: [PATCH 47/47] add io_pattern metric --- mooncake-store/src/master_service.cpp | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/mooncake-store/src/master_service.cpp b/mooncake-store/src/master_service.cpp index 791f1da96b..299903ce6d 100644 --- a/mooncake-store/src/master_service.cpp +++ b/mooncake-store/src/master_service.cpp @@ -686,6 +686,47 @@ MasterService::MasterService(const MasterServiceConfig& config) << "[IO-PATTERN-TIER-DOWN] candidate " << diag; } + if (tier_down_paved != 0 && io_pattern_runtime_) { + // The handler reads the authoritative metadata; the + // selector reads the collector snapshot. Log the + // snapshot's view so a disagreement is visible instead + // of looking like a driver that simply has nothing to + // do: metadata=paved with snapshot_paved=0 means the + // replica bit never reached the snapshot. + constexpr io_pattern::CacheTierMask kLowerTierBits = + static_cast( + io_pattern::CacheTierBit( + io_pattern::CacheTier::kLocalDisk) | + io_pattern::CacheTierBit( + io_pattern::CacheTier::kL2Segment) | + io_pattern::CacheTierBit( + io_pattern::CacheTier::kL3NofSsd)); + const auto snapshot = io_pattern_runtime_->Snapshot(); + size_t snapshot_paved = 0; + for (const auto& candidate : plan.candidates) { + if (candidate.action != + io_pattern::EvictionAction::kTierDown) { + continue; + } + const auto it = std::find_if( + snapshot.keys.begin(), snapshot.keys.end(), + [&candidate]( + const io_pattern::KeyMetrics& key) { + return key.object == candidate.object; + }); + if (it != snapshot.keys.end() && + (it->replica_tiers & kLowerTierBits) != 0) { + ++snapshot_paved; + } + } + LOG(WARNING) + << "[IO-PATTERN-TIER-DOWN] metadata=paved " + << tier_down_paved << " of " + << tier_down_attempts + << ", snapshot=paved " << snapshot_paved + << " (0 means the replica bit never reached " + "the snapshot)"; + } } for (const auto& [tenant, target] : targets) { const auto result = EvictTenantMemoryForQuota( @@ -8443,6 +8484,19 @@ MasterService::TierDownOutcome MasterService::TryQueueTierDown( // copy. Reported separately from kQueued because a driver that keeps // selecting paved keys is not making progress. if (metadata.HasReplica(&Replica::fn_is_local_disk_replica)) { + // Teach the collector what the metadata already knows. The selector can + // only exclude paved keys through the snapshot's replica bits, and those + // bits are recorded on the offload-completion path -- which can miss a key + // that was re-registered before its first access observation, or have them + // overwritten by a merged snapshot. Re-asserting it here is idempotent and + // makes the selection converge after one no-op cycle instead of spinning + // on the same keys forever. + if (io_pattern_runtime_) { + io_pattern_runtime_->RecordTierEvent(io_pattern::CacheEvent{ + .type = io_pattern::CacheEventType::kInserted, + .object = {object_id.tenant_id, object_id.user_key}, + .target_tier = io_pattern::CacheTier::kL3NofSsd}); + } return TierDownOutcome::kAlreadyPaved; } // A copy is already in flight. Re-pushing would only fail with