From 29b749ba5c152c45a3d2ad6acededb18e4a5d4e0 Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:31:00 +0000 Subject: [PATCH 01/13] refactor: extract ModeHLL into its own files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move store HLL off memory.go and split engine HLL into public API, owner apply, and cluster hops (same layout as TopK/CMS). Unexport hllx.merge — install is LWW replace. No contract change. --- docs/design/2026-09-02-refactor-hll.md | 42 +++++++ docs/design/README.md | 1 + pkg/engine/hll.go | 105 +----------------- pkg/engine/hll_apply.go | 59 ++++++++++ pkg/engine/hll_cluster.go | 65 +++++++++++ pkg/hllx/hllx.go | 5 +- pkg/hllx/hllx_test.go | 4 +- pkg/store/hll.go | 146 +++++++++++++++++++++++++ pkg/store/memory.go | 133 ---------------------- 9 files changed, 320 insertions(+), 240 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-hll.md create mode 100644 pkg/engine/hll_apply.go create mode 100644 pkg/engine/hll_cluster.go create mode 100644 pkg/store/hll.go diff --git a/docs/design/2026-09-02-refactor-hll.md b/docs/design/2026-09-02-refactor-hll.md new file mode 100644 index 0000000..da08f6e --- /dev/null +++ b/docs/design/2026-09-02-refactor-hll.md @@ -0,0 +1,42 @@ +# Refactor ModeHLL layout (no contract change) + +**Status:** approved (chat: start with HLL) +**Branch:** `feat/refactor-hll` +**Date:** 2026-09-02 + +## Problem + +HLL store methods still live in `pkg/store/memory.go` (~2400 lines). TopK and CMS already have their own files. Engine HLL is one 190-line file mixing public API, owner apply, and cluster hops. `pkg/hllx.Merge` is exported but unused on the data path (install is LWW replace). + +## Non-goals + +- No API / proto / flag / version / fan-out / hintID change +- No `hllx` algorithm change +- No decoded cache / Get-Peek flush helper +- Not extracting other modes + +## Contract + +Unchanged: `HLLAdd` / `HLLCount` / `Delete(name)`, `FlagHLL` + inbox `FlagHLLAdd`, snapshot fan-out, version under store mutex, present-bit, empty-until-delete. + +`pkg/hllx.Merge` becomes unexported (`merge`). Engine never called it. Package tests still cover per-register max. + +## Approach + +Standard Go file layout (one concern per file, same package): + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` HLL* | `pkg/store/hll.go` (same as `cms.go` / `topk.go`) | +| `pkg/engine/hll.go` (all) | `hll.go` public verbs; `hll_apply.go` owner write; `hll_cluster.go` inbox/fan-out/GetOrLoad | +| `pkg/hllx.Merge` | `merge` (unexported) | + +Rejected: rewrite registers; split `hllx` (150 lines is one file); touch `ApplyPut` order. + +## Tests (already exist; keep them) + +Existing `pkg/hllx`, `pkg/store` HLL, `pkg/engine` HLL unit + cluster + hint-after-down. No new behavior tests. + +## Bench risk + +Hot path? No new Get/Peek logic. Extra file split must not add allocs. Local `BenchmarkEngineGetHit` / `BenchmarkStoreGetHit` allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..cce26e8 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-hll.md](./2026-09-02-refactor-hll.md) | ModeHLL file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/hll.go b/pkg/engine/hll.go index e37d650..8ae63a2 100644 --- a/pkg/engine/hll.go +++ b/pkg/engine/hll.go @@ -6,10 +6,10 @@ import ( "github.com/Code0987/supercache/pkg/hllx" "github.com/Code0987/supercache/pkg/keyspace" - "github.com/Code0987/supercache/pkg/store" ) -// HLLAdd hashes item into a ModeHLL sketch (Redis PFADD). ACK-only. +// HLLAdd hashes item into a ModeHLL sketch (ACK-only; no Redis changed-bool). +// Non-owners forward an inbox FlagHLLAdd; the owner applies and fans a snapshot. func (e *Engine) HLLAdd(ctx context.Context, keyspaceName, name string, item []byte) error { if err := ctx.Err(); err != nil { return err @@ -87,104 +87,3 @@ func (e *Engine) hllNeed(ks *ksRuntime) error { } return nil } - -func (e *Engine) hllMutViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: append([]byte(nil), item...), Flags: store.FlagHLLAdd, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - applied, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - if err != nil { - return err - } - if !applied { - return fmt.Errorf("%w: hll add rejected", ErrInvalidArgument) - } - return nil -} - -func (e *Engine) hllAddLocal(ks *ksRuntime, name string, item []byte) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - applied, tooLarge := ks.store.HLLAdd(name, item, gate, expire, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf("%w: hll add rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.hllReplicateSnapshot(ks, name, ver, expire) - return nil -} - -func (e *Engine) hllReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { - ent, ok := ks.store.Peek(name) - if !ok || !ent.IsHLL() { - return - } - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: ent.Value, - Version: ver, - ExpireAt: expire, - Flags: store.FlagHLL, - }, false) -} - -func (e *Engine) hllFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsHLL() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.HLLInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyHLLAdd(ks *ksRuntime, name string, item []byte, expireAt int64) bool { - if len(item) == 0 { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.HLLAdd(name, item, gate, expireAt, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.hllReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyHLLInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HLLInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/hll_apply.go b/pkg/engine/hll_apply.go new file mode 100644 index 0000000..04b38ac --- /dev/null +++ b/pkg/engine/hll_apply.go @@ -0,0 +1,59 @@ +package engine + +import "fmt" + +// hllAddLocal is the owner / single-node write path. +// The store assigns the stored version; we fan PeekVersion after the write. +func (e *Engine) hllAddLocal(ks *ksRuntime, name string, item []byte) error { + expire := e.expireAt(ks.cfg.TTL) + max := e.maxValueSize + if ks.cfg.MaxValueSize > 0 { + max = ks.cfg.MaxValueSize + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + applied, tooLarge := ks.store.HLLAdd(name, item, gate, expire, max) + if tooLarge { + return ErrValueTooLarge + } + if !applied { + return fmt.Errorf("%w: hll add rejected", ErrInvalidArgument) + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.hllReplicateSnapshot(ks, name, ver, expire) + return nil +} + +// applyHLLAdd is the owner-inbox ApplyPut of FlagHLLAdd (raw item). +// Non-owners must not reach here (ApplyPut returns applied=false first). +func (e *Engine) applyHLLAdd(ks *ksRuntime, name string, item []byte, expireAt int64) bool { + if len(item) == 0 { + return false + } + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + max := e.maxValueSize + if ks.cfg.MaxValueSize > 0 { + max = ks.cfg.MaxValueSize + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + ok, tooLarge := ks.store.HLLAdd(name, item, gate, expireAt, max) + if !ok || tooLarge { + return false + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.hllReplicateSnapshot(ks, name, ver, expireAt) + return true +} + +// applyHLLInstall is replica / handoff ApplyPut of FlagHLL (LWW replace). +func (e *Engine) applyHLLInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.HLLInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/hll_cluster.go b/pkg/engine/hll_cluster.go new file mode 100644 index 0000000..830fb45 --- /dev/null +++ b/pkg/engine/hll_cluster.go @@ -0,0 +1,65 @@ +package engine + +import ( + "context" + "fmt" + + "github.com/Code0987/supercache/pkg/store" +) + +// hllMutViaOwner sends an inbox FlagHLLAdd to the ring owner. +// ACK-only: a rejected apply surfaces as InvalidArgument (no return payload). +func (e *Engine) hllMutViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: append([]byte(nil), item...), Flags: store.FlagHLLAdd, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + applied, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + if err != nil { + return err + } + if !applied { + return fmt.Errorf("%w: hll add rejected", ErrInvalidArgument) + } + return nil +} + +// hllReplicateSnapshot fans the post-write FlagHLL blob to RF−1 replicas. +// hintID is (ks, name), so a later snapshot replaces a pending hint — both items stay. +func (e *Engine) hllReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { + ent, ok := ks.store.Peek(name) + if !ok || !ent.IsHLL() { + return + } + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagHLL, + }, false) +} + +// hllFetchOwner loads a missing local name from the owner (GetOrLoad). +// RPC / !Found / wrong type → miss + nil error (do not return Unavailable). +// Replicas may install the snapshot; non-replicas do not. +func (e *Engine) hllFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsHLL() { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.HLLInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/hllx/hllx.go b/pkg/hllx/hllx.go index f1bc451..32ce875 100644 --- a/pkg/hllx/hllx.go +++ b/pkg/hllx/hllx.go @@ -103,8 +103,9 @@ func Count(regs []byte) uint64 { return uint64(math.Floor(e + 0.5)) } -// Merge writes per-register max(dst, src) into dst. -func Merge(dst, src []byte) error { +// merge writes per-register max(dst, src) into dst. +// Not used on install (handoff is LWW replace, not max-merge). +func merge(dst, src []byte) error { if len(dst) != DenseSize || len(src) != DenseSize { return ErrSize } diff --git a/pkg/hllx/hllx_test.go b/pkg/hllx/hllx_test.go index 44e9824..97a35e8 100644 --- a/pkg/hllx/hllx_test.go +++ b/pkg/hllx/hllx_test.go @@ -92,7 +92,7 @@ func TestNewSize(t *testing.T) { if len(New()) != DenseSize || DenseSize != 12288 { t.Fatal(len(New()), DenseSize) } - if err := Merge(New(), []byte{1}); err != ErrSize { + if err := merge(New(), []byte{1}); err != ErrSize { t.Fatal(err) } } @@ -101,7 +101,7 @@ func TestMergeMax(t *testing.T) { a, b := New(), New() Add(a, []byte("alice")) Add(b, []byte("bob")) - if err := Merge(a, b); err != nil { + if err := merge(a, b); err != nil { t.Fatal(err) } n := Count(a) diff --git a/pkg/store/hll.go b/pkg/store/hll.go new file mode 100644 index 0000000..9d8228b --- /dev/null +++ b/pkg/store/hll.go @@ -0,0 +1,146 @@ +package store + +import ( + "container/list" + + "github.com/Code0987/supercache/pkg/hllx" +) + +// HLLAdd hashes item into the named sketch (creates if missing). +// +// version is only the tombstone-gate floor. The stored version is 1 on create +// or local+1 on a live/tombstone replace — never the inbound number. +func (m *Memory) HLLAdd(key string, item []byte, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { + m.mu.Lock() + defer m.mu.Unlock() + + if maxValue > 0 && hllx.DenseSize > maxValue { + return false, true + } + + if el, exists := m.items[key]; exists { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false, false + } + return m.hllCommitLocked(el, it, key, nil, item, it.entry.Version+1, expireAt) + } + if !it.entry.IsHLL() { + return false, false + } + return m.hllCommitLocked(el, it, key, it.entry.Value, item, it.entry.Version+1, expireAt) + } + m.removeElement(el) + } + return m.hllInsertLocked(key, item, 1, expireAt) +} + +func (m *Memory) hllCommitLocked(el *list.Element, it *lruItem, key string, cur, item []byte, stored uint64, expireAt int64) (applied, tooLarge bool) { + var work []byte + if len(cur) == hllx.DenseSize { + hllx.Add(cur, item) + work = cur + } else { + work = hllx.New() + hllx.Add(work, item) + } + oldCost := it.cost + it.entry.Version = stored + it.entry.Flags = FlagHLL + it.entry.Value = work + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.cost = entryCost(key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still, false +} + +func (m *Memory) hllInsertLocked(key string, item []byte, stored uint64, expireAt int64) (applied, tooLarge bool) { + regs := hllx.New() + hllx.Add(regs, item) + ent := Entry{Value: regs, Version: stored, ExpireAt: expireAt, Flags: FlagHLL} + return m.insertHLLLocked(key, ent), false +} + +// HLLCount is the sketch estimate. Missing → ok=false. +func (m *Memory) HLLCount(key string) (uint64, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.hasHLLLocked(key) { + return 0, false + } + el := m.items[key] + it := el.Value.(*lruItem) + return hllx.Count(it.entry.Value), true +} + +// HasHLL is the present-bit: live, unexpired, FlagHLL. +func (m *Memory) HasHLL(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.hasHLLLocked(key) +} + +func (m *Memory) hasHLLLocked(key string) bool { + el, ok := m.items[key] + if !ok { + return false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return false + } + return !it.entry.IsTombstone() && it.entry.IsHLL() +} + +// HLLInstall is LWW snapshot handoff: keep blob if version > local. +func (m *Memory) HLLInstall(key string, blob []byte, version uint64, expireAt int64) bool { + if len(blob) != hllx.DenseSize { + return false + } + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsHLL() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagHLL} + return m.insertHLLLocked(key, ent) +} + +func (m *Memory) insertHLLLocked(key string, ent Entry) bool { + cost := entryCost(key, ent) + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..0e174ed 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -11,7 +11,6 @@ import ( "github.com/Code0987/supercache/pkg/counter" "github.com/Code0987/supercache/pkg/geo" "github.com/Code0987/supercache/pkg/hashx" - "github.com/Code0987/supercache/pkg/hllx" "github.com/Code0987/supercache/pkg/jsonx" "github.com/Code0987/supercache/pkg/listx" "github.com/Code0987/supercache/pkg/set" @@ -2240,138 +2239,6 @@ func (m *Memory) BInstall(key string, blob []byte, version uint64, expireAt int6 return m.insertBitmapLocked(key, ent) } -func (m *Memory) HLLAdd(key string, item []byte, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { - m.mu.Lock() - defer m.mu.Unlock() - - if maxValue > 0 && hllx.DenseSize > maxValue { - return false, true - } - - if el, exists := m.items[key]; exists { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false, false - } - return m.hllCommitLocked(el, it, key, nil, item, it.entry.Version+1, expireAt) - } - if !it.entry.IsHLL() { - return false, false - } - return m.hllCommitLocked(el, it, key, it.entry.Value, item, it.entry.Version+1, expireAt) - } - m.removeElement(el) - } - return m.hllInsertLocked(key, item, 1, expireAt) -} - -func (m *Memory) hllCommitLocked(el *list.Element, it *lruItem, key string, cur, item []byte, stored uint64, expireAt int64) (applied, tooLarge bool) { - var work []byte - if len(cur) == hllx.DenseSize { - hllx.Add(cur, item) - work = cur - } else { - work = hllx.New() - hllx.Add(work, item) - } - oldCost := it.cost - it.entry.Version = stored - it.entry.Flags = FlagHLL - it.entry.Value = work - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.cost = entryCost(key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still, false -} - -func (m *Memory) hllInsertLocked(key string, item []byte, stored uint64, expireAt int64) (applied, tooLarge bool) { - regs := hllx.New() - hllx.Add(regs, item) - ent := Entry{Value: regs, Version: stored, ExpireAt: expireAt, Flags: FlagHLL} - return m.insertHLLLocked(key, ent), false -} - -func (m *Memory) HLLCount(key string) (uint64, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if !m.hasHLLLocked(key) { - return 0, false - } - el := m.items[key] - it := el.Value.(*lruItem) - return hllx.Count(it.entry.Value), true -} - -func (m *Memory) HasHLL(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.hasHLLLocked(key) -} - -func (m *Memory) hasHLLLocked(key string) bool { - el, ok := m.items[key] - if !ok { - return false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return false - } - return !it.entry.IsTombstone() && it.entry.IsHLL() -} - -func (m *Memory) HLLInstall(key string, blob []byte, version uint64, expireAt int64) bool { - if len(blob) != hllx.DenseSize { - return false - } - m.mu.Lock() - defer m.mu.Unlock() - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsHLL() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagHLL} - return m.insertHLLLocked(key, ent) -} - -func (m *Memory) insertHLLLocked(key string, ent Entry) bool { - cost := entryCost(key, ent) - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) insertBitmapLocked(key string, ent Entry) bool { cost := entryCost(key, ent) it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} From 953a32b04c4680f15213bdae2936491c2c150f2c Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:33:27 +0000 Subject: [PATCH 02/13] refactor: extract ModeBitmap into its own files Move store Bitmap off memory.go and split engine Bitmap into public API, owner apply, and cluster hops. No contract change. --- docs/design/2026-09-02-refactor-bitmap.md | 38 +++++ docs/design/README.md | 1 + pkg/engine/bitmap.go | 104 +------------ pkg/engine/bitmap_apply.go | 64 ++++++++ pkg/engine/bitmap_cluster.go | 65 ++++++++ pkg/store/bitmap.go | 174 ++++++++++++++++++++++ pkg/store/memory.go | 159 -------------------- 7 files changed, 343 insertions(+), 262 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-bitmap.md create mode 100644 pkg/engine/bitmap_apply.go create mode 100644 pkg/engine/bitmap_cluster.go create mode 100644 pkg/store/bitmap.go diff --git a/docs/design/2026-09-02-refactor-bitmap.md b/docs/design/2026-09-02-refactor-bitmap.md new file mode 100644 index 0000000..7d988ac --- /dev/null +++ b/docs/design/2026-09-02-refactor-bitmap.md @@ -0,0 +1,38 @@ +# Refactor ModeBitmap layout (no contract change) + +**Status:** approved (chat: continue after HLL) +**Branch:** `feat/refactor-bitmap` +**Date:** 2026-09-02 + +## Problem + +Bitmap store methods still live in `pkg/store/memory.go`. Engine Bitmap is one ~250-line file mixing public verbs, owner apply, and cluster hops. + +## Non-goals + +- No API / proto / flag / version / fan-out / hintID change +- No `bitmapx` algorithm change +- Not extracting other modes (HLL is a separate PR) + +## Contract + +Unchanged: `BitSet` / `BitGet` / `BitCount` / `BitPos` / `Delete(name)`, `FlagBitmap` + inbox `FlagBitmapSet`, snapshot fan-out, version under store mutex, present-bit, empty-until-delete. + +## Approach + +Same file layout as HLL / TopK / CMS: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` B* | `pkg/store/bitmap.go` | +| `pkg/engine/bitmap.go` (all) | `bitmap.go` public verbs; `bitmap_apply.go` owner write; `bitmap_cluster.go` inbox/fan-out/GetOrLoad | + +Rejected: rewrite bit packing; split `bitmapx` (150 lines is one file). + +## Tests (already exist; keep them) + +Existing `pkg/bitmapx`, `pkg/store` bitmap, `pkg/engine` bitmap unit + cluster + hint-after-down. + +## Bench risk + +No new Get/Peek logic. Local `BenchmarkEngineGetHit` / `BenchmarkStoreGetHit` allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..b6a72d2 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-bitmap.md](./2026-09-02-refactor-bitmap.md) | ModeBitmap file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/bitmap.go b/pkg/engine/bitmap.go index 8e946b2..344a4a3 100644 --- a/pkg/engine/bitmap.go +++ b/pkg/engine/bitmap.go @@ -6,10 +6,10 @@ import ( "github.com/Code0987/supercache/pkg/bitmapx" "github.com/Code0987/supercache/pkg/keyspace" - "github.com/Code0987/supercache/pkg/store" ) // BitSet writes a bit on a ModeBitmap (Redis SETBIT). ACK-only. +// Non-owners forward an inbox FlagBitmapSet; the owner applies and fans a snapshot. func (e *Engine) BitSet(ctx context.Context, keyspaceName, name string, offset uint64, bit bool) error { if err := ctx.Err(); err != nil { return err @@ -144,105 +144,3 @@ func (e *Engine) bitmapNeed(ks *ksRuntime, offset uint64) error { } return nil } - -func (e *Engine) bMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: value, Flags: store.FlagBitmapSet, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - applied, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - if err != nil { - return err - } - if !applied { - return fmt.Errorf("%w: bitmap set rejected", ErrInvalidArgument) - } - return nil -} - -func (e *Engine) bSetLocal(ks *ksRuntime, name string, offset uint64, bit bool) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - applied, tooLarge := ks.store.BSet(name, offset, bit, gate, expire, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf("%w: bitmap set rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.bReplicateSnapshot(ks, name, ver, expire) - return nil -} - -func (e *Engine) bReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { - ent, ok := ks.store.Peek(name) - if !ok || !ent.IsBitmap() { - return - } - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: ent.Value, - Version: ver, - ExpireAt: expire, - Flags: store.FlagBitmap, - }, false) -} - -func (e *Engine) bFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsBitmap() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.BInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyBitmapSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { - offset, bit, err := bitmapx.DecodeSet(inbox) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.BSet(name, offset, bit, gate, expireAt, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.bReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyBitmapInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.BInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/bitmap_apply.go b/pkg/engine/bitmap_apply.go new file mode 100644 index 0000000..90dc556 --- /dev/null +++ b/pkg/engine/bitmap_apply.go @@ -0,0 +1,64 @@ +package engine + +import ( + "fmt" + + "github.com/Code0987/supercache/pkg/bitmapx" +) + +// bSetLocal is the owner / single-node write path. +// The store assigns the stored version; we fan PeekVersion after the write. +func (e *Engine) bSetLocal(ks *ksRuntime, name string, offset uint64, bit bool) error { + expire := e.expireAt(ks.cfg.TTL) + max := e.maxValueSize + if ks.cfg.MaxValueSize > 0 { + max = ks.cfg.MaxValueSize + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + applied, tooLarge := ks.store.BSet(name, offset, bit, gate, expire, max) + if tooLarge { + return ErrValueTooLarge + } + if !applied { + return fmt.Errorf("%w: bitmap set rejected", ErrInvalidArgument) + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.bReplicateSnapshot(ks, name, ver, expire) + return nil +} + +// applyBitmapSet is the owner-inbox ApplyPut of FlagBitmapSet. +// Non-owners must not reach here (ApplyPut returns applied=false first). +func (e *Engine) applyBitmapSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { + offset, bit, err := bitmapx.DecodeSet(inbox) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + max := e.maxValueSize + if ks.cfg.MaxValueSize > 0 { + max = ks.cfg.MaxValueSize + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + ok, tooLarge := ks.store.BSet(name, offset, bit, gate, expireAt, max) + if !ok || tooLarge { + return false + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.bReplicateSnapshot(ks, name, ver, expireAt) + return true +} + +// applyBitmapInstall is replica / handoff ApplyPut of FlagBitmap (LWW replace). +func (e *Engine) applyBitmapInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.BInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/bitmap_cluster.go b/pkg/engine/bitmap_cluster.go new file mode 100644 index 0000000..e9df40b --- /dev/null +++ b/pkg/engine/bitmap_cluster.go @@ -0,0 +1,65 @@ +package engine + +import ( + "context" + "fmt" + + "github.com/Code0987/supercache/pkg/store" +) + +// bMutViaOwner sends an inbox FlagBitmapSet to the ring owner. +// ACK-only: a rejected apply surfaces as InvalidArgument (no return payload). +func (e *Engine) bMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: value, Flags: store.FlagBitmapSet, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + applied, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + if err != nil { + return err + } + if !applied { + return fmt.Errorf("%w: bitmap set rejected", ErrInvalidArgument) + } + return nil +} + +// bReplicateSnapshot fans the post-write FlagBitmap blob to RF−1 replicas. +// hintID is (ks, name), so a later snapshot replaces a pending hint — both bits stay. +func (e *Engine) bReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { + ent, ok := ks.store.Peek(name) + if !ok || !ent.IsBitmap() { + return + } + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagBitmap, + }, false) +} + +// bFetchOwner loads a missing local name from the owner (GetOrLoad). +// RPC / !Found / wrong type → miss + nil error (do not return Unavailable). +// Replicas may install the snapshot; non-replicas do not. +func (e *Engine) bFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsBitmap() { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.BInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/store/bitmap.go b/pkg/store/bitmap.go new file mode 100644 index 0000000..0659428 --- /dev/null +++ b/pkg/store/bitmap.go @@ -0,0 +1,174 @@ +package store + +import ( + "container/list" + + "github.com/Code0987/supercache/pkg/bitmapx" +) + +// BSet writes a bit on the named bitmap (creates if missing). +// +// version is only the tombstone-gate floor. The stored version is 1 on create +// or local+1 on a live/tombstone replace — never the inbound number. +func (m *Memory) BSet(key string, offset uint64, bit bool, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { + m.mu.Lock() + defer m.mu.Unlock() + + need, ok := bitmapx.EncodedLen(offset) + if !ok { + return false, true + } + if maxValue > 0 && need > maxValue { + return false, true + } + + if el, exists := m.items[key]; exists { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false, false + } + return m.bCommitLocked(el, it, key, nil, offset, bit, it.entry.Version+1, expireAt, maxValue) + } + if !it.entry.IsBitmap() { + return false, false + } + return m.bCommitLocked(el, it, key, it.entry.Value, offset, bit, it.entry.Version+1, expireAt, maxValue) + } + m.removeElement(el) + } + return m.bInsertLocked(key, offset, bit, 1, expireAt, maxValue) +} + +func (m *Memory) bCommitLocked(el *list.Element, it *lruItem, key string, cur []byte, offset uint64, bit bool, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { + var work []byte + if cur != nil { + work = append([]byte(nil), cur...) + } + next := bitmapx.Set(work, offset, bit) + if maxValue > 0 && len(next) > maxValue { + return false, true + } + oldCost := it.cost + it.entry.Version = stored + it.entry.Flags = FlagBitmap + it.entry.Value = next + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.cost = entryCost(key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still, false +} + +func (m *Memory) bInsertLocked(key string, offset uint64, bit bool, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { + next := bitmapx.Set(nil, offset, bit) + if maxValue > 0 && len(next) > maxValue { + return false, true + } + ent := Entry{Value: next, Version: stored, ExpireAt: expireAt, Flags: FlagBitmap} + return m.insertBitmapLocked(key, ent), false +} + +// BGet returns the bit at offset. Missing → ok=false. +func (m *Memory) BGet(key string, offset uint64) (bool, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.hasBitmapLocked(key) { + return false, false + } + el := m.items[key] + it := el.Value.(*lruItem) + return bitmapx.Get(it.entry.Value, offset), true +} + +// BCount is BITCOUNT over a byte window. Missing → ok=false. +func (m *Memory) BCount(key string, start, end int) (int64, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.hasBitmapLocked(key) { + return 0, false + } + el := m.items[key] + it := el.Value.(*lruItem) + return bitmapx.Count(it.entry.Value, start, end), true +} + +// BPos is BITPOS over a byte window. Missing → ok=false. +func (m *Memory) BPos(key string, bit bool, start, end int) (int64, bool, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.hasBitmapLocked(key) { + return 0, false, false + } + el := m.items[key] + it := el.Value.(*lruItem) + pos, found := bitmapx.Pos(it.entry.Value, bit, start, end) + return pos, found, true +} + +// HasBitmap is the present-bit: live, unexpired, FlagBitmap. +func (m *Memory) HasBitmap(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.hasBitmapLocked(key) +} + +func (m *Memory) hasBitmapLocked(key string) bool { + el, ok := m.items[key] + if !ok { + return false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return false + } + return !it.entry.IsTombstone() && it.entry.IsBitmap() +} + +// BInstall is LWW snapshot handoff: keep blob if version > local. +func (m *Memory) BInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsBitmap() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagBitmap} + return m.insertBitmapLocked(key, ent) +} + +func (m *Memory) insertBitmapLocked(key string, ent Entry) bool { + cost := entryCost(key, ent) + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..4991914 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -6,7 +6,6 @@ import ( "sync/atomic" "time" - "github.com/Code0987/supercache/pkg/bitmapx" "github.com/Code0987/supercache/pkg/bloom" "github.com/Code0987/supercache/pkg/counter" "github.com/Code0987/supercache/pkg/geo" @@ -2093,153 +2092,6 @@ func (m *Memory) insertJSONLocked(key string, ent Entry, cache any, dirty bool) return ok } -func (m *Memory) BSet(key string, offset uint64, bit bool, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { - m.mu.Lock() - defer m.mu.Unlock() - - need, ok := bitmapx.EncodedLen(offset) - if !ok { - return false, true - } - if maxValue > 0 && need > maxValue { - return false, true - } - - if el, exists := m.items[key]; exists { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false, false - } - return m.bCommitLocked(el, it, key, nil, offset, bit, it.entry.Version+1, expireAt, maxValue) - } - if !it.entry.IsBitmap() { - return false, false - } - return m.bCommitLocked(el, it, key, it.entry.Value, offset, bit, it.entry.Version+1, expireAt, maxValue) - } - m.removeElement(el) - } - return m.bInsertLocked(key, offset, bit, 1, expireAt, maxValue) -} - -func (m *Memory) bCommitLocked(el *list.Element, it *lruItem, key string, cur []byte, offset uint64, bit bool, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { - var work []byte - if cur != nil { - work = append([]byte(nil), cur...) - } - next := bitmapx.Set(work, offset, bit) - if maxValue > 0 && len(next) > maxValue { - return false, true - } - oldCost := it.cost - it.entry.Version = stored - it.entry.Flags = FlagBitmap - it.entry.Value = next - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.cost = entryCost(key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still, false -} - -func (m *Memory) bInsertLocked(key string, offset uint64, bit bool, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { - next := bitmapx.Set(nil, offset, bit) - if maxValue > 0 && len(next) > maxValue { - return false, true - } - ent := Entry{Value: next, Version: stored, ExpireAt: expireAt, Flags: FlagBitmap} - return m.insertBitmapLocked(key, ent), false -} - -func (m *Memory) BGet(key string, offset uint64) (bool, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if !m.hasBitmapLocked(key) { - return false, false - } - el := m.items[key] - it := el.Value.(*lruItem) - return bitmapx.Get(it.entry.Value, offset), true -} - -func (m *Memory) BCount(key string, start, end int) (int64, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if !m.hasBitmapLocked(key) { - return 0, false - } - el := m.items[key] - it := el.Value.(*lruItem) - return bitmapx.Count(it.entry.Value, start, end), true -} - -func (m *Memory) BPos(key string, bit bool, start, end int) (int64, bool, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if !m.hasBitmapLocked(key) { - return 0, false, false - } - el := m.items[key] - it := el.Value.(*lruItem) - pos, found := bitmapx.Pos(it.entry.Value, bit, start, end) - return pos, found, true -} - -func (m *Memory) HasBitmap(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.hasBitmapLocked(key) -} - -func (m *Memory) hasBitmapLocked(key string) bool { - el, ok := m.items[key] - if !ok { - return false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return false - } - return !it.entry.IsTombstone() && it.entry.IsBitmap() -} - -func (m *Memory) BInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsBitmap() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagBitmap} - return m.insertBitmapLocked(key, ent) -} - func (m *Memory) HLLAdd(key string, item []byte, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { m.mu.Lock() defer m.mu.Unlock() @@ -2372,17 +2224,6 @@ func (m *Memory) insertHLLLocked(key string, ent Entry) bool { return ok } -func (m *Memory) insertBitmapLocked(key string, ent Entry) bool { - cost := entryCost(key, ent) - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) Close() { m.mu.Lock() defer m.mu.Unlock() From 1e84ed7d088c73836abc020ede79ae199f1bf2af Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:46:06 +0000 Subject: [PATCH 03/13] refactor: extract ModeCounter into its own files Move store Counter off memory.go and split engine Counter into public API, owner write, and GetOrLoad. Peer CounterIncr unchanged. --- docs/design/2026-09-02-refactor-counter.md | 38 ++++++ docs/design/README.md | 1 + pkg/engine/counter.go | 63 +-------- pkg/engine/counter_apply.go | 32 +++++ pkg/engine/counter_cluster.go | 44 ++++++ pkg/store/counter.go | 147 +++++++++++++++++++++ pkg/store/memory.go | 135 ------------------- 7 files changed, 264 insertions(+), 196 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-counter.md create mode 100644 pkg/engine/counter_apply.go create mode 100644 pkg/engine/counter_cluster.go create mode 100644 pkg/store/counter.go diff --git a/docs/design/2026-09-02-refactor-counter.md b/docs/design/2026-09-02-refactor-counter.md new file mode 100644 index 0000000..752e60e --- /dev/null +++ b/docs/design/2026-09-02-refactor-counter.md @@ -0,0 +1,38 @@ +# Refactor ModeCounter layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-counter` +**Date:** 2026-09-02 + +## Problem + +Counter store methods still live in `pkg/store/memory.go`. Engine Counter is one file mixing public verbs, owner write, and GetOrLoad. + +## Non-goals + +- No API / proto / Peer `CounterIncr` / version / fan-out change +- No `pkg/counter` algorithm change +- Not extracting other modes (HLL #40, Bitmap #41) + +## Contract + +Unchanged: `Incr` returns the new int64 (peer `CounterIncr` from non-owner). `CounterGet` present-bit. `FlagCounter` snapshot. Version under store mutex. Overflow → invalid argument, no wrap. + +## Approach + +Same file layout as HLL / Bitmap: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` C* | `pkg/store/counter.go` | +| `pkg/engine/counter.go` (all) | `counter.go` public verbs; `counter_apply.go` owner write + install; `counter_cluster.go` GetOrLoad / snapshot | + +Rejected: drop the Peer RPC (that is the return-*n* contract); split `pkg/counter` (39 lines). + +## Tests (already exist; keep them) + +Existing `pkg/counter`, `pkg/store` counter, `pkg/engine` counter unit + cluster. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..487b4e8 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-counter.md](./2026-09-02-refactor-counter.md) | ModeCounter file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/counter.go b/pkg/engine/counter.go index 3e675a3..f24bb04 100644 --- a/pkg/engine/counter.go +++ b/pkg/engine/counter.go @@ -6,10 +6,10 @@ import ( "github.com/Code0987/supercache/pkg/counter" "github.com/Code0987/supercache/pkg/keyspace" - "github.com/Code0987/supercache/pkg/store" ) // Incr adds delta to a ModeCounter and returns the new value. +// Non-owners use peer CounterIncr (the return payload needs a Peer RPC). func (e *Engine) Incr(ctx context.Context, keyspaceName, name string, delta int64) (int64, error) { if err := ctx.Err(); err != nil { return 0, err @@ -59,7 +59,7 @@ func (e *Engine) CounterGet(ctx context.Context, keyspaceName, name string) (int if err := e.validateKeyLen(ks, name); err != nil { return 0, false, err } - if e.hasCounterLocal(ks, name) { + if ks.store.HasCounter(name) { v, ok := ks.store.CGet(name) return v, ok, nil } @@ -73,62 +73,3 @@ func (e *Engine) CounterGet(ctx context.Context, keyspaceName, name string) (int } return v, true, nil } - -func (e *Engine) cIncrLocal(ks *ksRuntime, name string, delta int64, fanout bool) (int64, error) { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - n, applied, overflow := ks.store.CIncr(name, delta, gate, expire) - if overflow { - return 0, fmt.Errorf("%w: counter overflow", ErrInvalidArgument) - } - if !applied { - return 0, fmt.Errorf("%w: incr rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: counter.Encode(n), - Version: ver, - ExpireAt: expire, - Flags: store.FlagCounter, - }, false) - } - return n, nil -} - -func (e *Engine) hasCounterLocal(ks *ksRuntime, name string) bool { - return ks.store.HasCounter(name) -} - -func (e *Engine) cFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsCounter() { - return store.Entry{}, false, nil - } - if _, decErr := counter.Decode(res.Entry.Value); decErr != nil { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.CInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyCounterInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.CInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/counter_apply.go b/pkg/engine/counter_apply.go new file mode 100644 index 0000000..c892bc0 --- /dev/null +++ b/pkg/engine/counter_apply.go @@ -0,0 +1,32 @@ +package engine + +import "fmt" + +// cIncrLocal is the owner / single-node write path. +// The store assigns the stored version; we fan PeekVersion after the write. +func (e *Engine) cIncrLocal(ks *ksRuntime, name string, delta int64, fanout bool) (int64, error) { + expire := e.expireAt(ks.cfg.TTL) + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + n, applied, overflow := ks.store.CIncr(name, delta, gate, expire) + if overflow { + return 0, fmt.Errorf("%w: counter overflow", ErrInvalidArgument) + } + if !applied { + return 0, fmt.Errorf("%w: incr rejected", ErrInvalidArgument) + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + if fanout { + e.cReplicateSnapshot(ks, name, n, ver, expire) + } + return n, nil +} + +// applyCounterInstall is replica / handoff ApplyPut of FlagCounter (LWW replace). +func (e *Engine) applyCounterInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.CInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/counter_cluster.go b/pkg/engine/counter_cluster.go new file mode 100644 index 0000000..2748712 --- /dev/null +++ b/pkg/engine/counter_cluster.go @@ -0,0 +1,44 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/counter" + "github.com/Code0987/supercache/pkg/store" +) + +// cReplicateSnapshot fans the post-write FlagCounter blob to RF−1 replicas. +func (e *Engine) cReplicateSnapshot(ks *ksRuntime, name string, n int64, ver uint64, expire int64) { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: counter.Encode(n), + Version: ver, + ExpireAt: expire, + Flags: store.FlagCounter, + }, false) +} + +// cFetchOwner loads a missing local name from the owner (GetOrLoad). +// RPC / !Found / wrong type / bad blob → miss + nil error (do not return Unavailable). +func (e *Engine) cFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsCounter() { + return store.Entry{}, false, nil + } + if _, decErr := counter.Decode(res.Entry.Value); decErr != nil { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.CInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/store/counter.go b/pkg/store/counter.go new file mode 100644 index 0000000..9421892 --- /dev/null +++ b/pkg/store/counter.go @@ -0,0 +1,147 @@ +package store + +import ( + "github.com/Code0987/supercache/pkg/counter" +) + +// CIncr adds delta to the named counter (creates if missing). +// +// version is only the tombstone-gate floor. The stored version is 1 on create +// or local+1 on a live/tombstone replace — never the inbound number. +// Returns (value, applied, overflow). +func (m *Memory) CIncr(key string, delta int64, version uint64, expireAt int64) (int64, bool, bool) { + m.mu.Lock() + defer m.mu.Unlock() + + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return 0, false, false + } + stored := it.entry.Version + 1 + m.removeElement(el) + return m.cInsertLocked(key, delta, stored, expireAt) + } + if !it.entry.IsCounter() { + return 0, false, false + } + cur, err := counter.Decode(it.entry.Value) + if err != nil { + return 0, false, false + } + next, err := counter.Add(cur, delta) + if err != nil { + return cur, false, true + } + it.entry.Version = it.entry.Version + 1 + it.entry.Flags = FlagCounter + it.entry.Value = counter.Encode(next) + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + oldCost := it.cost + it.cost = entryCost(key, it.entry) + m.bytes += it.cost - oldCost + m.order.MoveToFront(el) + m.evictLocked() + return next, true, false + } + m.removeElement(el) + } + return m.cInsertLocked(key, delta, 1, expireAt) +} + +func (m *Memory) cInsertLocked(key string, val int64, version uint64, expireAt int64) (int64, bool, bool) { + ent := Entry{Value: counter.Encode(val), Version: version, ExpireAt: expireAt, Flags: FlagCounter} + if !m.insertCounterLocked(key, ent) { + return 0, false, false + } + return val, true, false +} + +// CGet returns the counter value. Missing → 0, ok=false. +func (m *Memory) CGet(key string) (int64, bool) { + m.mu.Lock() + defer m.mu.Unlock() + el, ok := m.items[key] + if !ok { + return 0, false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return 0, false + } + if it.entry.IsTombstone() || !it.entry.IsCounter() { + return 0, false + } + v, err := counter.Decode(it.entry.Value) + if err != nil { + return 0, false + } + return v, true +} + +// HasCounter is the present-bit: live, unexpired, FlagCounter, decodable. +func (m *Memory) HasCounter(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + el, ok := m.items[key] + if !ok { + return false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return false + } + if it.entry.IsTombstone() || !it.entry.IsCounter() { + return false + } + _, err := counter.Decode(it.entry.Value) + return err == nil +} + +// CInstall is LWW snapshot handoff: keep blob if version > local. +func (m *Memory) CInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + if _, err := counter.Decode(blob); err != nil { + return false + } + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsCounter() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagCounter} + return m.insertCounterLocked(key, ent) +} + +func (m *Memory) insertCounterLocked(key string, ent Entry) bool { + cost := entryCost(key, ent) + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..a94e263 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -8,7 +8,6 @@ import ( "github.com/Code0987/supercache/pkg/bitmapx" "github.com/Code0987/supercache/pkg/bloom" - "github.com/Code0987/supercache/pkg/counter" "github.com/Code0987/supercache/pkg/geo" "github.com/Code0987/supercache/pkg/hashx" "github.com/Code0987/supercache/pkg/hllx" @@ -1676,140 +1675,6 @@ func (m *Memory) insertHashLocked(key string, ent Entry, cache *hashx.Hash, dirt return ok } -func (m *Memory) CIncr(key string, delta int64, version uint64, expireAt int64) (int64, bool, bool) { - m.mu.Lock() - defer m.mu.Unlock() - - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return 0, false, false - } - stored := it.entry.Version + 1 - m.removeElement(el) - return m.cInsertLocked(key, delta, stored, expireAt) - } - if !it.entry.IsCounter() { - return 0, false, false - } - cur, err := counter.Decode(it.entry.Value) - if err != nil { - return 0, false, false - } - next, err := counter.Add(cur, delta) - if err != nil { - return cur, false, true - } - it.entry.Version = it.entry.Version + 1 - it.entry.Flags = FlagCounter - it.entry.Value = counter.Encode(next) - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - oldCost := it.cost - it.cost = entryCost(key, it.entry) - m.bytes += it.cost - oldCost - m.order.MoveToFront(el) - m.evictLocked() - return next, true, false - } - m.removeElement(el) - } - return m.cInsertLocked(key, delta, 1, expireAt) -} - -func (m *Memory) cInsertLocked(key string, val int64, version uint64, expireAt int64) (int64, bool, bool) { - ent := Entry{Value: counter.Encode(val), Version: version, ExpireAt: expireAt, Flags: FlagCounter} - if !m.insertCounterLocked(key, ent) { - return 0, false, false - } - return val, true, false -} - -func (m *Memory) CGet(key string) (int64, bool) { - m.mu.Lock() - defer m.mu.Unlock() - el, ok := m.items[key] - if !ok { - return 0, false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return 0, false - } - if it.entry.IsTombstone() || !it.entry.IsCounter() { - return 0, false - } - v, err := counter.Decode(it.entry.Value) - if err != nil { - return 0, false - } - return v, true -} - -func (m *Memory) HasCounter(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - el, ok := m.items[key] - if !ok { - return false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return false - } - if it.entry.IsTombstone() || !it.entry.IsCounter() { - return false - } - _, err := counter.Decode(it.entry.Value) - return err == nil -} - -func (m *Memory) CInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - if _, err := counter.Decode(blob); err != nil { - return false - } - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsCounter() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagCounter} - return m.insertCounterLocked(key, ent) -} - -func (m *Memory) insertCounterLocked(key string, ent Entry) bool { - cost := entryCost(key, ent) - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) JSet(key, path string, raw []byte, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { m.mu.Lock() defer m.mu.Unlock() From 6c9d4b387a96536707781734930ca83f13ffc02f Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:22:15 +0000 Subject: [PATCH 04/13] refactor: extract ModeGeo into its own files Move Geo store methods out of memory.go and split the engine file into public verbs, owner write/apply, and cluster forward. Same contract, including dirty gCache and FlagGeoAdd/FlagGeoRem item fan-out. --- docs/design/2026-09-02-refactor-geo.md | 40 +++++ docs/design/README.md | 1 + pkg/engine/geo.go | 141 +++------------ pkg/engine/geo_apply.go | 77 ++++++++ pkg/engine/geo_cluster.go | 38 ++++ pkg/store/geo.go | 238 +++++++++++++++++++++++++ pkg/store/memory.go | 233 ------------------------ 7 files changed, 422 insertions(+), 346 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-geo.md create mode 100644 pkg/engine/geo_apply.go create mode 100644 pkg/engine/geo_cluster.go create mode 100644 pkg/store/geo.go diff --git a/docs/design/2026-09-02-refactor-geo.md b/docs/design/2026-09-02-refactor-geo.md new file mode 100644 index 0000000..575177a --- /dev/null +++ b/docs/design/2026-09-02-refactor-geo.md @@ -0,0 +1,40 @@ +# Refactor ModeGeo layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-geo` +**Date:** 2026-09-02 + +## Problem + +Geo store methods still live in `pkg/store/memory.go`. Engine Geo is one file mixing public verbs, owner write, and item fan-out. + +## Non-goals + +- No API / proto / version / fan-out / dirty-cache change +- No `pkg/geo` algorithm change +- Not extracting other modes (ZSet / Set / Bloom / List / Hash / JSON stay on their own branches) + +## Contract + +Unchanged: GeoAdd/GeoRem ACK-only inbox (`FlagGeoAdd` / `FlagGeoRem` item fan-out). GeoRem of missing is a no-op. Invalid lon/lat and NaN/negative radius rejected. Dirty `gCache` flushed on Get/Peek only. + +## Approach + +Same file layout as ZSet: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` Geo* | `pkg/store/geo.go` (including `flushGeoValueLocked` / `insertGeoLocked`) | +| `pkg/engine/geo.go` (all) | `geo.go` public verbs; `geo_apply.go` owner write + inbox apply; `geo_cluster.go` owner forward / GetOrLoad | + +Geo-local error format strings live in a `const` block at the top of `geo.go` (same text as before). + +Rejected: rewrite dirty-cache / flush. Switch Geo to snapshot fan-out. Split `pkg/geo`. + +## Tests (already exist; keep them) + +Existing `pkg/geo`, `pkg/store` geo, `pkg/engine` geo unit + cluster. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..30d7e07 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-geo.md](./2026-09-02-refactor-geo.md) | ModeGeo file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/geo.go b/pkg/engine/geo.go index 124bbee..3d1a9c9 100644 --- a/pkg/engine/geo.go +++ b/pkg/engine/geo.go @@ -9,6 +9,21 @@ import ( "github.com/Code0987/supercache/pkg/store" ) +const ( + errEmptyGeoMember = "%w: empty geo member" + errInvalidLonLat = "%w: invalid lon/lat" + errInvalidRadius = "%w: invalid radius" + errGeoAddRequiresMode = "%w: GeoAdd requires ModeGeo" + errGeoRemRequiresMode = "%w: GeoRem requires ModeGeo" + errGeoPosRequiresMode = "%w: GeoPos requires ModeGeo" + errGeoCardRequiresMode = "%w: GeoCard requires ModeGeo" + errGeoDistRequiresMode = "%w: GeoDist requires ModeGeo" + errGeoRadiusRequiresMode = "%w: GeoRadius requires ModeGeo" + errGeoAddRejected = "%w: geoadd rejected" + errGeoRemRejected = "%w: georem rejected" + errOwnerNoAddress = "%w: owner %s has no address" +) + // GeoMember is a point plus optional distance from a query. type GeoMember struct { Member []byte @@ -26,17 +41,17 @@ func (e *Engine) GeoAdd(ctx context.Context, keyspaceName, name string, member [ return err } if len(member) == 0 { - return fmt.Errorf("%w: empty geo member", ErrInvalidArgument) + return fmt.Errorf(errEmptyGeoMember, ErrInvalidArgument) } if !geo.ValidCoord(lon, lat) { - return fmt.Errorf("%w: invalid lon/lat", ErrInvalidArgument) + return fmt.Errorf(errInvalidLonLat, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeGeo { - return fmt.Errorf("%w: GeoAdd requires ModeGeo", ErrInvalidArgument) + return fmt.Errorf(errGeoAddRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -48,7 +63,7 @@ func (e *Engine) GeoAdd(ctx context.Context, keyspaceName, name string, member [ if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.gMutViaOwner(ctx, ks, name, geo.EncodeAdd(member, lon, lat), store.FlagGeoAdd) } @@ -65,14 +80,14 @@ func (e *Engine) GeoRem(ctx context.Context, keyspaceName, name string, member [ return err } if len(member) == 0 { - return fmt.Errorf("%w: empty geo member", ErrInvalidArgument) + return fmt.Errorf(errEmptyGeoMember, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeGeo { - return fmt.Errorf("%w: GeoRem requires ModeGeo", ErrInvalidArgument) + return fmt.Errorf(errGeoRemRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -84,7 +99,7 @@ func (e *Engine) GeoRem(ctx context.Context, keyspaceName, name string, member [ if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.gMutViaOwner(ctx, ks, name, append([]byte(nil), member...), store.FlagGeoRem) } @@ -105,7 +120,7 @@ func (e *Engine) GeoPos(ctx context.Context, keyspaceName, name string, member [ return 0, 0, false, err } if ks.cfg.Mode != keyspace.ModeGeo { - return 0, 0, false, fmt.Errorf("%w: GeoPos requires ModeGeo", ErrInvalidArgument) + return 0, 0, false, fmt.Errorf(errGeoPosRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, 0, false, err @@ -141,7 +156,7 @@ func (e *Engine) GeoCard(ctx context.Context, keyspaceName, name string) (int, e return 0, err } if ks.cfg.Mode != keyspace.ModeGeo { - return 0, fmt.Errorf("%w: GeoCard requires ModeGeo", ErrInvalidArgument) + return 0, fmt.Errorf(errGeoCardRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, err @@ -173,7 +188,7 @@ func (e *Engine) GeoDist(ctx context.Context, keyspaceName, name string, a, b [] return 0, false, err } if ks.cfg.Mode != keyspace.ModeGeo { - return 0, false, fmt.Errorf("%w: GeoDist requires ModeGeo", ErrInvalidArgument) + return 0, false, fmt.Errorf(errGeoDistRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, false, err @@ -203,17 +218,17 @@ func (e *Engine) GeoRadius(ctx context.Context, keyspaceName, name string, lon, return nil, err } if !geo.ValidCoord(lon, lat) { - return nil, fmt.Errorf("%w: invalid lon/lat", ErrInvalidArgument) + return nil, fmt.Errorf(errInvalidLonLat, ErrInvalidArgument) } if radiusM < 0 || mathIsNaN(radiusM) { - return nil, fmt.Errorf("%w: invalid radius", ErrInvalidArgument) + return nil, fmt.Errorf(errInvalidRadius, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return nil, err } if ks.cfg.Mode != keyspace.ModeGeo { - return nil, fmt.Errorf("%w: GeoRadius requires ModeGeo", ErrInvalidArgument) + return nil, fmt.Errorf(errGeoRadiusRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, err @@ -236,110 +251,10 @@ func mathIsNaN(f float64) bool { return f != f } -func (e *Engine) gMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: value, Flags: flag, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - return err -} - -func (e *Engine) gAddLocal(ks *ksRuntime, name string, member []byte, lon, lat float64, fanout bool) error { - ver := e.gNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.GeoAdd(name, member, lon, lat, ver, expire) { - return fmt.Errorf("%w: geoadd rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: geo.EncodeAdd(member, lon, lat), - Version: ver, - ExpireAt: expire, - Flags: store.FlagGeoAdd, - }, false) - } - return nil -} - -func (e *Engine) gRemLocal(ks *ksRuntime, name string, member []byte, fanout bool) error { - ver := e.gNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.GeoRem(name, member, ver, expire) { - if !e.hasGeoLocal(ks, name) { - return nil - } - return fmt.Errorf("%w: georem rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), member...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagGeoRem, - }, false) - } - return nil -} - -func (e *Engine) gNextVersion(ks *ksRuntime, name string) uint64 { - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - func (e *Engine) hasGeoLocal(ks *ksRuntime, name string) bool { return ks.store.HasGeo(name) } -func (e *Engine) gFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsGeo() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.GeoInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyGeoAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { - member, lon, lat, err := geo.DecodeAdd(value) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.GeoAdd(name, member, lon, lat, version, expireAt) -} - -func (e *Engine) applyGeoRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.GeoRem(name, member, version, expireAt) -} - -func (e *Engine) applyGeoInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.GeoInstall(name, blob, version, expireAt) -} - func toEngineGeoMembers(in []store.GeoMember) []GeoMember { if len(in) == 0 { return nil diff --git a/pkg/engine/geo_apply.go b/pkg/engine/geo_apply.go new file mode 100644 index 0000000..ea8da21 --- /dev/null +++ b/pkg/engine/geo_apply.go @@ -0,0 +1,77 @@ +package engine + +import ( + "fmt" + + "github.com/Code0987/supercache/pkg/geo" + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) gAddLocal(ks *ksRuntime, name string, member []byte, lon, lat float64, fanout bool) error { + ver := e.gNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.GeoAdd(name, member, lon, lat, ver, expire) { + return fmt.Errorf(errGeoAddRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: geo.EncodeAdd(member, lon, lat), + Version: ver, + ExpireAt: expire, + Flags: store.FlagGeoAdd, + }, false) + } + return nil +} + +func (e *Engine) gRemLocal(ks *ksRuntime, name string, member []byte, fanout bool) error { + ver := e.gNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.GeoRem(name, member, ver, expire) { + if !e.hasGeoLocal(ks, name) { + return nil + } + return fmt.Errorf(errGeoRemRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: append([]byte(nil), member...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagGeoRem, + }, false) + } + return nil +} + +func (e *Engine) gNextVersion(ks *ksRuntime, name string) uint64 { + if ver, ok := ks.store.PeekVersion(name); ok { + return ks.nextVersion(name, ver) + } + return ks.nextVersion(name, 0) +} + +func (e *Engine) applyGeoAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { + member, lon, lat, err := geo.DecodeAdd(value) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.GeoAdd(name, member, lon, lat, version, expireAt) +} + +func (e *Engine) applyGeoRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.GeoRem(name, member, version, expireAt) +} + +func (e *Engine) applyGeoInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.GeoInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/geo_cluster.go b/pkg/engine/geo_cluster.go new file mode 100644 index 0000000..54b0ff8 --- /dev/null +++ b/pkg/engine/geo_cluster.go @@ -0,0 +1,38 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) gMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: value, Flags: flag, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + return err +} + +func (e *Engine) gFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsGeo() { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.GeoInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/store/geo.go b/pkg/store/geo.go new file mode 100644 index 0000000..dbf2265 --- /dev/null +++ b/pkg/store/geo.go @@ -0,0 +1,238 @@ +package store + +import ( + "github.com/Code0987/supercache/pkg/geo" +) + +func (m *Memory) GeoAdd(key string, member []byte, lon, lat float64, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.gMutateLocked(key, version, expireAt, func(g *geo.Index) error { + return g.Add(member, lon, lat) + }) +} + +func (m *Memory) GeoRem(key string, member []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + el, ok := m.items[key] + if !ok { + return true + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return true + } + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + return true + } + if !it.entry.IsGeo() { + return false + } + return m.gMutateLocked(key, version, expireAt, func(g *geo.Index) error { + g.Rem(member) + return nil + }) +} + +func (m *Memory) GeoPos(key string, member []byte) (float64, float64, bool) { + m.mu.Lock() + defer m.mu.Unlock() + g, ok := m.gPeekLocked(key) + if !ok { + return 0, 0, false + } + p, ok := g.Pos(member) + return p.Lon, p.Lat, ok +} + +func (m *Memory) GeoCard(key string) int { + m.mu.Lock() + defer m.mu.Unlock() + g, ok := m.gPeekLocked(key) + if !ok { + return 0 + } + return g.Card() +} + +func (m *Memory) HasGeo(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.gPeekLocked(key) + return ok +} + +func (m *Memory) GeoDist(key string, a, b []byte) (float64, bool) { + m.mu.Lock() + defer m.mu.Unlock() + g, ok := m.gPeekLocked(key) + if !ok { + return 0, false + } + return g.Dist(a, b) +} + +func (m *Memory) GeoRadius(key string, lon, lat, radiusM float64, limit int) []GeoMember { + m.mu.Lock() + defer m.mu.Unlock() + g, ok := m.gPeekLocked(key) + if !ok { + return nil + } + return toStoreGeoMembers(g.Radius(lon, lat, radiusM, limit)) +} + +func (m *Memory) GeoInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsGeo() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + g, err := geo.Decode(blob) + if err != nil { + return false + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagGeo} + return m.insertGeoLocked(key, ent, g, false) +} + +func toStoreGeoMembers(in []geo.Member) []GeoMember { + if len(in) == 0 { + return nil + } + out := make([]GeoMember, len(in)) + for i, m := range in { + out[i] = GeoMember{Member: m.Member, Lon: m.Lon, Lat: m.Lat, Dist: m.Dist} + } + return out +} + +func (m *Memory) gPeekLocked(key string) (*geo.Index, bool) { + el, ok := m.items[key] + if !ok { + return nil, false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return nil, false + } + if it.entry.IsTombstone() || !it.entry.IsGeo() { + return nil, false + } + if it.gCache != nil { + return it.gCache, true + } + g, err := geo.Decode(it.entry.Value) + if err != nil { + return nil, false + } + it.gCache = g + return g, true +} + +func (m *Memory) gMutateLocked(key string, version uint64, expireAt int64, mut func(*geo.Index) error) bool { + var g *geo.Index + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + m.removeElement(el) + g = geo.New() + } else if it.entry.IsGeo() { + g = it.gCache + if g == nil { + var err error + g, err = geo.Decode(it.entry.Value) + if err != nil { + return false + } + } + if err := mut(g); err != nil { + return false + } + oldCost := it.cost + it.entry.Version = version + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.entry.Flags = FlagGeo + it.gCache = g + it.gDirty = true + it.cost = int64(len(key)) + 64 + g.ApproxWireBytes() + m.bytes += it.cost - oldCost + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still + } else { + return false + } + } else { + m.removeElement(el) + g = geo.New() + } + } else { + g = geo.New() + } + if err := mut(g); err != nil { + return false + } + ent := Entry{Value: g.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagGeo} + return m.insertGeoLocked(key, ent, g, false) +} + +func (m *Memory) flushGeoValueLocked(it *lruItem) { + if it == nil || !it.gDirty || it.gCache == nil || !it.entry.IsGeo() { + return + } + blob := it.gCache.Encode() + oldCost := it.cost + it.entry.Value = blob + it.gDirty = false + it.cost = entryCost(it.key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } +} + +func (m *Memory) insertGeoLocked(key string, ent Entry, cache *geo.Index, dirty bool) bool { + cost := entryCost(key, ent) + if dirty && cache != nil { + cost = int64(len(key)) + 64 + cache.ApproxWireBytes() + } + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, gCache: cache, gDirty: dirty} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..78e7e4d 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -954,239 +954,6 @@ func (m *Memory) flushZSetValueLocked(it *lruItem) { } } -func (m *Memory) GeoAdd(key string, member []byte, lon, lat float64, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.gMutateLocked(key, version, expireAt, func(g *geo.Index) error { - return g.Add(member, lon, lat) - }) -} - -func (m *Memory) GeoRem(key string, member []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - el, ok := m.items[key] - if !ok { - return true - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return true - } - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - return true - } - if !it.entry.IsGeo() { - return false - } - return m.gMutateLocked(key, version, expireAt, func(g *geo.Index) error { - g.Rem(member) - return nil - }) -} - -func (m *Memory) GeoPos(key string, member []byte) (float64, float64, bool) { - m.mu.Lock() - defer m.mu.Unlock() - g, ok := m.gPeekLocked(key) - if !ok { - return 0, 0, false - } - p, ok := g.Pos(member) - return p.Lon, p.Lat, ok -} - -func (m *Memory) GeoCard(key string) int { - m.mu.Lock() - defer m.mu.Unlock() - g, ok := m.gPeekLocked(key) - if !ok { - return 0 - } - return g.Card() -} - -func (m *Memory) HasGeo(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.gPeekLocked(key) - return ok -} - -func (m *Memory) GeoDist(key string, a, b []byte) (float64, bool) { - m.mu.Lock() - defer m.mu.Unlock() - g, ok := m.gPeekLocked(key) - if !ok { - return 0, false - } - return g.Dist(a, b) -} - -func (m *Memory) GeoRadius(key string, lon, lat, radiusM float64, limit int) []GeoMember { - m.mu.Lock() - defer m.mu.Unlock() - g, ok := m.gPeekLocked(key) - if !ok { - return nil - } - return toStoreGeoMembers(g.Radius(lon, lat, radiusM, limit)) -} - -func (m *Memory) GeoInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsGeo() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - g, err := geo.Decode(blob) - if err != nil { - return false - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagGeo} - return m.insertGeoLocked(key, ent, g, false) -} - -func toStoreGeoMembers(in []geo.Member) []GeoMember { - if len(in) == 0 { - return nil - } - out := make([]GeoMember, len(in)) - for i, m := range in { - out[i] = GeoMember{Member: m.Member, Lon: m.Lon, Lat: m.Lat, Dist: m.Dist} - } - return out -} - -func (m *Memory) gPeekLocked(key string) (*geo.Index, bool) { - el, ok := m.items[key] - if !ok { - return nil, false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return nil, false - } - if it.entry.IsTombstone() || !it.entry.IsGeo() { - return nil, false - } - if it.gCache != nil { - return it.gCache, true - } - g, err := geo.Decode(it.entry.Value) - if err != nil { - return nil, false - } - it.gCache = g - return g, true -} - -func (m *Memory) gMutateLocked(key string, version uint64, expireAt int64, mut func(*geo.Index) error) bool { - var g *geo.Index - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - m.removeElement(el) - g = geo.New() - } else if it.entry.IsGeo() { - g = it.gCache - if g == nil { - var err error - g, err = geo.Decode(it.entry.Value) - if err != nil { - return false - } - } - if err := mut(g); err != nil { - return false - } - oldCost := it.cost - it.entry.Version = version - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.entry.Flags = FlagGeo - it.gCache = g - it.gDirty = true - it.cost = int64(len(key)) + 64 + g.ApproxWireBytes() - m.bytes += it.cost - oldCost - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still - } else { - return false - } - } else { - m.removeElement(el) - g = geo.New() - } - } else { - g = geo.New() - } - if err := mut(g); err != nil { - return false - } - ent := Entry{Value: g.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagGeo} - return m.insertGeoLocked(key, ent, g, false) -} - -func (m *Memory) flushGeoValueLocked(it *lruItem) { - if it == nil || !it.gDirty || it.gCache == nil || !it.entry.IsGeo() { - return - } - blob := it.gCache.Encode() - oldCost := it.cost - it.entry.Value = blob - it.gDirty = false - it.cost = entryCost(it.key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } -} - -func (m *Memory) insertGeoLocked(key string, ent Entry, cache *geo.Index, dirty bool) bool { - cost := entryCost(key, ent) - if dirty && cache != nil { - cost = int64(len(key)) + 64 + cache.ApproxWireBytes() - } - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, gCache: cache, gDirty: dirty} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) insertZSetLocked(key string, ent Entry, cache *zset.ZSet, dirty bool) bool { cost := entryCost(key, ent) if dirty && cache != nil { From ca97808b4ca337f814b5aafbcb9efdacb76a29eb Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:24:03 +0000 Subject: [PATCH 05/13] refactor: extract ModeZSet into its own files Move ZSet store methods out of memory.go and split the engine file into public verbs, owner write/apply, and cluster forward. Same contract, including dirty zCache and FlagZSetAdd/FlagZSetRem item fan-out. --- docs/design/2026-09-02-refactor-zset.md | 40 ++++ docs/design/README.md | 1 + pkg/engine/zset.go | 136 +++---------- pkg/engine/zset_apply.go | 77 ++++++++ pkg/engine/zset_cluster.go | 38 ++++ pkg/store/memory.go | 240 ----------------------- pkg/store/zset.go | 245 ++++++++++++++++++++++++ 7 files changed, 426 insertions(+), 351 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-zset.md create mode 100644 pkg/engine/zset_apply.go create mode 100644 pkg/engine/zset_cluster.go create mode 100644 pkg/store/zset.go diff --git a/docs/design/2026-09-02-refactor-zset.md b/docs/design/2026-09-02-refactor-zset.md new file mode 100644 index 0000000..3e56f25 --- /dev/null +++ b/docs/design/2026-09-02-refactor-zset.md @@ -0,0 +1,40 @@ +# Refactor ModeZSet layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-zset` +**Date:** 2026-09-02 + +## Problem + +ZSet store methods still live in `pkg/store/memory.go`. Engine ZSet is one file mixing public verbs, owner write, and item fan-out. + +## Non-goals + +- No API / proto / version / fan-out / dirty-cache change +- No `pkg/zset` algorithm change +- Not extracting other modes (Set / Bloom / List / Hash / JSON stay on their own branches) + +## Contract + +Unchanged: ZAdd/ZRem ACK-only inbox (`FlagZSetAdd` / `FlagZSetRem` item fan-out). ZRem of missing is a no-op. NaN score rejected. Dirty `zCache` flushed on Get/Peek only. + +## Approach + +Same file layout as Set: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` Z* | `pkg/store/zset.go` (including `flushZSetValueLocked` / `insertZSetLocked`) | +| `pkg/engine/zset.go` (all) | `zset.go` public verbs; `zset_apply.go` owner write + inbox apply; `zset_cluster.go` owner forward / GetOrLoad | + +ZSet-local error format strings live in a `const` block at the top of `zset.go` (same text as before). + +Rejected: rewrite dirty-cache / flush. Switch ZSet to snapshot fan-out. Split `pkg/zset`. + +## Tests (already exist; keep them) + +Existing `pkg/zset`, `pkg/store` zset, `pkg/engine` zset unit + cluster. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..1eb80fb 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-zset.md](./2026-09-02-refactor-zset.md) | ModeZSet file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/zset.go b/pkg/engine/zset.go index d03e6e6..7441de5 100644 --- a/pkg/engine/zset.go +++ b/pkg/engine/zset.go @@ -10,6 +10,20 @@ import ( "github.com/Code0987/supercache/pkg/zset" ) +const ( + errEmptyZSetMember = "%w: empty zset member" + errNaNScore = "%w: NaN score" + errZAddRequiresMode = "%w: ZAdd requires ModeZSet" + errZRemRequiresMode = "%w: ZRem requires ModeZSet" + errZScoreRequiresMode = "%w: ZScore requires ModeZSet" + errZCardRequiresMode = "%w: ZCard requires ModeZSet" + errZRangeRequiresMode = "%w: ZRange requires ModeZSet" + errZRangeByScoreRequiresMode = "%w: ZRangeByScore requires ModeZSet" + errZAddRejected = "%w: zadd rejected" + errZRemRejected = "%w: zrem rejected" + errOwnerNoAddress = "%w: owner %s has no address" +) + // ZMember is a scored sorted-set element. type ZMember struct { Member []byte @@ -25,17 +39,17 @@ func (e *Engine) ZAdd(ctx context.Context, keyspaceName, name string, member []b return err } if len(member) == 0 { - return fmt.Errorf("%w: empty zset member", ErrInvalidArgument) + return fmt.Errorf(errEmptyZSetMember, ErrInvalidArgument) } if math.IsNaN(score) { - return fmt.Errorf("%w: NaN score", ErrInvalidArgument) + return fmt.Errorf(errNaNScore, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeZSet { - return fmt.Errorf("%w: ZAdd requires ModeZSet", ErrInvalidArgument) + return fmt.Errorf(errZAddRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -47,7 +61,7 @@ func (e *Engine) ZAdd(ctx context.Context, keyspaceName, name string, member []b if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.zMutViaOwner(ctx, ks, name, zset.EncodeAdd(member, score), store.FlagZSetAdd) } @@ -64,14 +78,14 @@ func (e *Engine) ZRem(ctx context.Context, keyspaceName, name string, member []b return err } if len(member) == 0 { - return fmt.Errorf("%w: empty zset member", ErrInvalidArgument) + return fmt.Errorf(errEmptyZSetMember, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeZSet { - return fmt.Errorf("%w: ZRem requires ModeZSet", ErrInvalidArgument) + return fmt.Errorf(errZRemRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -83,7 +97,7 @@ func (e *Engine) ZRem(ctx context.Context, keyspaceName, name string, member []b if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.zMutViaOwner(ctx, ks, name, append([]byte(nil), member...), store.FlagZSetRem) } @@ -104,7 +118,7 @@ func (e *Engine) ZScore(ctx context.Context, keyspaceName, name string, member [ return 0, false, err } if ks.cfg.Mode != keyspace.ModeZSet { - return 0, false, fmt.Errorf("%w: ZScore requires ModeZSet", ErrInvalidArgument) + return 0, false, fmt.Errorf(errZScoreRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, false, err @@ -140,7 +154,7 @@ func (e *Engine) ZCard(ctx context.Context, keyspaceName, name string) (int, err return 0, err } if ks.cfg.Mode != keyspace.ModeZSet { - return 0, fmt.Errorf("%w: ZCard requires ModeZSet", ErrInvalidArgument) + return 0, fmt.Errorf(errZCardRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, err @@ -172,7 +186,7 @@ func (e *Engine) ZRange(ctx context.Context, keyspaceName, name string, start, s return nil, err } if ks.cfg.Mode != keyspace.ModeZSet { - return nil, fmt.Errorf("%w: ZRange requires ModeZSet", ErrInvalidArgument) + return nil, fmt.Errorf(errZRangeRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, err @@ -204,7 +218,7 @@ func (e *Engine) ZRangeByScore(ctx context.Context, keyspaceName, name string, m return nil, err } if ks.cfg.Mode != keyspace.ModeZSet { - return nil, fmt.Errorf("%w: ZRangeByScore requires ModeZSet", ErrInvalidArgument) + return nil, fmt.Errorf(errZRangeByScoreRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, err @@ -223,110 +237,10 @@ func (e *Engine) ZRangeByScore(ctx context.Context, keyspaceName, name string, m return toEngineZMembersFromPkg(z.RangeByScore(min, max)), nil } -func (e *Engine) zMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: value, Flags: flag, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - return err -} - -func (e *Engine) zAddLocal(ks *ksRuntime, name string, member []byte, score float64, fanout bool) error { - ver := e.zNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.ZAdd(name, member, score, ver, expire) { - return fmt.Errorf("%w: zadd rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: zset.EncodeAdd(member, score), - Version: ver, - ExpireAt: expire, - Flags: store.FlagZSetAdd, - }, false) - } - return nil -} - -func (e *Engine) zRemLocal(ks *ksRuntime, name string, member []byte, fanout bool) error { - ver := e.zNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.ZRem(name, member, ver, expire) { - if !e.hasZSetLocal(ks, name) { - return nil - } - return fmt.Errorf("%w: zrem rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), member...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagZSetRem, - }, false) - } - return nil -} - -func (e *Engine) zNextVersion(ks *ksRuntime, name string) uint64 { - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - func (e *Engine) hasZSetLocal(ks *ksRuntime, name string) bool { return ks.store.HasZSet(name) } -func (e *Engine) zFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsZSet() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.ZInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyZSetAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { - member, score, err := zset.DecodeAdd(value) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.ZAdd(name, member, score, version, expireAt) -} - -func (e *Engine) applyZSetRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.ZRem(name, member, version, expireAt) -} - -func (e *Engine) applyZSetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.ZInstall(name, blob, version, expireAt) -} - func toEngineZMembers(in []store.ZMember) []ZMember { if len(in) == 0 { return nil diff --git a/pkg/engine/zset_apply.go b/pkg/engine/zset_apply.go new file mode 100644 index 0000000..37d259c --- /dev/null +++ b/pkg/engine/zset_apply.go @@ -0,0 +1,77 @@ +package engine + +import ( + "fmt" + + "github.com/Code0987/supercache/pkg/store" + "github.com/Code0987/supercache/pkg/zset" +) + +func (e *Engine) zAddLocal(ks *ksRuntime, name string, member []byte, score float64, fanout bool) error { + ver := e.zNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.ZAdd(name, member, score, ver, expire) { + return fmt.Errorf(errZAddRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: zset.EncodeAdd(member, score), + Version: ver, + ExpireAt: expire, + Flags: store.FlagZSetAdd, + }, false) + } + return nil +} + +func (e *Engine) zRemLocal(ks *ksRuntime, name string, member []byte, fanout bool) error { + ver := e.zNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.ZRem(name, member, ver, expire) { + if !e.hasZSetLocal(ks, name) { + return nil + } + return fmt.Errorf(errZRemRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: append([]byte(nil), member...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagZSetRem, + }, false) + } + return nil +} + +func (e *Engine) zNextVersion(ks *ksRuntime, name string) uint64 { + if ver, ok := ks.store.PeekVersion(name); ok { + return ks.nextVersion(name, ver) + } + return ks.nextVersion(name, 0) +} + +func (e *Engine) applyZSetAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { + member, score, err := zset.DecodeAdd(value) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.ZAdd(name, member, score, version, expireAt) +} + +func (e *Engine) applyZSetRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.ZRem(name, member, version, expireAt) +} + +func (e *Engine) applyZSetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.ZInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/zset_cluster.go b/pkg/engine/zset_cluster.go new file mode 100644 index 0000000..4fe8788 --- /dev/null +++ b/pkg/engine/zset_cluster.go @@ -0,0 +1,38 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) zMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: value, Flags: flag, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + return err +} + +func (e *Engine) zFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsZSet() { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.ZInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..b5cd5c1 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -728,232 +728,6 @@ func (m *Memory) insertSetLocked(key string, ent Entry, cache *set.Set, dirty bo return ok } -// ZAdd upserts member/score. -func (m *Memory) ZAdd(key string, member []byte, score float64, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.zMutateLocked(key, version, expireAt, func(z *zset.ZSet) error { - return z.Add(member, score) - }) -} - -// ZRem removes a member (no-op if zset missing). -func (m *Memory) ZRem(key string, member []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - el, ok := m.items[key] - if !ok { - return true - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return true - } - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - return true - } - if !it.entry.IsZSet() { - return false - } - return m.zMutateLocked(key, version, expireAt, func(z *zset.ZSet) error { - z.Rem(member) - return nil - }) -} - -// ZScore returns score if present. -func (m *Memory) ZScore(key string, member []byte) (float64, bool) { - m.mu.Lock() - defer m.mu.Unlock() - z, ok := m.zPeekLocked(key) - if !ok { - return 0, false - } - return z.Score(member) -} - -// ZCard returns member count. -func (m *Memory) ZCard(key string) int { - m.mu.Lock() - defer m.mu.Unlock() - z, ok := m.zPeekLocked(key) - if !ok { - return 0 - } - return z.Card() -} - -// HasZSet reports a live FlagZSet entry. -func (m *Memory) HasZSet(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.zPeekLocked(key) - return ok -} - -// ZRange returns members by rank. -func (m *Memory) ZRange(key string, start, stop int) []ZMember { - m.mu.Lock() - defer m.mu.Unlock() - z, ok := m.zPeekLocked(key) - if !ok { - return nil - } - return toStoreZMembers(z.Range(start, stop)) -} - -// ZRangeByScore returns members in score window. -func (m *Memory) ZRangeByScore(key string, min, max float64) []ZMember { - m.mu.Lock() - defer m.mu.Unlock() - z, ok := m.zPeekLocked(key) - if !ok { - return nil - } - return toStoreZMembers(z.RangeByScore(min, max)) -} - -// ZInstall installs a versioned full snapshot. -func (m *Memory) ZInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsZSet() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - z, err := zset.Decode(blob) - if err != nil { - return false - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagZSet} - return m.insertZSetLocked(key, ent, z, false) -} - -func toStoreZMembers(in []zset.Member) []ZMember { - if len(in) == 0 { - return nil - } - out := make([]ZMember, len(in)) - for i, m := range in { - out[i] = ZMember{Member: m.Member, Score: m.Score} - } - return out -} - -func (m *Memory) zPeekLocked(key string) (*zset.ZSet, bool) { - el, ok := m.items[key] - if !ok { - return nil, false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return nil, false - } - if it.entry.IsTombstone() || !it.entry.IsZSet() { - return nil, false - } - if it.zCache != nil { - return it.zCache, true - } - z, err := zset.Decode(it.entry.Value) - if err != nil { - return nil, false - } - it.zCache = z - return z, true -} - -func (m *Memory) zMutateLocked(key string, version uint64, expireAt int64, mut func(*zset.ZSet) error) bool { - var z *zset.ZSet - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - m.removeElement(el) - z = zset.New() - } else if it.entry.IsZSet() { - z = it.zCache - if z == nil { - var err error - z, err = zset.Decode(it.entry.Value) - if err != nil { - return false - } - } - if err := mut(z); err != nil { - return false - } - oldCost := it.cost - it.entry.Version = version - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.entry.Flags = FlagZSet - it.zCache = z - it.zDirty = true - it.cost = int64(len(key)) + 64 + z.ApproxWireBytes() - m.bytes += it.cost - oldCost - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still - } else { - return false - } - } else { - m.removeElement(el) - z = zset.New() - } - } else { - z = zset.New() - } - if err := mut(z); err != nil { - return false - } - ent := Entry{Value: z.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagZSet} - return m.insertZSetLocked(key, ent, z, false) -} - -func (m *Memory) flushZSetValueLocked(it *lruItem) { - if it == nil || !it.zDirty || it.zCache == nil || !it.entry.IsZSet() { - return - } - blob := it.zCache.Encode() - oldCost := it.cost - it.entry.Value = blob - it.zDirty = false - it.cost = entryCost(it.key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } -} - func (m *Memory) GeoAdd(key string, member []byte, lon, lat float64, version uint64, expireAt int64) bool { m.mu.Lock() defer m.mu.Unlock() @@ -1187,20 +961,6 @@ func (m *Memory) insertGeoLocked(key string, ent Entry, cache *geo.Index, dirty return ok } -func (m *Memory) insertZSetLocked(key string, ent Entry, cache *zset.ZSet, dirty bool) bool { - cost := entryCost(key, ent) - if dirty && cache != nil { - cost = int64(len(key)) + 64 + cache.ApproxWireBytes() - } - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, zCache: cache, zDirty: dirty} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) LPush(key string, item []byte, version uint64, expireAt int64) bool { m.mu.Lock() defer m.mu.Unlock() diff --git a/pkg/store/zset.go b/pkg/store/zset.go new file mode 100644 index 0000000..59751ca --- /dev/null +++ b/pkg/store/zset.go @@ -0,0 +1,245 @@ +package store + +import ( + "github.com/Code0987/supercache/pkg/zset" +) + +// ZAdd upserts member/score. +func (m *Memory) ZAdd(key string, member []byte, score float64, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.zMutateLocked(key, version, expireAt, func(z *zset.ZSet) error { + return z.Add(member, score) + }) +} + +// ZRem removes a member (no-op if zset missing). +func (m *Memory) ZRem(key string, member []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + el, ok := m.items[key] + if !ok { + return true + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return true + } + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + return true + } + if !it.entry.IsZSet() { + return false + } + return m.zMutateLocked(key, version, expireAt, func(z *zset.ZSet) error { + z.Rem(member) + return nil + }) +} + +// ZScore returns score if present. +func (m *Memory) ZScore(key string, member []byte) (float64, bool) { + m.mu.Lock() + defer m.mu.Unlock() + z, ok := m.zPeekLocked(key) + if !ok { + return 0, false + } + return z.Score(member) +} + +// ZCard returns member count. +func (m *Memory) ZCard(key string) int { + m.mu.Lock() + defer m.mu.Unlock() + z, ok := m.zPeekLocked(key) + if !ok { + return 0 + } + return z.Card() +} + +// HasZSet reports a live FlagZSet entry. +func (m *Memory) HasZSet(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.zPeekLocked(key) + return ok +} + +// ZRange returns members by rank. +func (m *Memory) ZRange(key string, start, stop int) []ZMember { + m.mu.Lock() + defer m.mu.Unlock() + z, ok := m.zPeekLocked(key) + if !ok { + return nil + } + return toStoreZMembers(z.Range(start, stop)) +} + +// ZRangeByScore returns members in score window. +func (m *Memory) ZRangeByScore(key string, min, max float64) []ZMember { + m.mu.Lock() + defer m.mu.Unlock() + z, ok := m.zPeekLocked(key) + if !ok { + return nil + } + return toStoreZMembers(z.RangeByScore(min, max)) +} + +// ZInstall installs a versioned full snapshot. +func (m *Memory) ZInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsZSet() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + z, err := zset.Decode(blob) + if err != nil { + return false + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagZSet} + return m.insertZSetLocked(key, ent, z, false) +} + +func toStoreZMembers(in []zset.Member) []ZMember { + if len(in) == 0 { + return nil + } + out := make([]ZMember, len(in)) + for i, m := range in { + out[i] = ZMember{Member: m.Member, Score: m.Score} + } + return out +} + +func (m *Memory) zPeekLocked(key string) (*zset.ZSet, bool) { + el, ok := m.items[key] + if !ok { + return nil, false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return nil, false + } + if it.entry.IsTombstone() || !it.entry.IsZSet() { + return nil, false + } + if it.zCache != nil { + return it.zCache, true + } + z, err := zset.Decode(it.entry.Value) + if err != nil { + return nil, false + } + it.zCache = z + return z, true +} + +func (m *Memory) zMutateLocked(key string, version uint64, expireAt int64, mut func(*zset.ZSet) error) bool { + var z *zset.ZSet + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + m.removeElement(el) + z = zset.New() + } else if it.entry.IsZSet() { + z = it.zCache + if z == nil { + var err error + z, err = zset.Decode(it.entry.Value) + if err != nil { + return false + } + } + if err := mut(z); err != nil { + return false + } + oldCost := it.cost + it.entry.Version = version + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.entry.Flags = FlagZSet + it.zCache = z + it.zDirty = true + it.cost = int64(len(key)) + 64 + z.ApproxWireBytes() + m.bytes += it.cost - oldCost + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still + } else { + return false + } + } else { + m.removeElement(el) + z = zset.New() + } + } else { + z = zset.New() + } + if err := mut(z); err != nil { + return false + } + ent := Entry{Value: z.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagZSet} + return m.insertZSetLocked(key, ent, z, false) +} + +func (m *Memory) flushZSetValueLocked(it *lruItem) { + if it == nil || !it.zDirty || it.zCache == nil || !it.entry.IsZSet() { + return + } + blob := it.zCache.Encode() + oldCost := it.cost + it.entry.Value = blob + it.zDirty = false + it.cost = entryCost(it.key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } +} + +func (m *Memory) insertZSetLocked(key string, ent Entry, cache *zset.ZSet, dirty bool) bool { + cost := entryCost(key, ent) + if dirty && cache != nil { + cost = int64(len(key)) + 64 + cache.ApproxWireBytes() + } + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, zCache: cache, zDirty: dirty} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} From 6bd14038f82b12a465554931febb07ec21fde28a Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:25:57 +0000 Subject: [PATCH 06/13] refactor: extract ModeSet into its own files Move Set store methods out of memory.go and split the engine file into public verbs, owner write/apply, and cluster forward. Same contract, including dirty setCache and FlagSetAdd/FlagSetRemove item fan-out. PeekVersion stays in memory.go. --- docs/design/2026-09-02-refactor-set.md | 41 +++++ docs/design/README.md | 1 + pkg/engine/set.go | 109 +++--------- pkg/engine/set_apply.go | 76 ++++++++ pkg/engine/set_cluster.go | 17 ++ pkg/store/memory.go | 226 ------------------------ pkg/store/set.go | 231 +++++++++++++++++++++++++ 7 files changed, 387 insertions(+), 314 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-set.md create mode 100644 pkg/engine/set_apply.go create mode 100644 pkg/engine/set_cluster.go create mode 100644 pkg/store/set.go diff --git a/docs/design/2026-09-02-refactor-set.md b/docs/design/2026-09-02-refactor-set.md new file mode 100644 index 0000000..5076581 --- /dev/null +++ b/docs/design/2026-09-02-refactor-set.md @@ -0,0 +1,41 @@ +# Refactor ModeSet layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-set` +**Date:** 2026-09-02 + +## Problem + +Set store methods still live in `pkg/store/memory.go`. Engine Set is one file mixing public verbs, owner write, and item fan-out. + +## Non-goals + +- No API / proto / version / fan-out / dirty-cache change +- No `pkg/set` algorithm change +- Not extracting other modes (List / Hash / JSON / Bloom stay on their own branches) +- Not moving `PeekVersion` (generic; stays in `memory.go`) + +## Contract + +Unchanged: SetAdd/SetRemove ACK-only inbox (`FlagSetAdd` / `FlagSetRemove` item fan-out). SetRemove of missing is a no-op. Dirty `setCache` flushed on Get/Peek only. Replica GetOrLoad installs; non-replica decodes the blob once. + +## Approach + +Same file layout as Bloom: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` Set* | `pkg/store/set.go` (including `flushSetValueLocked` / `insertSetLocked`) | +| `pkg/engine/set.go` (verbs + apply) | `set.go` public verbs; `set_apply.go` owner write + inbox apply; `set_cluster.go` owner forward | + +`pkg/engine/set_blob.go` stays (non-replica decode helpers). Set-local error format strings live in a `const` block at the top of `set.go` (same text as before). + +Rejected: rewrite dirty-cache / flush. Switch Set to snapshot fan-out. Split `pkg/set`. + +## Tests (already exist; keep them) + +Existing `pkg/set`, `pkg/store` set, `pkg/engine` set unit + cluster. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..e0770c8 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-set.md](./2026-09-02-refactor-set.md) | ModeSet file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/set.go b/pkg/engine/set.go index e57ad7e..cb3c387 100644 --- a/pkg/engine/set.go +++ b/pkg/engine/set.go @@ -8,6 +8,18 @@ import ( "github.com/Code0987/supercache/pkg/store" ) +const ( + errEmptySetItem = "%w: empty set item" + errSetAddRequiresMode = "%w: SetAdd requires ModeSet" + errSetRemoveRequiresMode = "%w: SetRemove requires ModeSet" + errSetContainsRequiresMode = "%w: SetContains requires ModeSet" + errSetCardRequiresMode = "%w: SetCard requires ModeSet" + errSetMembersRequiresMode = "%w: SetMembers requires ModeSet" + errSetAddRejected = "%w: set add rejected" + errSetRemoveRejected = "%w: set remove rejected" + errOwnerNoAddress = "%w: owner %s has no address" +) + // SetAdd inserts item into the named set (ModeSet only). func (e *Engine) SetAdd(ctx context.Context, keyspaceName, name string, item []byte) error { if err := ctx.Err(); err != nil { @@ -17,14 +29,14 @@ func (e *Engine) SetAdd(ctx context.Context, keyspaceName, name string, item []b return err } if len(item) == 0 { - return fmt.Errorf("%w: empty set item", ErrInvalidArgument) + return fmt.Errorf(errEmptySetItem, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeSet { - return fmt.Errorf("%w: SetAdd requires ModeSet", ErrInvalidArgument) + return fmt.Errorf(errSetAddRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -36,7 +48,7 @@ func (e *Engine) SetAdd(ctx context.Context, keyspaceName, name string, item []b if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.setMutViaOwner(ctx, ks, name, item, store.FlagSetAdd) } @@ -53,14 +65,14 @@ func (e *Engine) SetRemove(ctx context.Context, keyspaceName, name string, item return err } if len(item) == 0 { - return fmt.Errorf("%w: empty set item", ErrInvalidArgument) + return fmt.Errorf(errEmptySetItem, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeSet { - return fmt.Errorf("%w: SetRemove requires ModeSet", ErrInvalidArgument) + return fmt.Errorf(errSetRemoveRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -72,7 +84,7 @@ func (e *Engine) SetRemove(ctx context.Context, keyspaceName, name string, item if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.setMutViaOwner(ctx, ks, name, item, store.FlagSetRemove) } @@ -93,7 +105,7 @@ func (e *Engine) SetContains(ctx context.Context, keyspaceName, name string, ite return false, err } if ks.cfg.Mode != keyspace.ModeSet { - return false, fmt.Errorf("%w: SetContains requires ModeSet", ErrInvalidArgument) + return false, fmt.Errorf(errSetContainsRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return false, err @@ -141,7 +153,7 @@ func (e *Engine) SetCard(ctx context.Context, keyspaceName, name string) (int, e return 0, err } if ks.cfg.Mode != keyspace.ModeSet { - return 0, fmt.Errorf("%w: SetCard requires ModeSet", ErrInvalidArgument) + return 0, fmt.Errorf(errSetCardRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, err @@ -183,7 +195,7 @@ func (e *Engine) SetMembers(ctx context.Context, keyspaceName, name string) ([][ return nil, err } if ks.cfg.Mode != keyspace.ModeSet { - return nil, fmt.Errorf("%w: SetMembers requires ModeSet", ErrInvalidArgument) + return nil, fmt.Errorf(errSetMembersRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, err @@ -212,85 +224,6 @@ func (e *Engine) SetMembers(ctx context.Context, keyspaceName, name string) ([][ return setMembersBlob(res.Entry.Value), nil } -func (e *Engine) setMutViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte, flag uint32) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: append([]byte(nil), item...), Flags: flag, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - return err -} - -func (e *Engine) setAddLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { - ver := e.setNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.SetAdd(name, item, ver, expire) { - return fmt.Errorf("%w: set add rejected", ErrInvalidArgument) - } - // Re-read version after mutate (store may keep same ver on in-place for bloom; we set ver). - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), item...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagSetAdd, - }, false) - } - return nil -} - -func (e *Engine) setRemoveLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { - ver := e.setNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.SetRemove(name, item, ver, expire) { - // Missing set or tombstone reject — remove of missing is success no-op if no entry. - if !e.hasSetLocal(ks, name) { - // If still no set, treat as success (design: no-op if missing). - return nil - } - return fmt.Errorf("%w: set remove rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), item...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagSetRemove, - }, false) - } - return nil -} - -func (e *Engine) setNextVersion(ks *ksRuntime, name string) uint64 { - // PeekVersion avoids flushing a dirty set blob on every Add/Remove. - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - func (e *Engine) hasSetLocal(ks *ksRuntime, name string) bool { return ks.store.HasSet(name) } - -func (e *Engine) applySetAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.SetAdd(name, item, version, expireAt) -} - -func (e *Engine) applySetRemove(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.SetRemove(name, item, version, expireAt) -} - -func (e *Engine) applySetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.SetInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/set_apply.go b/pkg/engine/set_apply.go new file mode 100644 index 0000000..43994d1 --- /dev/null +++ b/pkg/engine/set_apply.go @@ -0,0 +1,76 @@ +package engine + +import ( + "fmt" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) setAddLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { + ver := e.setNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.SetAdd(name, item, ver, expire) { + return fmt.Errorf(errSetAddRejected, ErrInvalidArgument) + } + // Re-read version after mutate (store may keep same ver on in-place for bloom; we set ver). + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: append([]byte(nil), item...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagSetAdd, + }, false) + } + return nil +} + +func (e *Engine) setRemoveLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { + ver := e.setNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.SetRemove(name, item, ver, expire) { + // Missing set or tombstone reject — remove of missing is success no-op if no entry. + if !e.hasSetLocal(ks, name) { + // If still no set, treat as success (design: no-op if missing). + return nil + } + return fmt.Errorf(errSetRemoveRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: append([]byte(nil), item...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagSetRemove, + }, false) + } + return nil +} + +func (e *Engine) setNextVersion(ks *ksRuntime, name string) uint64 { + // PeekVersion avoids flushing a dirty set blob on every Add/Remove. + if ver, ok := ks.store.PeekVersion(name); ok { + return ks.nextVersion(name, ver) + } + return ks.nextVersion(name, 0) +} + +func (e *Engine) applySetAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.SetAdd(name, item, version, expireAt) +} + +func (e *Engine) applySetRemove(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.SetRemove(name, item, version, expireAt) +} + +func (e *Engine) applySetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.SetInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/set_cluster.go b/pkg/engine/set_cluster.go new file mode 100644 index 0000000..ab5665e --- /dev/null +++ b/pkg/engine/set_cluster.go @@ -0,0 +1,17 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) setMutViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte, flag uint32) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: append([]byte(nil), item...), Flags: flag, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + return err +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..7c902c1 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -486,64 +486,6 @@ func (m *Memory) bloomMutateLocked(key string, mBits, k int, version uint64, exp return true } -// SetAdd inserts item into the named set (creates if missing). Version gates tombstones. -func (m *Memory) SetAdd(key string, item []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.setMutateLocked(key, version, expireAt, func(s *set.Set) { - s.Add(item) - }) -} - -// SetRemove removes item from the named set. No-op if set missing (does not create). -// Returns false only when blocked by a higher-version tombstone or non-set entry. -func (m *Memory) SetRemove(key string, item []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - el, ok := m.items[key] - if !ok { - return true // missing set: success no-op - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return true - } - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - // Higher than tombstone with remove only: still no set to remove from. - return true - } - if !it.entry.IsSet() { - return false - } - return m.setMutateLocked(key, version, expireAt, func(s *set.Set) { - s.Remove(item) - }) -} - -// SetContains reports exact membership. -func (m *Memory) SetContains(key string, item []byte) bool { - m.mu.Lock() - defer m.mu.Unlock() - s, ok := m.setPeekLocked(key) - if !ok { - return false - } - return s.Contains(item) -} - -// HasSet reports a live FlagSet entry without cloning the blob. -func (m *Memory) HasSet(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.setPeekLocked(key) - return ok -} - // PeekVersion returns the stored version without flushing dirty set blobs or cloning Value. func (m *Memory) PeekVersion(key string) (uint64, bool) { m.mu.Lock() @@ -560,174 +502,6 @@ func (m *Memory) PeekVersion(key string) (uint64, bool) { return it.entry.Version, true } -// SetCard returns the number of elements (0 if missing). -func (m *Memory) SetCard(key string) int { - m.mu.Lock() - defer m.mu.Unlock() - s, ok := m.setPeekLocked(key) - if !ok { - return 0 - } - return s.Len() -} - -// SetMembers returns defensive copies of all members (nil if missing/empty). -func (m *Memory) SetMembers(key string) [][]byte { - m.mu.Lock() - defer m.mu.Unlock() - s, ok := m.setPeekLocked(key) - if !ok { - return nil - } - return s.Members() -} - -// SetInstall installs a versioned full-set snapshot (handoff). Higher version wins. -func (m *Memory) SetInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsSet() { - if version < it.entry.Version { - m.staleSkip.Add(1) - return false - } - if version == it.entry.Version { - // Equal version: ignore (idempotent handoff). - return false - } - } else if !it.entry.IsTombstone() { - // Non-set live entry: only replace if version wins via remove path. - if version <= it.entry.Version { - return false - } - } - } - m.removeElement(el) - } - s, err := set.DecodeSet(blob) - if err != nil { - return false - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagSet} - return m.insertSetLocked(key, ent, s, false) -} - -func (m *Memory) setPeekLocked(key string) (*set.Set, bool) { - el, ok := m.items[key] - if !ok { - return nil, false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return nil, false - } - if it.entry.IsTombstone() || !it.entry.IsSet() { - return nil, false - } - if it.setCache != nil { - return it.setCache, true - } - s, err := set.DecodeSet(it.entry.Value) - if err != nil { - return nil, false - } - it.setCache = s - return s, true -} - -func (m *Memory) setMutateLocked(key string, version uint64, expireAt int64, mut func(*set.Set)) bool { - var s *set.Set - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - // Higher than tombstone: recreate empty then mut. - m.removeElement(el) - s = set.New() - } else if it.entry.IsSet() { - s = it.setCache - if s == nil { - var err error - s, err = set.DecodeSet(it.entry.Value) - if err != nil { - return false - } - } - mut(s) - oldCost := it.cost - it.entry.Version = version - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.entry.Flags = FlagSet - it.setCache = s - it.setDirty = true - // Defer Encode until Peek/Get; approximate cost for MaxBytes. - it.cost = int64(len(key)) + 64 + s.ApproxWireBytes() - m.bytes += it.cost - oldCost - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still - } else { - return false - } - } else { - m.removeElement(el) - s = set.New() - } - } else { - s = set.New() - } - mut(s) - // New set: encode once so Value is defined; later mutates stay dirty. - ent := Entry{Value: s.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagSet} - return m.insertSetLocked(key, ent, s, false) -} - -// flushSetValueLocked materializes entry.Value from setCache when dirty. -func (m *Memory) flushSetValueLocked(it *lruItem) { - if it == nil || !it.setDirty || it.setCache == nil || !it.entry.IsSet() { - return - } - blob := it.setCache.Encode() - oldCost := it.cost - it.entry.Value = blob - it.setDirty = false - it.cost = entryCost(it.key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } -} - -func (m *Memory) insertSetLocked(key string, ent Entry, cache *set.Set, dirty bool) bool { - cost := entryCost(key, ent) - if dirty && cache != nil { - cost = int64(len(key)) + 64 + cache.ApproxWireBytes() - } - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, setCache: cache, setDirty: dirty} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - // ZAdd upserts member/score. func (m *Memory) ZAdd(key string, member []byte, score float64, version uint64, expireAt int64) bool { m.mu.Lock() diff --git a/pkg/store/set.go b/pkg/store/set.go new file mode 100644 index 0000000..166eac0 --- /dev/null +++ b/pkg/store/set.go @@ -0,0 +1,231 @@ +package store + +import ( + "github.com/Code0987/supercache/pkg/set" +) + +// SetAdd inserts item into the named set (creates if missing). Version gates tombstones. +func (m *Memory) SetAdd(key string, item []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.setMutateLocked(key, version, expireAt, func(s *set.Set) { + s.Add(item) + }) +} + +// SetRemove removes item from the named set. No-op if set missing (does not create). +// Returns false only when blocked by a higher-version tombstone or non-set entry. +func (m *Memory) SetRemove(key string, item []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + el, ok := m.items[key] + if !ok { + return true // missing set: success no-op + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return true + } + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + // Higher than tombstone with remove only: still no set to remove from. + return true + } + if !it.entry.IsSet() { + return false + } + return m.setMutateLocked(key, version, expireAt, func(s *set.Set) { + s.Remove(item) + }) +} + +// SetContains reports exact membership. +func (m *Memory) SetContains(key string, item []byte) bool { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.setPeekLocked(key) + if !ok { + return false + } + return s.Contains(item) +} + +// HasSet reports a live FlagSet entry without cloning the blob. +func (m *Memory) HasSet(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.setPeekLocked(key) + return ok +} + +// SetCard returns the number of elements (0 if missing). +func (m *Memory) SetCard(key string) int { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.setPeekLocked(key) + if !ok { + return 0 + } + return s.Len() +} + +// SetMembers returns defensive copies of all members (nil if missing/empty). +func (m *Memory) SetMembers(key string) [][]byte { + m.mu.Lock() + defer m.mu.Unlock() + s, ok := m.setPeekLocked(key) + if !ok { + return nil + } + return s.Members() +} + +// SetInstall installs a versioned full-set snapshot (handoff). Higher version wins. +func (m *Memory) SetInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsSet() { + if version < it.entry.Version { + m.staleSkip.Add(1) + return false + } + if version == it.entry.Version { + // Equal version: ignore (idempotent handoff). + return false + } + } else if !it.entry.IsTombstone() { + // Non-set live entry: only replace if version wins via remove path. + if version <= it.entry.Version { + return false + } + } + } + m.removeElement(el) + } + s, err := set.DecodeSet(blob) + if err != nil { + return false + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagSet} + return m.insertSetLocked(key, ent, s, false) +} + +func (m *Memory) setPeekLocked(key string) (*set.Set, bool) { + el, ok := m.items[key] + if !ok { + return nil, false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return nil, false + } + if it.entry.IsTombstone() || !it.entry.IsSet() { + return nil, false + } + if it.setCache != nil { + return it.setCache, true + } + s, err := set.DecodeSet(it.entry.Value) + if err != nil { + return nil, false + } + it.setCache = s + return s, true +} + +func (m *Memory) setMutateLocked(key string, version uint64, expireAt int64, mut func(*set.Set)) bool { + var s *set.Set + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + // Higher than tombstone: recreate empty then mut. + m.removeElement(el) + s = set.New() + } else if it.entry.IsSet() { + s = it.setCache + if s == nil { + var err error + s, err = set.DecodeSet(it.entry.Value) + if err != nil { + return false + } + } + mut(s) + oldCost := it.cost + it.entry.Version = version + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.entry.Flags = FlagSet + it.setCache = s + it.setDirty = true + // Defer Encode until Peek/Get; approximate cost for MaxBytes. + it.cost = int64(len(key)) + 64 + s.ApproxWireBytes() + m.bytes += it.cost - oldCost + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still + } else { + return false + } + } else { + m.removeElement(el) + s = set.New() + } + } else { + s = set.New() + } + mut(s) + // New set: encode once so Value is defined; later mutates stay dirty. + ent := Entry{Value: s.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagSet} + return m.insertSetLocked(key, ent, s, false) +} + +// flushSetValueLocked materializes entry.Value from setCache when dirty. +func (m *Memory) flushSetValueLocked(it *lruItem) { + if it == nil || !it.setDirty || it.setCache == nil || !it.entry.IsSet() { + return + } + blob := it.setCache.Encode() + oldCost := it.cost + it.entry.Value = blob + it.setDirty = false + it.cost = entryCost(it.key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } +} + +func (m *Memory) insertSetLocked(key string, ent Entry, cache *set.Set, dirty bool) bool { + cost := entryCost(key, ent) + if dirty && cache != nil { + cost = int64(len(key)) + 64 + cache.ApproxWireBytes() + } + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, setCache: cache, setDirty: dirty} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} From 7197443229e9cfe5e035021f654602016a6265d7 Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:26:57 +0000 Subject: [PATCH 07/13] refactor: extract ModeBloom into its own files Move Bloom store methods out of memory.go and split the engine file into public verbs, owner write/apply, and cluster forward. Same contract, including FlagBloomAdd item-hint fan-out and no install on GetOrLoad. --- docs/design/2026-09-02-refactor-bloom.md | 40 ++++++++++ docs/design/README.md | 1 + pkg/engine/bloom.go | 67 +++-------------- pkg/engine/bloom_apply.go | 47 ++++++++++++ pkg/engine/bloom_cluster.go | 17 +++++ pkg/store/bloom.go | 94 ++++++++++++++++++++++++ pkg/store/memory.go | 90 ----------------------- 7 files changed, 211 insertions(+), 145 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-bloom.md create mode 100644 pkg/engine/bloom_apply.go create mode 100644 pkg/engine/bloom_cluster.go create mode 100644 pkg/store/bloom.go diff --git a/docs/design/2026-09-02-refactor-bloom.md b/docs/design/2026-09-02-refactor-bloom.md new file mode 100644 index 0000000..f5473bf --- /dev/null +++ b/docs/design/2026-09-02-refactor-bloom.md @@ -0,0 +1,40 @@ +# Refactor ModeBloom layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-bloom` +**Date:** 2026-09-02 + +## Problem + +Bloom store methods still live in `pkg/store/memory.go`. Engine Bloom is one file mixing public verbs, owner write, and item fan-out. + +## Non-goals + +- No API / proto / version / fan-out change +- No `pkg/bloom` algorithm change +- Not extracting other modes (List / Hash / JSON stay on their own branches) + +## Contract + +Unchanged: BloomAdd ACK-only inbox (`FlagBloomAdd` item hint — hintID includes the item). BloomTest present-bit + local miss does not install from GetOrLoad. Merge-OR on handoff. Version stays on a live filter (OR, not LWW bump). + +## Approach + +Same file layout as List / Hash / JSON: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` Bloom* | `pkg/store/bloom.go` | +| `pkg/engine/bloom.go` (all) | `bloom.go` public verbs; `bloom_apply.go` owner write + inbox apply; `bloom_cluster.go` owner forward | + +Bloom-local error format strings live in a `const` block at the top of `bloom.go` (same text as before). + +Rejected: switch BloomAdd to snapshot fan-out. Install-on-GetOrLoad. Split `pkg/bloom`. + +## Tests (already exist; keep them) + +Existing `pkg/bloom`, `pkg/store` bloom, `pkg/engine` bloom. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..93083ae 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-bloom.md](./2026-09-02-refactor-bloom.md) | ModeBloom file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/bloom.go b/pkg/engine/bloom.go index e9dcaff..2401158 100644 --- a/pkg/engine/bloom.go +++ b/pkg/engine/bloom.go @@ -6,7 +6,14 @@ import ( "github.com/Code0987/supercache/pkg/bloom" "github.com/Code0987/supercache/pkg/keyspace" - "github.com/Code0987/supercache/pkg/store" +) + +const ( + errEmptyBloomItem = "%w: empty bloom item" + errBloomAddRequiresMode = "%w: BloomAdd requires ModeBloom" + errBloomTestRequiresMode = "%w: BloomTest requires ModeBloom" + errBloomAddRejected = "%w: bloom add rejected" + errOwnerNoAddress = "%w: owner %s has no address" ) // BloomAdd inserts item into the named filter (ModeBloom only). @@ -18,14 +25,14 @@ func (e *Engine) BloomAdd(ctx context.Context, keyspaceName, name string, item [ return err } if len(item) == 0 { - return fmt.Errorf("%w: empty bloom item", ErrInvalidArgument) + return fmt.Errorf(errEmptyBloomItem, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeBloom { - return fmt.Errorf("%w: BloomAdd requires ModeBloom", ErrInvalidArgument) + return fmt.Errorf(errBloomAddRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -37,7 +44,7 @@ func (e *Engine) BloomAdd(ctx context.Context, keyspaceName, name string, item [ if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } // Forward as a flagged Apply via ForwardPut? Owner must BloomAdd. // Use peer ApplyPut with FlagBloomAdd after routing like Put. @@ -47,40 +54,6 @@ func (e *Engine) BloomAdd(ctx context.Context, keyspaceName, name string, item [ return e.bloomAddLocal(ks, name, item, true) } -func (e *Engine) bloomAddViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: append([]byte(nil), item...), Flags: store.FlagBloomAdd, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - return err -} - -func (e *Engine) bloomAddLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { - ver := uint64(1) - if cur, ok := ks.store.Peek(name); ok { - ver = ks.nextVersion(name, cur.Version) - if cur.IsBloom() && !cur.IsTombstone() { - ver = cur.Version - } - } - expire := e.expireAt(ks.cfg.TTL) - m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() - if !ks.store.BloomAdd(name, item, m, k, ver, expire) { - return fmt.Errorf("%w: bloom add rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), item...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagBloomAdd, - }, false) - } - return nil -} - // BloomTest reports whether item may be in the named filter. func (e *Engine) BloomTest(ctx context.Context, keyspaceName, name string, item []byte) (bool, error) { if err := ctx.Err(); err != nil { @@ -94,7 +67,7 @@ func (e *Engine) BloomTest(ctx context.Context, keyspaceName, name string, item return false, err } if ks.cfg.Mode != keyspace.ModeBloom { - return false, fmt.Errorf("%w: BloomTest requires ModeBloom", ErrInvalidArgument) + return false, fmt.Errorf(errBloomTestRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return false, err @@ -145,19 +118,3 @@ func (e *Engine) ApplyBloomMerge(keyspaceName, name string, bits []byte, version ks.observeVersion(name, version) return e.applyBloomMerge(ks, name, bits, version, 0), nil } - -func (e *Engine) applyBloomAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { - m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.BloomAdd(name, item, m, k, version, expireAt) -} - -func (e *Engine) applyBloomMerge(ks *ksRuntime, name string, bits []byte, version uint64, expireAt int64) bool { - m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.BloomMerge(name, bits, m, k, version, expireAt) -} diff --git a/pkg/engine/bloom_apply.go b/pkg/engine/bloom_apply.go new file mode 100644 index 0000000..bdf5dae --- /dev/null +++ b/pkg/engine/bloom_apply.go @@ -0,0 +1,47 @@ +package engine + +import ( + "fmt" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) bloomAddLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { + ver := uint64(1) + if cur, ok := ks.store.Peek(name); ok { + ver = ks.nextVersion(name, cur.Version) + if cur.IsBloom() && !cur.IsTombstone() { + ver = cur.Version + } + } + expire := e.expireAt(ks.cfg.TTL) + m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() + if !ks.store.BloomAdd(name, item, m, k, ver, expire) { + return fmt.Errorf(errBloomAddRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: append([]byte(nil), item...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagBloomAdd, + }, false) + } + return nil +} + +func (e *Engine) applyBloomAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { + m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.BloomAdd(name, item, m, k, version, expireAt) +} + +func (e *Engine) applyBloomMerge(ks *ksRuntime, name string, bits []byte, version uint64, expireAt int64) bool { + m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.BloomMerge(name, bits, m, k, version, expireAt) +} diff --git a/pkg/engine/bloom_cluster.go b/pkg/engine/bloom_cluster.go new file mode 100644 index 0000000..fd8351b --- /dev/null +++ b/pkg/engine/bloom_cluster.go @@ -0,0 +1,17 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) bloomAddViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: append([]byte(nil), item...), Flags: store.FlagBloomAdd, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + return err +} diff --git a/pkg/store/bloom.go b/pkg/store/bloom.go new file mode 100644 index 0000000..46573c0 --- /dev/null +++ b/pkg/store/bloom.go @@ -0,0 +1,94 @@ +package store + +import ( + "github.com/Code0987/supercache/pkg/bloom" +) + +func (m *Memory) BloomAdd(key string, item []byte, mBits, k int, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.bloomMutateLocked(key, mBits, k, version, expireAt, func(f *bloom.Filter) { + f.Add(item) + }) +} + +func (m *Memory) BloomTest(key string, item []byte, mBits, k int) bool { + m.mu.Lock() + defer m.mu.Unlock() + el, ok := m.items[key] + if !ok { + return false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return false + } + if it.entry.IsTombstone() || !it.entry.IsBloom() { + return false + } + f, err := bloom.Open(mBits, k, it.entry.Value) + if err != nil { + return false + } + return f.Test(item) +} + +func (m *Memory) BloomMerge(key string, bits []byte, mBits, k int, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + remote, err := bloom.Open(mBits, k, bits) + if err != nil { + return false + } + return m.bloomMutateLocked(key, mBits, k, version, expireAt, func(f *bloom.Filter) { + _ = f.Merge(remote) + }) +} + +func (m *Memory) bloomMutateLocked(key string, mBits, k int, version uint64, expireAt int64, mut func(*bloom.Filter)) bool { + need := (mBits + 7) / 8 + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsBloom() { + f, err := bloom.Open(mBits, k, it.entry.Value) + if err != nil { + return false + } + mut(f) + m.order.MoveToFront(el) + return true + } else { + return false + } + } + m.removeElement(el) + } + bits := make([]byte, need) + f, err := bloom.Open(mBits, k, bits) + if err != nil { + return false + } + mut(f) + ent := Entry{Value: bits, Version: version, ExpireAt: expireAt, Flags: FlagBloom} + cost := entryCost(key, ent) + if m.maxBytes > 0 && cost > m.maxBytes && m.order.Len() == 0 { + // Single filter larger than budget: still keep it (same as tombstone overshoot). + } + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + // evictLocked must not drop this live bloom if it is the only item. + if _, ok := m.items[key]; !ok { + return false + } + return true +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..0ce582b 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -7,7 +7,6 @@ import ( "time" "github.com/Code0987/supercache/pkg/bitmapx" - "github.com/Code0987/supercache/pkg/bloom" "github.com/Code0987/supercache/pkg/counter" "github.com/Code0987/supercache/pkg/geo" "github.com/Code0987/supercache/pkg/hashx" @@ -397,95 +396,6 @@ func (m *Memory) RangeAll(fn func(key string, e Entry) bool) { } } -func (m *Memory) BloomAdd(key string, item []byte, mBits, k int, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.bloomMutateLocked(key, mBits, k, version, expireAt, func(f *bloom.Filter) { - f.Add(item) - }) -} - -func (m *Memory) BloomTest(key string, item []byte, mBits, k int) bool { - m.mu.Lock() - defer m.mu.Unlock() - el, ok := m.items[key] - if !ok { - return false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return false - } - if it.entry.IsTombstone() || !it.entry.IsBloom() { - return false - } - f, err := bloom.Open(mBits, k, it.entry.Value) - if err != nil { - return false - } - return f.Test(item) -} - -func (m *Memory) BloomMerge(key string, bits []byte, mBits, k int, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - remote, err := bloom.Open(mBits, k, bits) - if err != nil { - return false - } - return m.bloomMutateLocked(key, mBits, k, version, expireAt, func(f *bloom.Filter) { - _ = f.Merge(remote) - }) -} - -func (m *Memory) bloomMutateLocked(key string, mBits, k int, version uint64, expireAt int64, mut func(*bloom.Filter)) bool { - need := (mBits + 7) / 8 - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsBloom() { - f, err := bloom.Open(mBits, k, it.entry.Value) - if err != nil { - return false - } - mut(f) - m.order.MoveToFront(el) - return true - } else { - return false - } - } - m.removeElement(el) - } - bits := make([]byte, need) - f, err := bloom.Open(mBits, k, bits) - if err != nil { - return false - } - mut(f) - ent := Entry{Value: bits, Version: version, ExpireAt: expireAt, Flags: FlagBloom} - cost := entryCost(key, ent) - if m.maxBytes > 0 && cost > m.maxBytes && m.order.Len() == 0 { - // Single filter larger than budget: still keep it (same as tombstone overshoot). - } - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - // evictLocked must not drop this live bloom if it is the only item. - if _, ok := m.items[key]; !ok { - return false - } - return true -} - // SetAdd inserts item into the named set (creates if missing). Version gates tombstones. func (m *Memory) SetAdd(key string, item []byte, version uint64, expireAt int64) bool { m.mu.Lock() From ca868d7c36f3d8f5efe43521dc79dce7f42b36aa Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:28:44 +0000 Subject: [PATCH 08/13] refactor: extract ModeJSON into its own files Move JSON store methods out of memory.go and split the engine file into public verbs, owner write/apply, and cluster forward. Same contract, including dirty jCache and FlagJSONSet/FlagJSONDel inbox then FlagJSON snapshot. --- docs/design/2026-09-02-refactor-json.md | 40 ++++ docs/design/README.md | 1 + pkg/engine/json.go | 171 ++------------ pkg/engine/json_apply.go | 95 ++++++++ pkg/engine/json_cluster.go | 62 +++++ pkg/store/json.go | 290 ++++++++++++++++++++++++ pkg/store/memory.go | 284 ----------------------- 7 files changed, 509 insertions(+), 434 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-json.md create mode 100644 pkg/engine/json_apply.go create mode 100644 pkg/engine/json_cluster.go create mode 100644 pkg/store/json.go diff --git a/docs/design/2026-09-02-refactor-json.md b/docs/design/2026-09-02-refactor-json.md new file mode 100644 index 0000000..dce12d4 --- /dev/null +++ b/docs/design/2026-09-02-refactor-json.md @@ -0,0 +1,40 @@ +# Refactor ModeJSON layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-json` +**Date:** 2026-09-02 + +## Problem + +JSON store methods still live in `pkg/store/memory.go`. Engine JSON is one file mixing public verbs, owner write, inbox apply, and GetOrLoad. + +## Non-goals + +- No API / proto / version / fan-out / dirty-cache change +- No `pkg/jsonx` algorithm change +- Not extracting other modes (List / Hash stay on their own branches) + +## Contract + +Unchanged: JsonSet/JsonDel ACK-only inbox (`FlagJSONSet` / `FlagJSONDel`) then `FlagJSON` snapshot fan-out. JsonGet present-bit. Version under store mutex. Dirty `jCache` flushed on Get/Peek only. + +## Approach + +Same file layout as List / Hash: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` J* | `pkg/store/json.go` (including `flushJSONValueLocked` / `insertJSONLocked`) | +| `pkg/engine/json.go` (all) | `json.go` public verbs; `json_apply.go` owner write + inbox apply; `json_cluster.go` owner forward / snapshot / GetOrLoad | + +JSON-local error format strings live in a `const` block at the top of `json.go` (same text as before). + +Rejected: rewrite dirty-cache / flush. Drop inbox and snapshot-only. Split `pkg/jsonx`. + +## Tests (already exist; keep them) + +Existing `pkg/jsonx`, `pkg/store` json, `pkg/engine` json unit + cluster. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..4a204c6 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-json.md](./2026-09-02-refactor-json.md) | ModeJSON file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/json.go b/pkg/engine/json.go index 783bb5e..9363b8b 100644 --- a/pkg/engine/json.go +++ b/pkg/engine/json.go @@ -9,6 +9,18 @@ import ( "github.com/Code0987/supercache/pkg/store" ) +const ( + errJSONSetRequiresMode = "%w: JsonSet requires ModeJSON" + errJSONGetRequiresMode = "%w: JsonGet requires ModeJSON" + errJSONDelRequiresMode = "%w: JsonDel requires ModeJSON" + errInvalidJSONPath = "%w: invalid json path" + errInvalidJSON = "%w: invalid json" + errJSONMutateRejected = "%w: json mutate rejected" + errJSONSetRejected = "%w: json set rejected" + errJSONDelRejected = "%w: json del rejected" + errOwnerNoAddress = "%w: owner %s has no address" +) + // JsonSet upserts JSON value at path on a ModeJSON document. func (e *Engine) JsonSet(ctx context.Context, keyspaceName, name, path string, value []byte) error { if err := ctx.Err(); err != nil { @@ -22,7 +34,7 @@ func (e *Engine) JsonSet(ctx context.Context, keyspaceName, name, path string, v return err } if ks.cfg.Mode != keyspace.ModeJSON { - return fmt.Errorf("%w: JsonSet requires ModeJSON", ErrInvalidArgument) + return fmt.Errorf(errJSONSetRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -31,19 +43,19 @@ func (e *Engine) JsonSet(ctx context.Context, keyspaceName, name, path string, v return ErrKeyTooLarge } if _, err := jsonx.ParsePath(path); err != nil { - return fmt.Errorf("%w: invalid json path", ErrInvalidArgument) + return fmt.Errorf(errInvalidJSONPath, ErrInvalidArgument) } if err := e.validateValueSize(ks, value); err != nil { return err } if _, err := jsonx.Decode(value); err != nil { - return fmt.Errorf("%w: invalid json", ErrInvalidArgument) + return fmt.Errorf(errInvalidJSON, ErrInvalidArgument) } c := e.clusterSnapshot() if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.jMutViaOwner(ctx, ks, name, jsonx.EncodeSet(path, value), store.FlagJSONSet) } @@ -64,7 +76,7 @@ func (e *Engine) JsonGet(ctx context.Context, keyspaceName, name, path string) ( return nil, false, err } if ks.cfg.Mode != keyspace.ModeJSON { - return nil, false, fmt.Errorf("%w: JsonGet requires ModeJSON", ErrInvalidArgument) + return nil, false, fmt.Errorf(errJSONGetRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, false, err @@ -73,7 +85,7 @@ func (e *Engine) JsonGet(ctx context.Context, keyspaceName, name, path string) ( return nil, false, ErrKeyTooLarge } if _, err := jsonx.ParsePath(path); err != nil { - return nil, false, fmt.Errorf("%w: invalid json path", ErrInvalidArgument) + return nil, false, fmt.Errorf(errInvalidJSONPath, ErrInvalidArgument) } if ks.store.HasJSON(name) { v, ok := ks.store.JGet(name, path) @@ -99,7 +111,7 @@ func (e *Engine) JsonDel(ctx context.Context, keyspaceName, name, path string) e return err } if ks.cfg.Mode != keyspace.ModeJSON { - return fmt.Errorf("%w: JsonDel requires ModeJSON", ErrInvalidArgument) + return fmt.Errorf(errJSONDelRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -108,13 +120,13 @@ func (e *Engine) JsonDel(ctx context.Context, keyspaceName, name, path string) e return ErrKeyTooLarge } if _, err := jsonx.ParsePath(path); err != nil { - return fmt.Errorf("%w: invalid json path", ErrInvalidArgument) + return fmt.Errorf(errInvalidJSONPath, ErrInvalidArgument) } c := e.clusterSnapshot() if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.jMutViaOwner(ctx, ks, name, []byte(path), store.FlagJSONDel) } @@ -122,147 +134,6 @@ func (e *Engine) JsonDel(ctx context.Context, keyspaceName, name, path string) e return e.jDelLocal(ks, name, path) } -func (e *Engine) jMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: value, Flags: flag, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - applied, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - if err != nil { - return err - } - if !applied { - return fmt.Errorf("%w: json mutate rejected", ErrInvalidArgument) - } - return nil -} - -func (e *Engine) jSetLocal(ks *ksRuntime, name, path string, value []byte) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - applied, tooLarge := ks.store.JSet(name, path, value, gate, expire, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf("%w: json set rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expire) - return nil -} - -func (e *Engine) jDelLocal(ks *ksRuntime, name, path string) error { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, mutated := ks.store.JDel(name, path, gate, expire) - if !ok { - return fmt.Errorf("%w: json del rejected", ErrInvalidArgument) - } - if !mutated { - return nil - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expire) - return nil -} - -func (e *Engine) jReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { - ent, ok := ks.store.Peek(name) - if !ok || !ent.IsJSON() { - return - } - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: ent.Value, - Version: ver, - ExpireAt: expire, - Flags: store.FlagJSON, - }, false) -} - -func (e *Engine) jFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsJSON() { - return store.Entry{}, false, nil - } - if _, decErr := jsonx.Decode(res.Entry.Value); decErr != nil { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.JInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyJSONSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { - path, raw, err := jsonx.DecodeSet(inbox) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.JSet(name, path, raw, gate, expireAt, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyJSONDel(ks *ksRuntime, name string, path []byte, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, mutated := ks.store.JDel(name, string(path), gate, expireAt) - if !ok { - return false - } - if !mutated { - return true - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyJSONInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.JInstall(name, blob, version, expireAt) -} - func jExtract(blob []byte, path string) ([]byte, bool, error) { doc, err := jsonx.Decode(blob) if err != nil { diff --git a/pkg/engine/json_apply.go b/pkg/engine/json_apply.go new file mode 100644 index 0000000..8c6ee40 --- /dev/null +++ b/pkg/engine/json_apply.go @@ -0,0 +1,95 @@ +package engine + +import ( + "fmt" + + "github.com/Code0987/supercache/pkg/jsonx" +) + +func (e *Engine) jSetLocal(ks *ksRuntime, name, path string, value []byte) error { + expire := e.expireAt(ks.cfg.TTL) + max := e.maxValueSize + if ks.cfg.MaxValueSize > 0 { + max = ks.cfg.MaxValueSize + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + applied, tooLarge := ks.store.JSet(name, path, value, gate, expire, max) + if tooLarge { + return ErrValueTooLarge + } + if !applied { + return fmt.Errorf(errJSONSetRejected, ErrInvalidArgument) + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.jReplicateSnapshot(ks, name, ver, expire) + return nil +} + +func (e *Engine) jDelLocal(ks *ksRuntime, name, path string) error { + expire := e.expireAt(ks.cfg.TTL) + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + ok, mutated := ks.store.JDel(name, path, gate, expire) + if !ok { + return fmt.Errorf(errJSONDelRejected, ErrInvalidArgument) + } + if !mutated { + return nil + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.jReplicateSnapshot(ks, name, ver, expire) + return nil +} + +func (e *Engine) applyJSONSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { + path, raw, err := jsonx.DecodeSet(inbox) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + max := e.maxValueSize + if ks.cfg.MaxValueSize > 0 { + max = ks.cfg.MaxValueSize + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + ok, tooLarge := ks.store.JSet(name, path, raw, gate, expireAt, max) + if !ok || tooLarge { + return false + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.jReplicateSnapshot(ks, name, ver, expireAt) + return true +} + +func (e *Engine) applyJSONDel(ks *ksRuntime, name string, path []byte, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + ok, mutated := ks.store.JDel(name, string(path), gate, expireAt) + if !ok { + return false + } + if !mutated { + return true + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.jReplicateSnapshot(ks, name, ver, expireAt) + return true +} + +func (e *Engine) applyJSONInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.JInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/json_cluster.go b/pkg/engine/json_cluster.go new file mode 100644 index 0000000..5fff06b --- /dev/null +++ b/pkg/engine/json_cluster.go @@ -0,0 +1,62 @@ +package engine + +import ( + "context" + "fmt" + + "github.com/Code0987/supercache/pkg/jsonx" + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) jMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: value, Flags: flag, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + applied, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + if err != nil { + return err + } + if !applied { + return fmt.Errorf(errJSONMutateRejected, ErrInvalidArgument) + } + return nil +} + +func (e *Engine) jReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { + ent, ok := ks.store.Peek(name) + if !ok || !ent.IsJSON() { + return + } + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagJSON, + }, false) +} + +func (e *Engine) jFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsJSON() { + return store.Entry{}, false, nil + } + if _, decErr := jsonx.Decode(res.Entry.Value); decErr != nil { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.JInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/store/json.go b/pkg/store/json.go new file mode 100644 index 0000000..83797bc --- /dev/null +++ b/pkg/store/json.go @@ -0,0 +1,290 @@ +package store + +import ( + "container/list" + + "github.com/Code0987/supercache/pkg/jsonx" +) + +func (m *Memory) JSet(key, path string, raw []byte, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { + m.mu.Lock() + defer m.mu.Unlock() + + val, err := jsonx.Decode(raw) + if err != nil { + return false, false + } + p, err := jsonx.ParsePath(path) + if err != nil { + return false, false + } + + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false, false + } + newDoc, err := m.jSetCreate(p, val) + if err != nil { + return false, false + } + return m.jCommitReplaceLocked(el, it, key, newDoc, it.entry.Version+1, expireAt, maxValue) + } + if !it.entry.IsJSON() { + return false, false + } + doc, ok := m.jDocLocked(it) + if !ok { + return false, false + } + newDoc, err := jsonx.Set(doc, p, val) + if err != nil { + return false, false + } + return m.jCommitReplaceLocked(el, it, key, newDoc, it.entry.Version+1, expireAt, maxValue) + } + m.removeElement(el) + } + newDoc, err := m.jSetCreate(p, val) + if err != nil { + return false, false + } + return m.jCommitInsertLocked(key, newDoc, 1, expireAt, maxValue) +} + +func (m *Memory) jSetCreate(p jsonx.Path, val any) (any, error) { + if len(p) == 0 { + return val, nil + } + return jsonx.Set(map[string]any{}, p, val) +} + +func (m *Memory) JDel(key, path string, version uint64, expireAt int64) (ok, mutated bool) { + m.mu.Lock() + defer m.mu.Unlock() + + el, exists := m.items[key] + if !exists { + return true, false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return true, false + } + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false, false + } + return false, false + } + if !it.entry.IsJSON() { + return false, false + } + p, err := jsonx.ParsePath(path) + if err != nil { + return false, false + } + doc, ok := m.jDocLocked(it) + if !ok { + return false, false + } + newDoc, removed, err := jsonx.Del(doc, p) + if err != nil { + return false, false + } + if !removed { + return true, false + } + blob, err := jsonx.Encode(newDoc) + if err != nil { + return false, false + } + oldCost := it.cost + it.entry.Version = it.entry.Version + 1 + it.entry.Flags = FlagJSON + it.entry.Value = blob + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.jCache = newDoc + it.jDirty = false + it.cost = entryCost(key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } + m.order.MoveToFront(el) + m.evictLocked() + return true, true +} + +func (m *Memory) JGet(key, path string) ([]byte, bool) { + m.mu.Lock() + defer m.mu.Unlock() + if !m.hasJSONLocked(key) { + return nil, false + } + p, err := jsonx.ParsePath(path) + if err != nil { + return nil, false + } + el := m.items[key] + it := el.Value.(*lruItem) + if len(p) == 0 && !it.jDirty { + return append([]byte(nil), it.entry.Value...), true + } + doc, ok := m.jDocLocked(it) + if !ok { + return nil, false + } + node, ok := jsonx.Get(doc, p) + if !ok { + return nil, false + } + blob, err := jsonx.Encode(node) + if err != nil { + return nil, false + } + return blob, true +} + +func (m *Memory) HasJSON(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.hasJSONLocked(key) +} + +func (m *Memory) hasJSONLocked(key string) bool { + el, ok := m.items[key] + if !ok { + return false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return false + } + return !it.entry.IsTombstone() && it.entry.IsJSON() +} + +func (m *Memory) JInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + doc, err := jsonx.Decode(blob) + if err != nil { + return false + } + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsJSON() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagJSON} + return m.insertJSONLocked(key, ent, doc, false) +} + +func (m *Memory) jDocLocked(it *lruItem) (any, bool) { + if it.jDirty { + return it.jCache, true + } + if it.jCache != nil { + return it.jCache, true + } + v, err := jsonx.Decode(it.entry.Value) + if err != nil { + return nil, false + } + it.jCache = v + return v, true +} + +func (m *Memory) jCommitReplaceLocked(el *list.Element, it *lruItem, key string, newDoc any, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { + blob, err := jsonx.Encode(newDoc) + if err != nil { + return false, false + } + if maxValue > 0 && len(blob) > maxValue { + return false, true + } + oldCost := it.cost + it.entry.Version = stored + it.entry.Flags = FlagJSON + it.entry.Value = blob + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.jCache = newDoc + it.jDirty = false + it.cost = entryCost(key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still, false +} + +func (m *Memory) jCommitInsertLocked(key string, newDoc any, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { + blob, err := jsonx.Encode(newDoc) + if err != nil { + return false, false + } + if maxValue > 0 && len(blob) > maxValue { + return false, true + } + ent := Entry{Value: blob, Version: stored, ExpireAt: expireAt, Flags: FlagJSON} + return m.insertJSONLocked(key, ent, newDoc, false), false +} + +func (m *Memory) flushJSONValueLocked(it *lruItem) { + if it == nil || !it.jDirty || !it.entry.IsJSON() { + return + } + blob, err := jsonx.Encode(it.jCache) + if err != nil { + return + } + oldCost := it.cost + it.entry.Value = blob + it.jDirty = false + it.cost = entryCost(it.key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } +} + +func (m *Memory) insertJSONLocked(key string, ent Entry, cache any, dirty bool) bool { + cost := entryCost(key, ent) + if dirty { + cost = int64(len(key)) + 64 + jsonx.ApproxWireBytes(cache) + } + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, jCache: cache, jDirty: dirty} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..a4a592d 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -12,7 +12,6 @@ import ( "github.com/Code0987/supercache/pkg/geo" "github.com/Code0987/supercache/pkg/hashx" "github.com/Code0987/supercache/pkg/hllx" - "github.com/Code0987/supercache/pkg/jsonx" "github.com/Code0987/supercache/pkg/listx" "github.com/Code0987/supercache/pkg/set" "github.com/Code0987/supercache/pkg/zset" @@ -1810,289 +1809,6 @@ func (m *Memory) insertCounterLocked(key string, ent Entry) bool { return ok } -func (m *Memory) JSet(key, path string, raw []byte, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { - m.mu.Lock() - defer m.mu.Unlock() - - val, err := jsonx.Decode(raw) - if err != nil { - return false, false - } - p, err := jsonx.ParsePath(path) - if err != nil { - return false, false - } - - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false, false - } - newDoc, err := m.jSetCreate(p, val) - if err != nil { - return false, false - } - return m.jCommitReplaceLocked(el, it, key, newDoc, it.entry.Version+1, expireAt, maxValue) - } - if !it.entry.IsJSON() { - return false, false - } - doc, ok := m.jDocLocked(it) - if !ok { - return false, false - } - newDoc, err := jsonx.Set(doc, p, val) - if err != nil { - return false, false - } - return m.jCommitReplaceLocked(el, it, key, newDoc, it.entry.Version+1, expireAt, maxValue) - } - m.removeElement(el) - } - newDoc, err := m.jSetCreate(p, val) - if err != nil { - return false, false - } - return m.jCommitInsertLocked(key, newDoc, 1, expireAt, maxValue) -} - -func (m *Memory) jSetCreate(p jsonx.Path, val any) (any, error) { - if len(p) == 0 { - return val, nil - } - return jsonx.Set(map[string]any{}, p, val) -} - -func (m *Memory) JDel(key, path string, version uint64, expireAt int64) (ok, mutated bool) { - m.mu.Lock() - defer m.mu.Unlock() - - el, exists := m.items[key] - if !exists { - return true, false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return true, false - } - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false, false - } - return false, false - } - if !it.entry.IsJSON() { - return false, false - } - p, err := jsonx.ParsePath(path) - if err != nil { - return false, false - } - doc, ok := m.jDocLocked(it) - if !ok { - return false, false - } - newDoc, removed, err := jsonx.Del(doc, p) - if err != nil { - return false, false - } - if !removed { - return true, false - } - blob, err := jsonx.Encode(newDoc) - if err != nil { - return false, false - } - oldCost := it.cost - it.entry.Version = it.entry.Version + 1 - it.entry.Flags = FlagJSON - it.entry.Value = blob - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.jCache = newDoc - it.jDirty = false - it.cost = entryCost(key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } - m.order.MoveToFront(el) - m.evictLocked() - return true, true -} - -func (m *Memory) JGet(key, path string) ([]byte, bool) { - m.mu.Lock() - defer m.mu.Unlock() - if !m.hasJSONLocked(key) { - return nil, false - } - p, err := jsonx.ParsePath(path) - if err != nil { - return nil, false - } - el := m.items[key] - it := el.Value.(*lruItem) - if len(p) == 0 && !it.jDirty { - return append([]byte(nil), it.entry.Value...), true - } - doc, ok := m.jDocLocked(it) - if !ok { - return nil, false - } - node, ok := jsonx.Get(doc, p) - if !ok { - return nil, false - } - blob, err := jsonx.Encode(node) - if err != nil { - return nil, false - } - return blob, true -} - -func (m *Memory) HasJSON(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.hasJSONLocked(key) -} - -func (m *Memory) hasJSONLocked(key string) bool { - el, ok := m.items[key] - if !ok { - return false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return false - } - return !it.entry.IsTombstone() && it.entry.IsJSON() -} - -func (m *Memory) JInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - doc, err := jsonx.Decode(blob) - if err != nil { - return false - } - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsJSON() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagJSON} - return m.insertJSONLocked(key, ent, doc, false) -} - -func (m *Memory) jDocLocked(it *lruItem) (any, bool) { - if it.jDirty { - return it.jCache, true - } - if it.jCache != nil { - return it.jCache, true - } - v, err := jsonx.Decode(it.entry.Value) - if err != nil { - return nil, false - } - it.jCache = v - return v, true -} - -func (m *Memory) jCommitReplaceLocked(el *list.Element, it *lruItem, key string, newDoc any, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { - blob, err := jsonx.Encode(newDoc) - if err != nil { - return false, false - } - if maxValue > 0 && len(blob) > maxValue { - return false, true - } - oldCost := it.cost - it.entry.Version = stored - it.entry.Flags = FlagJSON - it.entry.Value = blob - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.jCache = newDoc - it.jDirty = false - it.cost = entryCost(key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still, false -} - -func (m *Memory) jCommitInsertLocked(key string, newDoc any, stored uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { - blob, err := jsonx.Encode(newDoc) - if err != nil { - return false, false - } - if maxValue > 0 && len(blob) > maxValue { - return false, true - } - ent := Entry{Value: blob, Version: stored, ExpireAt: expireAt, Flags: FlagJSON} - return m.insertJSONLocked(key, ent, newDoc, false), false -} - -func (m *Memory) flushJSONValueLocked(it *lruItem) { - if it == nil || !it.jDirty || !it.entry.IsJSON() { - return - } - blob, err := jsonx.Encode(it.jCache) - if err != nil { - return - } - oldCost := it.cost - it.entry.Value = blob - it.jDirty = false - it.cost = entryCost(it.key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } -} - -func (m *Memory) insertJSONLocked(key string, ent Entry, cache any, dirty bool) bool { - cost := entryCost(key, ent) - if dirty { - cost = int64(len(key)) + 64 + jsonx.ApproxWireBytes(cache) - } - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, jCache: cache, jDirty: dirty} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) BSet(key string, offset uint64, bit bool, version uint64, expireAt int64, maxValue int) (applied, tooLarge bool) { m.mu.Lock() defer m.mu.Unlock() From 4191f4d746d9d3f4deec51174c41312b3499b89a Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:32:11 +0000 Subject: [PATCH 09/13] refactor: extract ModeHash into its own files Move Hash store methods out of memory.go and split the engine file into public verbs, owner write/apply, and cluster forward. Same contract, including dirty hCache and FlagHashSet/FlagHashDel item fan-out. --- docs/design/2026-09-02-refactor-hash.md | 40 +++++ docs/design/README.md | 1 + pkg/engine/hash.go | 131 +++----------- pkg/engine/hash_apply.go | 77 ++++++++ pkg/engine/hash_cluster.go | 38 ++++ pkg/store/hash.go | 229 ++++++++++++++++++++++++ pkg/store/memory.go | 224 ----------------------- 7 files changed, 407 insertions(+), 333 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-hash.md create mode 100644 pkg/engine/hash_apply.go create mode 100644 pkg/engine/hash_cluster.go create mode 100644 pkg/store/hash.go diff --git a/docs/design/2026-09-02-refactor-hash.md b/docs/design/2026-09-02-refactor-hash.md new file mode 100644 index 0000000..0282a4d --- /dev/null +++ b/docs/design/2026-09-02-refactor-hash.md @@ -0,0 +1,40 @@ +# Refactor ModeHash layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-hash` +**Date:** 2026-09-02 + +## Problem + +Hash store methods still live in `pkg/store/memory.go`. Engine Hash is one file mixing public verbs, owner write, item fan-out, and GetOrLoad. + +## Non-goals + +- No API / proto / version / fan-out / dirty-cache change +- No `pkg/hashx` algorithm change +- Not extracting other modes (List stays on `feat/refactor-list`) + +## Contract + +Unchanged: HSet/HDel ACK-only inbox (`FlagHashSet` / `FlagHashDel` item fan-out, not snapshot). HGet present-bit + field miss. Version via `nextVersion`. Dirty `hCache` flushed on Get/Peek only. + +## Approach + +Same file layout as List: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` H* | `pkg/store/hash.go` (including `flushHashValueLocked` / `insertHashLocked`) | +| `pkg/engine/hash.go` (all) | `hash.go` public verbs; `hash_apply.go` owner write + inbox apply; `hash_cluster.go` GetOrLoad / owner forward | + +Hash-local error format strings live in a `const` block at the top of `hash.go` (same text as before). + +Rejected: rewrite dirty-cache / flush. Switch Hash to snapshot fan-out. Split `pkg/hashx`. + +## Tests (already exist; keep them) + +Existing `pkg/hashx`, `pkg/store` hash, `pkg/engine` hash unit + cluster. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..27394ad 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-hash.md](./2026-09-02-refactor-hash.md) | ModeHash file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/hash.go b/pkg/engine/hash.go index 0a39d47..976dfd5 100644 --- a/pkg/engine/hash.go +++ b/pkg/engine/hash.go @@ -9,6 +9,19 @@ import ( "github.com/Code0987/supercache/pkg/store" ) +const ( + errEmptyHashField = "%w: empty hash field" + errHSetRequiresMode = "%w: HSet requires ModeHash" + errHDelRequiresMode = "%w: HDel requires ModeHash" + errHGetRequiresMode = "%w: HGet requires ModeHash" + errHExistsRequiresMode = "%w: HExists requires ModeHash" + errHLenRequiresMode = "%w: HLen requires ModeHash" + errHGetAllRequiresMode = "%w: HGetAll requires ModeHash" + errHSetRejected = "%w: hset rejected" + errHDelRejected = "%w: hdel rejected" + errOwnerNoAddress = "%w: owner %s has no address" +) + // HashField is one field/value pair returned by HGetAll. type HashField struct { Field []byte @@ -17,7 +30,7 @@ type HashField struct { func (e *Engine) checkHashField(field []byte) error { if len(field) == 0 { - return fmt.Errorf("%w: empty hash field", ErrInvalidArgument) + return fmt.Errorf(errEmptyHashField, ErrInvalidArgument) } if len(field) > e.maxKeyLen { return ErrKeyTooLarge @@ -41,7 +54,7 @@ func (e *Engine) HSet(ctx context.Context, keyspaceName, name string, field, val return err } if ks.cfg.Mode != keyspace.ModeHash { - return fmt.Errorf("%w: HSet requires ModeHash", ErrInvalidArgument) + return fmt.Errorf(errHSetRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -53,7 +66,7 @@ func (e *Engine) HSet(ctx context.Context, keyspaceName, name string, field, val if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.hMutViaOwner(ctx, ks, name, hashx.EncodeSet(field, value), store.FlagHashSet) } @@ -77,7 +90,7 @@ func (e *Engine) HDel(ctx context.Context, keyspaceName, name string, field []by return err } if ks.cfg.Mode != keyspace.ModeHash { - return fmt.Errorf("%w: HDel requires ModeHash", ErrInvalidArgument) + return fmt.Errorf(errHDelRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -86,7 +99,7 @@ func (e *Engine) HDel(ctx context.Context, keyspaceName, name string, field []by if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } return e.hMutViaOwner(ctx, ks, name, append([]byte(nil), field...), store.FlagHashDel) } @@ -110,7 +123,7 @@ func (e *Engine) HGet(ctx context.Context, keyspaceName, name string, field []by return nil, false, err } if ks.cfg.Mode != keyspace.ModeHash { - return nil, false, fmt.Errorf("%w: HGet requires ModeHash", ErrInvalidArgument) + return nil, false, fmt.Errorf(errHGetRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, false, err @@ -149,7 +162,7 @@ func (e *Engine) HExists(ctx context.Context, keyspaceName, name string, field [ return false, err } if ks.cfg.Mode != keyspace.ModeHash { - return false, fmt.Errorf("%w: HExists requires ModeHash", ErrInvalidArgument) + return false, fmt.Errorf(errHExistsRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return false, err @@ -181,7 +194,7 @@ func (e *Engine) HLen(ctx context.Context, keyspaceName, name string) (int, erro return 0, err } if ks.cfg.Mode != keyspace.ModeHash { - return 0, fmt.Errorf("%w: HLen requires ModeHash", ErrInvalidArgument) + return 0, fmt.Errorf(errHLenRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, err @@ -213,7 +226,7 @@ func (e *Engine) HGetAll(ctx context.Context, keyspaceName, name string) ([]Hash return nil, err } if ks.cfg.Mode != keyspace.ModeHash { - return nil, fmt.Errorf("%w: HGetAll requires ModeHash", ErrInvalidArgument) + return nil, fmt.Errorf(errHGetAllRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, err @@ -232,110 +245,10 @@ func (e *Engine) HGetAll(ctx context.Context, keyspaceName, name string) ([]Hash return toEngineHashFieldsFromPkg(h.All()), nil } -func (e *Engine) hMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: value, Flags: flag, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - return err -} - -func (e *Engine) hSetLocal(ks *ksRuntime, name string, field, value []byte, fanout bool) error { - ver := e.hNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.HSet(name, field, value, ver, expire) { - return fmt.Errorf("%w: hset rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: hashx.EncodeSet(field, value), - Version: ver, - ExpireAt: expire, - Flags: store.FlagHashSet, - }, false) - } - return nil -} - -func (e *Engine) hDelLocal(ks *ksRuntime, name string, field []byte, fanout bool) error { - ver := e.hNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.HDel(name, field, ver, expire) { - if !e.hasHashLocal(ks, name) { - return nil - } - return fmt.Errorf("%w: hdel rejected", ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), field...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagHashDel, - }, false) - } - return nil -} - -func (e *Engine) hNextVersion(ks *ksRuntime, name string) uint64 { - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - func (e *Engine) hasHashLocal(ks *ksRuntime, name string) bool { return ks.store.HasHash(name) } -func (e *Engine) hFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsHash() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.HInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyHashSet(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { - field, val, err := hashx.DecodeSet(value) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HSet(name, field, val, version, expireAt) -} - -func (e *Engine) applyHashDel(ks *ksRuntime, name string, field []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HDel(name, field, version, expireAt) -} - -func (e *Engine) applyHashInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HInstall(name, blob, version, expireAt) -} - func toEngineHashFields(in []store.HashField) []HashField { if len(in) == 0 { return nil diff --git a/pkg/engine/hash_apply.go b/pkg/engine/hash_apply.go new file mode 100644 index 0000000..babab36 --- /dev/null +++ b/pkg/engine/hash_apply.go @@ -0,0 +1,77 @@ +package engine + +import ( + "fmt" + + "github.com/Code0987/supercache/pkg/hashx" + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) hSetLocal(ks *ksRuntime, name string, field, value []byte, fanout bool) error { + ver := e.hNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.HSet(name, field, value, ver, expire) { + return fmt.Errorf(errHSetRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: hashx.EncodeSet(field, value), + Version: ver, + ExpireAt: expire, + Flags: store.FlagHashSet, + }, false) + } + return nil +} + +func (e *Engine) hDelLocal(ks *ksRuntime, name string, field []byte, fanout bool) error { + ver := e.hNextVersion(ks, name) + expire := e.expireAt(ks.cfg.TTL) + if !ks.store.HDel(name, field, ver, expire) { + if !e.hasHashLocal(ks, name) { + return nil + } + return fmt.Errorf(errHDelRejected, ErrInvalidArgument) + } + if fanout { + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: append([]byte(nil), field...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagHashDel, + }, false) + } + return nil +} + +func (e *Engine) hNextVersion(ks *ksRuntime, name string) uint64 { + if ver, ok := ks.store.PeekVersion(name); ok { + return ks.nextVersion(name, ver) + } + return ks.nextVersion(name, 0) +} + +func (e *Engine) applyHashSet(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { + field, val, err := hashx.DecodeSet(value) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.HSet(name, field, val, version, expireAt) +} + +func (e *Engine) applyHashDel(ks *ksRuntime, name string, field []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.HDel(name, field, version, expireAt) +} + +func (e *Engine) applyHashInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.HInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/hash_cluster.go b/pkg/engine/hash_cluster.go new file mode 100644 index 0000000..0b60bb7 --- /dev/null +++ b/pkg/engine/hash_cluster.go @@ -0,0 +1,38 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) hMutViaOwner(ctx context.Context, ks *ksRuntime, name string, value []byte, flag uint32) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: value, Flags: flag, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + return err +} + +func (e *Engine) hFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsHash() { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.HInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/store/hash.go b/pkg/store/hash.go new file mode 100644 index 0000000..8d43e48 --- /dev/null +++ b/pkg/store/hash.go @@ -0,0 +1,229 @@ +package store + +import ( + "github.com/Code0987/supercache/pkg/hashx" +) + +func (m *Memory) HSet(key string, field, value []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.hMutateLocked(key, version, expireAt, func(h *hashx.Hash) { + h.Set(field, value) + }) +} + +func (m *Memory) HDel(key string, field []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + el, ok := m.items[key] + if !ok { + return true + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return true + } + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + return true + } + if !it.entry.IsHash() { + return false + } + return m.hMutateLocked(key, version, expireAt, func(h *hashx.Hash) { + h.Del(field) + }) +} + +func (m *Memory) HGet(key string, field []byte) ([]byte, bool) { + m.mu.Lock() + defer m.mu.Unlock() + h, ok := m.hPeekLocked(key) + if !ok { + return nil, false + } + return h.Get(field) +} + +func (m *Memory) HExists(key string, field []byte) bool { + m.mu.Lock() + defer m.mu.Unlock() + h, ok := m.hPeekLocked(key) + if !ok { + return false + } + return h.Exists(field) +} + +func (m *Memory) HLen(key string) int { + m.mu.Lock() + defer m.mu.Unlock() + h, ok := m.hPeekLocked(key) + if !ok { + return 0 + } + return h.Len() +} + +func (m *Memory) HasHash(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.hPeekLocked(key) + return ok +} + +func (m *Memory) HGetAll(key string) []HashField { + m.mu.Lock() + defer m.mu.Unlock() + h, ok := m.hPeekLocked(key) + if !ok { + return nil + } + all := h.All() + if len(all) == 0 { + return nil + } + out := make([]HashField, len(all)) + for i, f := range all { + out[i] = HashField{Field: f.Field, Value: f.Value} + } + return out +} + +func (m *Memory) HInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsHash() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + h, err := hashx.Decode(blob) + if err != nil { + return false + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagHash} + return m.insertHashLocked(key, ent, h, false) +} + +func (m *Memory) hPeekLocked(key string) (*hashx.Hash, bool) { + el, ok := m.items[key] + if !ok { + return nil, false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return nil, false + } + if it.entry.IsTombstone() || !it.entry.IsHash() { + return nil, false + } + if it.hCache != nil { + return it.hCache, true + } + h, err := hashx.Decode(it.entry.Value) + if err != nil { + return nil, false + } + it.hCache = h + return h, true +} + +func (m *Memory) hMutateLocked(key string, version uint64, expireAt int64, mut func(*hashx.Hash)) bool { + var h *hashx.Hash + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + m.removeElement(el) + h = hashx.New() + } else if it.entry.IsHash() { + h = it.hCache + if h == nil { + var err error + h, err = hashx.Decode(it.entry.Value) + if err != nil { + return false + } + } + mut(h) + oldCost := it.cost + it.entry.Version = version + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.entry.Flags = FlagHash + it.hCache = h + it.hDirty = true + it.cost = int64(len(key)) + 64 + h.ApproxWireBytes() + m.bytes += it.cost - oldCost + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still + } else { + return false + } + } else { + m.removeElement(el) + h = hashx.New() + } + } else { + h = hashx.New() + } + mut(h) + ent := Entry{Value: h.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagHash} + return m.insertHashLocked(key, ent, h, false) +} + +func (m *Memory) flushHashValueLocked(it *lruItem) { + if it == nil || !it.hDirty || it.hCache == nil || !it.entry.IsHash() { + return + } + blob := it.hCache.Encode() + oldCost := it.cost + it.entry.Value = blob + it.hDirty = false + it.cost = entryCost(it.key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } +} + +func (m *Memory) insertHashLocked(key string, ent Entry, cache *hashx.Hash, dirty bool) bool { + cost := entryCost(key, ent) + if dirty && cache != nil { + cost = int64(len(key)) + 64 + cache.ApproxWireBytes() + } + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, hCache: cache, hDirty: dirty} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..8766783 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -1452,230 +1452,6 @@ func (m *Memory) insertListLocked(key string, ent Entry, cache *listx.List, dirt return ok } -func (m *Memory) HSet(key string, field, value []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.hMutateLocked(key, version, expireAt, func(h *hashx.Hash) { - h.Set(field, value) - }) -} - -func (m *Memory) HDel(key string, field []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - el, ok := m.items[key] - if !ok { - return true - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return true - } - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - return true - } - if !it.entry.IsHash() { - return false - } - return m.hMutateLocked(key, version, expireAt, func(h *hashx.Hash) { - h.Del(field) - }) -} - -func (m *Memory) HGet(key string, field []byte) ([]byte, bool) { - m.mu.Lock() - defer m.mu.Unlock() - h, ok := m.hPeekLocked(key) - if !ok { - return nil, false - } - return h.Get(field) -} - -func (m *Memory) HExists(key string, field []byte) bool { - m.mu.Lock() - defer m.mu.Unlock() - h, ok := m.hPeekLocked(key) - if !ok { - return false - } - return h.Exists(field) -} - -func (m *Memory) HLen(key string) int { - m.mu.Lock() - defer m.mu.Unlock() - h, ok := m.hPeekLocked(key) - if !ok { - return 0 - } - return h.Len() -} - -func (m *Memory) HasHash(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.hPeekLocked(key) - return ok -} - -func (m *Memory) HGetAll(key string) []HashField { - m.mu.Lock() - defer m.mu.Unlock() - h, ok := m.hPeekLocked(key) - if !ok { - return nil - } - all := h.All() - if len(all) == 0 { - return nil - } - out := make([]HashField, len(all)) - for i, f := range all { - out[i] = HashField{Field: f.Field, Value: f.Value} - } - return out -} - -func (m *Memory) HInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsHash() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - h, err := hashx.Decode(blob) - if err != nil { - return false - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagHash} - return m.insertHashLocked(key, ent, h, false) -} - -func (m *Memory) hPeekLocked(key string) (*hashx.Hash, bool) { - el, ok := m.items[key] - if !ok { - return nil, false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return nil, false - } - if it.entry.IsTombstone() || !it.entry.IsHash() { - return nil, false - } - if it.hCache != nil { - return it.hCache, true - } - h, err := hashx.Decode(it.entry.Value) - if err != nil { - return nil, false - } - it.hCache = h - return h, true -} - -func (m *Memory) hMutateLocked(key string, version uint64, expireAt int64, mut func(*hashx.Hash)) bool { - var h *hashx.Hash - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - m.removeElement(el) - h = hashx.New() - } else if it.entry.IsHash() { - h = it.hCache - if h == nil { - var err error - h, err = hashx.Decode(it.entry.Value) - if err != nil { - return false - } - } - mut(h) - oldCost := it.cost - it.entry.Version = version - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.entry.Flags = FlagHash - it.hCache = h - it.hDirty = true - it.cost = int64(len(key)) + 64 + h.ApproxWireBytes() - m.bytes += it.cost - oldCost - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still - } else { - return false - } - } else { - m.removeElement(el) - h = hashx.New() - } - } else { - h = hashx.New() - } - mut(h) - ent := Entry{Value: h.Encode(), Version: version, ExpireAt: expireAt, Flags: FlagHash} - return m.insertHashLocked(key, ent, h, false) -} - -func (m *Memory) flushHashValueLocked(it *lruItem) { - if it == nil || !it.hDirty || it.hCache == nil || !it.entry.IsHash() { - return - } - blob := it.hCache.Encode() - oldCost := it.cost - it.entry.Value = blob - it.hDirty = false - it.cost = entryCost(it.key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } -} - -func (m *Memory) insertHashLocked(key string, ent Entry, cache *hashx.Hash, dirty bool) bool { - cost := entryCost(key, ent) - if dirty && cache != nil { - cost = int64(len(key)) + 64 + cache.ApproxWireBytes() - } - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, hCache: cache, hDirty: dirty} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) CIncr(key string, delta int64, version uint64, expireAt int64) (int64, bool, bool) { m.mu.Lock() defer m.mu.Unlock() From 602b4233c1bd80c4dbd2da2b4a1ab5c14cc406ff Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:33:33 +0000 Subject: [PATCH 10/13] refactor: extract ModeList into its own files Move List store methods out of memory.go and split the engine file into public verbs, owner write/apply, and cluster forward. Same contract, including dirty lCache and FlagList snapshot fan-out. --- docs/design/2026-09-02-refactor-list.md | 40 ++++ docs/design/README.md | 1 + pkg/engine/list.go | 144 ++----------- pkg/engine/list_apply.go | 86 ++++++++ pkg/engine/list_cluster.go | 41 ++++ pkg/store/list.go | 256 ++++++++++++++++++++++++ pkg/store/memory.go | 251 ----------------------- 7 files changed, 443 insertions(+), 376 deletions(-) create mode 100644 docs/design/2026-09-02-refactor-list.md create mode 100644 pkg/engine/list_apply.go create mode 100644 pkg/engine/list_cluster.go create mode 100644 pkg/store/list.go diff --git a/docs/design/2026-09-02-refactor-list.md b/docs/design/2026-09-02-refactor-list.md new file mode 100644 index 0000000..d70b104 --- /dev/null +++ b/docs/design/2026-09-02-refactor-list.md @@ -0,0 +1,40 @@ +# Refactor ModeList layout (no contract change) + +**Status:** approved (chat: continue) +**Branch:** `feat/refactor-list` +**Date:** 2026-09-02 + +## Problem + +List store methods still live in `pkg/store/memory.go`. Engine List is one file mixing public verbs, owner write, and GetOrLoad. + +## Non-goals + +- No API / proto / version / fan-out / dirty-cache change +- No `pkg/listx` algorithm change +- Not extracting other modes (HLL / Bitmap / Counter stay on their own branches) + +## Contract + +Unchanged: LPush/RPush ACK-only inbox; LPop/RPop Peer `ListPop`; FlagList snapshot; version under store mutex; dirty `lCache` flushed on Get/Peek only. + +## Approach + +Same file layout as HLL / Bitmap / Counter: + +| Before | After | +|--------|--------| +| `pkg/store/memory.go` L* | `pkg/store/list.go` (including `flushListValueLocked` / `insertListLocked`) | +| `pkg/engine/list.go` (all) | `list.go` public verbs; `list_apply.go` owner write + inbox apply; `list_cluster.go` GetOrLoad / snapshot | + +List-local error format strings live in a `const` block at the top of `list.go` (same text as before). + +Rejected: rewrite dirty-cache / flush (JSON/Hash share that pattern; leave it). Split `pkg/listx` (166 lines). + +## Tests (already exist; keep them) + +Existing `pkg/listx`, `pkg/store` list, `pkg/engine` list unit + cluster. + +## Bench risk + +No new Get/Peek logic. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. diff --git a/docs/design/README.md b/docs/design/README.md index 7275235..1c3fd6a 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -22,6 +22,7 @@ Do not implement from a draft. | [2026-08-25-mode-hll.md](./2026-08-25-mode-hll.md) | `ModeHLL` | | [2026-08-31-mode-topk.md](./2026-08-31-mode-topk.md) | `ModeTopK` | | [2026-09-01-mode-cms.md](./2026-09-01-mode-cms.md) | `ModeCMS` | +| [2026-09-02-refactor-list.md](./2026-09-02-refactor-list.md) | ModeList file layout (no contract) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/engine/list.go b/pkg/engine/list.go index 19392b7..89902f1 100644 --- a/pkg/engine/list.go +++ b/pkg/engine/list.go @@ -9,6 +9,17 @@ import ( "github.com/Code0987/supercache/pkg/store" ) +const ( + errEmptyListItem = "%w: empty list item" + errListOpRequiresMode = "%w: list op requires ModeList" + errLLenRequiresMode = "%w: LLen requires ModeList" + errLIndexRequiresMode = "%w: LIndex requires ModeList" + errLRangeRequiresMode = "%w: LRange requires ModeList" + errListPushRejected = "%w: list push rejected" + errListPopRejected = "%w: list pop rejected" + errOwnerNoAddress = "%w: owner %s has no address" +) + // LPush prepends item on a ModeList. func (e *Engine) LPush(ctx context.Context, keyspaceName, name string, item []byte) error { return e.lPush(ctx, keyspaceName, name, item, true) @@ -27,14 +38,14 @@ func (e *Engine) lPush(ctx context.Context, keyspaceName, name string, item []by return err } if len(item) == 0 { - return fmt.Errorf("%w: empty list item", ErrInvalidArgument) + return fmt.Errorf(errEmptyListItem, ErrInvalidArgument) } ks, err := e.getKS(keyspaceName) if err != nil { return err } if ks.cfg.Mode != keyspace.ModeList { - return fmt.Errorf("%w: list op requires ModeList", ErrInvalidArgument) + return fmt.Errorf(errListOpRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return err @@ -50,7 +61,7 @@ func (e *Engine) lPush(ctx context.Context, keyspaceName, name string, item []by if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } ent := store.Entry{Value: append([]byte(nil), item...), Flags: flag, Version: 1} pctx, cancel := e.peerCtx(ctx, ks) @@ -84,7 +95,7 @@ func (e *Engine) lPop(ctx context.Context, keyspaceName, name string, left bool) return nil, false, err } if ks.cfg.Mode != keyspace.ModeList { - return nil, false, fmt.Errorf("%w: list op requires ModeList", ErrInvalidArgument) + return nil, false, fmt.Errorf(errListOpRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, false, err @@ -93,7 +104,7 @@ func (e *Engine) lPop(ctx context.Context, keyspaceName, name string, left bool) if c != nil && c.Ring != nil { if owner, ok := c.Ring.Owner(name); ok && owner.ID != "" && owner.ID != c.SelfID { if c.Transport == nil || owner.Addr == "" { - return nil, false, fmt.Errorf("%w: owner %s has no address", ErrUnavailable, owner.ID) + return nil, false, fmt.Errorf(errOwnerNoAddress, ErrUnavailable, owner.ID) } pctx, cancel := e.peerCtx(ctx, ks) defer cancel() @@ -116,7 +127,7 @@ func (e *Engine) LLen(ctx context.Context, keyspaceName, name string) (int, erro return 0, err } if ks.cfg.Mode != keyspace.ModeList { - return 0, fmt.Errorf("%w: LLen requires ModeList", ErrInvalidArgument) + return 0, fmt.Errorf(errLLenRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return 0, err @@ -148,7 +159,7 @@ func (e *Engine) LIndex(ctx context.Context, keyspaceName, name string, idx int) return nil, false, err } if ks.cfg.Mode != keyspace.ModeList { - return nil, false, fmt.Errorf("%w: LIndex requires ModeList", ErrInvalidArgument) + return nil, false, fmt.Errorf(errLIndexRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, false, err @@ -182,7 +193,7 @@ func (e *Engine) LRange(ctx context.Context, keyspaceName, name string, start, s return nil, err } if ks.cfg.Mode != keyspace.ModeList { - return nil, fmt.Errorf("%w: LRange requires ModeList", ErrInvalidArgument) + return nil, fmt.Errorf(errLRangeRequiresMode, ErrInvalidArgument) } if err := e.validateKeyLen(ks, name); err != nil { return nil, err @@ -201,123 +212,6 @@ func (e *Engine) LRange(ctx context.Context, keyspaceName, name string, start, s return l.Range(start, stop), nil } -func (e *Engine) lPushLocal(ks *ksRuntime, name string, item []byte, left, fanout bool) error { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok := false - if left { - ok = ks.store.LPush(name, item, gate, expire) - } else { - ok = ks.store.RPush(name, item, gate, expire) - } - if !ok { - return fmt.Errorf("%w: list push rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - if fanout { - e.lReplicateSnapshot(ks, name, ver, expire) - } - return nil -} - -func (e *Engine) lPopLocal(ks *ksRuntime, name string, left, fanout bool) ([]byte, bool, error) { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - var item []byte - var popped, applied bool - if left { - item, popped, applied = ks.store.LPop(name, gate, expire) - } else { - item, popped, applied = ks.store.RPop(name, gate, expire) - } - if !applied { - if !e.hasListLocal(ks, name) { - return nil, false, nil - } - return nil, false, fmt.Errorf("%w: list pop rejected", ErrInvalidArgument) - } - if popped && fanout { - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.lReplicateSnapshot(ks, name, ver, expire) - } - return item, popped, nil -} - -func (e *Engine) lReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { - ent, ok := ks.store.Peek(name) - if !ok || !ent.IsList() { - return - } - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: ent.Value, - Version: ver, - ExpireAt: expire, - Flags: store.FlagList, - }, false) -} - func (e *Engine) hasListLocal(ks *ksRuntime, name string) bool { return ks.store.HasList(name) } - -func (e *Engine) lFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsList() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.LInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - -func (e *Engine) applyListLPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - if !ks.store.LPush(name, item, gate, expireAt) { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.lReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyListRPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - if !ks.store.RPush(name, item, gate, expireAt) { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.lReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyListInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.LInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/list_apply.go b/pkg/engine/list_apply.go new file mode 100644 index 0000000..4415ca5 --- /dev/null +++ b/pkg/engine/list_apply.go @@ -0,0 +1,86 @@ +package engine + +import "fmt" + +func (e *Engine) lPushLocal(ks *ksRuntime, name string, item []byte, left, fanout bool) error { + expire := e.expireAt(ks.cfg.TTL) + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + ok := false + if left { + ok = ks.store.LPush(name, item, gate, expire) + } else { + ok = ks.store.RPush(name, item, gate, expire) + } + if !ok { + return fmt.Errorf(errListPushRejected, ErrInvalidArgument) + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + if fanout { + e.lReplicateSnapshot(ks, name, ver, expire) + } + return nil +} + +func (e *Engine) lPopLocal(ks *ksRuntime, name string, left, fanout bool) ([]byte, bool, error) { + expire := e.expireAt(ks.cfg.TTL) + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + var item []byte + var popped, applied bool + if left { + item, popped, applied = ks.store.LPop(name, gate, expire) + } else { + item, popped, applied = ks.store.RPop(name, gate, expire) + } + if !applied { + if !e.hasListLocal(ks, name) { + return nil, false, nil + } + return nil, false, fmt.Errorf(errListPopRejected, ErrInvalidArgument) + } + if popped && fanout { + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.lReplicateSnapshot(ks, name, ver, expire) + } + return item, popped, nil +} + +func (e *Engine) applyListLPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + if !ks.store.LPush(name, item, gate, expireAt) { + return false + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.lReplicateSnapshot(ks, name, ver, expireAt) + return true +} + +func (e *Engine) applyListRPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + cur, _ := ks.store.PeekVersion(name) + gate := cur + 1 + if !ks.store.RPush(name, item, gate, expireAt) { + return false + } + ver, _ := ks.store.PeekVersion(name) + ks.observeVersion(name, ver) + e.lReplicateSnapshot(ks, name, ver, expireAt) + return true +} + +func (e *Engine) applyListInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = e.expireAt(ks.cfg.TTL) + } + return ks.store.LInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/list_cluster.go b/pkg/engine/list_cluster.go new file mode 100644 index 0000000..edfed30 --- /dev/null +++ b/pkg/engine/list_cluster.go @@ -0,0 +1,41 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +func (e *Engine) lReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { + ent, ok := ks.store.Peek(name) + if !ok || !ent.IsList() { + return + } + e.replicate(ks.cfg.Name, name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagList, + }, false) +} + +func (e *Engine) lFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { + c := e.clusterSnapshot() + if c == nil || c.Ring == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) + if err != nil || !res.Found || !res.Entry.IsList() { + return store.Entry{}, false, nil + } + if e.holdsReplica(c, ks, name) { + _ = ks.store.LInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) + } + return res.Entry, true, nil +} diff --git a/pkg/store/list.go b/pkg/store/list.go new file mode 100644 index 0000000..93d4b78 --- /dev/null +++ b/pkg/store/list.go @@ -0,0 +1,256 @@ +package store + +import ( + "github.com/Code0987/supercache/pkg/listx" +) + +func (m *Memory) LPush(key string, item []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.lMutateLocked(key, version, expireAt, func(l *listx.List) { + l.LPush(item) + }) +} + +func (m *Memory) RPush(key string, item []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + return m.lMutateLocked(key, version, expireAt, func(l *listx.List) { + l.RPush(item) + }) +} + +func (m *Memory) LPop(key string, version uint64, expireAt int64) (item []byte, popped, applied bool) { + m.mu.Lock() + defer m.mu.Unlock() + return m.lPopLocked(key, version, expireAt, true) +} + +func (m *Memory) RPop(key string, version uint64, expireAt int64) (item []byte, popped, applied bool) { + m.mu.Lock() + defer m.mu.Unlock() + return m.lPopLocked(key, version, expireAt, false) +} + +func (m *Memory) LLen(key string) int { + m.mu.Lock() + defer m.mu.Unlock() + l, ok := m.lPeekLocked(key) + if !ok { + return 0 + } + return l.Len() +} + +func (m *Memory) HasList(key string) bool { + m.mu.Lock() + defer m.mu.Unlock() + _, ok := m.lPeekLocked(key) + return ok +} + +func (m *Memory) LIndex(key string, idx int) ([]byte, bool) { + m.mu.Lock() + defer m.mu.Unlock() + l, ok := m.lPeekLocked(key) + if !ok { + return nil, false + } + return l.Index(idx) +} + +func (m *Memory) LRange(key string, start, stop int) [][]byte { + m.mu.Lock() + defer m.mu.Unlock() + l, ok := m.lPeekLocked(key) + if !ok { + return nil + } + return l.Range(start, stop) +} + +func (m *Memory) LInstall(key string, blob []byte, version uint64, expireAt int64) bool { + m.mu.Lock() + defer m.mu.Unlock() + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if it.entry.IsList() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + } else if version <= it.entry.Version { + return false + } + } + m.removeElement(el) + } + l, err := listx.Decode(blob) + if err != nil { + return false + } + ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagList} + return m.insertListLocked(key, ent, l, false) +} + +func (m *Memory) lPeekLocked(key string) (*listx.List, bool) { + el, ok := m.items[key] + if !ok { + return nil, false + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return nil, false + } + if it.entry.IsTombstone() || !it.entry.IsList() { + return nil, false + } + if it.lCache != nil { + return it.lCache, true + } + l, err := listx.Decode(it.entry.Value) + if err != nil { + return nil, false + } + it.lCache = l + return l, true +} + +func (m *Memory) lMutateLocked(key string, version uint64, expireAt int64, mut func(*listx.List)) bool { + var l *listx.List + if el, ok := m.items[key]; ok { + it := el.Value.(*lruItem) + if !it.entry.Expired(m.now()) { + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return false + } + stored := it.entry.Version + 1 + m.removeElement(el) + l = listx.New() + mut(l) + ent := Entry{Value: l.Encode(), Version: stored, ExpireAt: expireAt, Flags: FlagList} + return m.insertListLocked(key, ent, l, false) + } else if it.entry.IsList() { + l = it.lCache + if l == nil { + var err error + l, err = listx.Decode(it.entry.Value) + if err != nil { + return false + } + } + mut(l) + oldCost := it.cost + it.entry.Version = it.entry.Version + 1 + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.entry.Flags = FlagList + it.lCache = l + it.lDirty = true + it.cost = int64(len(key)) + 64 + l.ApproxWireBytes() + m.bytes += it.cost - oldCost + m.order.MoveToFront(el) + m.evictLocked() + _, still := m.items[key] + return still + } else { + return false + } + } + m.removeElement(el) + } + l = listx.New() + mut(l) + ent := Entry{Value: l.Encode(), Version: 1, ExpireAt: expireAt, Flags: FlagList} + return m.insertListLocked(key, ent, l, false) +} + +func (m *Memory) lPopLocked(key string, version uint64, expireAt int64, left bool) (item []byte, popped, applied bool) { + el, ok := m.items[key] + if !ok { + return nil, false, true + } + it := el.Value.(*lruItem) + if it.entry.Expired(m.now()) { + m.removeElement(el) + return nil, false, true + } + if it.entry.IsTombstone() { + if version <= it.entry.Version { + m.staleSkip.Add(1) + return nil, false, false + } + return nil, false, true + } + if !it.entry.IsList() { + return nil, false, false + } + l := it.lCache + if l == nil { + var err error + l, err = listx.Decode(it.entry.Value) + if err != nil { + return nil, false, false + } + } + if left { + item, popped = l.LPop() + } else { + item, popped = l.RPop() + } + if !popped { + return nil, false, true + } + oldCost := it.cost + it.entry.Version = it.entry.Version + 1 + if expireAt != 0 { + it.entry.ExpireAt = expireAt + } + it.entry.Flags = FlagList + it.lCache = l + it.lDirty = true + it.cost = int64(len(key)) + 64 + l.ApproxWireBytes() + m.bytes += it.cost - oldCost + m.order.MoveToFront(el) + m.evictLocked() + return item, true, true +} + +func (m *Memory) flushListValueLocked(it *lruItem) { + if it == nil || !it.lDirty || it.lCache == nil || !it.entry.IsList() { + return + } + blob := it.lCache.Encode() + oldCost := it.cost + it.entry.Value = blob + it.lDirty = false + it.cost = entryCost(it.key, it.entry) + m.bytes += it.cost - oldCost + if m.bytes < 0 { + m.bytes = 0 + } +} + +func (m *Memory) insertListLocked(key string, ent Entry, cache *listx.List, dirty bool) bool { + cost := entryCost(key, ent) + if dirty && cache != nil { + cost = int64(len(key)) + 64 + cache.ApproxWireBytes() + } + it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, lCache: cache, lDirty: dirty} + el := m.order.PushFront(it) + m.items[key] = el + m.bytes += cost + m.evictLocked() + _, ok := m.items[key] + return ok +} diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 4647487..ed4205b 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -1201,257 +1201,6 @@ func (m *Memory) insertZSetLocked(key string, ent Entry, cache *zset.ZSet, dirty return ok } -func (m *Memory) LPush(key string, item []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.lMutateLocked(key, version, expireAt, func(l *listx.List) { - l.LPush(item) - }) -} - -func (m *Memory) RPush(key string, item []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - return m.lMutateLocked(key, version, expireAt, func(l *listx.List) { - l.RPush(item) - }) -} - -func (m *Memory) LPop(key string, version uint64, expireAt int64) (item []byte, popped, applied bool) { - m.mu.Lock() - defer m.mu.Unlock() - return m.lPopLocked(key, version, expireAt, true) -} - -func (m *Memory) RPop(key string, version uint64, expireAt int64) (item []byte, popped, applied bool) { - m.mu.Lock() - defer m.mu.Unlock() - return m.lPopLocked(key, version, expireAt, false) -} - -func (m *Memory) LLen(key string) int { - m.mu.Lock() - defer m.mu.Unlock() - l, ok := m.lPeekLocked(key) - if !ok { - return 0 - } - return l.Len() -} - -func (m *Memory) HasList(key string) bool { - m.mu.Lock() - defer m.mu.Unlock() - _, ok := m.lPeekLocked(key) - return ok -} - -func (m *Memory) LIndex(key string, idx int) ([]byte, bool) { - m.mu.Lock() - defer m.mu.Unlock() - l, ok := m.lPeekLocked(key) - if !ok { - return nil, false - } - return l.Index(idx) -} - -func (m *Memory) LRange(key string, start, stop int) [][]byte { - m.mu.Lock() - defer m.mu.Unlock() - l, ok := m.lPeekLocked(key) - if !ok { - return nil - } - return l.Range(start, stop) -} - -func (m *Memory) LInstall(key string, blob []byte, version uint64, expireAt int64) bool { - m.mu.Lock() - defer m.mu.Unlock() - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if it.entry.IsList() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - } else if version <= it.entry.Version { - return false - } - } - m.removeElement(el) - } - l, err := listx.Decode(blob) - if err != nil { - return false - } - ent := Entry{Value: append([]byte(nil), blob...), Version: version, ExpireAt: expireAt, Flags: FlagList} - return m.insertListLocked(key, ent, l, false) -} - -func (m *Memory) lPeekLocked(key string) (*listx.List, bool) { - el, ok := m.items[key] - if !ok { - return nil, false - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return nil, false - } - if it.entry.IsTombstone() || !it.entry.IsList() { - return nil, false - } - if it.lCache != nil { - return it.lCache, true - } - l, err := listx.Decode(it.entry.Value) - if err != nil { - return nil, false - } - it.lCache = l - return l, true -} - -func (m *Memory) lMutateLocked(key string, version uint64, expireAt int64, mut func(*listx.List)) bool { - var l *listx.List - if el, ok := m.items[key]; ok { - it := el.Value.(*lruItem) - if !it.entry.Expired(m.now()) { - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return false - } - stored := it.entry.Version + 1 - m.removeElement(el) - l = listx.New() - mut(l) - ent := Entry{Value: l.Encode(), Version: stored, ExpireAt: expireAt, Flags: FlagList} - return m.insertListLocked(key, ent, l, false) - } else if it.entry.IsList() { - l = it.lCache - if l == nil { - var err error - l, err = listx.Decode(it.entry.Value) - if err != nil { - return false - } - } - mut(l) - oldCost := it.cost - it.entry.Version = it.entry.Version + 1 - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.entry.Flags = FlagList - it.lCache = l - it.lDirty = true - it.cost = int64(len(key)) + 64 + l.ApproxWireBytes() - m.bytes += it.cost - oldCost - m.order.MoveToFront(el) - m.evictLocked() - _, still := m.items[key] - return still - } else { - return false - } - } - m.removeElement(el) - } - l = listx.New() - mut(l) - ent := Entry{Value: l.Encode(), Version: 1, ExpireAt: expireAt, Flags: FlagList} - return m.insertListLocked(key, ent, l, false) -} - -func (m *Memory) lPopLocked(key string, version uint64, expireAt int64, left bool) (item []byte, popped, applied bool) { - el, ok := m.items[key] - if !ok { - return nil, false, true - } - it := el.Value.(*lruItem) - if it.entry.Expired(m.now()) { - m.removeElement(el) - return nil, false, true - } - if it.entry.IsTombstone() { - if version <= it.entry.Version { - m.staleSkip.Add(1) - return nil, false, false - } - return nil, false, true - } - if !it.entry.IsList() { - return nil, false, false - } - l := it.lCache - if l == nil { - var err error - l, err = listx.Decode(it.entry.Value) - if err != nil { - return nil, false, false - } - } - if left { - item, popped = l.LPop() - } else { - item, popped = l.RPop() - } - if !popped { - return nil, false, true - } - oldCost := it.cost - it.entry.Version = it.entry.Version + 1 - if expireAt != 0 { - it.entry.ExpireAt = expireAt - } - it.entry.Flags = FlagList - it.lCache = l - it.lDirty = true - it.cost = int64(len(key)) + 64 + l.ApproxWireBytes() - m.bytes += it.cost - oldCost - m.order.MoveToFront(el) - m.evictLocked() - return item, true, true -} - -func (m *Memory) flushListValueLocked(it *lruItem) { - if it == nil || !it.lDirty || it.lCache == nil || !it.entry.IsList() { - return - } - blob := it.lCache.Encode() - oldCost := it.cost - it.entry.Value = blob - it.lDirty = false - it.cost = entryCost(it.key, it.entry) - m.bytes += it.cost - oldCost - if m.bytes < 0 { - m.bytes = 0 - } -} - -func (m *Memory) insertListLocked(key string, ent Entry, cache *listx.List, dirty bool) bool { - cost := entryCost(key, ent) - if dirty && cache != nil { - cost = int64(len(key)) + 64 + cache.ApproxWireBytes() - } - it := &lruItem{key: key, entry: copyEntry(ent), cost: cost, lCache: cache, lDirty: dirty} - el := m.order.PushFront(it) - m.items[key] = el - m.bytes += cost - m.evictLocked() - _, ok := m.items[key] - return ok -} - func (m *Memory) HSet(key string, field, value []byte, version uint64, expireAt int64) bool { m.mu.Lock() defer m.mu.Unlock() From 6aaeefc283fd81fafdcc46d6f19c8391cc58c700 Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Wed, 2 Sep 2026 15:40:28 +0000 Subject: [PATCH 11/13] fix: drop unused store imports and share owner-address error text After stacking the mode extracts, memory.go no longer references bloom or jsonx. errOwnerNoAddress is declared once so the engine package builds with every extract merged. --- pkg/engine/bloom.go | 1 - pkg/engine/errors.go | 2 ++ pkg/engine/geo.go | 1 - pkg/engine/hash.go | 1 - pkg/engine/json.go | 1 - pkg/engine/list.go | 1 - pkg/engine/set.go | 1 - pkg/engine/zset.go | 1 - pkg/store/memory.go | 4 ---- 9 files changed, 2 insertions(+), 11 deletions(-) diff --git a/pkg/engine/bloom.go b/pkg/engine/bloom.go index 2401158..03eaa1d 100644 --- a/pkg/engine/bloom.go +++ b/pkg/engine/bloom.go @@ -13,7 +13,6 @@ const ( errBloomAddRequiresMode = "%w: BloomAdd requires ModeBloom" errBloomTestRequiresMode = "%w: BloomTest requires ModeBloom" errBloomAddRejected = "%w: bloom add rejected" - errOwnerNoAddress = "%w: owner %s has no address" ) // BloomAdd inserts item into the named filter (ModeBloom only). diff --git a/pkg/engine/errors.go b/pkg/engine/errors.go index 2ac10af..52cd50d 100644 --- a/pkg/engine/errors.go +++ b/pkg/engine/errors.go @@ -27,6 +27,8 @@ var ( ErrBatchTooLarge = errors.New("supercache: batch too large") ) +const errOwnerNoAddress = "%w: owner %s has no address" + // PeerError is a failure talking to one peer (cluster mode). type PeerError struct { PeerID string diff --git a/pkg/engine/geo.go b/pkg/engine/geo.go index 3d1a9c9..8ee6d7c 100644 --- a/pkg/engine/geo.go +++ b/pkg/engine/geo.go @@ -21,7 +21,6 @@ const ( errGeoRadiusRequiresMode = "%w: GeoRadius requires ModeGeo" errGeoAddRejected = "%w: geoadd rejected" errGeoRemRejected = "%w: georem rejected" - errOwnerNoAddress = "%w: owner %s has no address" ) // GeoMember is a point plus optional distance from a query. diff --git a/pkg/engine/hash.go b/pkg/engine/hash.go index 976dfd5..a15c014 100644 --- a/pkg/engine/hash.go +++ b/pkg/engine/hash.go @@ -19,7 +19,6 @@ const ( errHGetAllRequiresMode = "%w: HGetAll requires ModeHash" errHSetRejected = "%w: hset rejected" errHDelRejected = "%w: hdel rejected" - errOwnerNoAddress = "%w: owner %s has no address" ) // HashField is one field/value pair returned by HGetAll. diff --git a/pkg/engine/json.go b/pkg/engine/json.go index 9363b8b..a4f6dfd 100644 --- a/pkg/engine/json.go +++ b/pkg/engine/json.go @@ -18,7 +18,6 @@ const ( errJSONMutateRejected = "%w: json mutate rejected" errJSONSetRejected = "%w: json set rejected" errJSONDelRejected = "%w: json del rejected" - errOwnerNoAddress = "%w: owner %s has no address" ) // JsonSet upserts JSON value at path on a ModeJSON document. diff --git a/pkg/engine/list.go b/pkg/engine/list.go index 89902f1..a59a5ac 100644 --- a/pkg/engine/list.go +++ b/pkg/engine/list.go @@ -17,7 +17,6 @@ const ( errLRangeRequiresMode = "%w: LRange requires ModeList" errListPushRejected = "%w: list push rejected" errListPopRejected = "%w: list pop rejected" - errOwnerNoAddress = "%w: owner %s has no address" ) // LPush prepends item on a ModeList. diff --git a/pkg/engine/set.go b/pkg/engine/set.go index cb3c387..2203dd7 100644 --- a/pkg/engine/set.go +++ b/pkg/engine/set.go @@ -17,7 +17,6 @@ const ( errSetMembersRequiresMode = "%w: SetMembers requires ModeSet" errSetAddRejected = "%w: set add rejected" errSetRemoveRejected = "%w: set remove rejected" - errOwnerNoAddress = "%w: owner %s has no address" ) // SetAdd inserts item into the named set (ModeSet only). diff --git a/pkg/engine/zset.go b/pkg/engine/zset.go index 7441de5..755c393 100644 --- a/pkg/engine/zset.go +++ b/pkg/engine/zset.go @@ -21,7 +21,6 @@ const ( errZRangeByScoreRequiresMode = "%w: ZRangeByScore requires ModeZSet" errZAddRejected = "%w: zadd rejected" errZRemRejected = "%w: zrem rejected" - errOwnerNoAddress = "%w: owner %s has no address" ) // ZMember is a scored sorted-set element. diff --git a/pkg/store/memory.go b/pkg/store/memory.go index e996980..1c58fe1 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -6,10 +6,8 @@ import ( "sync/atomic" "time" - "github.com/Code0987/supercache/pkg/bloom" "github.com/Code0987/supercache/pkg/geo" "github.com/Code0987/supercache/pkg/hashx" - "github.com/Code0987/supercache/pkg/jsonx" "github.com/Code0987/supercache/pkg/listx" "github.com/Code0987/supercache/pkg/set" "github.com/Code0987/supercache/pkg/zset" @@ -394,7 +392,6 @@ func (m *Memory) RangeAll(fn func(key string, e Entry) bool) { } } - // PeekVersion returns the stored version without flushing dirty set blobs or cloning Value. func (m *Memory) PeekVersion(key string) (uint64, bool) { m.mu.Lock() @@ -411,7 +408,6 @@ func (m *Memory) PeekVersion(key string) (uint64, bool) { return it.entry.Version, true } - func (m *Memory) Close() { m.mu.Lock() defer m.mu.Unlock() From 76cda555272159337f7715bbe5bd49f4308c0316 Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Fri, 4 Sep 2026 06:44:59 +0000 Subject: [PATCH 12/13] refactor: move mode logic into per-feature folders Engine keeps thin public verbs and a shared modeHost. Owner write, snapshot/GetOrLoad, and mode tests live under pkg//eng and the feature package. listx becomes pkg/list. Same Engine/Store API. --- README.md | 2 +- docs/design/2026-09-02-feature-folders.md | 57 +++++++ docs/design/README.md | 1 + pkg/bitmapx/eng/local.go | 101 +++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../bitmap_test.go => bitmapx/engine_test.go} | 2 +- pkg/bloom/eng/host.go | 14 ++ pkg/bloom/eng/local.go | 49 ++++++ .../bloom_test.go => bloom/engine_test.go} | 2 +- pkg/cmsx/eng/local.go | 101 +++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../cms_test.go => cmsx/engine_test.go} | 2 +- pkg/counter/eng/host.go | 19 +++ pkg/counter/eng/local.go | 69 +++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../engine_test.go} | 2 +- pkg/engine/bitmap.go | 21 ++- pkg/engine/bitmap_apply.go | 64 -------- pkg/engine/bloom.go | 30 +++- pkg/engine/bloom_apply.go | 47 ------ pkg/engine/bloom_cluster.go | 17 --- pkg/engine/cms.go | 92 ++---------- pkg/engine/counter.go | 16 +- pkg/engine/counter_apply.go | 32 ---- pkg/engine/counter_cluster.go | 44 ------ pkg/engine/geo.go | 29 +++- pkg/engine/geo_apply.go | 77 ---------- pkg/engine/hash.go | 29 +++- pkg/engine/hash_apply.go | 77 ---------- pkg/engine/hll.go | 17 ++- pkg/engine/hll_apply.go | 59 -------- pkg/engine/json.go | 25 +++- pkg/engine/json_apply.go | 95 ------------ pkg/engine/list.go | 59 ++++---- pkg/engine/list_apply.go | 86 ----------- pkg/engine/list_cluster.go | 41 ----- pkg/engine/modehost.go | 98 ++++++++++++ pkg/engine/set.go | 27 +++- pkg/engine/set_apply.go | 76 ---------- pkg/engine/set_blob.go | 27 ---- pkg/engine/topk.go | 17 ++- pkg/engine/topk_apply.go | 61 -------- pkg/engine/zset.go | 29 +++- pkg/engine/zset_apply.go | 77 ---------- pkg/geo/eng/host.go | 21 +++ pkg/geo/eng/local.go | 90 +++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../geo_test.go => geo/engine_test.go} | 2 +- pkg/hashx/eng/local.go | 103 +++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../hash_test.go => hashx/engine_test.go} | 2 +- pkg/hllx/eng/local.go | 99 +++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../hll_test.go => hllx/engine_test.go} | 2 +- pkg/jsonx/eng/local.go | 140 ++++++++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../json_test.go => jsonx/engine_test.go} | 2 +- pkg/{listx/listx.go => list/codec.go} | 8 +- .../listx_test.go => list/codec_test.go} | 2 +- pkg/list/eng/cluster.go | 68 +++++++++ pkg/list/eng/host.go | 21 +++ pkg/list/eng/local.go | 94 ++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../list_test.go => list/engine_test.go} | 6 +- pkg/set/eng/cluster.go | 24 +++ pkg/set/eng/host.go | 21 +++ pkg/set/eng/local.go | 92 ++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../set_test.go => set/engine_test.go} | 2 +- pkg/store/list.go | 2 +- pkg/store/memory.go | 2 +- pkg/topkx/eng/local.go | 101 +++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../topk_test.go => topkx/engine_test.go} | 2 +- pkg/zset/eng/local.go | 103 +++++++++++++ .../engine_bench_test.go} | 2 +- .../engine_cluster_test.go} | 2 +- .../zset_test.go => zset/engine_test.go} | 2 +- 88 files changed, 1773 insertions(+), 1072 deletions(-) create mode 100644 docs/design/2026-09-02-feature-folders.md create mode 100644 pkg/bitmapx/eng/local.go rename pkg/{engine/bitmap_bench_test.go => bitmapx/engine_bench_test.go} (99%) rename pkg/{engine/bitmap_cluster_test.go => bitmapx/engine_cluster_test.go} (99%) rename pkg/{engine/bitmap_test.go => bitmapx/engine_test.go} (99%) create mode 100644 pkg/bloom/eng/host.go create mode 100644 pkg/bloom/eng/local.go rename pkg/{engine/bloom_test.go => bloom/engine_test.go} (99%) create mode 100644 pkg/cmsx/eng/local.go rename pkg/{engine/cms_bench_test.go => cmsx/engine_bench_test.go} (99%) rename pkg/{engine/cms_cluster_test.go => cmsx/engine_cluster_test.go} (99%) rename pkg/{engine/cms_test.go => cmsx/engine_test.go} (99%) create mode 100644 pkg/counter/eng/host.go create mode 100644 pkg/counter/eng/local.go rename pkg/{engine/counter_bench_test.go => counter/engine_bench_test.go} (98%) rename pkg/{engine/counter_cluster_test.go => counter/engine_cluster_test.go} (99%) rename pkg/{engine/counter_test.go => counter/engine_test.go} (99%) delete mode 100644 pkg/engine/bitmap_apply.go delete mode 100644 pkg/engine/bloom_apply.go delete mode 100644 pkg/engine/bloom_cluster.go delete mode 100644 pkg/engine/counter_apply.go delete mode 100644 pkg/engine/counter_cluster.go delete mode 100644 pkg/engine/geo_apply.go delete mode 100644 pkg/engine/hash_apply.go delete mode 100644 pkg/engine/hll_apply.go delete mode 100644 pkg/engine/json_apply.go delete mode 100644 pkg/engine/list_apply.go delete mode 100644 pkg/engine/list_cluster.go create mode 100644 pkg/engine/modehost.go delete mode 100644 pkg/engine/set_apply.go delete mode 100644 pkg/engine/set_blob.go delete mode 100644 pkg/engine/topk_apply.go delete mode 100644 pkg/engine/zset_apply.go create mode 100644 pkg/geo/eng/host.go create mode 100644 pkg/geo/eng/local.go rename pkg/{engine/geo_bench_test.go => geo/engine_bench_test.go} (99%) rename pkg/{engine/geo_cluster_test.go => geo/engine_cluster_test.go} (99%) rename pkg/{engine/geo_test.go => geo/engine_test.go} (99%) create mode 100644 pkg/hashx/eng/local.go rename pkg/{engine/hash_bench_test.go => hashx/engine_bench_test.go} (99%) rename pkg/{engine/hash_cluster_test.go => hashx/engine_cluster_test.go} (99%) rename pkg/{engine/hash_test.go => hashx/engine_test.go} (99%) create mode 100644 pkg/hllx/eng/local.go rename pkg/{engine/hll_bench_test.go => hllx/engine_bench_test.go} (99%) rename pkg/{engine/hll_cluster_test.go => hllx/engine_cluster_test.go} (99%) rename pkg/{engine/hll_test.go => hllx/engine_test.go} (99%) create mode 100644 pkg/jsonx/eng/local.go rename pkg/{engine/json_bench_test.go => jsonx/engine_bench_test.go} (99%) rename pkg/{engine/json_cluster_test.go => jsonx/engine_cluster_test.go} (99%) rename pkg/{engine/json_test.go => jsonx/engine_test.go} (99%) rename pkg/{listx/listx.go => list/codec.go} (93%) rename pkg/{listx/listx_test.go => list/codec_test.go} (98%) create mode 100644 pkg/list/eng/cluster.go create mode 100644 pkg/list/eng/host.go create mode 100644 pkg/list/eng/local.go rename pkg/{engine/list_bench_test.go => list/engine_bench_test.go} (99%) rename pkg/{engine/list_cluster_test.go => list/engine_cluster_test.go} (99%) rename pkg/{engine/list_test.go => list/engine_test.go} (98%) create mode 100644 pkg/set/eng/cluster.go create mode 100644 pkg/set/eng/host.go create mode 100644 pkg/set/eng/local.go rename pkg/{engine/set_bench_test.go => set/engine_bench_test.go} (99%) rename pkg/{engine/set_cluster_test.go => set/engine_cluster_test.go} (99%) rename pkg/{engine/set_test.go => set/engine_test.go} (99%) create mode 100644 pkg/topkx/eng/local.go rename pkg/{engine/topk_bench_test.go => topkx/engine_bench_test.go} (99%) rename pkg/{engine/topk_cluster_test.go => topkx/engine_cluster_test.go} (99%) rename pkg/{engine/topk_test.go => topkx/engine_test.go} (99%) create mode 100644 pkg/zset/eng/local.go rename pkg/{engine/zset_bench_test.go => zset/engine_bench_test.go} (99%) rename pkg/{engine/zset_cluster_test.go => zset/engine_cluster_test.go} (99%) rename pkg/{engine/zset_test.go => zset/engine_test.go} (99%) diff --git a/README.md b/README.md index fd522da..f074280 100644 --- a/README.md +++ b/README.md @@ -191,7 +191,7 @@ Apps: `client.DialTLS` with `pkg/tlsconfig.ClientFiles`. See [docs/OPERATIONS.md | `pkg/set` | Exact set encode/decode for `ModeSet` | | `pkg/zset` | Sorted-set encode/decode for `ModeZSet` | | `pkg/geo` | Point index encode/haversine for `ModeGeo` | -| `pkg/listx` | Ordered list encode for `ModeList` | +| `pkg/list` | ModeList codec + owner/cluster helpers | | `pkg/hashx` | Field-map encode for `ModeHash` | | `pkg/counter` | int64 encode/add for `ModeCounter` | | `pkg/jsonx` | Nested JSON path encode for `ModeJSON` | diff --git a/docs/design/2026-09-02-feature-folders.md b/docs/design/2026-09-02-feature-folders.md new file mode 100644 index 0000000..7e049e0 --- /dev/null +++ b/docs/design/2026-09-02-feature-folders.md @@ -0,0 +1,57 @@ +# Feature folders (same contract) + +**Status:** approved (chat: this sounds good) +**Branch (later):** `feat/feature-folders` +**Date:** 2026-09-02 + +## Problem + +`pkg/engine` is ~84 files. Most are per-mode (`list.go`, `list_apply.go`, `list_cluster.go`, tests, benches) sitting next to the cluster/Get/Put core. + +Go still forbids `pkg/engine/list/` as `package engine` (one directory = one package). Unexported `ksRuntime` cannot be seen from another folder. + +## Non-goals + +- No API / proto / flag / version / fan-out / dirty-cache change +- Callers still use `engine.Engine.LPush` (not `list.LPush`) +- Not moving `Memory` methods out of `pkg/store` in this design (`lruItem` / dirty cache stay unexported there) +- Not exporting `ksRuntime` + +## Contract + +Unchanged public surface: `pkg/engine`, `pkg/store`, `pkg/client`. Codec import paths change (`pkg/listx` → `pkg/list`, same for the other `*x` packages). + +## Approach + +Each mode gets a top-level feature package. Engine keeps **thin public verbs + ApplyPut switch**. Mode logic lives in the feature package and talks to engine through a small **Host** interface defined **in the feature package** (so `pkg/list` does not import `pkg/engine` — no cycle). `*Engine` implements those interfaces. + +``` +pkg/list/ # package list — codec only (store imports this; no store import here) + codec.go + codec_test.go + engine_test.go # moved engine list tests (package list_test) + eng/ # package eng — owner write + GetOrLoad (imports store; engine imports eng) + host.go + local.go + cluster.go + +pkg/engine/list.go # LPush/RPush/LPop/… only: validate, route, call list.* +pkg/engine/engine.go # ApplyPut: list.ApplyLPush(host, store, …) +pkg/store/list.go # unchanged (Memory receivers) +``` + +`pkg/engine` after a full roll-out: core (`engine.go`, `cluster.go`, `errors.go`, shared tests) plus **one file per mode**. Apply/cluster/mode tests leave. + +Same map for the other modes (`pkg/hash`, `pkg/hll`, `pkg/bitmap`, `pkg/cms`, `pkg/topk`; `pkg/bloom` / `set` / `zset` / `geo` / `counter` already exist — add engine helpers there). + +**First PR = List only** (this design). Other modes copy the pattern. + +Rejected: `pkg/engine/list/` same package (illegal). Codec-only rename (does not shrink engine). Export `ksRuntime` so feature packages import engine (cycle + contract leak). + +## Tests (already exist; keep them) + +Move `pkg/engine/list_*test.go` into `pkg/list` as `package list_test` (still `engine.New()`, same asserts). Existing `pkg/store` list tests stay. `go test ./...`. + +## Bench risk + +Hot path is Get/Put, not List. Local Get-hit / StoreGetHit allocs/op must stay 15 / 2. No extra alloc on the Host interface if it is a concrete `struct { e *Engine; ks *ksRuntime }` passed by value — verify List micros if they exist. diff --git a/docs/design/README.md b/docs/design/README.md index 5f86b2b..9c8c463 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -32,6 +32,7 @@ Do not implement from a draft. | [2026-09-02-refactor-set.md](./2026-09-02-refactor-set.md) | ModeSet file layout (no contract) | | [2026-09-02-refactor-zset.md](./2026-09-02-refactor-zset.md) | ModeZSet file layout (no contract) | | [2026-09-02-refactor-geo.md](./2026-09-02-refactor-geo.md) | ModeGeo file layout (no contract) | +| [2026-09-02-feature-folders.md](./2026-09-02-feature-folders.md) | Feature folders (List first) | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/pkg/bitmapx/eng/local.go b/pkg/bitmapx/eng/local.go new file mode 100644 index 0000000..40ca381 --- /dev/null +++ b/pkg/bitmapx/eng/local.go @@ -0,0 +1,101 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/bitmapx" + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + MaxValue() int + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} + +var ( + ErrRejected = errors.New("rejected") + ErrTooLarge = errors.New("too large") +) + +func ReplicateSnapshot(h Host, name string, ver uint64, expire int64) { + ent, ok := h.Store().Peek(name) + if !ok || !ent.IsBitmap() { + return + } + h.Replicate(name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagBitmap, + }) +} + +func SetLocal(h Host, name string, offset uint64, bit bool) error { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + applied, tooLarge := h.Store().BSet(name, offset, bit, gate, expire, h.MaxValue()) + if tooLarge { + return ErrTooLarge + } + if !applied { + return ErrRejected + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expire) + return nil +} + +func ApplySet(h Host, name string, inbox []byte, expireAt int64) bool { + offset, bit, err := bitmapx.DecodeSet(inbox) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok, tooLarge := h.Store().BSet(name, offset, bit, gate, expireAt, h.MaxValue()) + if !ok || tooLarge { + return false + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().BInstall(name, blob, version, expireAt) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsBitmap() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().BInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/bitmap_bench_test.go b/pkg/bitmapx/engine_bench_test.go similarity index 99% rename from pkg/engine/bitmap_bench_test.go rename to pkg/bitmapx/engine_bench_test.go index 4484fc0..d2b29d3 100644 --- a/pkg/engine/bitmap_bench_test.go +++ b/pkg/bitmapx/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package bitmapx_test import ( "context" diff --git a/pkg/engine/bitmap_cluster_test.go b/pkg/bitmapx/engine_cluster_test.go similarity index 99% rename from pkg/engine/bitmap_cluster_test.go rename to pkg/bitmapx/engine_cluster_test.go index 5e5ac5c..d01f61c 100644 --- a/pkg/engine/bitmap_cluster_test.go +++ b/pkg/bitmapx/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package bitmapx_test import ( "context" diff --git a/pkg/engine/bitmap_test.go b/pkg/bitmapx/engine_test.go similarity index 99% rename from pkg/engine/bitmap_test.go rename to pkg/bitmapx/engine_test.go index 39dfb64..c481a23 100644 --- a/pkg/engine/bitmap_test.go +++ b/pkg/bitmapx/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package bitmapx_test import ( "context" diff --git a/pkg/bloom/eng/host.go b/pkg/bloom/eng/host.go new file mode 100644 index 0000000..3d23599 --- /dev/null +++ b/pkg/bloom/eng/host.go @@ -0,0 +1,14 @@ +package eng + +import ( + "github.com/Code0987/supercache/pkg/store" +) + +// Host is engine-owned wiring for ModeBloom. +type Host interface { + Store() store.Store + ExpireAt() int64 + Replicate(name string, ent store.Entry) + BloomMK() (mBits, k int) + NextVersion(name string) uint64 +} diff --git a/pkg/bloom/eng/local.go b/pkg/bloom/eng/local.go new file mode 100644 index 0000000..68f64af --- /dev/null +++ b/pkg/bloom/eng/local.go @@ -0,0 +1,49 @@ +package eng + +import ( + "errors" + + "github.com/Code0987/supercache/pkg/store" +) + +var ErrRejected = errors.New("rejected") + +func AddLocal(h Host, name string, item []byte, fanout bool) error { + ver := uint64(1) + if cur, ok := h.Store().Peek(name); ok { + ver = h.NextVersion(name) + if cur.IsBloom() && !cur.IsTombstone() { + ver = cur.Version + } + } + expire := h.ExpireAt() + m, k := h.BloomMK() + if !h.Store().BloomAdd(name, item, m, k, ver, expire) { + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: append([]byte(nil), item...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagBloomAdd, + }) + } + return nil +} + +func ApplyAdd(h Host, name string, item []byte, version uint64, expireAt int64) bool { + m, k := h.BloomMK() + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().BloomAdd(name, item, m, k, version, expireAt) +} + +func ApplyMerge(h Host, name string, bits []byte, version uint64, expireAt int64) bool { + m, k := h.BloomMK() + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().BloomMerge(name, bits, m, k, version, expireAt) +} diff --git a/pkg/engine/bloom_test.go b/pkg/bloom/engine_test.go similarity index 99% rename from pkg/engine/bloom_test.go rename to pkg/bloom/engine_test.go index d810f14..9114a3d 100644 --- a/pkg/engine/bloom_test.go +++ b/pkg/bloom/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package bloom_test import ( "context" diff --git a/pkg/cmsx/eng/local.go b/pkg/cmsx/eng/local.go new file mode 100644 index 0000000..68e3170 --- /dev/null +++ b/pkg/cmsx/eng/local.go @@ -0,0 +1,101 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/cmsx" + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + MaxValue() int + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} + +var ( + ErrRejected = errors.New("rejected") + ErrTooLarge = errors.New("too large") +) + +func ReplicateSnapshot(h Host, name string, ver uint64, expire int64) { + ent, ok := h.Store().Peek(name) + if !ok || !ent.IsCMS() { + return + } + h.Replicate(name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagCMS, + }) +} + +func IncrLocal(h Host, name string, item []byte, n uint64) error { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + applied, tooLarge := h.Store().CMSIncr(name, item, n, gate, expire, h.MaxValue()) + if tooLarge { + return ErrTooLarge + } + if !applied { + return ErrRejected + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expire) + return nil +} + +func ApplyIncr(h Host, name string, inbox []byte, expireAt int64) bool { + n, item, err := cmsx.DecodeInbox(inbox) + if err != nil || len(item) == 0 { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok, tooLarge := h.Store().CMSIncr(name, item, n, gate, expireAt, h.MaxValue()) + if !ok || tooLarge { + return false + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().CMSInstall(name, blob, version, expireAt) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsCMS() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().CMSInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/cms_bench_test.go b/pkg/cmsx/engine_bench_test.go similarity index 99% rename from pkg/engine/cms_bench_test.go rename to pkg/cmsx/engine_bench_test.go index 8a4263a..6d86503 100644 --- a/pkg/engine/cms_bench_test.go +++ b/pkg/cmsx/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package cmsx_test import ( "context" diff --git a/pkg/engine/cms_cluster_test.go b/pkg/cmsx/engine_cluster_test.go similarity index 99% rename from pkg/engine/cms_cluster_test.go rename to pkg/cmsx/engine_cluster_test.go index 6aa5a3b..af37dcf 100644 --- a/pkg/engine/cms_cluster_test.go +++ b/pkg/cmsx/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package cmsx_test import ( "context" diff --git a/pkg/engine/cms_test.go b/pkg/cmsx/engine_test.go similarity index 99% rename from pkg/engine/cms_test.go rename to pkg/cmsx/engine_test.go index 7785caf..3f3fd0c 100644 --- a/pkg/engine/cms_test.go +++ b/pkg/cmsx/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package cmsx_test import ( "context" diff --git a/pkg/counter/eng/host.go b/pkg/counter/eng/host.go new file mode 100644 index 0000000..4ca1b8b --- /dev/null +++ b/pkg/counter/eng/host.go @@ -0,0 +1,19 @@ +package eng + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} diff --git a/pkg/counter/eng/local.go b/pkg/counter/eng/local.go new file mode 100644 index 0000000..9a1cbc4 --- /dev/null +++ b/pkg/counter/eng/local.go @@ -0,0 +1,69 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/counter" + "github.com/Code0987/supercache/pkg/store" +) + +var ( + ErrRejected = errors.New("rejected") + ErrOverflow = errors.New("overflow") +) + +func IncrLocal(h Host, name string, delta int64, fanout bool) (int64, error) { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + n, applied, overflow := h.Store().CIncr(name, delta, gate, expire) + if overflow { + return 0, ErrOverflow + } + if !applied { + return 0, ErrRejected + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + if fanout { + ReplicateSnapshot(h, name, n, ver, expire) + } + return n, nil +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().CInstall(name, blob, version, expireAt) +} + +func ReplicateSnapshot(h Host, name string, n int64, ver uint64, expire int64) { + h.Replicate(name, store.Entry{ + Value: counter.Encode(n), + Version: ver, + ExpireAt: expire, + Flags: store.FlagCounter, + }) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsCounter() { + return store.Entry{}, false, nil + } + if _, decErr := counter.Decode(ent.Value); decErr != nil { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().CInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/counter_bench_test.go b/pkg/counter/engine_bench_test.go similarity index 98% rename from pkg/engine/counter_bench_test.go rename to pkg/counter/engine_bench_test.go index 21ad447..6663848 100644 --- a/pkg/engine/counter_bench_test.go +++ b/pkg/counter/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package counter_test import ( "context" diff --git a/pkg/engine/counter_cluster_test.go b/pkg/counter/engine_cluster_test.go similarity index 99% rename from pkg/engine/counter_cluster_test.go rename to pkg/counter/engine_cluster_test.go index 361999d..c13abd9 100644 --- a/pkg/engine/counter_cluster_test.go +++ b/pkg/counter/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package counter_test import ( "context" diff --git a/pkg/engine/counter_test.go b/pkg/counter/engine_test.go similarity index 99% rename from pkg/engine/counter_test.go rename to pkg/counter/engine_test.go index e01f042..c74895d 100644 --- a/pkg/engine/counter_test.go +++ b/pkg/counter/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package counter_test import ( "context" diff --git a/pkg/engine/bitmap.go b/pkg/engine/bitmap.go index 344a4a3..1508ab6 100644 --- a/pkg/engine/bitmap.go +++ b/pkg/engine/bitmap.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/bitmapx" + bitmapeng "github.com/Code0987/supercache/pkg/bitmapx/eng" "github.com/Code0987/supercache/pkg/keyspace" ) @@ -39,7 +40,12 @@ func (e *Engine) BitSet(ctx context.Context, keyspaceName, name string, offset u return e.bMutViaOwner(ctx, ks, name, bitmapx.EncodeSet(offset, bit)) } } - return e.bSetLocal(ks, name, offset, bit) + if err := bitmapeng.SetLocal(e.modeHost(ks), name, offset, bit); err == bitmapeng.ErrTooLarge { + return ErrValueTooLarge + } else if err != nil { + return fmt.Errorf("%w: bitmap set rejected", ErrInvalidArgument) + } + return nil } // BitGet returns the bit at offset. Missing name → ok=false. @@ -64,7 +70,7 @@ func (e *Engine) BitGet(ctx context.Context, keyspaceName, name string, offset u bit, ok := ks.store.BGet(name, offset) return bit, ok, nil } - ent, found, err := e.bFetchOwner(ctx, ks, name) + ent, found, err := bitmapeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return false, false, err } @@ -93,7 +99,7 @@ func (e *Engine) BitCount(ctx context.Context, keyspaceName, name string, start, n, _ := ks.store.BCount(name, start, end) return n, nil } - ent, found, err := e.bFetchOwner(ctx, ks, name) + ent, found, err := bitmapeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return 0, err } @@ -122,7 +128,7 @@ func (e *Engine) BitPos(ctx context.Context, keyspaceName, name string, bit bool pos, found, _ := ks.store.BPos(name, bit, start, end) return pos, found, nil } - ent, found, err := e.bFetchOwner(ctx, ks, name) + ent, found, err := bitmapeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return 0, false, err } @@ -144,3 +150,10 @@ func (e *Engine) bitmapNeed(ks *ksRuntime, offset uint64) error { } return nil } + +func (e *Engine) applyBitmapSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { + return bitmapeng.ApplySet(e.modeHost(ks), name, inbox, expireAt) +} +func (e *Engine) applyBitmapInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return bitmapeng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/bitmap_apply.go b/pkg/engine/bitmap_apply.go deleted file mode 100644 index 90dc556..0000000 --- a/pkg/engine/bitmap_apply.go +++ /dev/null @@ -1,64 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/Code0987/supercache/pkg/bitmapx" -) - -// bSetLocal is the owner / single-node write path. -// The store assigns the stored version; we fan PeekVersion after the write. -func (e *Engine) bSetLocal(ks *ksRuntime, name string, offset uint64, bit bool) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - applied, tooLarge := ks.store.BSet(name, offset, bit, gate, expire, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf("%w: bitmap set rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.bReplicateSnapshot(ks, name, ver, expire) - return nil -} - -// applyBitmapSet is the owner-inbox ApplyPut of FlagBitmapSet. -// Non-owners must not reach here (ApplyPut returns applied=false first). -func (e *Engine) applyBitmapSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { - offset, bit, err := bitmapx.DecodeSet(inbox) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.BSet(name, offset, bit, gate, expireAt, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.bReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -// applyBitmapInstall is replica / handoff ApplyPut of FlagBitmap (LWW replace). -func (e *Engine) applyBitmapInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.BInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/bloom.go b/pkg/engine/bloom.go index 03eaa1d..a8e7f61 100644 --- a/pkg/engine/bloom.go +++ b/pkg/engine/bloom.go @@ -5,7 +5,9 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/bloom" + bloomeng "github.com/Code0987/supercache/pkg/bloom/eng" "github.com/Code0987/supercache/pkg/keyspace" + "github.com/Code0987/supercache/pkg/store" ) const ( @@ -50,7 +52,10 @@ func (e *Engine) BloomAdd(ctx context.Context, keyspaceName, name string, item [ return e.bloomAddViaOwner(ctx, ks, name, item) } } - return e.bloomAddLocal(ks, name, item, true) + if err := bloomeng.AddLocal(e.modeHost(ks), name, item, true); err != nil { + return fmt.Errorf(errBloomAddRejected, ErrInvalidArgument) + } + return nil } // BloomTest reports whether item may be in the named filter. @@ -101,8 +106,7 @@ func (e *Engine) BloomTest(ctx context.Context, keyspaceName, name string, item } func (e *Engine) hasBloomLocal(ks *ksRuntime, name string) bool { - ent, ok := ks.store.Peek(name) - return ok && ent.IsBloom() && !ent.IsTombstone() + return e.modeHost(ks).HasBloom(name) } // ApplyBloomMerge ORs a snapshot into the named filter (handoff / tests). @@ -115,5 +119,23 @@ func (e *Engine) ApplyBloomMerge(keyspaceName, name string, bits []byte, version return false, err } ks.observeVersion(name, version) - return e.applyBloomMerge(ks, name, bits, version, 0), nil + return bloomeng.ApplyMerge(e.modeHost(ks), name, bits, version, 0), nil +} + +func (e *Engine) bloomAddViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte) error { + c := e.clusterSnapshot() + owner, _ := c.Ring.Owner(name) + ent := store.Entry{Value: append([]byte(nil), item...), Flags: store.FlagBloomAdd, Version: 1} + pctx, cancel := e.peerCtx(ctx, ks) + defer cancel() + _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) + return err +} + +func (e *Engine) applyBloomAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { + return bloomeng.ApplyAdd(e.modeHost(ks), name, item, version, expireAt) +} + +func (e *Engine) applyBloomMerge(ks *ksRuntime, name string, bits []byte, version uint64, expireAt int64) bool { + return bloomeng.ApplyMerge(e.modeHost(ks), name, bits, version, expireAt) } diff --git a/pkg/engine/bloom_apply.go b/pkg/engine/bloom_apply.go deleted file mode 100644 index bdf5dae..0000000 --- a/pkg/engine/bloom_apply.go +++ /dev/null @@ -1,47 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/Code0987/supercache/pkg/store" -) - -func (e *Engine) bloomAddLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { - ver := uint64(1) - if cur, ok := ks.store.Peek(name); ok { - ver = ks.nextVersion(name, cur.Version) - if cur.IsBloom() && !cur.IsTombstone() { - ver = cur.Version - } - } - expire := e.expireAt(ks.cfg.TTL) - m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() - if !ks.store.BloomAdd(name, item, m, k, ver, expire) { - return fmt.Errorf(errBloomAddRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), item...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagBloomAdd, - }, false) - } - return nil -} - -func (e *Engine) applyBloomAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { - m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.BloomAdd(name, item, m, k, version, expireAt) -} - -func (e *Engine) applyBloomMerge(ks *ksRuntime, name string, bits []byte, version uint64, expireAt int64) bool { - m, k := ks.cfg.EffectiveBloomBits(), ks.cfg.EffectiveBloomHashes() - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.BloomMerge(name, bits, m, k, version, expireAt) -} diff --git a/pkg/engine/bloom_cluster.go b/pkg/engine/bloom_cluster.go deleted file mode 100644 index fd8351b..0000000 --- a/pkg/engine/bloom_cluster.go +++ /dev/null @@ -1,17 +0,0 @@ -package engine - -import ( - "context" - - "github.com/Code0987/supercache/pkg/store" -) - -func (e *Engine) bloomAddViaOwner(ctx context.Context, ks *ksRuntime, name string, item []byte) error { - c := e.clusterSnapshot() - owner, _ := c.Ring.Owner(name) - ent := store.Entry{Value: append([]byte(nil), item...), Flags: store.FlagBloomAdd, Version: 1} - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - _, err := c.Transport.ApplyPut(pctx, owner.Addr, ks.cfg.Name, name, ent, c.Ring.Generation()) - return err -} diff --git a/pkg/engine/cms.go b/pkg/engine/cms.go index 2433d57..343a435 100644 --- a/pkg/engine/cms.go +++ b/pkg/engine/cms.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/cmsx" + cmseng "github.com/Code0987/supercache/pkg/cmsx/eng" "github.com/Code0987/supercache/pkg/keyspace" "github.com/Code0987/supercache/pkg/store" ) @@ -45,7 +46,12 @@ func (e *Engine) CMSIncr(ctx context.Context, keyspaceName, name string, item [] return e.cmsMutViaOwner(ctx, ks, name, item, n) } } - return e.cmsIncrLocal(ks, name, item, n) + if err := cmseng.IncrLocal(e.modeHost(ks), name, item, n); err == cmseng.ErrTooLarge { + return ErrValueTooLarge + } else if err != nil { + return fmt.Errorf("%w: cms incr rejected", ErrInvalidArgument) + } + return nil } // CMSQuery is min-of-d for item. Missing name → ok=false. @@ -73,7 +79,7 @@ func (e *Engine) CMSQuery(ctx context.Context, keyspaceName, name string, item [ n, ok := ks.store.CMSQuery(name, item) return n, ok, nil } - ent, found, err := e.cmsFetchOwner(ctx, ks, name) + ent, found, err := cmseng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return 0, false, err } @@ -111,88 +117,10 @@ func (e *Engine) cmsMutViaOwner(ctx context.Context, ks *ksRuntime, name string, return nil } -func (e *Engine) cmsIncrLocal(ks *ksRuntime, name string, item []byte, n uint64) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - applied, tooLarge := ks.store.CMSIncr(name, item, n, gate, expire, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf("%w: cms incr rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.cmsReplicateSnapshot(ks, name, ver, expire) - return nil -} - -func (e *Engine) cmsReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { - ent, ok := ks.store.Peek(name) - if !ok || !ent.IsCMS() { - return - } - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: ent.Value, - Version: ver, - ExpireAt: expire, - Flags: store.FlagCMS, - }, false) -} - -func (e *Engine) cmsFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsCMS() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.CMSInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} - func (e *Engine) applyCMSIncr(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { - n, item, err := cmsx.DecodeInbox(inbox) - if err != nil || len(item) == 0 { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.CMSIncr(name, item, n, gate, expireAt, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.cmsReplicateSnapshot(ks, name, ver, expireAt) - return true + return cmseng.ApplyIncr(e.modeHost(ks), name, inbox, expireAt) } func (e *Engine) applyCMSInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.CMSInstall(name, blob, version, expireAt) + return cmseng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) } diff --git a/pkg/engine/counter.go b/pkg/engine/counter.go index f24bb04..16c5667 100644 --- a/pkg/engine/counter.go +++ b/pkg/engine/counter.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/counter" + countereng "github.com/Code0987/supercache/pkg/counter/eng" "github.com/Code0987/supercache/pkg/keyspace" ) @@ -38,7 +39,14 @@ func (e *Engine) Incr(ctx context.Context, keyspaceName, name string, delta int6 return c.Transport.CounterIncr(pctx, owner.Addr, ks.cfg.Name, name, delta) } } - return e.cIncrLocal(ks, name, delta, true) + n, err := countereng.IncrLocal(e.modeHost(ks), name, delta, true) + if err == countereng.ErrOverflow { + return 0, fmt.Errorf("%w: counter overflow", ErrInvalidArgument) + } + if err != nil { + return 0, fmt.Errorf("%w: incr rejected", ErrInvalidArgument) + } + return n, nil } // CounterGet returns the counter value. Missing → 0, ok=false. @@ -63,7 +71,7 @@ func (e *Engine) CounterGet(ctx context.Context, keyspaceName, name string) (int v, ok := ks.store.CGet(name) return v, ok, nil } - ent, found, err := e.cFetchOwner(ctx, ks, name) + ent, found, err := countereng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return 0, false, err } @@ -73,3 +81,7 @@ func (e *Engine) CounterGet(ctx context.Context, keyspaceName, name string) (int } return v, true, nil } + +func (e *Engine) applyCounterInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return countereng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/counter_apply.go b/pkg/engine/counter_apply.go deleted file mode 100644 index c892bc0..0000000 --- a/pkg/engine/counter_apply.go +++ /dev/null @@ -1,32 +0,0 @@ -package engine - -import "fmt" - -// cIncrLocal is the owner / single-node write path. -// The store assigns the stored version; we fan PeekVersion after the write. -func (e *Engine) cIncrLocal(ks *ksRuntime, name string, delta int64, fanout bool) (int64, error) { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - n, applied, overflow := ks.store.CIncr(name, delta, gate, expire) - if overflow { - return 0, fmt.Errorf("%w: counter overflow", ErrInvalidArgument) - } - if !applied { - return 0, fmt.Errorf("%w: incr rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - if fanout { - e.cReplicateSnapshot(ks, name, n, ver, expire) - } - return n, nil -} - -// applyCounterInstall is replica / handoff ApplyPut of FlagCounter (LWW replace). -func (e *Engine) applyCounterInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.CInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/counter_cluster.go b/pkg/engine/counter_cluster.go deleted file mode 100644 index 2748712..0000000 --- a/pkg/engine/counter_cluster.go +++ /dev/null @@ -1,44 +0,0 @@ -package engine - -import ( - "context" - - "github.com/Code0987/supercache/pkg/counter" - "github.com/Code0987/supercache/pkg/store" -) - -// cReplicateSnapshot fans the post-write FlagCounter blob to RF−1 replicas. -func (e *Engine) cReplicateSnapshot(ks *ksRuntime, name string, n int64, ver uint64, expire int64) { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: counter.Encode(n), - Version: ver, - ExpireAt: expire, - Flags: store.FlagCounter, - }, false) -} - -// cFetchOwner loads a missing local name from the owner (GetOrLoad). -// RPC / !Found / wrong type / bad blob → miss + nil error (do not return Unavailable). -func (e *Engine) cFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsCounter() { - return store.Entry{}, false, nil - } - if _, decErr := counter.Decode(res.Entry.Value); decErr != nil { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.CInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} diff --git a/pkg/engine/geo.go b/pkg/engine/geo.go index 8ee6d7c..dbcd050 100644 --- a/pkg/engine/geo.go +++ b/pkg/engine/geo.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/geo" + geoeng "github.com/Code0987/supercache/pkg/geo/eng" "github.com/Code0987/supercache/pkg/keyspace" "github.com/Code0987/supercache/pkg/store" ) @@ -67,7 +68,10 @@ func (e *Engine) GeoAdd(ctx context.Context, keyspaceName, name string, member [ return e.gMutViaOwner(ctx, ks, name, geo.EncodeAdd(member, lon, lat), store.FlagGeoAdd) } } - return e.gAddLocal(ks, name, member, lon, lat, true) + if err := geoeng.AddLocal(e.modeHost(ks), name, member, lon, lat, true); err != nil { + return fmt.Errorf(errGeoAddRejected, ErrInvalidArgument) + } + return nil } // GeoRem removes a member from a ModeGeo index. @@ -103,7 +107,10 @@ func (e *Engine) GeoRem(ctx context.Context, keyspaceName, name string, member [ return e.gMutViaOwner(ctx, ks, name, append([]byte(nil), member...), store.FlagGeoRem) } } - return e.gRemLocal(ks, name, member, true) + if err := geoeng.RemLocal(e.modeHost(ks), name, member, true); err != nil { + return fmt.Errorf(errGeoRemRejected, ErrInvalidArgument) + } + return nil } // GeoPos returns lon/lat if the member is present. @@ -130,7 +137,7 @@ func (e *Engine) GeoPos(ctx context.Context, keyspaceName, name string, member [ if e.hasGeoLocal(ks, name) { return 0, 0, false, nil } - ent, found, err := e.gFetchOwner(ctx, ks, name) + ent, found, err := geoeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return 0, 0, false, err } @@ -163,7 +170,7 @@ func (e *Engine) GeoCard(ctx context.Context, keyspaceName, name string) (int, e if e.hasGeoLocal(ks, name) { return ks.store.GeoCard(name), nil } - ent, ok, err := e.gFetchOwner(ctx, ks, name) + ent, ok, err := geoeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return 0, err } @@ -196,7 +203,7 @@ func (e *Engine) GeoDist(ctx context.Context, keyspaceName, name string, a, b [] d, ok := ks.store.GeoDist(name, a, b) return d, ok, nil } - ent, found, err := e.gFetchOwner(ctx, ks, name) + ent, found, err := geoeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return 0, false, err } @@ -235,7 +242,7 @@ func (e *Engine) GeoRadius(ctx context.Context, keyspaceName, name string, lon, if e.hasGeoLocal(ks, name) { return toEngineGeoMembers(ks.store.GeoRadius(name, lon, lat, radiusM, limit)), nil } - ent, ok, err := e.gFetchOwner(ctx, ks, name) + ent, ok, err := geoeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return nil, err } @@ -275,3 +282,13 @@ func toEngineGeoMembersFromPkg(in []geo.Member) []GeoMember { } return out } + +func (e *Engine) applyGeoAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { + return geoeng.ApplyAdd(e.modeHost(ks), name, value, version, expireAt) +} +func (e *Engine) applyGeoRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { + return geoeng.ApplyRem(e.modeHost(ks), name, member, version, expireAt) +} +func (e *Engine) applyGeoInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return geoeng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/geo_apply.go b/pkg/engine/geo_apply.go deleted file mode 100644 index ea8da21..0000000 --- a/pkg/engine/geo_apply.go +++ /dev/null @@ -1,77 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/Code0987/supercache/pkg/geo" - "github.com/Code0987/supercache/pkg/store" -) - -func (e *Engine) gAddLocal(ks *ksRuntime, name string, member []byte, lon, lat float64, fanout bool) error { - ver := e.gNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.GeoAdd(name, member, lon, lat, ver, expire) { - return fmt.Errorf(errGeoAddRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: geo.EncodeAdd(member, lon, lat), - Version: ver, - ExpireAt: expire, - Flags: store.FlagGeoAdd, - }, false) - } - return nil -} - -func (e *Engine) gRemLocal(ks *ksRuntime, name string, member []byte, fanout bool) error { - ver := e.gNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.GeoRem(name, member, ver, expire) { - if !e.hasGeoLocal(ks, name) { - return nil - } - return fmt.Errorf(errGeoRemRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), member...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagGeoRem, - }, false) - } - return nil -} - -func (e *Engine) gNextVersion(ks *ksRuntime, name string) uint64 { - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - -func (e *Engine) applyGeoAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { - member, lon, lat, err := geo.DecodeAdd(value) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.GeoAdd(name, member, lon, lat, version, expireAt) -} - -func (e *Engine) applyGeoRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.GeoRem(name, member, version, expireAt) -} - -func (e *Engine) applyGeoInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.GeoInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/hash.go b/pkg/engine/hash.go index a15c014..348837b 100644 --- a/pkg/engine/hash.go +++ b/pkg/engine/hash.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/hashx" + hasheng "github.com/Code0987/supercache/pkg/hashx/eng" "github.com/Code0987/supercache/pkg/keyspace" "github.com/Code0987/supercache/pkg/store" ) @@ -70,7 +71,10 @@ func (e *Engine) HSet(ctx context.Context, keyspaceName, name string, field, val return e.hMutViaOwner(ctx, ks, name, hashx.EncodeSet(field, value), store.FlagHashSet) } } - return e.hSetLocal(ks, name, field, value, true) + if err := hasheng.SetLocal(e.modeHost(ks), name, field, value, true); err != nil { + return fmt.Errorf(errHSetRejected, ErrInvalidArgument) + } + return nil } // HDel removes a field from a ModeHash map. @@ -103,7 +107,10 @@ func (e *Engine) HDel(ctx context.Context, keyspaceName, name string, field []by return e.hMutViaOwner(ctx, ks, name, append([]byte(nil), field...), store.FlagHashDel) } } - return e.hDelLocal(ks, name, field, true) + if err := hasheng.DelLocal(e.modeHost(ks), name, field, true); err != nil { + return fmt.Errorf(errHDelRejected, ErrInvalidArgument) + } + return nil } // HGet returns a copy of the field value. @@ -133,7 +140,7 @@ func (e *Engine) HGet(ctx context.Context, keyspaceName, name string, field []by if e.hasHashLocal(ks, name) { return nil, false, nil } - ent, found, err := e.hFetchOwner(ctx, ks, name) + ent, found, err := hasheng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return nil, false, err } @@ -169,7 +176,7 @@ func (e *Engine) HExists(ctx context.Context, keyspaceName, name string, field [ if e.hasHashLocal(ks, name) { return ks.store.HExists(name, field), nil } - ent, found, err := e.hFetchOwner(ctx, ks, name) + ent, found, err := hasheng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return false, err } @@ -201,7 +208,7 @@ func (e *Engine) HLen(ctx context.Context, keyspaceName, name string) (int, erro if e.hasHashLocal(ks, name) { return ks.store.HLen(name), nil } - ent, ok, err := e.hFetchOwner(ctx, ks, name) + ent, ok, err := hasheng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return 0, err } @@ -233,7 +240,7 @@ func (e *Engine) HGetAll(ctx context.Context, keyspaceName, name string) ([]Hash if e.hasHashLocal(ks, name) { return toEngineHashFields(ks.store.HGetAll(name)), nil } - ent, ok, err := e.hFetchOwner(ctx, ks, name) + ent, ok, err := hasheng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return nil, err } @@ -269,3 +276,13 @@ func toEngineHashFieldsFromPkg(in []hashx.Field) []HashField { } return out } + +func (e *Engine) applyHashSet(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { + return hasheng.ApplySet(e.modeHost(ks), name, value, version, expireAt) +} +func (e *Engine) applyHashDel(ks *ksRuntime, name string, field []byte, version uint64, expireAt int64) bool { + return hasheng.ApplyDel(e.modeHost(ks), name, field, version, expireAt) +} +func (e *Engine) applyHashInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return hasheng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/hash_apply.go b/pkg/engine/hash_apply.go deleted file mode 100644 index babab36..0000000 --- a/pkg/engine/hash_apply.go +++ /dev/null @@ -1,77 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/Code0987/supercache/pkg/hashx" - "github.com/Code0987/supercache/pkg/store" -) - -func (e *Engine) hSetLocal(ks *ksRuntime, name string, field, value []byte, fanout bool) error { - ver := e.hNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.HSet(name, field, value, ver, expire) { - return fmt.Errorf(errHSetRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: hashx.EncodeSet(field, value), - Version: ver, - ExpireAt: expire, - Flags: store.FlagHashSet, - }, false) - } - return nil -} - -func (e *Engine) hDelLocal(ks *ksRuntime, name string, field []byte, fanout bool) error { - ver := e.hNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.HDel(name, field, ver, expire) { - if !e.hasHashLocal(ks, name) { - return nil - } - return fmt.Errorf(errHDelRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), field...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagHashDel, - }, false) - } - return nil -} - -func (e *Engine) hNextVersion(ks *ksRuntime, name string) uint64 { - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - -func (e *Engine) applyHashSet(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { - field, val, err := hashx.DecodeSet(value) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HSet(name, field, val, version, expireAt) -} - -func (e *Engine) applyHashDel(ks *ksRuntime, name string, field []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HDel(name, field, version, expireAt) -} - -func (e *Engine) applyHashInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/hll.go b/pkg/engine/hll.go index 8ae63a2..7f8bf7d 100644 --- a/pkg/engine/hll.go +++ b/pkg/engine/hll.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/hllx" + hlleng "github.com/Code0987/supercache/pkg/hllx/eng" "github.com/Code0987/supercache/pkg/keyspace" ) @@ -45,7 +46,12 @@ func (e *Engine) HLLAdd(ctx context.Context, keyspaceName, name string, item []b return e.hllMutViaOwner(ctx, ks, name, item) } } - return e.hllAddLocal(ks, name, item) + if err := hlleng.AddLocal(e.modeHost(ks), name, item); err == hlleng.ErrTooLarge { + return ErrValueTooLarge + } else if err != nil { + return fmt.Errorf("%w: hll add rejected", ErrInvalidArgument) + } + return nil } // HLLCount is the sketch estimate. Missing name → ok=false. @@ -70,7 +76,7 @@ func (e *Engine) HLLCount(ctx context.Context, keyspaceName, name string) (uint6 n, ok := ks.store.HLLCount(name) return n, ok, nil } - ent, found, err := e.hllFetchOwner(ctx, ks, name) + ent, found, err := hlleng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return 0, false, err } @@ -87,3 +93,10 @@ func (e *Engine) hllNeed(ks *ksRuntime) error { } return nil } + +func (e *Engine) applyHLLAdd(ks *ksRuntime, name string, item []byte, expireAt int64) bool { + return hlleng.ApplyAdd(e.modeHost(ks), name, item, expireAt) +} +func (e *Engine) applyHLLInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return hlleng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/hll_apply.go b/pkg/engine/hll_apply.go deleted file mode 100644 index 04b38ac..0000000 --- a/pkg/engine/hll_apply.go +++ /dev/null @@ -1,59 +0,0 @@ -package engine - -import "fmt" - -// hllAddLocal is the owner / single-node write path. -// The store assigns the stored version; we fan PeekVersion after the write. -func (e *Engine) hllAddLocal(ks *ksRuntime, name string, item []byte) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - applied, tooLarge := ks.store.HLLAdd(name, item, gate, expire, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf("%w: hll add rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.hllReplicateSnapshot(ks, name, ver, expire) - return nil -} - -// applyHLLAdd is the owner-inbox ApplyPut of FlagHLLAdd (raw item). -// Non-owners must not reach here (ApplyPut returns applied=false first). -func (e *Engine) applyHLLAdd(ks *ksRuntime, name string, item []byte, expireAt int64) bool { - if len(item) == 0 { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.HLLAdd(name, item, gate, expireAt, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.hllReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -// applyHLLInstall is replica / handoff ApplyPut of FlagHLL (LWW replace). -func (e *Engine) applyHLLInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.HLLInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/json.go b/pkg/engine/json.go index a4f6dfd..b6f7f22 100644 --- a/pkg/engine/json.go +++ b/pkg/engine/json.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/jsonx" + jsoneng "github.com/Code0987/supercache/pkg/jsonx/eng" "github.com/Code0987/supercache/pkg/keyspace" "github.com/Code0987/supercache/pkg/store" ) @@ -59,7 +60,12 @@ func (e *Engine) JsonSet(ctx context.Context, keyspaceName, name, path string, v return e.jMutViaOwner(ctx, ks, name, jsonx.EncodeSet(path, value), store.FlagJSONSet) } } - return e.jSetLocal(ks, name, path, value) + if err := jsoneng.SetLocal(e.modeHost(ks), name, path, value); err == jsoneng.ErrTooLarge { + return ErrValueTooLarge + } else if err != nil { + return fmt.Errorf(errJSONSetRejected, ErrInvalidArgument) + } + return nil } // JsonGet returns a copy of the JSON at path. Missing doc or path → ok=false. @@ -90,7 +96,7 @@ func (e *Engine) JsonGet(ctx context.Context, keyspaceName, name, path string) ( v, ok := ks.store.JGet(name, path) return v, ok, nil } - ent, found, err := e.jFetchOwner(ctx, ks, name) + ent, found, err := jsoneng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return nil, false, err } @@ -130,7 +136,10 @@ func (e *Engine) JsonDel(ctx context.Context, keyspaceName, name, path string) e return e.jMutViaOwner(ctx, ks, name, []byte(path), store.FlagJSONDel) } } - return e.jDelLocal(ks, name, path) + if err := jsoneng.DelLocal(e.modeHost(ks), name, path); err != nil { + return fmt.Errorf(errJSONDelRejected, ErrInvalidArgument) + } + return nil } func jExtract(blob []byte, path string) ([]byte, bool, error) { @@ -152,3 +161,13 @@ func jExtract(blob []byte, path string) ([]byte, bool, error) { } return out, true, nil } + +func (e *Engine) applyJSONSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { + return jsoneng.ApplySet(e.modeHost(ks), name, inbox, expireAt) +} +func (e *Engine) applyJSONDel(ks *ksRuntime, name string, path []byte, expireAt int64) bool { + return jsoneng.ApplyDel(e.modeHost(ks), name, path, expireAt) +} +func (e *Engine) applyJSONInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return jsoneng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/json_apply.go b/pkg/engine/json_apply.go deleted file mode 100644 index 8c6ee40..0000000 --- a/pkg/engine/json_apply.go +++ /dev/null @@ -1,95 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/Code0987/supercache/pkg/jsonx" -) - -func (e *Engine) jSetLocal(ks *ksRuntime, name, path string, value []byte) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - applied, tooLarge := ks.store.JSet(name, path, value, gate, expire, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf(errJSONSetRejected, ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expire) - return nil -} - -func (e *Engine) jDelLocal(ks *ksRuntime, name, path string) error { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, mutated := ks.store.JDel(name, path, gate, expire) - if !ok { - return fmt.Errorf(errJSONDelRejected, ErrInvalidArgument) - } - if !mutated { - return nil - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expire) - return nil -} - -func (e *Engine) applyJSONSet(ks *ksRuntime, name string, inbox []byte, expireAt int64) bool { - path, raw, err := jsonx.DecodeSet(inbox) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.JSet(name, path, raw, gate, expireAt, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyJSONDel(ks *ksRuntime, name string, path []byte, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, mutated := ks.store.JDel(name, string(path), gate, expireAt) - if !ok { - return false - } - if !mutated { - return true - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.jReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyJSONInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.JInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/list.go b/pkg/engine/list.go index a59a5ac..b9b4601 100644 --- a/pkg/engine/list.go +++ b/pkg/engine/list.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/keyspace" - "github.com/Code0987/supercache/pkg/listx" + listeng "github.com/Code0987/supercache/pkg/list/eng" "github.com/Code0987/supercache/pkg/store" ) @@ -69,7 +69,10 @@ func (e *Engine) lPush(ctx context.Context, keyspaceName, name string, item []by return err } } - return e.lPushLocal(ks, name, item, left, true) + if err := listeng.Push(e.modeHost(ks), name, item, left, true); err != nil { + return fmt.Errorf(errListPushRejected, ErrInvalidArgument) + } + return nil } // LPop removes and returns the head. @@ -110,7 +113,11 @@ func (e *Engine) lPop(ctx context.Context, keyspaceName, name string, left bool) return c.Transport.ListPop(pctx, owner.Addr, ks.cfg.Name, name, left) } } - return e.lPopLocal(ks, name, left, true) + item, popped, err := listeng.Pop(e.modeHost(ks), name, left, true) + if err != nil { + return nil, false, fmt.Errorf(errListPopRejected, ErrInvalidArgument) + } + return item, popped, nil } // LLen returns list length. @@ -131,18 +138,16 @@ func (e *Engine) LLen(ctx context.Context, keyspaceName, name string) (int, erro if err := e.validateKeyLen(ks, name); err != nil { return 0, err } - if e.hasListLocal(ks, name) { + h := e.modeHost(ks) + if h.HasList(name) { return ks.store.LLen(name), nil } - ent, ok, err := e.lFetchOwner(ctx, ks, name) + ent, ok, err := listeng.FetchOwner(ctx, h, name) if err != nil || !ok { return 0, err } - l, err := listx.Decode(ent.Value) - if err != nil { - return 0, nil - } - return l.Len(), nil + n, _ := listeng.LenFromBlob(ent.Value) + return n, nil } // LIndex returns a copy of the element at idx. @@ -163,19 +168,16 @@ func (e *Engine) LIndex(ctx context.Context, keyspaceName, name string, idx int) if err := e.validateKeyLen(ks, name); err != nil { return nil, false, err } - if e.hasListLocal(ks, name) { + h := e.modeHost(ks) + if h.HasList(name) { it, ok := ks.store.LIndex(name, idx) return it, ok, nil } - ent, found, err := e.lFetchOwner(ctx, ks, name) + ent, found, err := listeng.FetchOwner(ctx, h, name) if err != nil || !found { return nil, false, err } - l, err := listx.Decode(ent.Value) - if err != nil { - return nil, false, nil - } - it, ok := l.Index(idx) + it, ok := listeng.IndexFromBlob(ent.Value, idx) return it, ok, nil } @@ -197,20 +199,25 @@ func (e *Engine) LRange(ctx context.Context, keyspaceName, name string, start, s if err := e.validateKeyLen(ks, name); err != nil { return nil, err } - if e.hasListLocal(ks, name) { + h := e.modeHost(ks) + if h.HasList(name) { return ks.store.LRange(name, start, stop), nil } - ent, ok, err := e.lFetchOwner(ctx, ks, name) + ent, ok, err := listeng.FetchOwner(ctx, h, name) if err != nil || !ok { return nil, err } - l, err := listx.Decode(ent.Value) - if err != nil { - return nil, nil - } - return l.Range(start, stop), nil + return listeng.RangeFromBlob(ent.Value, start, stop), nil +} + +func (e *Engine) applyListLPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { + return listeng.ApplyLPush(e.modeHost(ks), name, item, expireAt) +} + +func (e *Engine) applyListRPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { + return listeng.ApplyRPush(e.modeHost(ks), name, item, expireAt) } -func (e *Engine) hasListLocal(ks *ksRuntime, name string) bool { - return ks.store.HasList(name) +func (e *Engine) applyListInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return listeng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) } diff --git a/pkg/engine/list_apply.go b/pkg/engine/list_apply.go deleted file mode 100644 index 4415ca5..0000000 --- a/pkg/engine/list_apply.go +++ /dev/null @@ -1,86 +0,0 @@ -package engine - -import "fmt" - -func (e *Engine) lPushLocal(ks *ksRuntime, name string, item []byte, left, fanout bool) error { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok := false - if left { - ok = ks.store.LPush(name, item, gate, expire) - } else { - ok = ks.store.RPush(name, item, gate, expire) - } - if !ok { - return fmt.Errorf(errListPushRejected, ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - if fanout { - e.lReplicateSnapshot(ks, name, ver, expire) - } - return nil -} - -func (e *Engine) lPopLocal(ks *ksRuntime, name string, left, fanout bool) ([]byte, bool, error) { - expire := e.expireAt(ks.cfg.TTL) - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - var item []byte - var popped, applied bool - if left { - item, popped, applied = ks.store.LPop(name, gate, expire) - } else { - item, popped, applied = ks.store.RPop(name, gate, expire) - } - if !applied { - if !e.hasListLocal(ks, name) { - return nil, false, nil - } - return nil, false, fmt.Errorf(errListPopRejected, ErrInvalidArgument) - } - if popped && fanout { - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.lReplicateSnapshot(ks, name, ver, expire) - } - return item, popped, nil -} - -func (e *Engine) applyListLPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - if !ks.store.LPush(name, item, gate, expireAt) { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.lReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyListRPush(ks *ksRuntime, name string, item []byte, _ uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - if !ks.store.RPush(name, item, gate, expireAt) { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.lReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -func (e *Engine) applyListInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.LInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/list_cluster.go b/pkg/engine/list_cluster.go deleted file mode 100644 index edfed30..0000000 --- a/pkg/engine/list_cluster.go +++ /dev/null @@ -1,41 +0,0 @@ -package engine - -import ( - "context" - - "github.com/Code0987/supercache/pkg/store" -) - -func (e *Engine) lReplicateSnapshot(ks *ksRuntime, name string, ver uint64, expire int64) { - ent, ok := ks.store.Peek(name) - if !ok || !ent.IsList() { - return - } - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: ent.Value, - Version: ver, - ExpireAt: expire, - Flags: store.FlagList, - }, false) -} - -func (e *Engine) lFetchOwner(ctx context.Context, ks *ksRuntime, name string) (store.Entry, bool, error) { - c := e.clusterSnapshot() - if c == nil || c.Ring == nil || c.Transport == nil { - return store.Entry{}, false, nil - } - owner, ok := c.Ring.Owner(name) - if !ok || owner.ID == "" || owner.ID == c.SelfID || owner.Addr == "" { - return store.Entry{}, false, nil - } - pctx, cancel := e.peerCtx(ctx, ks) - defer cancel() - res, err := c.Transport.GetOrLoad(pctx, owner.Addr, ks.cfg.Name, name) - if err != nil || !res.Found || !res.Entry.IsList() { - return store.Entry{}, false, nil - } - if e.holdsReplica(c, ks, name) { - _ = ks.store.LInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) - } - return res.Entry, true, nil -} diff --git a/pkg/engine/modehost.go b/pkg/engine/modehost.go new file mode 100644 index 0000000..be41e2d --- /dev/null +++ b/pkg/engine/modehost.go @@ -0,0 +1,98 @@ +package engine + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +// modeHost is the engine-owned wiring every feature package's Host needs. +type modeHost struct { + e *Engine + ks *ksRuntime +} + +func (e *Engine) modeHost(ks *ksRuntime) modeHost { + return modeHost{e: e, ks: ks} +} + +func (h modeHost) Store() store.Store { return h.ks.store } +func (h modeHost) KeyspaceName() string { return h.ks.cfg.Name } +func (h modeHost) ExpireAt() int64 { return h.e.expireAt(h.ks.cfg.TTL) } +func (h modeHost) ObserveVersion(name string, ver uint64) { + h.ks.observeVersion(name, ver) +} +func (h modeHost) Replicate(name string, ent store.Entry) { + h.e.replicate(h.ks.cfg.Name, name, ent, false) +} +func (h modeHost) PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) { + return h.e.peerCtx(ctx, h.ks) +} +func (h modeHost) Owner(name string) (id, addr string, isSelf bool, ok bool) { + c := h.e.clusterSnapshot() + if c == nil || c.Ring == nil { + return "", "", false, false + } + owner, ok := c.Ring.Owner(name) + if !ok || owner.ID == "" { + return "", "", false, false + } + return owner.ID, owner.Addr, owner.ID == c.SelfID, true +} +func (h modeHost) GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) { + c := h.e.clusterSnapshot() + if c == nil || c.Transport == nil { + return store.Entry{}, false, nil + } + res, err := c.Transport.GetOrLoad(ctx, addr, keyspace, name) + if err != nil || !res.Found { + return store.Entry{}, false, err + } + return res.Entry, true, nil +} +func (h modeHost) HoldsReplica(name string) bool { + c := h.e.clusterSnapshot() + if c == nil { + return false + } + return h.e.holdsReplica(c, h.ks, name) +} +func (h modeHost) NextVersion(name string) uint64 { + if ver, ok := h.ks.store.PeekVersion(name); ok { + return h.ks.nextVersion(name, ver) + } + return h.ks.nextVersion(name, 0) +} +func (h modeHost) MaxValue() int { + max := h.e.maxValueSize + if h.ks.cfg.MaxValueSize > 0 { + max = h.ks.cfg.MaxValueSize + } + return max +} +func (h modeHost) BloomMK() (mBits, k int) { + return h.ks.cfg.EffectiveBloomBits(), h.ks.cfg.EffectiveBloomHashes() +} +func (h modeHost) TopKSize() int { return h.ks.cfg.EffectiveTopKSize() } +func (h modeHost) ItemMax() int { + max := h.e.maxKeyLen + if h.ks.cfg.MaxKeyLen > 0 { + max = h.ks.cfg.MaxKeyLen + } + return max +} +func (h modeHost) HasList(name string) bool { return h.ks.store.HasList(name) } +func (h modeHost) HasHash(name string) bool { return h.ks.store.HasHash(name) } +func (h modeHost) HasSet(name string) bool { return h.ks.store.HasSet(name) } +func (h modeHost) HasZSet(name string) bool { return h.ks.store.HasZSet(name) } +func (h modeHost) HasGeo(name string) bool { return h.ks.store.HasGeo(name) } +func (h modeHost) HasCounter(name string) bool { return h.ks.store.HasCounter(name) } +func (h modeHost) HasJSON(name string) bool { return h.ks.store.HasJSON(name) } +func (h modeHost) HasBitmap(name string) bool { return h.ks.store.HasBitmap(name) } +func (h modeHost) HasHLL(name string) bool { return h.ks.store.HasHLL(name) } +func (h modeHost) HasTopK(name string) bool { return h.ks.store.HasTopK(name) } +func (h modeHost) HasCMS(name string) bool { return h.ks.store.HasCMS(name) } +func (h modeHost) HasBloom(name string) bool { + ent, ok := h.ks.store.Peek(name) + return ok && ent.IsBloom() && !ent.IsTombstone() +} diff --git a/pkg/engine/set.go b/pkg/engine/set.go index 2203dd7..c12e076 100644 --- a/pkg/engine/set.go +++ b/pkg/engine/set.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/Code0987/supercache/pkg/keyspace" + "github.com/Code0987/supercache/pkg/set/eng" "github.com/Code0987/supercache/pkg/store" ) @@ -52,7 +53,10 @@ func (e *Engine) SetAdd(ctx context.Context, keyspaceName, name string, item []b return e.setMutViaOwner(ctx, ks, name, item, store.FlagSetAdd) } } - return e.setAddLocal(ks, name, item, true) + if err := eng.AddLocal(e.modeHost(ks), name, item, true); err != nil { + return fmt.Errorf(errSetAddRejected, ErrInvalidArgument) + } + return nil } // SetRemove removes item from the named set (ModeSet only). @@ -88,7 +92,10 @@ func (e *Engine) SetRemove(ctx context.Context, keyspaceName, name string, item return e.setMutViaOwner(ctx, ks, name, item, store.FlagSetRemove) } } - return e.setRemoveLocal(ks, name, item, true) + if err := eng.RemoveLocal(e.modeHost(ks), name, item, true); err != nil { + return fmt.Errorf(errSetRemoveRejected, ErrInvalidArgument) + } + return nil } // SetContains reports exact membership. @@ -136,7 +143,7 @@ func (e *Engine) SetContains(ctx context.Context, keyspaceName, name string, ite return ks.store.SetContains(name, item), nil } // Non-replica: decode once without storing. - return setContainsBlob(res.Entry.Value, item), nil + return eng.ContainsBlob(res.Entry.Value, item), nil } // SetCard returns the number of elements (0 if missing). @@ -178,7 +185,7 @@ func (e *Engine) SetCard(ctx context.Context, keyspaceName, name string) (int, e _ = ks.store.SetInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) return ks.store.SetCard(name), nil } - return setCardBlob(res.Entry.Value), nil + return eng.CardBlob(res.Entry.Value), nil } // SetMembers returns all members (defensive copies). Missing → empty. @@ -220,9 +227,19 @@ func (e *Engine) SetMembers(ctx context.Context, keyspaceName, name string) ([][ _ = ks.store.SetInstall(name, res.Entry.Value, res.Entry.Version, res.Entry.ExpireAt) return ks.store.SetMembers(name), nil } - return setMembersBlob(res.Entry.Value), nil + return eng.MembersBlob(res.Entry.Value), nil } func (e *Engine) hasSetLocal(ks *ksRuntime, name string) bool { return ks.store.HasSet(name) } + +func (e *Engine) applySetAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { + return eng.ApplyAdd(e.modeHost(ks), name, item, version, expireAt) +} +func (e *Engine) applySetRemove(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { + return eng.ApplyRemove(e.modeHost(ks), name, item, version, expireAt) +} +func (e *Engine) applySetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return eng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/set_apply.go b/pkg/engine/set_apply.go deleted file mode 100644 index 43994d1..0000000 --- a/pkg/engine/set_apply.go +++ /dev/null @@ -1,76 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/Code0987/supercache/pkg/store" -) - -func (e *Engine) setAddLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { - ver := e.setNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.SetAdd(name, item, ver, expire) { - return fmt.Errorf(errSetAddRejected, ErrInvalidArgument) - } - // Re-read version after mutate (store may keep same ver on in-place for bloom; we set ver). - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), item...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagSetAdd, - }, false) - } - return nil -} - -func (e *Engine) setRemoveLocal(ks *ksRuntime, name string, item []byte, fanout bool) error { - ver := e.setNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.SetRemove(name, item, ver, expire) { - // Missing set or tombstone reject — remove of missing is success no-op if no entry. - if !e.hasSetLocal(ks, name) { - // If still no set, treat as success (design: no-op if missing). - return nil - } - return fmt.Errorf(errSetRemoveRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), item...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagSetRemove, - }, false) - } - return nil -} - -func (e *Engine) setNextVersion(ks *ksRuntime, name string) uint64 { - // PeekVersion avoids flushing a dirty set blob on every Add/Remove. - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - -func (e *Engine) applySetAdd(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.SetAdd(name, item, version, expireAt) -} - -func (e *Engine) applySetRemove(ks *ksRuntime, name string, item []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.SetRemove(name, item, version, expireAt) -} - -func (e *Engine) applySetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.SetInstall(name, blob, version, expireAt) -} diff --git a/pkg/engine/set_blob.go b/pkg/engine/set_blob.go deleted file mode 100644 index 78b8de2..0000000 --- a/pkg/engine/set_blob.go +++ /dev/null @@ -1,27 +0,0 @@ -package engine - -import "github.com/Code0987/supercache/pkg/set" - -func setContainsBlob(blob, item []byte) bool { - s, err := set.DecodeSet(blob) - if err != nil { - return false - } - return s.Contains(item) -} - -func setCardBlob(blob []byte) int { - s, err := set.DecodeSet(blob) - if err != nil { - return 0 - } - return s.Len() -} - -func setMembersBlob(blob []byte) [][]byte { - s, err := set.DecodeSet(blob) - if err != nil { - return nil - } - return s.Members() -} diff --git a/pkg/engine/topk.go b/pkg/engine/topk.go index e997ac4..10ca7ad 100644 --- a/pkg/engine/topk.go +++ b/pkg/engine/topk.go @@ -7,6 +7,7 @@ import ( "github.com/Code0987/supercache/pkg/keyspace" "github.com/Code0987/supercache/pkg/store" "github.com/Code0987/supercache/pkg/topkx" + topkeng "github.com/Code0987/supercache/pkg/topkx/eng" ) // TopKEntry is one chart row (same shape as topkx.Entry; clients must not import topkx). @@ -52,7 +53,12 @@ func (e *Engine) TopKAdd(ctx context.Context, keyspaceName, name string, item [] return e.topkMutViaOwner(ctx, ks, name, item) } } - return e.topkAddLocal(ks, name, item) + if err := topkeng.AddLocal(e.modeHost(ks), name, item); err == topkeng.ErrTooLarge { + return ErrValueTooLarge + } else if err != nil { + return fmt.Errorf("%w: topk add rejected", ErrInvalidArgument) + } + return nil } // TopKList is the current chart. Missing name → ok=false, nil entries, nil error. @@ -78,7 +84,7 @@ func (e *Engine) TopKList(ctx context.Context, keyspaceName, name string) ([]Top rows, ok := ks.store.TopKList(name, k) return storeToEngineTopK(rows), ok, nil } - ent, found, err := e.topkFetchOwner(ctx, ks, name) + ent, found, err := topkeng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !found { return nil, false, err } @@ -133,3 +139,10 @@ func topkxToEngineTopK(rows []topkx.Entry) []TopKEntry { } return out } + +func (e *Engine) applyTopKAdd(ks *ksRuntime, name string, item []byte, expireAt int64) bool { + return topkeng.ApplyAdd(e.modeHost(ks), name, item, expireAt) +} +func (e *Engine) applyTopKInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return topkeng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/topk_apply.go b/pkg/engine/topk_apply.go deleted file mode 100644 index 2533d72..0000000 --- a/pkg/engine/topk_apply.go +++ /dev/null @@ -1,61 +0,0 @@ -package engine - -import "fmt" - -// topkAddLocal is the owner / single-node write path. -// The store assigns the stored version; we fan PeekVersion after the write. -func (e *Engine) topkAddLocal(ks *ksRuntime, name string, item []byte) error { - expire := e.expireAt(ks.cfg.TTL) - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - k := ks.cfg.EffectiveTopKSize() - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 // tombstone floor only - applied, tooLarge := ks.store.TopKAdd(name, item, gate, expire, k, max) - if tooLarge { - return ErrValueTooLarge - } - if !applied { - return fmt.Errorf("%w: topk add rejected", ErrInvalidArgument) - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.topkReplicateSnapshot(ks, name, ver, expire) - return nil -} - -// applyTopKAdd is the owner-inbox ApplyPut of FlagTopKAdd (raw item). -// Non-owners must not reach here (ApplyPut returns applied=false first). -func (e *Engine) applyTopKAdd(ks *ksRuntime, name string, item []byte, expireAt int64) bool { - if len(item) == 0 || len(item) > e.itemMax(ks) { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - max := e.maxValueSize - if ks.cfg.MaxValueSize > 0 { - max = ks.cfg.MaxValueSize - } - k := ks.cfg.EffectiveTopKSize() - cur, _ := ks.store.PeekVersion(name) - gate := cur + 1 - ok, tooLarge := ks.store.TopKAdd(name, item, gate, expireAt, k, max) - if !ok || tooLarge { - return false - } - ver, _ := ks.store.PeekVersion(name) - ks.observeVersion(name, ver) - e.topkReplicateSnapshot(ks, name, ver, expireAt) - return true -} - -// applyTopKInstall is replica snapshot handoff (incoming > local). -func (e *Engine) applyTopKInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.TopKInstall(name, blob, version, expireAt, ks.cfg.EffectiveTopKSize()) -} diff --git a/pkg/engine/zset.go b/pkg/engine/zset.go index 755c393..5c13880 100644 --- a/pkg/engine/zset.go +++ b/pkg/engine/zset.go @@ -8,6 +8,7 @@ import ( "github.com/Code0987/supercache/pkg/keyspace" "github.com/Code0987/supercache/pkg/store" "github.com/Code0987/supercache/pkg/zset" + zseteng "github.com/Code0987/supercache/pkg/zset/eng" ) const ( @@ -65,7 +66,10 @@ func (e *Engine) ZAdd(ctx context.Context, keyspaceName, name string, member []b return e.zMutViaOwner(ctx, ks, name, zset.EncodeAdd(member, score), store.FlagZSetAdd) } } - return e.zAddLocal(ks, name, member, score, true) + if err := zseteng.AddLocal(e.modeHost(ks), name, member, score, true); err != nil { + return fmt.Errorf(errZAddRejected, ErrInvalidArgument) + } + return nil } // ZRem removes a member from a ModeZSet. @@ -101,7 +105,10 @@ func (e *Engine) ZRem(ctx context.Context, keyspaceName, name string, member []b return e.zMutViaOwner(ctx, ks, name, append([]byte(nil), member...), store.FlagZSetRem) } } - return e.zRemLocal(ks, name, member, true) + if err := zseteng.RemLocal(e.modeHost(ks), name, member, true); err != nil { + return fmt.Errorf(errZRemRejected, ErrInvalidArgument) + } + return nil } // ZScore returns the score if the member is present. @@ -128,7 +135,7 @@ func (e *Engine) ZScore(ctx context.Context, keyspaceName, name string, member [ if e.hasZSetLocal(ks, name) { return 0, false, nil } - ent, ok, err := e.zFetchOwner(ctx, ks, name) + ent, ok, err := zseteng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return 0, false, err } @@ -161,7 +168,7 @@ func (e *Engine) ZCard(ctx context.Context, keyspaceName, name string) (int, err if e.hasZSetLocal(ks, name) { return ks.store.ZCard(name), nil } - ent, ok, err := e.zFetchOwner(ctx, ks, name) + ent, ok, err := zseteng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return 0, err } @@ -193,7 +200,7 @@ func (e *Engine) ZRange(ctx context.Context, keyspaceName, name string, start, s if e.hasZSetLocal(ks, name) { return toEngineZMembers(ks.store.ZRange(name, start, stop)), nil } - ent, ok, err := e.zFetchOwner(ctx, ks, name) + ent, ok, err := zseteng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return nil, err } @@ -225,7 +232,7 @@ func (e *Engine) ZRangeByScore(ctx context.Context, keyspaceName, name string, m if e.hasZSetLocal(ks, name) { return toEngineZMembers(ks.store.ZRangeByScore(name, min, max)), nil } - ent, ok, err := e.zFetchOwner(ctx, ks, name) + ent, ok, err := zseteng.FetchOwner(ctx, e.modeHost(ks), name) if err != nil || !ok { return nil, err } @@ -261,3 +268,13 @@ func toEngineZMembersFromPkg(in []zset.Member) []ZMember { } return out } + +func (e *Engine) applyZSetAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { + return zseteng.ApplyAdd(e.modeHost(ks), name, value, version, expireAt) +} +func (e *Engine) applyZSetRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { + return zseteng.ApplyRem(e.modeHost(ks), name, member, version, expireAt) +} +func (e *Engine) applyZSetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { + return zseteng.ApplyInstall(e.modeHost(ks), name, blob, version, expireAt) +} diff --git a/pkg/engine/zset_apply.go b/pkg/engine/zset_apply.go deleted file mode 100644 index 37d259c..0000000 --- a/pkg/engine/zset_apply.go +++ /dev/null @@ -1,77 +0,0 @@ -package engine - -import ( - "fmt" - - "github.com/Code0987/supercache/pkg/store" - "github.com/Code0987/supercache/pkg/zset" -) - -func (e *Engine) zAddLocal(ks *ksRuntime, name string, member []byte, score float64, fanout bool) error { - ver := e.zNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.ZAdd(name, member, score, ver, expire) { - return fmt.Errorf(errZAddRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: zset.EncodeAdd(member, score), - Version: ver, - ExpireAt: expire, - Flags: store.FlagZSetAdd, - }, false) - } - return nil -} - -func (e *Engine) zRemLocal(ks *ksRuntime, name string, member []byte, fanout bool) error { - ver := e.zNextVersion(ks, name) - expire := e.expireAt(ks.cfg.TTL) - if !ks.store.ZRem(name, member, ver, expire) { - if !e.hasZSetLocal(ks, name) { - return nil - } - return fmt.Errorf(errZRemRejected, ErrInvalidArgument) - } - if fanout { - e.replicate(ks.cfg.Name, name, store.Entry{ - Value: append([]byte(nil), member...), - Version: ver, - ExpireAt: expire, - Flags: store.FlagZSetRem, - }, false) - } - return nil -} - -func (e *Engine) zNextVersion(ks *ksRuntime, name string) uint64 { - if ver, ok := ks.store.PeekVersion(name); ok { - return ks.nextVersion(name, ver) - } - return ks.nextVersion(name, 0) -} - -func (e *Engine) applyZSetAdd(ks *ksRuntime, name string, value []byte, version uint64, expireAt int64) bool { - member, score, err := zset.DecodeAdd(value) - if err != nil { - return false - } - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.ZAdd(name, member, score, version, expireAt) -} - -func (e *Engine) applyZSetRem(ks *ksRuntime, name string, member []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.ZRem(name, member, version, expireAt) -} - -func (e *Engine) applyZSetInstall(ks *ksRuntime, name string, blob []byte, version uint64, expireAt int64) bool { - if expireAt == 0 { - expireAt = e.expireAt(ks.cfg.TTL) - } - return ks.store.ZInstall(name, blob, version, expireAt) -} diff --git a/pkg/geo/eng/host.go b/pkg/geo/eng/host.go new file mode 100644 index 0000000..87c7ab2 --- /dev/null +++ b/pkg/geo/eng/host.go @@ -0,0 +1,21 @@ +package eng + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + HasGeo(name string) bool + NextVersion(name string) uint64 + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} diff --git a/pkg/geo/eng/local.go b/pkg/geo/eng/local.go new file mode 100644 index 0000000..e8a97e1 --- /dev/null +++ b/pkg/geo/eng/local.go @@ -0,0 +1,90 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/geo" + "github.com/Code0987/supercache/pkg/store" +) + +var ErrRejected = errors.New("rejected") + +func AddLocal(h Host, name string, member []byte, lon, lat float64, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().GeoAdd(name, member, lon, lat, ver, expire) { + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: geo.EncodeAdd(member, lon, lat), + Version: ver, + ExpireAt: expire, + Flags: store.FlagGeoAdd, + }) + } + return nil +} + +func RemLocal(h Host, name string, member []byte, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().GeoRem(name, member, ver, expire) { + if !h.HasGeo(name) { + return nil + } + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: append([]byte(nil), member...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagGeoRem, + }) + } + return nil +} + +func ApplyAdd(h Host, name string, value []byte, version uint64, expireAt int64) bool { + member, lon, lat, err := geo.DecodeAdd(value) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().GeoAdd(name, member, lon, lat, version, expireAt) +} + +func ApplyRem(h Host, name string, member []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().GeoRem(name, member, version, expireAt) +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().GeoInstall(name, blob, version, expireAt) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsGeo() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().GeoInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/geo_bench_test.go b/pkg/geo/engine_bench_test.go similarity index 99% rename from pkg/engine/geo_bench_test.go rename to pkg/geo/engine_bench_test.go index 8f9b322..ad7b11d 100644 --- a/pkg/engine/geo_bench_test.go +++ b/pkg/geo/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package geo_test import ( "context" diff --git a/pkg/engine/geo_cluster_test.go b/pkg/geo/engine_cluster_test.go similarity index 99% rename from pkg/engine/geo_cluster_test.go rename to pkg/geo/engine_cluster_test.go index 91d31df..65dd7d9 100644 --- a/pkg/engine/geo_cluster_test.go +++ b/pkg/geo/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package geo_test import ( "context" diff --git a/pkg/engine/geo_test.go b/pkg/geo/engine_test.go similarity index 99% rename from pkg/engine/geo_test.go rename to pkg/geo/engine_test.go index 453fbbb..d47e185 100644 --- a/pkg/engine/geo_test.go +++ b/pkg/geo/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package geo_test import ( "context" diff --git a/pkg/hashx/eng/local.go b/pkg/hashx/eng/local.go new file mode 100644 index 0000000..77d1726 --- /dev/null +++ b/pkg/hashx/eng/local.go @@ -0,0 +1,103 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/hashx" + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + HasHash(name string) bool + NextVersion(name string) uint64 + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} + +var ErrRejected = errors.New("rejected") + +func SetLocal(h Host, name string, field, value []byte, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().HSet(name, field, value, ver, expire) { + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: hashx.EncodeSet(field, value), + Version: ver, + ExpireAt: expire, + Flags: store.FlagHashSet, + }) + } + return nil +} + +func DelLocal(h Host, name string, field []byte, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().HDel(name, field, ver, expire) { + if !h.HasHash(name) { + return nil + } + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: append([]byte(nil), field...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagHashDel, + }) + } + return nil +} + +func ApplySet(h Host, name string, value []byte, version uint64, expireAt int64) bool { + field, val, err := hashx.DecodeSet(value) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().HSet(name, field, val, version, expireAt) +} + +func ApplyDel(h Host, name string, field []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().HDel(name, field, version, expireAt) +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().HInstall(name, blob, version, expireAt) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsHash() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().HInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/hash_bench_test.go b/pkg/hashx/engine_bench_test.go similarity index 99% rename from pkg/engine/hash_bench_test.go rename to pkg/hashx/engine_bench_test.go index d164f82..050bb00 100644 --- a/pkg/engine/hash_bench_test.go +++ b/pkg/hashx/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package hashx_test import ( "context" diff --git a/pkg/engine/hash_cluster_test.go b/pkg/hashx/engine_cluster_test.go similarity index 99% rename from pkg/engine/hash_cluster_test.go rename to pkg/hashx/engine_cluster_test.go index c371864..0f63ff2 100644 --- a/pkg/engine/hash_cluster_test.go +++ b/pkg/hashx/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package hashx_test import ( "context" diff --git a/pkg/engine/hash_test.go b/pkg/hashx/engine_test.go similarity index 99% rename from pkg/engine/hash_test.go rename to pkg/hashx/engine_test.go index d7418a0..b620dab 100644 --- a/pkg/engine/hash_test.go +++ b/pkg/hashx/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package hashx_test import ( "context" diff --git a/pkg/hllx/eng/local.go b/pkg/hllx/eng/local.go new file mode 100644 index 0000000..af7de42 --- /dev/null +++ b/pkg/hllx/eng/local.go @@ -0,0 +1,99 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + MaxValue() int + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} + +var ( + ErrRejected = errors.New("rejected") + ErrTooLarge = errors.New("too large") +) + +func ReplicateSnapshot(h Host, name string, ver uint64, expire int64) { + ent, ok := h.Store().Peek(name) + if !ok || !ent.IsHLL() { + return + } + h.Replicate(name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagHLL, + }) +} + +func AddLocal(h Host, name string, item []byte) error { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + applied, tooLarge := h.Store().HLLAdd(name, item, gate, expire, h.MaxValue()) + if tooLarge { + return ErrTooLarge + } + if !applied { + return ErrRejected + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expire) + return nil +} + +func ApplyAdd(h Host, name string, item []byte, expireAt int64) bool { + if len(item) == 0 { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok, tooLarge := h.Store().HLLAdd(name, item, gate, expireAt, h.MaxValue()) + if !ok || tooLarge { + return false + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().HLLInstall(name, blob, version, expireAt) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsHLL() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().HLLInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/hll_bench_test.go b/pkg/hllx/engine_bench_test.go similarity index 99% rename from pkg/engine/hll_bench_test.go rename to pkg/hllx/engine_bench_test.go index 7169183..513d957 100644 --- a/pkg/engine/hll_bench_test.go +++ b/pkg/hllx/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package hllx_test import ( "context" diff --git a/pkg/engine/hll_cluster_test.go b/pkg/hllx/engine_cluster_test.go similarity index 99% rename from pkg/engine/hll_cluster_test.go rename to pkg/hllx/engine_cluster_test.go index fe663a5..d0e7366 100644 --- a/pkg/engine/hll_cluster_test.go +++ b/pkg/hllx/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package hllx_test import ( "context" diff --git a/pkg/engine/hll_test.go b/pkg/hllx/engine_test.go similarity index 99% rename from pkg/engine/hll_test.go rename to pkg/hllx/engine_test.go index 87b1b22..d5d4b02 100644 --- a/pkg/engine/hll_test.go +++ b/pkg/hllx/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package hllx_test import ( "context" diff --git a/pkg/jsonx/eng/local.go b/pkg/jsonx/eng/local.go new file mode 100644 index 0000000..09db5e2 --- /dev/null +++ b/pkg/jsonx/eng/local.go @@ -0,0 +1,140 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/jsonx" + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + MaxValue() int + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} + +var ( + ErrRejected = errors.New("rejected") + ErrTooLarge = errors.New("too large") +) + +func ReplicateSnapshot(h Host, name string, ver uint64, expire int64) { + ent, ok := h.Store().Peek(name) + if !ok || !ent.IsJSON() { + return + } + h.Replicate(name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagJSON, + }) +} + +func SetLocal(h Host, name, path string, value []byte) error { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + applied, tooLarge := h.Store().JSet(name, path, value, gate, expire, h.MaxValue()) + if tooLarge { + return ErrTooLarge + } + if !applied { + return ErrRejected + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expire) + return nil +} + +func DelLocal(h Host, name, path string) error { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok, mutated := h.Store().JDel(name, path, gate, expire) + if !ok { + return ErrRejected + } + if !mutated { + return nil + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expire) + return nil +} + +func ApplySet(h Host, name string, inbox []byte, expireAt int64) bool { + path, raw, err := jsonx.DecodeSet(inbox) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok, tooLarge := h.Store().JSet(name, path, raw, gate, expireAt, h.MaxValue()) + if !ok || tooLarge { + return false + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +func ApplyDel(h Host, name string, path []byte, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok, mutated := h.Store().JDel(name, string(path), gate, expireAt) + if !ok { + return false + } + if !mutated { + return true + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().JInstall(name, blob, version, expireAt) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsJSON() { + return store.Entry{}, false, nil + } + if _, decErr := jsonx.Decode(ent.Value); decErr != nil { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().JInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/json_bench_test.go b/pkg/jsonx/engine_bench_test.go similarity index 99% rename from pkg/engine/json_bench_test.go rename to pkg/jsonx/engine_bench_test.go index 04d5215..59c1037 100644 --- a/pkg/engine/json_bench_test.go +++ b/pkg/jsonx/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package jsonx_test import ( "context" diff --git a/pkg/engine/json_cluster_test.go b/pkg/jsonx/engine_cluster_test.go similarity index 99% rename from pkg/engine/json_cluster_test.go rename to pkg/jsonx/engine_cluster_test.go index 0d138b7..eddc8e0 100644 --- a/pkg/engine/json_cluster_test.go +++ b/pkg/jsonx/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package jsonx_test import ( "context" diff --git a/pkg/engine/json_test.go b/pkg/jsonx/engine_test.go similarity index 99% rename from pkg/engine/json_test.go rename to pkg/jsonx/engine_test.go index fc6a73d..3bb2e32 100644 --- a/pkg/engine/json_test.go +++ b/pkg/jsonx/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package jsonx_test import ( "context" diff --git a/pkg/listx/listx.go b/pkg/list/codec.go similarity index 93% rename from pkg/listx/listx.go rename to pkg/list/codec.go index 7c37a47..5b28295 100644 --- a/pkg/listx/listx.go +++ b/pkg/list/codec.go @@ -1,5 +1,5 @@ -// Package listx encodes ordered lists for ModeList. -package listx +// Package list is the ModeList feature package: codec, owner write, and cluster fetch. +package list import ( "bytes" @@ -153,11 +153,11 @@ func Decode(b []byte) (*List, error) { for len(b) > 0 { n, w := binary.Uvarint(b) if w <= 0 { - return nil, fmt.Errorf("listx: bad uvarint") + return nil, fmt.Errorf("list: bad uvarint") } b = b[w:] if uint64(len(b)) < n { - return nil, fmt.Errorf("listx: truncated item") + return nil, fmt.Errorf("list: truncated item") } l.items = append(l.items, append([]byte(nil), b[:n]...)) b = b[n:] diff --git a/pkg/listx/listx_test.go b/pkg/list/codec_test.go similarity index 98% rename from pkg/listx/listx_test.go rename to pkg/list/codec_test.go index c147374..eedd6c3 100644 --- a/pkg/listx/listx_test.go +++ b/pkg/list/codec_test.go @@ -1,4 +1,4 @@ -package listx +package list import ( "bytes" diff --git a/pkg/list/eng/cluster.go b/pkg/list/eng/cluster.go new file mode 100644 index 0000000..e730e83 --- /dev/null +++ b/pkg/list/eng/cluster.go @@ -0,0 +1,68 @@ +package eng + +import ( + "context" + + "github.com/Code0987/supercache/pkg/list" + "github.com/Code0987/supercache/pkg/store" +) + +// ReplicateSnapshot fans the post-write FlagList blob to RF−1 replicas. +func ReplicateSnapshot(h Host, name string, ver uint64, expire int64) { + ent, ok := h.Store().Peek(name) + if !ok || !ent.IsList() { + return + } + h.Replicate(name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagList, + }) +} + +// FetchOwner loads a missing local name from the owner (GetOrLoad). +// RPC / !Found / wrong type → miss + nil error. +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsList() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().LInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} + +// LenFromBlob decodes a snapshot. Bad blob → 0, false. +func LenFromBlob(blob []byte) (int, bool) { + l, err := list.Decode(blob) + if err != nil { + return 0, false + } + return l.Len(), true +} + +// IndexFromBlob decodes one element. Bad blob → miss. +func IndexFromBlob(blob []byte, idx int) ([]byte, bool) { + l, err := list.Decode(blob) + if err != nil { + return nil, false + } + return l.Index(idx) +} + +// RangeFromBlob decodes a window. Bad blob → nil. +func RangeFromBlob(blob []byte, start, stop int) [][]byte { + l, err := list.Decode(blob) + if err != nil { + return nil + } + return l.Range(start, stop) +} diff --git a/pkg/list/eng/host.go b/pkg/list/eng/host.go new file mode 100644 index 0000000..ca7f5ab --- /dev/null +++ b/pkg/list/eng/host.go @@ -0,0 +1,21 @@ +package eng + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +// Host is engine-owned wiring. Implemented by engine; this package does not import engine. +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + HasList(name string) bool + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} diff --git a/pkg/list/eng/local.go b/pkg/list/eng/local.go new file mode 100644 index 0000000..7405d36 --- /dev/null +++ b/pkg/list/eng/local.go @@ -0,0 +1,94 @@ +package eng + +import "errors" + +// ErrRejected means the store refused the mutate (wrong type / tombstone gate). +var ErrRejected = errors.New("rejected") + +// Push writes one item on the owner / single-node path. +func Push(h Host, name string, item []byte, left, fanout bool) error { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok := false + if left { + ok = h.Store().LPush(name, item, gate, expire) + } else { + ok = h.Store().RPush(name, item, gate, expire) + } + if !ok { + return ErrRejected + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + if fanout { + ReplicateSnapshot(h, name, ver, expire) + } + return nil +} + +// Pop removes one end on the owner / single-node path. +func Pop(h Host, name string, left, fanout bool) ([]byte, bool, error) { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + var item []byte + var popped, applied bool + if left { + item, popped, applied = h.Store().LPop(name, gate, expire) + } else { + item, popped, applied = h.Store().RPop(name, gate, expire) + } + if !applied { + if !h.HasList(name) { + return nil, false, nil + } + return nil, false, ErrRejected + } + if popped && fanout { + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expire) + } + return item, popped, nil +} + +// ApplyLPush is inbox FlagListLPush on the owner. +func ApplyLPush(h Host, name string, item []byte, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + if !h.Store().LPush(name, item, gate, expireAt) { + return false + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +// ApplyRPush is inbox FlagListRPush on the owner. +func ApplyRPush(h Host, name string, item []byte, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + if !h.Store().RPush(name, item, gate, expireAt) { + return false + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +// ApplyInstall is replica / handoff FlagList snapshot. +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().LInstall(name, blob, version, expireAt) +} diff --git a/pkg/engine/list_bench_test.go b/pkg/list/engine_bench_test.go similarity index 99% rename from pkg/engine/list_bench_test.go rename to pkg/list/engine_bench_test.go index a190877..fd6002f 100644 --- a/pkg/engine/list_bench_test.go +++ b/pkg/list/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package list_test import ( "context" diff --git a/pkg/engine/list_cluster_test.go b/pkg/list/engine_cluster_test.go similarity index 99% rename from pkg/engine/list_cluster_test.go rename to pkg/list/engine_cluster_test.go index 6badfd4..bf5ad03 100644 --- a/pkg/engine/list_cluster_test.go +++ b/pkg/list/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package list_test import ( "context" diff --git a/pkg/engine/list_test.go b/pkg/list/engine_test.go similarity index 98% rename from pkg/engine/list_test.go rename to pkg/list/engine_test.go index afcac9a..dbba2f8 100644 --- a/pkg/engine/list_test.go +++ b/pkg/list/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package list_test import ( "context" @@ -9,7 +9,7 @@ import ( "github.com/Code0987/supercache/pkg/engine" "github.com/Code0987/supercache/pkg/keyspace" - "github.com/Code0987/supercache/pkg/listx" + "github.com/Code0987/supercache/pkg/list" "github.com/Code0987/supercache/pkg/store" ) @@ -126,7 +126,7 @@ func TestListDeleteTombstone(t *testing.T) { if n != 0 { t.Fatal(n) } - l := listx.New() + l := list.New() l.RPush([]byte("a")) applied, err := e.ApplyPut("ls", "q", store.Entry{Value: l.Encode(), Version: 1, Flags: store.FlagList}) if err != nil { diff --git a/pkg/set/eng/cluster.go b/pkg/set/eng/cluster.go new file mode 100644 index 0000000..ef911b9 --- /dev/null +++ b/pkg/set/eng/cluster.go @@ -0,0 +1,24 @@ +package eng + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsSet() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().SetInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/set/eng/host.go b/pkg/set/eng/host.go new file mode 100644 index 0000000..06415ee --- /dev/null +++ b/pkg/set/eng/host.go @@ -0,0 +1,21 @@ +package eng + +import ( + "context" + + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + ExpireAt() int64 + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + HasSet(name string) bool + NextVersion(name string) uint64 + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool + KeyspaceName() string +} diff --git a/pkg/set/eng/local.go b/pkg/set/eng/local.go new file mode 100644 index 0000000..664fffe --- /dev/null +++ b/pkg/set/eng/local.go @@ -0,0 +1,92 @@ +package eng + +import ( + "errors" + + "github.com/Code0987/supercache/pkg/set" + "github.com/Code0987/supercache/pkg/store" +) + +var ErrRejected = errors.New("rejected") + +func AddLocal(h Host, name string, item []byte, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().SetAdd(name, item, ver, expire) { + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: append([]byte(nil), item...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagSetAdd, + }) + } + return nil +} + +func RemoveLocal(h Host, name string, item []byte, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().SetRemove(name, item, ver, expire) { + if !h.HasSet(name) { + return nil + } + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: append([]byte(nil), item...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagSetRemove, + }) + } + return nil +} + +func ApplyAdd(h Host, name string, item []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().SetAdd(name, item, version, expireAt) +} + +func ApplyRemove(h Host, name string, item []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().SetRemove(name, item, version, expireAt) +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().SetInstall(name, blob, version, expireAt) +} + +func ContainsBlob(blob, item []byte) bool { + s, err := set.DecodeSet(blob) + if err != nil { + return false + } + return s.Contains(item) +} + +func CardBlob(blob []byte) int { + s, err := set.DecodeSet(blob) + if err != nil { + return 0 + } + return s.Len() +} + +func MembersBlob(blob []byte) [][]byte { + s, err := set.DecodeSet(blob) + if err != nil { + return nil + } + return s.Members() +} diff --git a/pkg/engine/set_bench_test.go b/pkg/set/engine_bench_test.go similarity index 99% rename from pkg/engine/set_bench_test.go rename to pkg/set/engine_bench_test.go index 3eaa4e1..92ac360 100644 --- a/pkg/engine/set_bench_test.go +++ b/pkg/set/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package set_test import ( "context" diff --git a/pkg/engine/set_cluster_test.go b/pkg/set/engine_cluster_test.go similarity index 99% rename from pkg/engine/set_cluster_test.go rename to pkg/set/engine_cluster_test.go index aca4693..1fc3da8 100644 --- a/pkg/engine/set_cluster_test.go +++ b/pkg/set/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package set_test import ( "context" diff --git a/pkg/engine/set_test.go b/pkg/set/engine_test.go similarity index 99% rename from pkg/engine/set_test.go rename to pkg/set/engine_test.go index 5a69c5f..9dbf644 100644 --- a/pkg/engine/set_test.go +++ b/pkg/set/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package set_test import ( "context" diff --git a/pkg/store/list.go b/pkg/store/list.go index 93d4b78..4635d0a 100644 --- a/pkg/store/list.go +++ b/pkg/store/list.go @@ -1,7 +1,7 @@ package store import ( - "github.com/Code0987/supercache/pkg/listx" + listx "github.com/Code0987/supercache/pkg/list" ) func (m *Memory) LPush(key string, item []byte, version uint64, expireAt int64) bool { diff --git a/pkg/store/memory.go b/pkg/store/memory.go index 1c58fe1..aca43f2 100644 --- a/pkg/store/memory.go +++ b/pkg/store/memory.go @@ -8,7 +8,7 @@ import ( "github.com/Code0987/supercache/pkg/geo" "github.com/Code0987/supercache/pkg/hashx" - "github.com/Code0987/supercache/pkg/listx" + listx "github.com/Code0987/supercache/pkg/list" "github.com/Code0987/supercache/pkg/set" "github.com/Code0987/supercache/pkg/zset" ) diff --git a/pkg/topkx/eng/local.go b/pkg/topkx/eng/local.go new file mode 100644 index 0000000..49cfdd1 --- /dev/null +++ b/pkg/topkx/eng/local.go @@ -0,0 +1,101 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/store" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + MaxValue() int + TopKSize() int + ItemMax() int + ObserveVersion(name string, ver uint64) + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} + +var ( + ErrRejected = errors.New("rejected") + ErrTooLarge = errors.New("too large") +) + +func ReplicateSnapshot(h Host, name string, ver uint64, expire int64) { + ent, ok := h.Store().Peek(name) + if !ok || !ent.IsTopK() { + return + } + h.Replicate(name, store.Entry{ + Value: ent.Value, + Version: ver, + ExpireAt: expire, + Flags: store.FlagTopK, + }) +} + +func AddLocal(h Host, name string, item []byte) error { + expire := h.ExpireAt() + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + applied, tooLarge := h.Store().TopKAdd(name, item, gate, expire, h.TopKSize(), h.MaxValue()) + if tooLarge { + return ErrTooLarge + } + if !applied { + return ErrRejected + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expire) + return nil +} + +func ApplyAdd(h Host, name string, item []byte, expireAt int64) bool { + if len(item) == 0 || len(item) > h.ItemMax() { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + cur, _ := h.Store().PeekVersion(name) + gate := cur + 1 + ok, tooLarge := h.Store().TopKAdd(name, item, gate, expireAt, h.TopKSize(), h.MaxValue()) + if !ok || tooLarge { + return false + } + ver, _ := h.Store().PeekVersion(name) + h.ObserveVersion(name, ver) + ReplicateSnapshot(h, name, ver, expireAt) + return true +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().TopKInstall(name, blob, version, expireAt, h.TopKSize()) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsTopK() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().TopKInstall(name, ent.Value, ent.Version, ent.ExpireAt, h.TopKSize()) + } + return ent, true, nil +} diff --git a/pkg/engine/topk_bench_test.go b/pkg/topkx/engine_bench_test.go similarity index 99% rename from pkg/engine/topk_bench_test.go rename to pkg/topkx/engine_bench_test.go index b4b49ed..08b652b 100644 --- a/pkg/engine/topk_bench_test.go +++ b/pkg/topkx/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package topkx_test import ( "context" diff --git a/pkg/engine/topk_cluster_test.go b/pkg/topkx/engine_cluster_test.go similarity index 99% rename from pkg/engine/topk_cluster_test.go rename to pkg/topkx/engine_cluster_test.go index f273856..9408f5f 100644 --- a/pkg/engine/topk_cluster_test.go +++ b/pkg/topkx/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package topkx_test import ( "context" diff --git a/pkg/engine/topk_test.go b/pkg/topkx/engine_test.go similarity index 99% rename from pkg/engine/topk_test.go rename to pkg/topkx/engine_test.go index b722789..6232b48 100644 --- a/pkg/engine/topk_test.go +++ b/pkg/topkx/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package topkx_test import ( "context" diff --git a/pkg/zset/eng/local.go b/pkg/zset/eng/local.go new file mode 100644 index 0000000..347e511 --- /dev/null +++ b/pkg/zset/eng/local.go @@ -0,0 +1,103 @@ +package eng + +import ( + "context" + "errors" + + "github.com/Code0987/supercache/pkg/store" + "github.com/Code0987/supercache/pkg/zset" +) + +type Host interface { + Store() store.Store + KeyspaceName() string + ExpireAt() int64 + HasZSet(name string) bool + NextVersion(name string) uint64 + Replicate(name string, ent store.Entry) + PeerCtx(ctx context.Context) (context.Context, context.CancelFunc) + Owner(name string) (id, addr string, isSelf bool, ok bool) + GetOrLoad(ctx context.Context, addr, keyspace, name string) (store.Entry, bool, error) + HoldsReplica(name string) bool +} + +var ErrRejected = errors.New("rejected") + +func AddLocal(h Host, name string, member []byte, score float64, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().ZAdd(name, member, score, ver, expire) { + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: zset.EncodeAdd(member, score), + Version: ver, + ExpireAt: expire, + Flags: store.FlagZSetAdd, + }) + } + return nil +} + +func RemLocal(h Host, name string, member []byte, fanout bool) error { + ver := h.NextVersion(name) + expire := h.ExpireAt() + if !h.Store().ZRem(name, member, ver, expire) { + if !h.HasZSet(name) { + return nil + } + return ErrRejected + } + if fanout { + h.Replicate(name, store.Entry{ + Value: append([]byte(nil), member...), + Version: ver, + ExpireAt: expire, + Flags: store.FlagZSetRem, + }) + } + return nil +} + +func ApplyAdd(h Host, name string, value []byte, version uint64, expireAt int64) bool { + member, score, err := zset.DecodeAdd(value) + if err != nil { + return false + } + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().ZAdd(name, member, score, version, expireAt) +} + +func ApplyRem(h Host, name string, member []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().ZRem(name, member, version, expireAt) +} + +func ApplyInstall(h Host, name string, blob []byte, version uint64, expireAt int64) bool { + if expireAt == 0 { + expireAt = h.ExpireAt() + } + return h.Store().ZInstall(name, blob, version, expireAt) +} + +func FetchOwner(ctx context.Context, h Host, name string) (store.Entry, bool, error) { + _, addr, isSelf, ok := h.Owner(name) + if !ok || isSelf || addr == "" { + return store.Entry{}, false, nil + } + pctx, cancel := h.PeerCtx(ctx) + defer cancel() + ent, found, err := h.GetOrLoad(pctx, addr, h.KeyspaceName(), name) + if err != nil || !found || !ent.IsZSet() { + return store.Entry{}, false, nil + } + if h.HoldsReplica(name) { + _ = h.Store().ZInstall(name, ent.Value, ent.Version, ent.ExpireAt) + } + return ent, true, nil +} diff --git a/pkg/engine/zset_bench_test.go b/pkg/zset/engine_bench_test.go similarity index 99% rename from pkg/engine/zset_bench_test.go rename to pkg/zset/engine_bench_test.go index 17c609e..fc00dba 100644 --- a/pkg/engine/zset_bench_test.go +++ b/pkg/zset/engine_bench_test.go @@ -1,4 +1,4 @@ -package engine_test +package zset_test import ( "context" diff --git a/pkg/engine/zset_cluster_test.go b/pkg/zset/engine_cluster_test.go similarity index 99% rename from pkg/engine/zset_cluster_test.go rename to pkg/zset/engine_cluster_test.go index bbbcd4a..6e62a1e 100644 --- a/pkg/engine/zset_cluster_test.go +++ b/pkg/zset/engine_cluster_test.go @@ -1,4 +1,4 @@ -package engine_test +package zset_test import ( "context" diff --git a/pkg/engine/zset_test.go b/pkg/zset/engine_test.go similarity index 99% rename from pkg/engine/zset_test.go rename to pkg/zset/engine_test.go index 0ccca60..8bbfe7c 100644 --- a/pkg/engine/zset_test.go +++ b/pkg/zset/engine_test.go @@ -1,4 +1,4 @@ -package engine_test +package zset_test import ( "context" From 47e6531f551e3b53bbc833a962e7e7d2693928cb Mon Sep 17 00:00:00 2001 From: Code0987 <1825861+Code0987@users.noreply.github.com> Date: Fri, 4 Sep 2026 06:54:23 +0000 Subject: [PATCH 13/13] fix: stop warmup before fanout close in join tests Go LIFO defers closed FanoutPool while warmup workers still called Submit. CI -race failed on TestJoinHandoffAvoidsDataSourceReload. Test-only teardown order; production Submit/Close left unchanged. --- pkg/engine/cluster_test.go | 16 ++++++++++------ pkg/engine/join_handoff_test.go | 16 ++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/pkg/engine/cluster_test.go b/pkg/engine/cluster_test.go index 19959b7..7b04fd4 100644 --- a/pkg/engine/cluster_test.go +++ b/pkg/engine/cluster_test.go @@ -987,9 +987,6 @@ func TestJoinTopologyHandoffFillsNewNode(t *testing.T) { wmA.Start(bg) wmB.Start(bg) wmC.Start(bg) - defer wmA.Stop() - defer wmB.Stop() - defer wmC.Stop() gsA, _, err := peerserver.ListenAndServe(addrA, engA) if err != nil { @@ -1024,6 +1021,11 @@ func TestJoinTopologyHandoffFillsNewNode(t *testing.T) { defer foA.Close() defer foB.Close() defer foC.Close() + // Stop warmup after fanout Close is deferred so LIFO teardown waits for + // workers before closechan (Submit vs Close race under -race). + defer wmA.Stop() + defer wmB.Stop() + defer wmC.Stop() engA.AttachCluster(&engine.Cluster{SelfID: "a", Ring: rA, Transport: trA, Fanout: foA}) engB.AttachCluster(&engine.Cluster{SelfID: "b", Ring: rB, Transport: trB, Fanout: foB}) @@ -1141,9 +1143,6 @@ func TestJoinHandoffAvoidsDataSourceReload(t *testing.T) { wmA.Start(bg) wmB.Start(bg) wmC.Start(bg) - defer wmA.Stop() - defer wmB.Stop() - defer wmC.Stop() gsA, _, err := peerserver.ListenAndServe(addrA, engA) if err != nil { @@ -1178,6 +1177,11 @@ func TestJoinHandoffAvoidsDataSourceReload(t *testing.T) { defer foA.Close() defer foB.Close() defer foC.Close() + // Stop warmup after fanout Close is deferred so LIFO teardown waits for + // workers before closechan (Submit vs Close race under -race). + defer wmA.Stop() + defer wmB.Stop() + defer wmC.Stop() engA.AttachCluster(&engine.Cluster{SelfID: "a", Ring: rA, Transport: trA, Fanout: foA}) engB.AttachCluster(&engine.Cluster{SelfID: "b", Ring: rB, Transport: trB, Fanout: foB}) diff --git a/pkg/engine/join_handoff_test.go b/pkg/engine/join_handoff_test.go index 4f47b2c..8cb8569 100644 --- a/pkg/engine/join_handoff_test.go +++ b/pkg/engine/join_handoff_test.go @@ -68,9 +68,6 @@ func TestJoinHandoffEndToEnd(t *testing.T) { wmA.Start(bg) wmB.Start(bg) wmC.Start(bg) - defer wmA.Stop() - defer wmB.Stop() - defer wmC.Stop() gsA, _, err := peerserver.ListenAndServe(addrA, engA) if err != nil { @@ -105,6 +102,11 @@ func TestJoinHandoffEndToEnd(t *testing.T) { defer foA.Close() defer foB.Close() defer foC.Close() + // Stop warmup after fanout Close is deferred so LIFO teardown waits for + // workers before closechan (Submit vs Close race under -race). + defer wmA.Stop() + defer wmB.Stop() + defer wmC.Stop() engA.AttachCluster(&engine.Cluster{SelfID: "a", Ring: rA, Transport: trA, Fanout: foA}) engB.AttachCluster(&engine.Cluster{SelfID: "b", Ring: rB, Transport: trB, Fanout: foB}) @@ -289,9 +291,6 @@ func TestJoinWithoutHandoffStaysCold(t *testing.T) { wmA.Start(bg) wmB.Start(bg) wmC.Start(bg) - defer wmA.Stop() - defer wmB.Stop() - defer wmC.Stop() gsA, _, err := peerserver.ListenAndServe(addrA, engA) if err != nil { @@ -325,6 +324,11 @@ func TestJoinWithoutHandoffStaysCold(t *testing.T) { defer foA.Close() defer foB.Close() defer foC.Close() + // Stop warmup after fanout Close is deferred so LIFO teardown waits for + // workers before closechan (Submit vs Close race under -race). + defer wmA.Stop() + defer wmB.Stop() + defer wmC.Stop() engA.AttachCluster(&engine.Cluster{SelfID: "a", Ring: rA, Transport: trA, Fanout: foA}) engB.AttachCluster(&engine.Cluster{SelfID: "b", Ring: rB, Transport: trB, Fanout: foB})