diff --git a/README.md b/README.md index 66a6861..050f0bb 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Optional: `-gossip-secret `. ### CLI (`sc`) ```bash -# With supercache-node running on defaults (-demo-keyspace: demo + tags + board + profile + doc + flags): +# With supercache-node running on defaults (-demo-keyspace: demo + tags + board + profile + doc + flags + embeddings): go run ./cmd/sc put greeting "hello" go run ./cmd/sc get greeting go run ./cmd/sc del greeting @@ -84,6 +84,11 @@ go run ./cmd/sc -keyspace doc jsonset user $.name '"Ada"' go run ./cmd/sc -keyspace doc jsonget user $.name go run ./cmd/sc -keyspace flags bitset seen 0 1 go run ./cmd/sc -keyspace flags bitget seen 0 +go run ./cmd/sc -keyspace embeddings vadd products east 1,0 +go run ./cmd/sc -keyspace embeddings vadd products north 0,1 +go run ./cmd/sc -keyspace embeddings vadd products ne 0.7,0.7 +go run ./cmd/sc -keyspace embeddings vcard products +go run ./cmd/sc -keyspace embeddings vsim products 1,0.05 3 go run ./cmd/sc -keyspace seen bloom add users alice # ModeBloom keyspace # ModeTopK (register a ModeTopK keyspace, or use examples/billboard plays/hot): # go run ./cmd/sc -keyspace plays topkadd hot t001 @@ -208,7 +213,7 @@ Apps: `client.DialTLS` with `pkg/tlsconfig.ClientFiles`. See [docs/OPERATIONS.md | `pkg/warmup` | Hot keys, topology handoff (hot then rest), refresh-ahead | | `pkg/client` | Application gRPC client (KV + Bloom + Set + ZSet + Geo + List + Hash + Counter + JSON + Bitmap + HLL + TopK) | | `pkg/tlsconfig` | TLS/mTLS config from PEM files | -| `cmd/supercache-node` | Node binary (`-demo-keyspace`: demo / tags / board / profile / doc / flags) | +| `cmd/supercache-node` | Node binary (`-demo-keyspace`: demo / tags / board / profile / doc / flags / embeddings) | | `cmd/sc` | CLI: get/put/del, bloom, sadd*, z*, geo*, l*, h*, incr/cget, json*, bit*, hlladd/hllcount, topkadd/topklist, admin diagnostics | | `cmd/scbench` | SuperCache vs Redis load harness + in-process matrix | diff --git a/cmd/supercache-node/main.go b/cmd/supercache-node/main.go index 4bdd85b..47c60a9 100644 --- a/cmd/supercache-node/main.go +++ b/cmd/supercache-node/main.go @@ -44,7 +44,7 @@ func main() { gossipAdv = flag.String("gossip-advertise", "127.0.0.1", "gossip advertise address") seeds = flag.String("seeds", "", "comma-separated gossip seeds host:port") gossipSecret = flag.String("gossip-secret", "", "optional gossip shared secret") - demoKS = flag.Bool("demo-keyspace", true, "register demo CacheOnly keyspace") + demoKS = flag.Bool("demo-keyspace", true, "register stock demo keyspaces (demo/tags/board/profile/doc/flags/embeddings)") globalRPS = flag.Float64("global-rps", 0, "global DataSource rate limit (0=off)") cluster = flag.Bool("cluster", false, "enable gossip membership + peer fan-out") showVersion = flag.Bool("version", false, "print version and exit") @@ -87,60 +87,10 @@ func main() { defer wm.Stop() if *demoKS { - if err := eng.UpdateKeySpace(keyspace.Config{ - Name: "demo", - Mode: keyspace.ModeCacheOnly, - MaxBytes: 64 << 20, - TTL: 5 * time.Minute, - }); err != nil { + if err := registerDemoKeyspaces(eng); err != nil { log.Fatal(err) } - // ModeSet for exact membership demos (feature tags, allow-lists, …). - if err := eng.UpdateKeySpace(keyspace.Config{ - Name: "tags", - Mode: keyspace.ModeSet, - MaxBytes: 16 << 20, - TTL: 30 * time.Minute, - }); err != nil { - log.Fatal(err) - } - // ModeZSet for scored rankings (leaderboards, time-ordered feeds, …). - if err := eng.UpdateKeySpace(keyspace.Config{ - Name: "board", - Mode: keyspace.ModeZSet, - MaxBytes: 16 << 20, - TTL: 30 * time.Minute, - }); err != nil { - log.Fatal(err) - } - // ModeHash for per-field maps (user profiles, session attrs, …). - if err := eng.UpdateKeySpace(keyspace.Config{ - Name: "profile", - Mode: keyspace.ModeHash, - MaxBytes: 16 << 20, - TTL: 30 * time.Minute, - }); err != nil { - log.Fatal(err) - } - // ModeJSON for nested documents (path set/get/del). - if err := eng.UpdateKeySpace(keyspace.Config{ - Name: "doc", - Mode: keyspace.ModeJSON, - MaxBytes: 16 << 20, - TTL: 30 * time.Minute, - }); err != nil { - log.Fatal(err) - } - // ModeBitmap for packed flags (SETBIT / GETBIT / BITCOUNT / BITPOS). - if err := eng.UpdateKeySpace(keyspace.Config{ - Name: "flags", - Mode: keyspace.ModeBitmap, - MaxBytes: 16 << 20, - TTL: 30 * time.Minute, - }); err != nil { - log.Fatal(err) - } - log.Printf("demo keyspaces: demo=CacheOnly tags=ModeSet board=ModeZSet profile=ModeHash doc=ModeJSON flags=ModeBitmap") + log.Printf("demo keyspaces: demo=CacheOnly tags=ModeSet board=ModeZSet profile=ModeHash doc=ModeJSON flags=ModeBitmap embeddings=ModeVectorSet") } cacheSrvOpts, peerSrvOpts, peerDialTLS, err := buildTLS( @@ -334,3 +284,24 @@ func buildTLS( } return cacheOpts, peerOpts, peerDial, nil } + +// registerDemoKeyspaces installs the stock -demo-keyspace set. +func registerDemoKeyspaces(eng *engine.Engine) error { + type ks struct { + cfg keyspace.Config + } + for _, item := range []ks{ + {keyspace.Config{Name: "demo", Mode: keyspace.ModeCacheOnly, MaxBytes: 64 << 20, TTL: 5 * time.Minute}}, + {keyspace.Config{Name: "tags", Mode: keyspace.ModeSet, MaxBytes: 16 << 20, TTL: 30 * time.Minute}}, + {keyspace.Config{Name: "board", Mode: keyspace.ModeZSet, MaxBytes: 16 << 20, TTL: 30 * time.Minute}}, + {keyspace.Config{Name: "profile", Mode: keyspace.ModeHash, MaxBytes: 16 << 20, TTL: 30 * time.Minute}}, + {keyspace.Config{Name: "doc", Mode: keyspace.ModeJSON, MaxBytes: 16 << 20, TTL: 30 * time.Minute}}, + {keyspace.Config{Name: "flags", Mode: keyspace.ModeBitmap, MaxBytes: 16 << 20, TTL: 30 * time.Minute}}, + {keyspace.Config{Name: "embeddings", Mode: keyspace.ModeVectorSet, MaxBytes: 16 << 20, TTL: 30 * time.Minute}}, + } { + if err := eng.UpdateKeySpace(item.cfg); err != nil { + return err + } + } + return nil +} diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 3b0196e..e8ce488 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -78,7 +78,7 @@ Peer mesh with mTLS: every node uses the same CA; each node presents a cert sign Wrong verb for the mode → invalid argument. Configure the same modes on every node (see rollout above). -Demo node (`-demo-keyspace`): registers `demo` (CacheOnly), `tags` (ModeSet), `board` (ModeZSet), `profile` (ModeHash), `doc` (ModeJSON), `flags` (ModeBitmap). Geo/List/Counter/HLL/TopK keyspaces are configured by the app. Live plays billboard: [examples/billboard](../examples/billboard/README.md). Hash walkthrough: [examples/hash](../examples/hash/README.md). Rate limiter: [examples/ratelimit](../examples/ratelimit/README.md). JSON document: [examples/json](../examples/json/README.md). Bitmap flags: [examples/bitmap](../examples/bitmap/README.md). HLL sketch: [examples/hll](../examples/hll/README.md). +Demo node (`-demo-keyspace`): registers `demo` (CacheOnly), `tags` (ModeSet), `board` (ModeZSet), `profile` (ModeHash), `doc` (ModeJSON), `flags` (ModeBitmap), `embeddings` (ModeVectorSet). Geo/List/Counter/HLL/TopK/CMS keyspaces are configured by the app. Live plays billboard: [examples/billboard](../examples/billboard/README.md). Hash walkthrough: [examples/hash](../examples/hash/README.md). Rate limiter: [examples/ratelimit](../examples/ratelimit/README.md). JSON document: [examples/json](../examples/json/README.md). Bitmap flags: [examples/bitmap](../examples/bitmap/README.md). HLL sketch: [examples/hll](../examples/hll/README.md). ## Consistency cheatsheet diff --git a/docs/design/README.md b/docs/design/README.md index a38b5be..a1f6b8f 100644 --- a/docs/design/README.md +++ b/docs/design/README.md @@ -6,12 +6,6 @@ Use the template in [WORKFLOW.md](../WORKFLOW.md). Status is `draft` until revie Do not implement from a draft. -## Drafts (not approved) - -| Design | Feature | -|--------|---------| -| [2026-09-04-mode-vector-set.md](./2026-09-04-mode-vector-set.md) | `ModeVectorSet` (approved in chat; implementing on `feat/mode-vector-set`) | - ## Shipped (reference) | Design | Feature | @@ -39,6 +33,7 @@ Do not implement from a draft. | [2026-09-02-refactor-zset.md](./2026-09-02-refactor-zset.md) | ModeZSet file layout (no contract) | | [2026-09-02-refactor-geo.md](./2026-09-02-refactor-geo.md) | ModeGeo file layout (no contract) | | [2026-09-02-feature-folders.md](./2026-09-02-feature-folders.md) | Feature folders (List first) | +| [2026-09-04-mode-vector-set.md](./2026-09-04-mode-vector-set.md) | `ModeVectorSet` | | [2026-08-25-list-counter-version.md](./2026-08-25-list-counter-version.md) | List/Counter snapshot version | | [2026-08-13-unify-grpc-error-map.md](./2026-08-13-unify-grpc-error-map.md) | grpcmap | diff --git a/examples/cluster3/README.md b/examples/cluster3/README.md index 426051b..7d1f0c8 100644 --- a/examples/cluster3/README.md +++ b/examples/cluster3/README.md @@ -2,7 +2,7 @@ Exercises **CacheOnly** (`demo`: sessions / KV), **ModeSet** (`tags`: feature flags), and **ModeZSet** (`board`: leaderboard via `ZAdd` / `ZScore` / `ZRange` / `ZRem`). -Requires `supercache-node` with `-demo-keyspace` (default), which registers `demo`, `tags`, `board`, `profile`, `doc`, and `flags` (this example uses the first three). +Requires `supercache-node` with `-demo-keyspace` (default), which registers `demo`, `tags`, `board`, `profile`, `doc`, `flags`, and `embeddings` (this example uses the first three). Start three nodes (separate terminals), then run this example. diff --git a/examples/vecset/README.md b/examples/vecset/README.md new file mode 100644 index 0000000..98de5de --- /dev/null +++ b/examples/vecset/README.md @@ -0,0 +1,59 @@ +# ModeVectorSet example — similar items + +A **self-contained 3-node** walkthrough of `ModeVectorSet`: a named set of +`float32` embeddings with brute-force K-NN. Metric is a **keyspace** knob +(cosine / L2 / IP), not an argument on `VSim`. + +## What it shows + +| Step | SuperCache behavior | +|------|---------------------| +| Why not ZSet / Put | ZSet score is a scalar you write. A JSON blob `Put` is one LWW value | +| `VAdd` | Three members on cosine `items`; first add locks dim=2 | +| `VCard` / `VDim` / `VEmb` | Count, locked dim, stored vector copy | +| RF=2 + `VSim` | Every node returns the same cosine order; non-replica does not install | +| Replace | `VAdd` same id overwrites the vector; card stays 3 | +| Dim lock / zero | Wrong dim and cosine `{0,0}` are invalid; Get/Put rejected | +| L2 vs IP | Same two members: L2 picks nearest, IP picks largest dot | +| `k` / missing | `k>card` returns card hits; missing name is empty / `present=false` | +| Last `VRem` | Empty live set keeps dim; `Delete` is what drops the name | +| Recreate | `Delete` + `VAdd` makes a new set | + +## Run + +```bash +# from repo root — starts 3 nodes, prints the walkthrough, exits 0 on success +go run ./examples/vecset +``` + +CI covers the same path: `go test ./examples/vecset`. + +## Manual `sc` against a stock node + +`supercache-node -demo-keyspace` (the default) registers **`embeddings`** +(`ModeVectorSet`, cosine, dim unlocked until first `vadd`). + +```bash +# terminal 1 +go run ./cmd/supercache-node \ + -cache 127.0.0.1:9000 -peer 127.0.0.1:9001 -admin 127.0.0.1:8080 + +# terminal 2 — add, inspect, search +go run ./cmd/sc -keyspace embeddings vadd products east 1,0 +go run ./cmd/sc -keyspace embeddings vadd products north 0,1 +go run ./cmd/sc -keyspace embeddings vadd products ne 0.7,0.7 +go run ./cmd/sc -keyspace embeddings vcard products +go run ./cmd/sc -keyspace embeddings vdim products +go run ./cmd/sc -keyspace embeddings vemb products east +go run ./cmd/sc -keyspace embeddings vsim products 1,0.05 3 + +# replace, rem, missing +go run ./cmd/sc -keyspace embeddings vadd products east 0,1 +go run ./cmd/sc -keyspace embeddings vsim products 1,0 3 +go run ./cmd/sc -keyspace embeddings vrem products north +go run ./cmd/sc -keyspace embeddings vcard products +go run ./cmd/sc -keyspace embeddings vdim missing # prints missing, exit 1 + +# cosine rejects a zero vector (L2/IP would allow it on another keyspace) +# go run ./cmd/sc -keyspace embeddings vadd products origin 0,0 +``` diff --git a/examples/vecset/demo.go b/examples/vecset/demo.go index 1a8825f..233bd87 100644 --- a/examples/vecset/demo.go +++ b/examples/vecset/demo.go @@ -2,25 +2,33 @@ package main import ( "context" + "errors" "fmt" "io" + "strings" "time" "github.com/Code0987/supercache/internal/testcluster" "github.com/Code0987/supercache/pkg/client" + "github.com/Code0987/supercache/pkg/engine" "github.com/Code0987/supercache/pkg/keyspace" ) -const ks = "items" +const ( + ksCosine = "items" + ksL2 = "items-l2" + ksIP = "items-ip" + setName = "set" +) func runDemo(out io.Writer) error { c, err := testcluster.Start(testcluster.Config{ Nodes: 3, - Keyspaces: []keyspace.Config{{ - Name: ks, Mode: keyspace.ModeVectorSet, - MaxBytes: 4 << 20, TTL: time.Hour, ReplicationFactor: 2, - VectorDim: 2, VectorMetric: keyspace.VectorMetricCosine, - }}, + Keyspaces: []keyspace.Config{ + {Name: ksCosine, Mode: keyspace.ModeVectorSet, MaxBytes: 4 << 20, TTL: time.Hour, ReplicationFactor: 2, VectorDim: 2, VectorMetric: keyspace.VectorMetricCosine}, + {Name: ksL2, Mode: keyspace.ModeVectorSet, MaxBytes: 4 << 20, TTL: time.Hour, ReplicationFactor: 2, VectorDim: 2, VectorMetric: keyspace.VectorMetricL2}, + {Name: ksIP, Mode: keyspace.ModeVectorSet, MaxBytes: 4 << 20, TTL: time.Hour, ReplicationFactor: 2, VectorDim: 2, VectorMetric: keyspace.VectorMetricIP}, + }, }) if err != nil { return err @@ -28,50 +36,276 @@ func runDemo(out io.Writer) error { defer c.Close() ctx := context.Background() - cli, err := client.Dial(ctx, c.Nodes()[0].CacheAddr) + nodes := c.Nodes() + clis := make([]*client.Client, 0, len(nodes)) + for _, n := range nodes { + cli, err := client.Dial(ctx, n.CacheAddr) + if err != nil { + return fmt.Errorf("dial %s: %w", n.CacheAddr, err) + } + defer cli.Close() + clis = append(clis, cli) + } + cli := clis[0] + p := func(format string, args ...any) { fmt.Fprintf(out, format+"\n", args...) } + + p("ModeVectorSet example — named embeddings + K-NN (not ZSet scores)") + p("3 in-process nodes, RF=2. Cosine / L2 / IP are *keyspace* knobs.") + p("") + + p("=== 1) Why not ZSet or Put(blob)? ===") + p(" ZSet score is a scalar the caller writes. Put of [{id,vec}] is one LWW") + p(" blob — two VAdds of different members would clobber each other.") + p("") + + p("=== 2) VAdd three members (cosine keyspace, dim locked at 2) ===") + seed := []struct { + id string + vec []float32 + }{ + {"east", []float32{1, 0}}, + {"north", []float32{0, 1}}, + {"ne", []float32{0.7, 0.7}}, + } + for _, m := range seed { + if err := cli.VAdd(ctx, ksCosine, setName, []byte(m.id), m.vec); err != nil { + return fmt.Errorf("VAdd %s: %w", m.id, err) + } + p(" VAdd %s %v", m.id, m.vec) + } + if err := waitLocals(nodes, ksCosine, setName, 2, 2*time.Second); err != nil { + return err + } + n, ok, err := cli.VCard(ctx, ksCosine, setName) + if err != nil || !ok || n != 3 { + return fmt.Errorf("VCard: %d %v %v", n, ok, err) + } + dim, ok, err := cli.VDim(ctx, ksCosine, setName) + if err != nil || !ok || dim != 2 { + return fmt.Errorf("VDim: %d %v %v", dim, ok, err) + } + emb, ok, err := cli.VEmb(ctx, ksCosine, setName, []byte("east")) + if err != nil || !ok || emb[0] != 1 || emb[1] != 0 { + return fmt.Errorf("VEmb east: %v %v %v", emb, ok, err) + } + p(" VCard=%d VDim=%d VEmb(east)=%v copies: %s", n, dim, emb, localSummary(nodes, ksCosine, setName)) + + p("") + p("=== 3) VSim cosine from every node (best first) ===") + for i, c := range clis { + hits, err := waitSim(c, ksCosine, []float32{1, 0.05}, 3, "east", 2*time.Second) + if err != nil { + return fmt.Errorf("n%d VSim: %w", i, err) + } + if len(hits) != 3 || string(hits[0].Member) != "east" || string(hits[2].Member) != "north" { + return fmt.Errorf("n%d order: %s", i, formatHits(hits)) + } + p(" n%d VSim ~east → %s local=%v", i, formatHits(hits), nodes[i].Engine.HasLocal(ksCosine, setName)) + } + p(" (non-replica GetOrLoads the snapshot and scores locally; does not install)") + + p("") + p("=== 4) Replace member — VAdd same id overwrites the vector ===") + if err := cli.VAdd(ctx, ksCosine, setName, []byte("east"), []float32{0, 1}); err != nil { + return fmt.Errorf("replace east: %w", err) + } + hits, err := waitSim(cli, ksCosine, []float32{1, 0}, 3, "ne", 2*time.Second) if err != nil { return err } - defer cli.Close() + n, ok, err = cli.VCard(ctx, ksCosine, setName) + if err != nil || !ok || n != 3 { + return fmt.Errorf("replace grew/shrunk card=%d", n) + } + p(" VAdd east={0,1}; VSim ~east now %s (card still %d)", formatHits(hits), n) - p := func(format string, args ...any) { fmt.Fprintf(out, format+"\n", args...) } - p("ModeVectorSet example — cosine neighbors (not ZSet scores)") - if err := cli.VAdd(ctx, ks, "set", []byte("east"), []float32{1, 0}); err != nil { + p("") + p("=== 5) Dim lock + cosine rejects zero ===") + if err := cli.VAdd(ctx, ksCosine, setName, []byte("bad"), []float32{1, 0, 0}); err == nil { + return fmt.Errorf("dim-3 VAdd should fail") + } + p(" VAdd dim=3 → invalid") + if err := cli.VAdd(ctx, ksCosine, setName, []byte("zero"), []float32{0, 0}); err == nil { + return fmt.Errorf("zero VAdd should fail on cosine") + } + p(" VAdd {0,0} on cosine → invalid") + if _, err := cli.Get(ctx, ksCosine, setName); !isInvalid(err) { + return fmt.Errorf("Get: %v", err) + } + if err := cli.Put(ctx, ksCosine, setName, []byte("nope")); err == nil { + return fmt.Errorf("Put should fail") + } + p(" Get/Put on ModeVectorSet → invalid (use VEmb / VAdd)") + + p("") + p("=== 6) L2 vs IP — same members, different keyspace metric ===") + for _, pair := range []struct { + ks string + id string + vec []float32 + note string + }{ + {ksL2, "origin", []float32{0, 0}, "zero allowed on L2"}, + {ksL2, "far", []float32{10, 0}, ""}, + {ksIP, "origin", []float32{0, 0}, "zero allowed on IP"}, + {ksIP, "far", []float32{10, 0}, ""}, + } { + if err := cli.VAdd(ctx, pair.ks, setName, []byte(pair.id), pair.vec); err != nil { + return fmt.Errorf("VAdd %s/%s: %w", pair.ks, pair.id, err) + } + if pair.note != "" { + p(" %s", pair.note) + } + } + l2, err := waitSim(cli, ksL2, []float32{1, 0}, 1, "origin", 2*time.Second) + if err != nil { + return fmt.Errorf("L2: %w", err) + } + if l2[0].Score < 0 { + return fmt.Errorf("L2 score %g want >=0", l2[0].Score) + } + ip, err := waitSim(cli, ksIP, []float32{1, 0}, 1, "far", 2*time.Second) + if err != nil { + return fmt.Errorf("IP: %w", err) + } + p(" L2 VSim {1,0} → %s (nearest; lower distance first)", formatHits(l2)) + p(" IP VSim {1,0} → %s (largest dot first)", formatHits(ip)) + + p("") + p("=== 7) k > card, ties, missing name ===") + wide, err := cli.VSim(ctx, ksCosine, setName, []float32{0, 1}, 50) + if err != nil || len(wide) != 3 { + return fmt.Errorf("k>card: %d %v", len(wide), err) + } + p(" VSim k=50 → %d hits (min(k,card))", len(wide)) + miss, err := cli.VSim(ctx, ksCosine, "no-such", []float32{1, 0}, 3) + if err != nil || len(miss) != 0 { + return fmt.Errorf("missing VSim: %v %v", miss, err) + } + _, present, err := cli.VDim(ctx, ksCosine, "no-such") + if err != nil || present { + return fmt.Errorf("missing VDim: %v %v", present, err) + } + p(" missing name: VSim empty, VDim present=false") + + p("") + p("=== 8) Last VRem keeps empty set (dim stays locked) ===") + for _, id := range []string{"east", "north", "ne"} { + if err := cli.VRem(ctx, ksCosine, setName, []byte(id)); err != nil { + return fmt.Errorf("VRem %s: %w", id, err) + } + } + if err := waitCard(cli, ksCosine, setName, 0, true, 2*time.Second); err != nil { + return err + } + dim, ok, err = cli.VDim(ctx, ksCosine, setName) + if err != nil || !ok || dim != 2 { + return fmt.Errorf("empty VDim: %d %v %v", dim, ok, err) + } + empty, err := cli.VSim(ctx, ksCosine, setName, []float32{1, 0}, 3) + if err != nil || len(empty) != 0 { + return fmt.Errorf("empty VSim: %v %v", empty, err) + } + p(" after last VRem: VCard=0 VDim=2 VSim=[] (Delete is what drops the name)") + + p("") + p("=== 9) Delete tombstone then recreate ===") + if err := cli.Delete(ctx, ksCosine, setName); err != nil { return err } - if err := cli.VAdd(ctx, ks, "set", []byte("north"), []float32{0, 1}); err != nil { + if err := waitCard(cli, ksCosine, setName, 0, false, 2*time.Second); err != nil { return err } - if err := cli.VAdd(ctx, ks, "set", []byte("ne"), []float32{0.7, 0.7}); err != nil { + if err := cli.VAdd(ctx, ksCosine, setName, []byte("west"), []float32{-1, 0}); err != nil { + return fmt.Errorf("recreate: %w", err) + } + if err := waitCard(cli, ksCosine, setName, 1, true, 2*time.Second); err != nil { + return err + } + if err := waitLocals(nodes, ksCosine, setName, 2, 2*time.Second); err != nil { return err } - var hits []struct { - Member []byte - Score float32 + p(" Delete + VAdd west; copies: %s", localSummary(nodes, ksCosine, setName)) + + p("") + p("OK: ModeVectorSet walkthrough passed") + return nil +} + +func isInvalid(err error) bool { + return err != nil && (errors.Is(err, engine.ErrInvalidArgument) || + strings.Contains(err.Error(), "InvalidArgument") || + strings.Contains(err.Error(), "use VEmb") || + strings.Contains(err.Error(), "use VAdd")) +} + +func formatHits(hits []engine.VSimHit) string { + parts := make([]string, len(hits)) + for i, h := range hits { + parts[i] = fmt.Sprintf("%s=%.3f", h.Member, h.Score) } - deadline := time.Now().Add(2 * time.Second) + return strings.Join(parts, " ") +} + +func waitSim(cli *client.Client, ks string, q []float32, k int, wantFirst string, d time.Duration) ([]engine.VSimHit, error) { + ctx := context.Background() + deadline := time.Now().Add(d) + var last []engine.VSimHit for time.Now().Before(deadline) { - got, err := cli.VSim(ctx, ks, "set", []float32{1, 0.05}, 2) + hits, err := cli.VSim(ctx, ks, setName, q, k) if err != nil { - return err + return nil, err + } + last = hits + if len(hits) > 0 && string(hits[0].Member) == wantFirst { + return hits, nil + } + time.Sleep(15 * time.Millisecond) + } + return last, fmt.Errorf("VSim want first=%s got %s", wantFirst, formatHits(last)) +} + +func waitCard(cli *client.Client, ks, name string, want int, wantPresent bool, d time.Duration) error { + ctx := context.Background() + deadline := time.Now().Add(d) + var n int + var ok bool + for time.Now().Before(deadline) { + var err error + n, ok, err = cli.VCard(ctx, ks, name) + if err == nil && ok == wantPresent && n == want { + return nil } - if len(got) >= 1 && string(got[0].Member) == "east" { - hits = make([]struct { - Member []byte - Score float32 - }, len(got)) - for i, h := range got { - hits[i].Member = h.Member - hits[i].Score = h.Score + time.Sleep(15 * time.Millisecond) + } + return fmt.Errorf("VCard want %d present=%v got %d present=%v", want, wantPresent, n, ok) +} + +func waitLocals(nodes []testcluster.Node, ks, name string, want int, d time.Duration) error { + deadline := time.Now().Add(d) + var n int + for time.Now().Before(deadline) { + n = 0 + for _, node := range nodes { + if node.Engine.HasLocal(ks, name) { + n++ } - break } - time.Sleep(20 * time.Millisecond) + if n == want { + return nil + } + time.Sleep(15 * time.Millisecond) } - if len(hits) == 0 || string(hits[0].Member) != "east" { - return fmt.Errorf("expected east first, got %+v", hits) + return fmt.Errorf("local copies=%d want %d (%s)", n, want, localSummary(nodes, ks, name)) +} + +func localSummary(nodes []testcluster.Node, ks, name string) string { + s := "" + for i, n := range nodes { + if i > 0 { + s += " " + } + s += fmt.Sprintf("%s=%v", n.ID, n.Engine.HasLocal(ks, name)) } - p(" VSim ~east → %s (score=%g)", hits[0].Member, hits[0].Score) - p("OK: ModeVectorSet walkthrough passed") - return nil + return s } diff --git a/examples/vecset/demo_test.go b/examples/vecset/demo_test.go index 0024db0..d4a740f 100644 --- a/examples/vecset/demo_test.go +++ b/examples/vecset/demo_test.go @@ -11,7 +11,17 @@ func TestExampleVecSet(t *testing.T) { if err := runDemo(&buf); err != nil { t.Fatalf("%v\n%s", err, buf.String()) } - if !strings.Contains(buf.String(), "OK: ModeVectorSet walkthrough passed") { - t.Fatalf("missing OK line:\n%s", buf.String()) + got := buf.String() + for _, want := range []string{ + "OK: ModeVectorSet walkthrough passed", + "=== 3) VSim cosine", + "=== 4) Replace member", + "=== 6) L2 vs IP", + "=== 8) Last VRem", + "=== 9) Delete tombstone", + } { + if !strings.Contains(got, want) { + t.Fatalf("missing %q:\n%s", want, got) + } } }