From fa794245911651a4cb0d73d3baf17d1f57993873 Mon Sep 17 00:00:00 2001 From: tonic Date: Tue, 11 Aug 2026 15:49:06 +0800 Subject: [PATCH 1/3] feat: schedule snapshots by CPU compatibility Render snapshot compatibility as an independent hard node selector, preserve it across wake flows, and adopt pre-feature pods without restarting running VMs. Pin the merged cocoon-common contract. --- cocoonset/pods.go | 24 +++++--- cocoonset/pods_test.go | 55 +++++++++++++++++++ cocoonset/slotrelease_test.go | 4 ++ .../cocoonset.cocoonstack.io_cocoonsets.yaml | 11 ++++ docs/cocoonset.md | 2 + go.mod | 2 +- go.sum | 4 +- 7 files changed, 92 insertions(+), 10 deletions(-) diff --git a/cocoonset/pods.go b/cocoonset/pods.go index c8b5d8c..c30a301 100644 --- a/cocoonset/pods.go +++ b/cocoonset/pods.go @@ -172,6 +172,12 @@ func toolboxPodName(csName, tbName string) string { func newManagedPod(cs *cocoonv1.CocoonSet, podName, role, slotLabel string, scheme *runtime.Scheme) (*corev1.Pod, error) { one := int64(1) pool := cmp.Or(cs.Spec.NodePool, meta.DefaultNodePool) + nodeSelector := map[string]string{ + meta.LabelNodePool: pool, + } + if class := cs.Spec.SnapshotCompatibilityClass; class != "" { + nodeSelector[meta.LabelSnapshotCompatibilityClass] = class + } pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, @@ -193,9 +199,7 @@ func newManagedPod(cs *cocoonv1.CocoonSet, podName, role, slotLabel string, sche {Key: corev1.TaintNodeNotReady, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}, {Key: corev1.TaintNodeUnreachable, Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute}, }, - NodeSelector: map[string]string{ - meta.LabelNodePool: pool, - }, + NodeSelector: nodeSelector, Containers: []corev1.Container{ { Name: agentContainerName, @@ -233,7 +237,7 @@ func podSpecMatchesAgent(pod *corev1.Pod, cs *cocoonv1.CocoonSet, slot int32) bo if !equality.Semantic.DeepEqual(pod.Spec.Containers[0].EnvFrom, cs.Spec.Agent.EnvFrom) { return false } - if !nodePoolMatches(pod, cs) { + if !schedulingMatches(pod, cs) { return false } return true @@ -247,7 +251,7 @@ func podSpecMatchesToolbox(pod *corev1.Pod, cs *cocoonv1.CocoonSet, tb cocoonv1. if !vmSpecMatches(current, want) || !resourcesMatch(pod, tb.Resources) { return false } - if !nodePoolMatches(pod, cs) { + if !schedulingMatches(pod, cs) { return false } if tb.Mode == cocoonv1.ToolboxModeStatic { @@ -298,8 +302,14 @@ func quantityEqual(a, b corev1.ResourceList, name corev1.ResourceName) bool { return qa.Cmp(qb) == 0 } -func nodePoolMatches(pod *corev1.Pod, cs *cocoonv1.CocoonSet) bool { - return meta.PodNodePool(pod) == cmp.Or(cs.Spec.NodePool, meta.DefaultNodePool) +func schedulingMatches(pod *corev1.Pod, cs *cocoonv1.CocoonSet) bool { + if meta.PodNodePool(pod) != cmp.Or(cs.Spec.NodePool, meta.DefaultNodePool) { + return false + } + class := pod.Spec.NodeSelector[meta.LabelSnapshotCompatibilityClass] + // Pod scheduling fields are immutable. Adopt pre-feature pods without the + // selector; their next normal recreate will carry the CocoonSet's class. + return class == "" || class == cs.Spec.SnapshotCompatibilityClass } // applyStorageRequest propagates the VMOptions.Storage quantity into the diff --git a/cocoonset/pods_test.go b/cocoonset/pods_test.go index caa2822..2dd4053 100644 --- a/cocoonset/pods_test.go +++ b/cocoonset/pods_test.go @@ -176,6 +176,27 @@ func TestNewManagedPodHasOwnerReference(t *testing.T) { } } +func TestNewManagedPodSelectsPoolAndSnapshotCompatibilityClass(t *testing.T) { + cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { + cs.Spec.NodePool = "purpose-a" + cs.Spec.SnapshotCompatibilityClass = "n2-cascade-lake-v1" + }) + pod := mustNewManagedPod(t, cs, "demo-0", meta.RoleMain, "0", testScheme(t)) + if got := pod.Spec.NodeSelector[meta.LabelNodePool]; got != "purpose-a" { + t.Errorf("node pool selector = %q, want purpose-a", got) + } + if got := pod.Spec.NodeSelector[meta.LabelSnapshotCompatibilityClass]; got != "n2-cascade-lake-v1" { + t.Errorf("snapshot compatibility selector = %q, want n2-cascade-lake-v1", got) + } +} + +func TestNewManagedPodOmitsEmptySnapshotCompatibilityClass(t *testing.T) { + pod := mustNewManagedPod(t, newCocoonSet("demo"), "demo-0", meta.RoleMain, "0", testScheme(t)) + if _, ok := pod.Spec.NodeSelector[meta.LabelSnapshotCompatibilityClass]; ok { + t.Error("legacy CocoonSet must not gain a snapshot compatibility selector") + } +} + func TestNewManagedPodCarriesCocoonToleration(t *testing.T) { cs := newCocoonSet("demo") pod := mustNewManagedPod(t, cs, "demo-0", meta.RoleMain, "0", testScheme(t)) @@ -421,6 +442,28 @@ func TestPodSpecMatchesAgentNodePoolDefaultFallback(t *testing.T) { } } +func TestPodSpecMatchesAgentDetectsSnapshotCompatibilityClassDrift(t *testing.T) { + cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { + cs.Spec.SnapshotCompatibilityClass = "n2-cascade-lake-v1" + }) + pod := mustBuildAgentPod(t, cs, 0, "", "", testScheme(t)) + pod.Spec.NodeSelector[meta.LabelSnapshotCompatibilityClass] = "n4-emerald-rapids-v1" + if podSpecMatchesAgent(pod, cs, 0) { + t.Error("agent pod with the wrong snapshot compatibility selector must not match") + } +} + +func TestPodSpecMatchesAgentAdoptsLegacyMissingSnapshotCompatibilityClass(t *testing.T) { + cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { + cs.Spec.SnapshotCompatibilityClass = "n2-cascade-lake-v1" + }) + pod := mustBuildAgentPod(t, cs, 0, "", "", testScheme(t)) + delete(pod.Spec.NodeSelector, meta.LabelSnapshotCompatibilityClass) + if !podSpecMatchesAgent(pod, cs, 0) { + t.Error("pre-feature agent pod must be adopted until its next normal recreate") + } +} + func TestPodSpecMatchesToolboxDetectsStaticVMIDDrift(t *testing.T) { cs := newCocoonSet("demo") scheme := testScheme(t) @@ -525,6 +568,18 @@ func TestPodSpecMatchesToolboxDetectsNodePoolDrift(t *testing.T) { } } +func TestPodSpecMatchesToolboxAdoptsLegacyMissingSnapshotCompatibilityClass(t *testing.T) { + cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { + cs.Spec.SnapshotCompatibilityClass = "n2-cascade-lake-v1" + }) + tb := cocoonv1.ToolboxSpec{Name: "tb", Image: "image", Mode: cocoonv1.ToolboxModeRun} + pod := mustBuildToolboxPod(t, cs, tb, testScheme(t)) + delete(pod.Spec.NodeSelector, meta.LabelSnapshotCompatibilityClass) + if !podSpecMatchesToolbox(pod, cs, tb) { + t.Error("pre-feature toolbox pod must be adopted until its next normal recreate") + } +} + func TestBuildAgentPodMainPinnedViaHostnameAffinity(t *testing.T) { cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { cs.Spec.NodeName = "node-b" diff --git a/cocoonset/slotrelease_test.go b/cocoonset/slotrelease_test.go index 1fd98b4..acd0148 100644 --- a/cocoonset/slotrelease_test.go +++ b/cocoonset/slotrelease_test.go @@ -164,6 +164,7 @@ func TestSuspendReleaseFreesSeatWhenFlagPatchFails(t *testing.T) { func TestWakeRecreatesWithRestoreAndPreferredAffinity(t *testing.T) { cs := relCocoonSet(func(cs *cocoonv1.CocoonSet) { cs.Spec.Suspend = false + cs.Spec.SnapshotCompatibilityClass = "n2-cascade-lake-v1" cs.Status.Phase = cocoonv1.CocoonSetPhaseSuspended cs.Annotations = map[string]string{meta.AnnotationHibernatedOnNode: "node-a"} }) @@ -185,6 +186,9 @@ func TestWakeRecreatesWithRestoreAndPreferredAffinity(t *testing.T) { if got.Spec.NodeName != "" { t.Errorf("wake must not hard-pin a node, got %q", got.Spec.NodeName) } + if got.Spec.NodeSelector[meta.LabelSnapshotCompatibilityClass] != "n2-cascade-lake-v1" { + t.Errorf("wake must preserve snapshot compatibility selector, got %v", got.Spec.NodeSelector) + } na := got.Spec.Affinity if na == nil || na.NodeAffinity == nil || len(na.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 || na.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Preference.MatchExpressions[0].Values[0] != "node-a" { diff --git a/config/crd/bases/cocoonset.cocoonstack.io_cocoonsets.yaml b/config/crd/bases/cocoonset.cocoonstack.io_cocoonsets.yaml index 68c738d..c365e2c 100644 --- a/config/crd/bases/cocoonset.cocoonstack.io_cocoonsets.yaml +++ b/config/crd/bases/cocoonset.cocoonstack.io_cocoonsets.yaml @@ -267,6 +267,14 @@ spec: nodePool: default: default type: string + snapshotCompatibilityClass: + description: |- + SnapshotCompatibilityClass selects nodes that expose a guest-visible CPU + ABI compatible with this set's memory snapshots. It is independent from + NodePool so multiple workload pools can share one snapshot class. + maxLength: 63 + pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ + type: string snapshotPolicy: default: always description: SnapshotPolicy defines when VM snapshots are taken. @@ -436,6 +444,9 @@ spec: rule: '!has(self.hibernatePolicy) || self.hibernatePolicy != ''release'' || ((!has(self.agent.replicas) || self.agent.replicas == 0) && (!has(self.toolboxes) || size(self.toolboxes) == 0))' + - message: snapshotCompatibilityClass is immutable once set + rule: '!has(oldSelf.snapshotCompatibilityClass) || (has(self.snapshotCompatibilityClass) + && self.snapshotCompatibilityClass == oldSelf.snapshotCompatibilityClass)' status: description: CocoonSetStatus represents the observed state of a CocoonSet. properties: diff --git a/docs/cocoonset.md b/docs/cocoonset.md index 2f331e0..73f1a79 100644 --- a/docs/cocoonset.md +++ b/docs/cocoonset.md @@ -16,6 +16,8 @@ Pods are constructed via `meta.FromAgentSpec` / `meta.FromToolboxSpec` factory helpers so the operator never touches the annotation map directly. These factories propagate the full `VMOptions` surface (OS, Backend, ConnType, Network, ForcePull, NoDirectIO, ProbePort, Storage, Resources) into the pod annotations that vk-cocoon consumes. The `For` watch uses `predicate.GenerationChangedPredicate` so reconciles only fire when the spec actually changes — status-only patches the operator makes itself never loop back. The `Owns` side filters pod events to creation, deletion, and meaningful transitions (phase change, readiness flip, label/annotation mutation) via a `podRelevantChange` predicate so pure VK status churn does not trigger reconcile storms. +`spec.nodePool` and `spec.snapshotCompatibilityClass` are independent hard placement dimensions. Every managed pod selects `cocoonstack.io/pool=`; a non-empty snapshot class also selects `cocoonstack.io/snapshot-cpu-class=`. The class identifies a certified guest-visible CPU ABI, not merely `amd64`, and remains in force across release-policy wake, rebuild, and migration. Hostname affinity stays a locality preference or explicit migration target and cannot relax either selector. Pre-feature pods whose immutable `nodeSelector` lacks the class are adopted rather than drift-deleted; their next normal recreate or released-seat wake converges them onto the hard selector. + See [Observability](observability.md) for the Event reasons and metrics this loop emits, and [Configuration](configuration.md) for the operator's environment variables. diff --git a/go.mod b/go.mod index d55fc94..ab7f75a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cocoonstack/cocoon-operator go 1.26.5 require ( - github.com/cocoonstack/cocoon-common v0.2.9-0.20260731042413-9ca4f1c8fc0f + github.com/cocoonstack/cocoon-common v0.2.10-0.20260811071104-34821ee9f5ef github.com/go-logr/logr v1.4.3 github.com/google/go-containerregistry v0.21.7 github.com/projecteru2/core v0.0.0-20241016125006-ff909eefe04c diff --git a/go.sum b/go.sum index 7d5893e..7933556 100644 --- a/go.sum +++ b/go.sum @@ -29,8 +29,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cocoonstack/cocoon-common v0.2.9-0.20260731042413-9ca4f1c8fc0f h1:mcHAOv4VUN1cc3CSAeoWyKh/iwPQ4WithgB2sKAVi0Y= -github.com/cocoonstack/cocoon-common v0.2.9-0.20260731042413-9ca4f1c8fc0f/go.mod h1:VSfgYiWxoHRnWybzQNaxmK4kVoLT9ffiMKll5/i92CM= +github.com/cocoonstack/cocoon-common v0.2.10-0.20260811071104-34821ee9f5ef h1:715n/BrVpsRBzPvj+GQyU2xrVV7LiLOf1qBhRuRDN6U= +github.com/cocoonstack/cocoon-common v0.2.10-0.20260811071104-34821ee9f5ef/go.mod h1:VSfgYiWxoHRnWybzQNaxmK4kVoLT9ffiMKll5/i92CM= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= From ddeeabd44d39c4857c4868cb4150a1f1cffd385c Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 11 Aug 2026 19:30:55 +0800 Subject: [PATCH 2/3] review: collapse selector literal, tighten adoption comment --- cocoonset/pods.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cocoonset/pods.go b/cocoonset/pods.go index c30a301..d6db172 100644 --- a/cocoonset/pods.go +++ b/cocoonset/pods.go @@ -172,9 +172,7 @@ func toolboxPodName(csName, tbName string) string { func newManagedPod(cs *cocoonv1.CocoonSet, podName, role, slotLabel string, scheme *runtime.Scheme) (*corev1.Pod, error) { one := int64(1) pool := cmp.Or(cs.Spec.NodePool, meta.DefaultNodePool) - nodeSelector := map[string]string{ - meta.LabelNodePool: pool, - } + nodeSelector := map[string]string{meta.LabelNodePool: pool} if class := cs.Spec.SnapshotCompatibilityClass; class != "" { nodeSelector[meta.LabelSnapshotCompatibilityClass] = class } @@ -307,8 +305,7 @@ func schedulingMatches(pod *corev1.Pod, cs *cocoonv1.CocoonSet) bool { return false } class := pod.Spec.NodeSelector[meta.LabelSnapshotCompatibilityClass] - // Pod scheduling fields are immutable. Adopt pre-feature pods without the - // selector; their next normal recreate will carry the CocoonSet's class. + // NodeSelector is immutable; adopt pre-feature pods until their next recreate. return class == "" || class == cs.Spec.SnapshotCompatibilityClass } From 99014f6e8771e855514820ddc4a4dc79c3642483 Mon Sep 17 00:00:00 2001 From: CMGS Date: Tue, 11 Aug 2026 19:54:33 +0800 Subject: [PATCH 3/3] fix: rebuild terminal legacy pods on class backfill --- cocoonset/pods.go | 4 ++-- cocoonset/pods_test.go | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cocoonset/pods.go b/cocoonset/pods.go index d6db172..ad40fa7 100644 --- a/cocoonset/pods.go +++ b/cocoonset/pods.go @@ -305,8 +305,8 @@ func schedulingMatches(pod *corev1.Pod, cs *cocoonv1.CocoonSet) bool { return false } class := pod.Spec.NodeSelector[meta.LabelSnapshotCompatibilityClass] - // NodeSelector is immutable; adopt pre-feature pods until their next recreate. - return class == "" || class == cs.Spec.SnapshotCompatibilityClass + // NodeSelector is immutable; adopt live pre-feature pods until their next recreate. + return class == cs.Spec.SnapshotCompatibilityClass || (class == "" && !podIsTerminal(pod)) } // applyStorageRequest propagates the VMOptions.Storage quantity into the diff --git a/cocoonset/pods_test.go b/cocoonset/pods_test.go index 2dd4053..49097a0 100644 --- a/cocoonset/pods_test.go +++ b/cocoonset/pods_test.go @@ -453,14 +453,18 @@ func TestPodSpecMatchesAgentDetectsSnapshotCompatibilityClassDrift(t *testing.T) } } -func TestPodSpecMatchesAgentAdoptsLegacyMissingSnapshotCompatibilityClass(t *testing.T) { +func TestPodSpecMatchesAgentAdoptsLiveLegacyMissingSnapshotCompatibilityClass(t *testing.T) { cs := newCocoonSet("demo", func(cs *cocoonv1.CocoonSet) { cs.Spec.SnapshotCompatibilityClass = "n2-cascade-lake-v1" }) pod := mustBuildAgentPod(t, cs, 0, "", "", testScheme(t)) delete(pod.Spec.NodeSelector, meta.LabelSnapshotCompatibilityClass) if !podSpecMatchesAgent(pod, cs, 0) { - t.Error("pre-feature agent pod must be adopted until its next normal recreate") + t.Error("live pre-feature agent pod must be adopted until its next normal recreate") + } + pod.Status.Phase = corev1.PodFailed + if podSpecMatchesAgent(pod, cs, 0) { + t.Error("terminal pre-feature agent pod must be recreated with the compatibility selector") } }