feat: derive VPC identifier from the network's fabric identity

cmd/main.go

@@ -112,7 +112,7 @@ func main() {
 	}
 
 	if err := (&controller.NetworkContextReconciler{
-		Client: mgr.GetClient(), Scheme: mgr.GetScheme(),
+		Client: mgr.GetClient(), Scheme: mgr.GetScheme(), APIReader: mgr.GetAPIReader(),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "NetworkContext")
 		os.Exit(1)

internal/controller/networkcontext_controller.go

@@ -25,6 +25,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -40,7 +41,8 @@ import (
 // identifier the whole galactic fabric keys on.
 type NetworkContextReconciler struct {
 	client.Client
-	Scheme *runtime.Scheme
+	Scheme    *runtime.Scheme
+	APIReader client.Reader
 }
 
 // +kubebuilder:rbac:groups=networking.datumapis.com,resources=networkcontexts,verbs=get;list;watch
@@ -83,7 +85,7 @@ func (r *NetworkContextReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 	}
 
 	if vpc.Status.VPC == "" {
-		allocated, err := r.allocateVPCIdentifier(ctx)
+		allocated, err := r.vpcIdentifier(ctx, &networkContext)
 		if err != nil {
 			return ctrl.Result{}, err
 		}
@@ -142,6 +144,76 @@ func subnetRange(subnet *networkingv1alpha.Subnet) (string, int32, bool) {
 	return "", 0, false
 }
 
+// vpcIdentifier resolves the identifier for a VPC that does not have one yet.
+// A network whose fabric identity has been allocated for it derives its
+// identifier from that value, so every location holding the same network
+// arrives at the same one. A network with no allocated identity keeps the
+// original behaviour and draws a random identifier for this cell.
+func (r *NetworkContextReconciler) vpcIdentifier(
+	ctx context.Context, networkContext *networkingv1alpha.NetworkContext,
+) (string, error) {
+	fabricIdentity, err := r.fabricIdentity(ctx, networkContext)
+	if err != nil {
+		return "", err
+	}
+	if fabricIdentity == 0 {
+		return r.allocateVPCIdentifier(ctx)
+	}
+	return vpcIdentifierFor(fabricIdentity)
+}
+
+// vpcIdentifierFor renders an allocated fabric identity as the base62 VPC
+// identifier the galactic data plane keys on. The rendering is total and
+// deterministic: the same identity yields the same identifier in every cell.
+func vpcIdentifierFor(fabricIdentity int64) (string, error) {
+	if fabricIdentity < 0 {
+		return "", fmt.Errorf("fabric identity %d is negative", fabricIdentity)
+	}
+	rendered, err := identifier.VPCBase62(uint64(fabricIdentity))
+	if err != nil {
+		return "", fmt.Errorf("render fabric identity %d: %w", fabricIdentity, err)
+	}
+	return rendered, nil
+}
+
+// fabricIdentity reads the identity allocated for the network this context
+// belongs to, or zero when none has been allocated.
+//
+// The field is read untyped because the Go type in the pinned
+// network-services-operator release does not carry it yet, and a typed client
+// discards fields its struct does not know: the value would read as absent
+// every time. Reading it directly means this cell picks the identity up as
+// soon as it is published, with no release ordering between the two repos.
+// Once a release carrying the field is pinned, this collapses to a field read.
+func (r *NetworkContextReconciler) fabricIdentity(
+	ctx context.Context, networkContext *networkingv1alpha.NetworkContext,
+) (int64, error) {
+	reader := r.APIReader
+	if reader == nil {
+		reader = r.Client
+	}
+
+	raw := &unstructured.Unstructured{}
+	raw.SetGroupVersionKind(networkingv1alpha.GroupVersion.WithKind("NetworkContext"))
+	if err := reader.Get(ctx, client.ObjectKeyFromObject(networkContext), raw); err != nil {
+		return 0, fmt.Errorf("read NetworkContext %s: %w", networkContext.Name, err)
+	}
+	return fabricIdentityFrom(raw.Object)
+}
+
+// fabricIdentityFrom extracts spec.fabricIdentity from an untyped
+// NetworkContext, treating an absent field as no allocated identity.
+func fabricIdentityFrom(object map[string]any) (int64, error) {
+	value, found, err := unstructured.NestedInt64(object, "spec", "fabricIdentity")
+	if err != nil {
+		return 0, fmt.Errorf("read spec.fabricIdentity: %w", err)
+	}
+	if !found {
+		return 0, nil
+	}
+	return value, nil
+}
+
 // allocateVPCIdentifier draws a random 48-bit identifier not already in use.
 // A single leader-elected controller is the only writer, so a list plus a
 // collision check serializes correctly.

internal/controller/networkcontext_controller_test.go

@@ -18,8 +18,19 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 package controller
 
 import (
+	"context"
 	"testing"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+
+	cloudv1alpha1 "go.datum.net/cloud/api/v1alpha1"
+	"go.datum.net/cloud/internal/identifier"
 	networkingv1alpha "go.datum.net/network-services-operator/api/v1alpha"
 )
 
@@ -50,3 +61,215 @@ func TestSubnetRangePrefersStatusAndFallsBackToSpec(t *testing.T) {
 		t.Fatal("an unallocated subnet should yield nothing")
 	}
 }
+
+// A network allocated a fabric identity must resolve to the same VPC
+// identifier in every cell that holds it. Two cells reaching different
+// identifiers is the defect this replaces: the edge VRF device is named from
+// the VPC alone, and the Route Target is derived from it, so a network whose
+// locations disagree neither shares a device nor exchanges routes.
+func TestFabricIdentityDerivesTheSameVPCIdentifierInEveryCell(t *testing.T) {
+	const allocated int64 = 0x1A2B3C4D5E6F
+
+	first, err := vpcIdentifierFor(allocated)
+	if err != nil {
+		t.Fatalf("derive in the first cell: %v", err)
+	}
+	second, err := vpcIdentifierFor(allocated)
+	if err != nil {
+		t.Fatalf("derive in the second cell: %v", err)
+	}
+	if first != second {
+		t.Fatalf("two cells derived %q and %q from the same identity", first, second)
+	}
+
+	other, err := vpcIdentifierFor(allocated + 1)
+	if err != nil {
+		t.Fatalf("derive a neighbouring identity: %v", err)
+	}
+	if other == first {
+		t.Fatalf("distinct identities both derived %q", first)
+	}
+}
+
+// The identifier lands in a nine-character slot in the galactic VRF device
+// name, so nothing the allocator can hand out may render wider than that.
+func TestDerivedVPCIdentifierFitsTheVRFDeviceName(t *testing.T) {
+	for _, allocated := range []int64{1, 2, 1000, 1 << 24, int64(identifier.MaxVPC) - 1} {
+		derived, err := vpcIdentifierFor(allocated)
+		if err != nil {
+			t.Fatalf("derive %d: %v", allocated, err)
+		}
+		if len(derived) > 9 {
+			t.Fatalf("identity %d rendered %d characters, wider than the slot holds", allocated, len(derived))
+		}
+	}
+}
+
+// Values the fabric cannot represent are refused rather than folded into
+// something that collides with another network's identifier.
+func TestUnrepresentableFabricIdentityIsRefused(t *testing.T) {
+	for _, allocated := range []int64{-1, int64(identifier.MaxVPC), int64(identifier.MaxVPC) + 1} {
+		if _, err := vpcIdentifierFor(allocated); err == nil {
+			t.Fatalf("identity %d should have been refused", allocated)
+		}
+	}
+}
+
+// Networks that predate the allocator carry no identity, and a context written
+// before the field existed carries nothing either. Both read as unallocated.
+func TestAbsentFabricIdentityReadsAsUnallocated(t *testing.T) {
+	absent, err := fabricIdentityFrom(map[string]any{"spec": map[string]any{}})
+	if err != nil || absent != 0 {
+		t.Fatalf("an unset field should read as 0, got %d err=%v", absent, err)
+	}
+
+	noSpec, err := fabricIdentityFrom(map[string]any{})
+	if err != nil || noSpec != 0 {
+		t.Fatalf("a context with no spec should read as 0, got %d err=%v", noSpec, err)
+	}
+
+	present, err := fabricIdentityFrom(map[string]any{
+		"spec": map[string]any{"fabricIdentity": int64(4242)},
+	})
+	if err != nil || present != 4242 {
+		t.Fatalf("a projected identity should read back, got %d err=%v", present, err)
+	}
+
+	if _, err := fabricIdentityFrom(map[string]any{
+		"spec": map[string]any{"fabricIdentity": "not-a-number"},
+	}); err == nil {
+		t.Fatal("a non-integer identity should be refused")
+	}
+}
+
+func fabricTestScheme(t *testing.T) *runtime.Scheme {
+	t.Helper()
+	s := runtime.NewScheme()
+	if err := cloudv1alpha1.AddToScheme(s); err != nil {
+		t.Fatalf("register cloud types: %v", err)
+	}
+	if err := networkingv1alpha.AddToScheme(s); err != nil {
+		t.Fatalf("register networking types: %v", err)
+	}
+	return s
+}
+
+// projectedIdentityReader serves the NetworkContext as it appears on the wire,
+// carrying the projected identity. The fake client stores objects through their
+// registered Go type, which discards a field that type does not carry yet —
+// the very reason the reconciler reads this field untyped.
+type projectedIdentityReader struct {
+	client.Reader
+	fabricIdentity int64
+}
+
+func (r projectedIdentityReader) Get(
+	ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption,
+) error {
+	raw, ok := obj.(*unstructured.Unstructured)
+	if !ok {
+		return r.Reader.Get(ctx, key, obj, opts...)
+	}
+	raw.Object = map[string]any{"spec": map[string]any{}}
+	if r.fabricIdentity != 0 {
+		raw.Object["spec"] = map[string]any{"fabricIdentity": r.fabricIdentity}
+	}
+	return nil
+}
+
+func networkContextWithSubnet() []client.Object {
+	networkContext := &networkingv1alpha.NetworkContext{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "default-us-central-1",
+			Namespace: "project-a",
+			UID:       "0c1d2e3f-4a5b-6c7d-8e9f-0a1b2c3d4e5f",
+		},
+		Spec: networkingv1alpha.NetworkContextSpec{
+			Network: networkingv1alpha.LocalNetworkRef{Name: "default"},
+		},
+	}
+
+	subnet := &networkingv1alpha.Subnet{
+		ObjectMeta: metav1.ObjectMeta{Name: "default-v6", Namespace: "project-a"},
+		Spec: networkingv1alpha.SubnetSpec{
+			NetworkContext: networkingv1alpha.LocalNetworkContextRef{Name: "default-us-central-1"},
+			StartAddress:   "fd00::",
+			PrefixLength:   48,
+		},
+	}
+	return []client.Object{networkContext, subnet}
+}
+
+func reconcileNetworkContext(
+	t *testing.T, fabricIdentity int64, objects ...client.Object,
+) *cloudv1alpha1.VPC {
+	t.Helper()
+	s := fabricTestScheme(t)
+	c := fake.NewClientBuilder().
+		WithScheme(s).
+		WithObjects(objects...).
+		WithStatusSubresource(&cloudv1alpha1.VPC{}).
+		Build()
+
+	r := &NetworkContextReconciler{
+		Client: c, Scheme: s,
+		APIReader: projectedIdentityReader{Reader: c, fabricIdentity: fabricIdentity},
+	}
+	request := ctrl.Request{NamespacedName: types.NamespacedName{
+		Name: "default-us-central-1", Namespace: "project-a",
+	}}
+	if _, err := r.Reconcile(context.Background(), request); err != nil {
+		t.Fatalf("reconcile: %v", err)
+	}
+
+	vpc := &cloudv1alpha1.VPC{}
+	if err := c.Get(context.Background(), request.NamespacedName, vpc); err != nil {
+		t.Fatalf("read back the VPC: %v", err)
+	}
+	return vpc
+}
+
+// The allocated identity is what the VPC ends up carrying, which is the whole
+// point: every cell reconciling this network reaches the same identifier.
+func TestReconcileUsesTheAllocatedFabricIdentity(t *testing.T) {
+	const allocated int64 = 0x1A2B3C4D5E6F
+
+	expected, err := vpcIdentifierFor(allocated)
+	if err != nil {
+		t.Fatalf("derive the expected identifier: %v", err)
+	}
+
+	vpc := reconcileNetworkContext(t, allocated, networkContextWithSubnet()...)
+	if vpc.Status.VPC != expected {
+		t.Fatalf("VPC carries %q, want the derived %q", vpc.Status.VPC, expected)
+	}
+}
+
+// Nothing has allocated identities yet and existing networks have none, so a
+// context without one must still get an identifier exactly as it does today.
+func TestReconcileFallsBackToARandomIdentifierWithoutAnAllocatedIdentity(t *testing.T) {
+	vpc := reconcileNetworkContext(t, 0, networkContextWithSubnet()...)
+	if vpc.Status.VPC == "" {
+		t.Fatal("a context with no allocated identity should still receive an identifier")
+	}
+	if _, err := identifier.Base62ToHex(vpc.Status.VPC); err != nil {
+		t.Fatalf("identifier %q is not base62: %v", vpc.Status.VPC, err)
+	}
+}
+
+// A VPC already carrying an identifier keeps it. Rewriting one renames the
+// edge VRF device and changes the Route Target under running traffic.
+func TestReconcileLeavesAnAlreadyAllocatedVPCAlone(t *testing.T) {
+	const existing = "3fA2bQ71x"
+
+	objects := append(networkContextWithSubnet(), &cloudv1alpha1.VPC{
+		ObjectMeta: metav1.ObjectMeta{Name: "default-us-central-1", Namespace: "project-a"},
+		Spec:       cloudv1alpha1.VPCSpec{Networks: []cloudv1alpha1.Network{"fd00::/48"}},
+		Status:     cloudv1alpha1.VPCStatus{VPC: existing},
+	})
+
+	vpc := reconcileNetworkContext(t, 0x1A2B3C4D5E6F, objects...)
+	if vpc.Status.VPC != existing {
+		t.Fatalf("a live VPC was renumbered from %q to %q", existing, vpc.Status.VPC)
+	}
+}

internal/identifier/identifier.go

@@ -80,6 +80,17 @@ func Base62ToHex(value string) (string, error) {
 	return baseconv.Convert(value, baseconv.Digits62, baseconv.DigitsHex)
 }
 
+// VPCBase62 renders a specific 48-bit VPC identifier in base62. Callers that
+// hold an identifier allocated elsewhere use this instead of drawing a random
+// one, so every cell rendering the same value produces the same identifier.
+func VPCBase62(value uint64) (string, error) {
+	hex, err := Hex(value, MaxVPC)
+	if err != nil {
+		return "", err
+	}
+	return HexToBase62(hex)
+}
+
 // RandomVPCBase62 returns a random VPC identifier in base62.
 func RandomVPCBase62() (string, error) {
 	hex, err := RandomVPC()