diff --git a/api/common/v1alpha1/common_types.go b/api/common/v1alpha1/common_types.go index 9932d0fee..65ffe0a59 100644 --- a/api/common/v1alpha1/common_types.go +++ b/api/common/v1alpha1/common_types.go @@ -36,6 +36,25 @@ const ( DefaultEphemeralManager = "ephemeral-manager" ) +// TopologyLabel represents a topology label that can be configured on machinepoollet, volumepoollet, and bucketpoollet, +// which set them on MachinePool, VolumePool, and BucketPool resources. +// These labels are managed exclusively by the respective poollet controllers (machinepoollet, volumepoollet, bucketpoollet). +// Any manual changes to these labels will be overwritten by the poollet controllers. +// The intent is similar to Kubernetes' topology labels. +type TopologyLabel string + +const ( + // TopologyLabelRegion is a label applied to MachinePool, VolumePool, and BucketPool resources. + // Machines, Volumes, and Buckets can use this label in their pool selectors. + // The intent is similar to Kubernetes' topology labels (e.g., `topology.kubernetes.io/region`). + TopologyLabelRegion TopologyLabel = "topology.ironcore.dev/region" + + // TopologyLabelZone is a label applied to MachinePool, VolumePool, and BucketPool resources. + // Machines, Volumes, and Buckets can use this label in their pool selectors. + // The intent is similar to Kubernetes' topology labels (e.g., `topology.kubernetes.io/zone`). + TopologyLabelZone TopologyLabel = "topology.ironcore.dev/zone" +) + // ConfigMapKeySelector is a reference to a specific 'key' within a ConfigMap resource. // In some instances, `key` is a required field. // +structType=atomic diff --git a/docs/api-reference/common.md b/docs/api-reference/common.md index 295bc27d1..06bee8fc3 100644 --- a/docs/api-reference/common.md +++ b/docs/api-reference/common.md @@ -393,6 +393,34 @@ When specified, allowed values are NoSchedule.

+

TopologyLabel +(string alias)

+
+

TopologyLabel represents a topology label that can be configured on machinepoollet, volumepoollet, and bucketpoollet, +which set them on MachinePool, VolumePool, and BucketPool resources. +These labels are managed exclusively by the respective poollet controllers (machinepoollet, volumepoollet, bucketpoollet). +Any manual changes to these labels will be overwritten by the poollet controllers. +The intent is similar to Kubernetes’ topology labels.

+
+ + + + + + + + + + + + +
ValueDescription

"topology.ironcore.dev/region"

TopologyLabelRegion is a label applied to MachinePool, VolumePool, and BucketPool resources. +Machines, Volumes, and Buckets can use this label in their pool selectors. +The intent is similar to Kubernetes’ topology labels (e.g., topology.kubernetes.io/region).

+

"topology.ironcore.dev/zone"

TopologyLabelZone is a label applied to MachinePool, VolumePool, and BucketPool resources. +Machines, Volumes, and Buckets can use this label in their pool selectors. +The intent is similar to Kubernetes’ topology labels (e.g., topology.kubernetes.io/zone).

+

UIDReference

diff --git a/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go b/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go index 2e9cbd5ce..da6d4b948 100644 --- a/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go +++ b/poollet/bucketpoollet/cmd/bucketpoollet/app/app.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" @@ -66,6 +67,8 @@ type Options struct { BucketDownwardAPILabels map[string]string BucketDownwardAPIAnnotations map[string]string BucketPoolName string + TopologyRegionLabel string + TopologyZoneLabel string ProviderID string BucketRuntimeEndpoint string DialTimeout time.Duration @@ -104,6 +107,8 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringToStringVar(&o.BucketDownwardAPILabels, "bucket-downward-api-label", o.BucketDownwardAPILabels, "Downward-API labels to set on the IRI bucket.") fs.StringToStringVar(&o.BucketDownwardAPIAnnotations, "bucket-downward-api-annotation", o.BucketDownwardAPIAnnotations, "Downward-API annotations to set on the IRI bucket.") fs.StringVar(&o.BucketPoolName, "bucket-pool-name", o.BucketPoolName, "Name of the bucket pool to announce / watch") + fs.StringVar(&o.TopologyRegionLabel, "topology-region-label", "", "Label to use for the region topology information.") + fs.StringVar(&o.TopologyZoneLabel, "topology-zone-label", "", "Label to use for the zone topology information.") fs.StringVar(&o.ProviderID, "provider-id", "", "Provider id to announce on the bucket pool.") fs.StringVar(&o.BucketRuntimeEndpoint, "bucket-runtime-endpoint", o.BucketRuntimeEndpoint, "Endpoint of the remote bucket runtime service.") fs.DurationVar(&o.DialTimeout, "dial-timeout", 1*time.Second, "Timeout for dialing to the bucket runtime endpoint.") @@ -157,6 +162,14 @@ func Run(ctx context.Context, opts Options) error { logger := ctrl.LoggerFrom(ctx) setupLog := ctrl.Log.WithName("setup") + topologyLabels := map[commonv1alpha1.TopologyLabel]string{} + if opts.TopologyRegionLabel != "" { + topologyLabels[commonv1alpha1.TopologyLabelRegion] = opts.TopologyRegionLabel + } + if opts.TopologyZoneLabel != "" { + topologyLabels[commonv1alpha1.TopologyLabelZone] = opts.TopologyZoneLabel + } + getter, err := bucketpoolletconfig.NewGetter(opts.BucketPoolName) if err != nil { setupLog.Error(err, "Error creating new getter") @@ -342,6 +355,7 @@ func Run(ctx context.Context, opts Options) error { BucketPoolName: opts.BucketPoolName, BucketClassMapper: bucketClassMapper, BucketRuntime: bucketRuntime, + TopologyLabels: topologyLabels, }).SetupWithManager(mgr); err != nil { return fmt.Errorf("error setting up bucket pool reconciler with manager: %w", err) } @@ -353,6 +367,7 @@ func Run(ctx context.Context, opts Options) error { Client: mgr.GetClient(), BucketPoolName: opts.BucketPoolName, ProviderID: opts.ProviderID, + TopologyLabels: topologyLabels, OnInitialized: onInitialized, }).SetupWithManager(mgr); err != nil { return fmt.Errorf("error setting up bucket pool init with manager: %w", err) diff --git a/poollet/bucketpoollet/controllers/bucketpool_controller.go b/poollet/bucketpoollet/controllers/bucketpool_controller.go index 5389f8af1..6402ea424 100644 --- a/poollet/bucketpoollet/controllers/bucketpool_controller.go +++ b/poollet/bucketpoollet/controllers/bucketpool_controller.go @@ -9,9 +9,11 @@ import ( "fmt" "github.com/go-logr/logr" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" iriBucket "github.com/ironcore-dev/ironcore/iri/apis/bucket" "github.com/ironcore-dev/ironcore/poollet/bucketpoollet/bcm" + poolletutils "github.com/ironcore-dev/ironcore/poollet/common/utils" corev1 "k8s.io/api/core/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" @@ -25,6 +27,8 @@ type BucketPoolReconciler struct { BucketPoolName string BucketRuntime iriBucket.RuntimeService BucketClassMapper bcm.BucketClassMapper + + TopologyLabels map[commonv1alpha1.TopologyLabel]string } //+kubebuilder:rbac:groups=storage.ironcore.dev,resources=bucketpools,verbs=get;list;watch;update;patch @@ -70,6 +74,11 @@ func (r *BucketPoolReconciler) supportsBucketClass(ctx context.Context, bucketCl func (r *BucketPoolReconciler) reconcile(ctx context.Context, log logr.Logger, bucketPool *storagev1alpha1.BucketPool) (ctrl.Result, error) { log.V(1).Info("Reconcile") + log.V(1).Info("Enforcing configured topology labels") + if err := r.enforceOriginalTopologyLabels(ctx, log, bucketPool); err != nil { + return ctrl.Result{}, fmt.Errorf("error enforcing original topology labels: %w", err) + } + log.V(1).Info("Listing bucket classes") bucketClassList := &storagev1alpha1.BucketClassList{} if err := r.List(ctx, bucketClassList); err != nil { @@ -101,6 +110,14 @@ func (r *BucketPoolReconciler) reconcile(ctx context.Context, log logr.Logger, b return ctrl.Result{}, nil } +func (r *BucketPoolReconciler) enforceOriginalTopologyLabels(ctx context.Context, log logr.Logger, bucketPool *storagev1alpha1.BucketPool) error { + base := bucketPool.DeepCopy() + + poolletutils.SetTopologyLabels(log, &bucketPool.ObjectMeta, r.TopologyLabels) + + return r.Patch(ctx, bucketPool, client.MergeFrom(base)) +} + func (r *BucketPoolReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For( diff --git a/poollet/bucketpoollet/controllers/bucketpool_controller_test.go b/poollet/bucketpoollet/controllers/bucketpool_controller_test.go index 39269bc25..a778d5c89 100644 --- a/poollet/bucketpoollet/controllers/bucketpool_controller_test.go +++ b/poollet/bucketpoollet/controllers/bucketpool_controller_test.go @@ -73,4 +73,21 @@ var _ = Describe("BucketPoolController", func() { })), )) }) + + It("should enforce topology labels", func(ctx SpecContext) { + By("patching the bucket pool with incorrect topology labels") + Eventually(Update(bucketPool, func() { + if bucketPool.Labels == nil { + bucketPool.Labels = make(map[string]string) + } + bucketPool.Labels["topology.ironcore.dev/region"] = "wrong-region" + bucketPool.Labels["topology.ironcore.dev/zone"] = "wrong-zone" + })).Should(Succeed()) + + By("checking if the reconciler resets the topology labels to its original values") + Eventually(Object(bucketPool)).Should(SatisfyAll( + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/region", "test-region-1")), + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/zone", "test-zone-1")), + )) + }) }) diff --git a/poollet/bucketpoollet/controllers/bucketpool_init.go b/poollet/bucketpoollet/controllers/bucketpool_init.go index 60269f49b..3696928c1 100644 --- a/poollet/bucketpoollet/controllers/bucketpool_init.go +++ b/poollet/bucketpoollet/controllers/bucketpool_init.go @@ -7,8 +7,10 @@ import ( "context" "fmt" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" bucketpoolletv1alpha1 "github.com/ironcore-dev/ironcore/poollet/bucketpoollet/api/v1alpha1" + poolletutils "github.com/ironcore-dev/ironcore/poollet/common/utils" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -20,6 +22,8 @@ type BucketPoolInit struct { BucketPoolName string ProviderID string + TopologyLabels map[commonv1alpha1.TopologyLabel]string + OnInitialized func(ctx context.Context) error OnFailed func(ctx context.Context, reason error) error } @@ -42,6 +46,10 @@ func (i *BucketPoolInit) Start(ctx context.Context) error { ProviderID: i.ProviderID, }, } + + log.V(1).Info("Initially setting topology labels") + poolletutils.SetTopologyLabels(log, &bucketPool.ObjectMeta, i.TopologyLabels) + if err := i.Patch(ctx, bucketPool, client.Apply, client.ForceOwnership, client.FieldOwner(bucketpoolletv1alpha1.FieldOwner)); err != nil { if i.OnFailed != nil { log.V(1).Info("Failed applying, calling OnFailed callback", "Error", err) diff --git a/poollet/bucketpoollet/controllers/bucketpool_init_test.go b/poollet/bucketpoollet/controllers/bucketpool_init_test.go new file mode 100644 index 000000000..853e74da1 --- /dev/null +++ b/poollet/bucketpoollet/controllers/bucketpool_init_test.go @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package controllers_test + +import ( + "context" + + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" + storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" + "github.com/ironcore-dev/ironcore/poollet/bucketpoollet/controllers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +var _ = Describe("BucketPoolInit", func() { + It("should set topology labels", func(ctx SpecContext) { + initializedCalled := false + + bpi := &controllers.BucketPoolInit{ + Client: k8sClient, + BucketPoolName: "test-pool", + ProviderID: "provider-123", + TopologyLabels: map[commonv1alpha1.TopologyLabel]string{ + commonv1alpha1.TopologyLabelRegion: "foo-region-1", + commonv1alpha1.TopologyLabelZone: "foo-zone-1", + }, + OnInitialized: func(ctx context.Context) error { + initializedCalled = true + return nil + }, + } + + Expect(bpi.Start(ctx)).To(Succeed()) + + pool := &storagev1alpha1.BucketPool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pool", + }, + } + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), pool)).To(Succeed()) + DeferCleanup(k8sClient.Delete, pool) + + Expect(initializedCalled).To(BeTrue(), "OnInitialized should have been called") + + Eventually(Object(pool)).Should(SatisfyAll( + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/region", "foo-region-1")), + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/zone", "foo-zone-1")), + )) + }) +}) diff --git a/poollet/bucketpoollet/controllers/controllers_suite_test.go b/poollet/bucketpoollet/controllers/controllers_suite_test.go index ff5b19859..a74f1c459 100644 --- a/poollet/bucketpoollet/controllers/controllers_suite_test.go +++ b/poollet/bucketpoollet/controllers/controllers_suite_test.go @@ -11,6 +11,7 @@ import ( "testing" "time" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" storageclient "github.com/ironcore-dev/ironcore/internal/client/storage" @@ -158,6 +159,10 @@ func SetupTest() (*corev1.Namespace, *storagev1alpha1.BucketPool, *storagev1alph *bp = storagev1alpha1.BucketPool{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-bp-", + Labels: map[string]string{ + string(commonv1alpha1.TopologyLabelRegion): "test-region-1", + string(commonv1alpha1.TopologyLabelZone): "test-zone-1", + }, }, } Expect(k8sClient.Create(ctx, bp)).To(Succeed(), "failed to create test bucket pool") @@ -239,6 +244,10 @@ func SetupTest() (*corev1.Namespace, *storagev1alpha1.BucketPool, *storagev1alph BucketRuntime: srv, BucketClassMapper: bucketClassMapper, BucketPoolName: bp.Name, + TopologyLabels: map[commonv1alpha1.TopologyLabel]string{ + commonv1alpha1.TopologyLabelRegion: "test-region-1", + commonv1alpha1.TopologyLabelZone: "test-zone-1", + }, }).SetupWithManager(k8sManager)).To(Succeed()) go func() { diff --git a/poollet/common/utils/topology.go b/poollet/common/utils/topology.go new file mode 100644 index 000000000..c6b260697 --- /dev/null +++ b/poollet/common/utils/topology.go @@ -0,0 +1,25 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package utils + +import ( + "github.com/go-logr/logr" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func SetTopologyLabels(log logr.Logger, om *v1.ObjectMeta, labels map[commonv1alpha1.TopologyLabel]string) { + if len(labels) == 0 { + return + } + + if om.Labels == nil { + om.Labels = make(map[string]string) + } + + for key, val := range labels { + log.V(1).Info("Setting topology label", "Label", key, "Value", val) + om.Labels[string(key)] = val + } +} diff --git a/poollet/machinepoollet/cmd/machinepoollet/app/app.go b/poollet/machinepoollet/cmd/machinepoollet/app/app.go index e688652ba..b26ce64c2 100644 --- a/poollet/machinepoollet/cmd/machinepoollet/app/app.go +++ b/poollet/machinepoollet/cmd/machinepoollet/app/app.go @@ -17,6 +17,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" computev1alpha1 "github.com/ironcore-dev/ironcore/api/compute/v1alpha1" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" @@ -87,6 +88,9 @@ type Options struct { NetworkDownwardAPILabels map[string]string NetworkDownwardAPIAnnotations map[string]string + TopologyRegionLabel string + TopologyZoneLabel string + ProviderID string MachineRuntimeEndpoint string MachineRuntimeSocketDiscoveryTimeout time.Duration @@ -136,6 +140,9 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringToStringVar(&o.NetworkDownwardAPILabels, "network-downward-api-label", o.NetworkDownwardAPILabels, "Downward-API labels to set on the iri network.") fs.StringToStringVar(&o.NetworkDownwardAPIAnnotations, "network-downward-api-annotation", o.NetworkDownwardAPIAnnotations, "Downward-API annotations to set on the iri network.") + fs.StringVar(&o.TopologyRegionLabel, "topology-region-label", "", "Label to use for the region topology information.") + fs.StringVar(&o.TopologyZoneLabel, "topology-zone-label", "", "Label to use for the zone topology information.") + fs.StringVar(&o.ProviderID, "provider-id", "", "Provider id to announce on the machine pool.") fs.StringVar(&o.MachineRuntimeEndpoint, "machine-runtime-endpoint", o.MachineRuntimeEndpoint, "Endpoint of the remote machine runtime service.") fs.DurationVar(&o.MachineRuntimeSocketDiscoveryTimeout, "machine-runtime-socket-discovery-timeout", 20*time.Second, "Timeout for discovering the machine runtime socket.") @@ -320,6 +327,14 @@ func Run(ctx context.Context, opts Options) error { }) } + topologyLabels := map[commonv1alpha1.TopologyLabel]string{} + if opts.TopologyRegionLabel != "" { + topologyLabels[commonv1alpha1.TopologyLabelRegion] = opts.TopologyRegionLabel + } + if opts.TopologyZoneLabel != "" { + topologyLabels[commonv1alpha1.TopologyLabelZone] = opts.TopologyZoneLabel + } + mgr, err := ctrl.NewManager(cfg, ctrl.Options{ Logger: logger, Scheme: scheme, @@ -462,6 +477,7 @@ func Run(ctx context.Context, opts Options) error { Port: port, MachineRuntime: machineRuntime, MachineClassMapper: machineClassMapper, + TopologyLabels: topologyLabels, }).SetupWithManager(mgr); err != nil { return fmt.Errorf("error setting up machine pool reconciler with manager: %w", err) } @@ -481,6 +497,7 @@ func Run(ctx context.Context, opts Options) error { Client: mgr.GetClient(), MachinePoolName: opts.MachinePoolName, ProviderID: opts.ProviderID, + TopologyLabels: topologyLabels, OnInitialized: onInitialized, }).SetupWithManager(mgr); err != nil { return fmt.Errorf("error setting up machine pool init with manager: %w", err) diff --git a/poollet/machinepoollet/controllers/controllers_suite_test.go b/poollet/machinepoollet/controllers/controllers_suite_test.go index dd593849e..2ee306ddc 100644 --- a/poollet/machinepoollet/controllers/controllers_suite_test.go +++ b/poollet/machinepoollet/controllers/controllers_suite_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/ironcore-dev/controller-utils/buildutils" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" computev1alpha1 "github.com/ironcore-dev/ironcore/api/compute/v1alpha1" corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" @@ -167,6 +168,10 @@ func SetupTest() (*corev1.Namespace, *computev1alpha1.MachinePool, *computev1alp *mp = computev1alpha1.MachinePool{ ObjectMeta: metav1.ObjectMeta{ GenerateName: "test-mp-", + Labels: map[string]string{ + string(commonv1alpha1.TopologyLabelRegion): "foo-region-1", + string(commonv1alpha1.TopologyLabelZone): "foo-zone-1", + }, }, } Expect(k8sClient.Create(ctx, mp)).To(Succeed(), "failed to create test machine pool") @@ -269,6 +274,10 @@ func SetupTest() (*corev1.Namespace, *computev1alpha1.MachinePool, *computev1alp MachineRuntime: srv, MachineClassMapper: machineClassMapper, MachinePoolName: mp.Name, + TopologyLabels: map[commonv1alpha1.TopologyLabel]string{ + commonv1alpha1.TopologyLabelRegion: "foo-region-1", + commonv1alpha1.TopologyLabelZone: "foo-zone-1", + }, }).SetupWithManager(k8sManager)).To(Succeed()) Expect((&controllers.MachinePoolAnnotatorReconciler{ diff --git a/poollet/machinepoollet/controllers/machinepool_controller.go b/poollet/machinepoollet/controllers/machinepool_controller.go index e055dd763..b6603b050 100644 --- a/poollet/machinepoollet/controllers/machinepool_controller.go +++ b/poollet/machinepoollet/controllers/machinepool_controller.go @@ -9,11 +9,13 @@ import ( "fmt" "github.com/go-logr/logr" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" computev1alpha1 "github.com/ironcore-dev/ironcore/api/compute/v1alpha1" corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1" computeclient "github.com/ironcore-dev/ironcore/internal/client/compute" "github.com/ironcore-dev/ironcore/iri/apis/machine" iri "github.com/ironcore-dev/ironcore/iri/apis/machine/v1alpha1" + poolletutils "github.com/ironcore-dev/ironcore/poollet/common/utils" "github.com/ironcore-dev/ironcore/poollet/machinepoollet/mcm" ironcoreclient "github.com/ironcore-dev/ironcore/utils/client" "github.com/ironcore-dev/ironcore/utils/quota" @@ -38,6 +40,8 @@ type MachinePoolReconciler struct { MachineRuntime machine.RuntimeService MachineClassMapper mcm.MachineClassMapper + + TopologyLabels map[commonv1alpha1.TopologyLabel]string } //+kubebuilder:rbac:groups=compute.ironcore.dev,resources=machinepools,verbs=get;list;watch;update;patch @@ -151,6 +155,11 @@ func (r *MachinePoolReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{RequeueAfter: 1}, nil } + log.V(1).Info("Enforcing configured topology labels") + if err := r.enforceOriginalTopologyLabels(ctx, log, machinePool); err != nil { + return ctrl.Result{}, fmt.Errorf("error enforcing original topology labels: %w", err) + } + log.V(1).Info("Listing machine classes") machineClassList := &computev1alpha1.MachineClassList{} if err := r.List(ctx, machineClassList); err != nil { @@ -174,6 +183,14 @@ func (r *MachinePoolReconciler) reconcile(ctx context.Context, log logr.Logger, return ctrl.Result{}, nil } +func (r *MachinePoolReconciler) enforceOriginalTopologyLabels(ctx context.Context, log logr.Logger, machinePool *computev1alpha1.MachinePool) error { + base := machinePool.DeepCopy() + + poolletutils.SetTopologyLabels(log, &machinePool.ObjectMeta, r.TopologyLabels) + + return r.Patch(ctx, machinePool, client.MergeFrom(base)) +} + func (r *MachinePoolReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For( diff --git a/poollet/machinepoollet/controllers/machinepool_controller_test.go b/poollet/machinepoollet/controllers/machinepool_controller_test.go index 3117ac019..4742b231e 100644 --- a/poollet/machinepoollet/controllers/machinepool_controller_test.go +++ b/poollet/machinepoollet/controllers/machinepool_controller_test.go @@ -238,4 +238,20 @@ var _ = Describe("MachinePoolController", func() { ))), ) }) + + It("should enforce topology labels", func(ctx SpecContext) { + By("patching the machine pool with incorrect topology labels") + Eventually(Update(machinePool, func() { + machinePool.Labels = map[string]string{ + "topology.ironcore.dev/region": "wrong-region", + "topology.ironcore.dev/zone": "wrong-zone", + } + })).Should(Succeed()) + + By("checking if the reconciler resets the topology labels to its original values") + Eventually(Object(machinePool)).Should(SatisfyAll( + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/region", "foo-region-1")), + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/zone", "foo-zone-1")), + )) + }) }) diff --git a/poollet/machinepoollet/controllers/machinepool_init.go b/poollet/machinepoollet/controllers/machinepool_init.go index c9ea4b7fe..c8a3bc0f9 100644 --- a/poollet/machinepoollet/controllers/machinepool_init.go +++ b/poollet/machinepoollet/controllers/machinepool_init.go @@ -7,7 +7,9 @@ import ( "context" "fmt" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" computev1alpha1 "github.com/ironcore-dev/ironcore/api/compute/v1alpha1" + poolletutils "github.com/ironcore-dev/ironcore/poollet/common/utils" machinepoolletv1alpha1 "github.com/ironcore-dev/ironcore/poollet/machinepoollet/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -20,6 +22,8 @@ type MachinePoolInit struct { MachinePoolName string ProviderID string + TopologyLabels map[commonv1alpha1.TopologyLabel]string + // TODO: Remove OnInitialized / OnFailed as soon as the controller-runtime provides support for pre-start hooks: // https://github.com/kubernetes-sigs/controller-runtime/pull/2044 @@ -45,6 +49,10 @@ func (i *MachinePoolInit) Start(ctx context.Context) error { ProviderID: i.ProviderID, }, } + + log.V(1).Info("Initially setting topology labels") + poolletutils.SetTopologyLabels(log, &machinePool.ObjectMeta, i.TopologyLabels) + if err := i.Patch(ctx, machinePool, client.Apply, client.ForceOwnership, client.FieldOwner(machinepoolletv1alpha1.FieldOwner)); err != nil { if i.OnFailed != nil { log.V(1).Info("Failed applying, calling OnFailed callback", "Error", err) diff --git a/poollet/machinepoollet/controllers/machinepool_init_test.go b/poollet/machinepoollet/controllers/machinepool_init_test.go new file mode 100644 index 000000000..6e710083f --- /dev/null +++ b/poollet/machinepoollet/controllers/machinepool_init_test.go @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2025 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package controllers_test + +import ( + "context" + + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" + computev1alpha1 "github.com/ironcore-dev/ironcore/api/compute/v1alpha1" + "github.com/ironcore-dev/ironcore/poollet/machinepoollet/controllers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +var _ = Describe("MachinePoolInit", func() { + It("should set topology labels", func(ctx SpecContext) { + initializedCalled := false + + mpi := &controllers.MachinePoolInit{ + Client: k8sClient, + MachinePoolName: "test-pool", + ProviderID: "provider-123", + TopologyLabels: map[commonv1alpha1.TopologyLabel]string{ + commonv1alpha1.TopologyLabelRegion: "foo-region-1", + commonv1alpha1.TopologyLabelZone: "foo-zone-1", + }, + OnInitialized: func(ctx context.Context) error { + initializedCalled = true + return nil + }, + } + + Expect(mpi.Start(ctx)).ToNot(HaveOccurred()) + Expect(initializedCalled).To(BeTrue()) + + pool := &computev1alpha1.MachinePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pool", + }, + } + + Eventually(Object(pool)).Should(SatisfyAll( + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/region", "foo-region-1")), + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/zone", "foo-zone-1")), + )) + }) +}) diff --git a/poollet/volumepoollet/cmd/volumepoollet/app/app.go b/poollet/volumepoollet/cmd/volumepoollet/app/app.go index 28e7e8d42..8f3b7b8cf 100644 --- a/poollet/volumepoollet/cmd/volumepoollet/app/app.go +++ b/poollet/volumepoollet/cmd/volumepoollet/app/app.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" ipamv1alpha1 "github.com/ironcore-dev/ironcore/api/ipam/v1alpha1" networkingv1alpha1 "github.com/ironcore-dev/ironcore/api/networking/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" @@ -64,9 +65,13 @@ type Options struct { ProbeAddr string PprofAddr string - VolumePoolName string - VolumeDownwardAPILabels map[string]string - VolumeDownwardAPIAnnotations map[string]string + VolumePoolName string + VolumeDownwardAPILabels map[string]string + VolumeDownwardAPIAnnotations map[string]string + + TopologyRegionLabel string + TopologyZoneLabel string + ProviderID string VolumeRuntimeEndpoint string DialTimeout time.Duration @@ -110,6 +115,10 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) { fs.StringToStringVar(&o.VolumeDownwardAPIAnnotations, "volume-downward-api-annotation", o.VolumeDownwardAPIAnnotations, "Downward-API annotations to set on the IRI volume.") fs.StringToStringVar(&o.VolumeSnapshotDownwardAPILabels, "volume-snapshot-downward-api-label", o.VolumeSnapshotDownwardAPILabels, "Downward-API labels to set on IRI volume snapshot.") fs.StringToStringVar(&o.VolumeSnapshotDownwardAPIAnnotations, "volume-snapshot-downward-api-annotation", o.VolumeSnapshotDownwardAPIAnnotations, "Downward-API annotations to set on the IRI volume snapshot.") + + fs.StringVar(&o.TopologyRegionLabel, "topology-region-label", "", "Label to use for the region topology information.") + fs.StringVar(&o.TopologyZoneLabel, "topology-zone-label", "", "Label to use for the zone topology information.") + fs.StringVar(&o.ProviderID, "provider-id", "", "Provider id to announce on the volume pool.") fs.StringVar(&o.VolumeRuntimeEndpoint, "volume-runtime-endpoint", o.VolumeRuntimeEndpoint, "Endpoint of the remote volume runtime service.") fs.DurationVar(&o.DialTimeout, "dial-timeout", 1*time.Second, "Timeout for dialing to the volume runtime endpoint.") @@ -163,6 +172,14 @@ func Run(ctx context.Context, opts Options) error { logger := ctrl.LoggerFrom(ctx) setupLog := ctrl.Log.WithName("setup") + topologyLabels := map[commonv1alpha1.TopologyLabel]string{} + if opts.TopologyRegionLabel != "" { + topologyLabels[commonv1alpha1.TopologyLabelRegion] = opts.TopologyRegionLabel + } + if opts.TopologyZoneLabel != "" { + topologyLabels[commonv1alpha1.TopologyLabelZone] = opts.TopologyZoneLabel + } + getter, err := volumepoolletconfig.NewGetter(opts.VolumePoolName) if err != nil { setupLog.Error(err, "Error creating new getter") @@ -395,6 +412,7 @@ func Run(ctx context.Context, opts Options) error { VolumePoolName: opts.VolumePoolName, VolumeClassMapper: volumeClassMapper, VolumeRuntime: volumeRuntime, + TopologyLabels: topologyLabels, }).SetupWithManager(mgr); err != nil { return fmt.Errorf("error setting up volume pool reconciler with manager: %w", err) } @@ -414,6 +432,7 @@ func Run(ctx context.Context, opts Options) error { Client: mgr.GetClient(), VolumePoolName: opts.VolumePoolName, ProviderID: opts.ProviderID, + TopologyLabels: topologyLabels, OnInitialized: onInitialized, }).SetupWithManager(mgr); err != nil { return fmt.Errorf("error setting up volume pool init with manager: %w", err) diff --git a/poollet/volumepoollet/controllers/volumepool_controller.go b/poollet/volumepoollet/controllers/volumepool_controller.go index 5e17ccfdb..323db94b1 100644 --- a/poollet/volumepoollet/controllers/volumepool_controller.go +++ b/poollet/volumepoollet/controllers/volumepool_controller.go @@ -14,9 +14,11 @@ import ( "k8s.io/apimachinery/pkg/api/resource" "github.com/go-logr/logr" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" "github.com/ironcore-dev/ironcore/iri/apis/volume" iri "github.com/ironcore-dev/ironcore/iri/apis/volume/v1alpha1" + poolletutils "github.com/ironcore-dev/ironcore/poollet/common/utils" "github.com/ironcore-dev/ironcore/poollet/volumepoollet/vcm" ironcoreclient "github.com/ironcore-dev/ironcore/utils/client" corev1 "k8s.io/api/core/v1" @@ -32,6 +34,8 @@ type VolumePoolReconciler struct { VolumePoolName string VolumeRuntime volume.RuntimeService VolumeClassMapper vcm.VolumeClassMapper + + TopologyLabels map[commonv1alpha1.TopologyLabel]string } //+kubebuilder:rbac:groups=storage.ironcore.dev,resources=volumepools,verbs=get;list;watch;update;patch @@ -143,6 +147,11 @@ func (r *VolumePoolReconciler) reconcile(ctx context.Context, log logr.Logger, v return ctrl.Result{RequeueAfter: 1}, nil } + log.V(1).Info("Enforcing configured topology labels") + if err := r.enforceOriginalTopologyLabels(ctx, log, volumePool); err != nil { + return ctrl.Result{}, fmt.Errorf("error enforcing original topology labels: %w", err) + } + log.V(1).Info("Listing volume classes") volumeClassList := &storagev1alpha1.VolumeClassList{} if err := r.List(ctx, volumeClassList); err != nil { @@ -166,6 +175,14 @@ func (r *VolumePoolReconciler) reconcile(ctx context.Context, log logr.Logger, v return ctrl.Result{}, nil } +func (r *VolumePoolReconciler) enforceOriginalTopologyLabels(ctx context.Context, log logr.Logger, volumePool *storagev1alpha1.VolumePool) error { + base := volumePool.DeepCopy() + + poolletutils.SetTopologyLabels(log, &volumePool.ObjectMeta, r.TopologyLabels) + + return r.Patch(ctx, volumePool, client.MergeFrom(base)) +} + func (r *VolumePoolReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). For( diff --git a/poollet/volumepoollet/controllers/volumepool_controller_test.go b/poollet/volumepoollet/controllers/volumepool_controller_test.go index c992fbb80..159adea35 100644 --- a/poollet/volumepoollet/controllers/volumepool_controller_test.go +++ b/poollet/volumepoollet/controllers/volumepool_controller_test.go @@ -4,16 +4,23 @@ package controllers_test import ( + "time" + + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" corev1alpha1 "github.com/ironcore-dev/ironcore/api/core/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" iri "github.com/ironcore-dev/ironcore/iri/apis/volume/v1alpha1" "github.com/ironcore-dev/ironcore/iri/testing/volume" + "github.com/ironcore-dev/ironcore/poollet/volumepoollet/controllers" + "github.com/ironcore-dev/ironcore/poollet/volumepoollet/vcm" "github.com/ironcore-dev/ironcore/utils/quota" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" ) @@ -284,4 +291,49 @@ var _ = Describe("VolumePoolController", func() { })), )) }) + + It("should enforce topology labels", func(ctx SpecContext) { + By("creating a volume pool with topology labels") + topologyVolumePool := &storagev1alpha1.VolumePool{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "test-topology-vp-", + Labels: map[string]string{ + "topology.ironcore.dev/region": "test-region-1", + "topology.ironcore.dev/zone": "test-zone-1", + }, + }, + } + Expect(k8sClient.Create(ctx, topologyVolumePool)).To(Succeed(), "failed to create topology volume pool") + DeferCleanup(k8sClient.Delete, topologyVolumePool) + + By("setting up a reconciler with topology labels") + topologyReconciler := &controllers.VolumePoolReconciler{ + Client: k8sClient, + VolumeRuntime: srv, + VolumeClassMapper: vcm.NewGeneric(srv, vcm.GenericOptions{RelistPeriod: 2 * time.Second}), + VolumePoolName: topologyVolumePool.Name, + TopologyLabels: map[commonv1alpha1.TopologyLabel]string{ + commonv1alpha1.TopologyLabelRegion: "test-region-1", + commonv1alpha1.TopologyLabelZone: "test-zone-1", + }, + } + + By("patching the volume pool with incorrect topology labels") + Eventually(Update(topologyVolumePool, func() { + topologyVolumePool.Labels = map[string]string{ + "topology.ironcore.dev/region": "wrong-region", + "topology.ironcore.dev/zone": "wrong-zone", + } + })).Should(Succeed()) + + By("manually triggering reconciliation") + _, err := topologyReconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKey{Name: topologyVolumePool.Name}}) + Expect(err).NotTo(HaveOccurred()) + + By("checking if the reconciler resets the topology labels to its original values") + Eventually(Object(topologyVolumePool)).Should(SatisfyAll( + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/region", "test-region-1")), + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/zone", "test-zone-1")), + )) + }) }) diff --git a/poollet/volumepoollet/controllers/volumepool_init.go b/poollet/volumepoollet/controllers/volumepool_init.go index 58cc427ae..35d4ef8d9 100644 --- a/poollet/volumepoollet/controllers/volumepool_init.go +++ b/poollet/volumepoollet/controllers/volumepool_init.go @@ -7,7 +7,9 @@ import ( "context" "fmt" + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" + poolletutils "github.com/ironcore-dev/ironcore/poollet/common/utils" volumepoolletv1alpha1 "github.com/ironcore-dev/ironcore/poollet/volumepoollet/api/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ctrl "sigs.k8s.io/controller-runtime" @@ -20,6 +22,8 @@ type VolumePoolInit struct { VolumePoolName string ProviderID string + TopologyLabels map[commonv1alpha1.TopologyLabel]string + OnInitialized func(ctx context.Context) error OnFailed func(ctx context.Context, reason error) error } @@ -42,6 +46,10 @@ func (i *VolumePoolInit) Start(ctx context.Context) error { ProviderID: i.ProviderID, }, } + + log.V(1).Info("Initially setting topology labels") + poolletutils.SetTopologyLabels(log, &volumePool.ObjectMeta, i.TopologyLabels) + if err := i.Patch(ctx, volumePool, client.Apply, client.ForceOwnership, client.FieldOwner(volumepoolletv1alpha1.FieldOwner)); err != nil { if i.OnFailed != nil { log.V(1).Info("Failed applying, calling OnFailed callback", "Error", err) diff --git a/poollet/volumepoollet/controllers/volumepool_init_test.go b/poollet/volumepoollet/controllers/volumepool_init_test.go new file mode 100644 index 000000000..2c2d95ae3 --- /dev/null +++ b/poollet/volumepoollet/controllers/volumepool_init_test.go @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2023 SAP SE or an SAP affiliate company and IronCore contributors +// SPDX-License-Identifier: Apache-2.0 + +package controllers_test + +import ( + "context" + + commonv1alpha1 "github.com/ironcore-dev/ironcore/api/common/v1alpha1" + storagev1alpha1 "github.com/ironcore-dev/ironcore/api/storage/v1alpha1" + "github.com/ironcore-dev/ironcore/poollet/volumepoollet/controllers" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + . "sigs.k8s.io/controller-runtime/pkg/envtest/komega" +) + +var _ = Describe("VolumePoolInit", func() { + It("should set topology labels", func(ctx SpecContext) { + initializedCalled := false + + vpi := &controllers.VolumePoolInit{ + Client: k8sClient, + VolumePoolName: "test-pool", + ProviderID: "provider-123", + TopologyLabels: map[commonv1alpha1.TopologyLabel]string{ + commonv1alpha1.TopologyLabelRegion: "foo-region-1", + commonv1alpha1.TopologyLabelZone: "foo-zone-1", + }, + OnInitialized: func(ctx context.Context) error { + initializedCalled = true + return nil + }, + } + + Expect(vpi.Start(ctx)).To(Succeed()) + + pool := &storagev1alpha1.VolumePool{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pool", + }, + } + Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(pool), pool)).To(Succeed()) + DeferCleanup(k8sClient.Delete, pool) + + Expect(initializedCalled).To(BeTrue(), "OnInitialized should have been called") + + Eventually(Object(pool)).Should(SatisfyAll( + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/region", "foo-region-1")), + HaveField("ObjectMeta.Labels", HaveKeyWithValue("topology.ironcore.dev/zone", "foo-zone-1")), + )) + }) +})