IPPool sync to address set

charts/kube-ovn-v2/crds/kube-ovn-crd.yaml

@@ -2682,6 +2682,9 @@ spec:
         - name: Subnet
           type: string
           jsonPath: .spec.subnet
+        - name: enableAddressSet
+          type: boolean
+          jsonPath: .spec.enableAddressSet
         - name: IPs
           type: string
           jsonPath: .spec.ips
@@ -2726,6 +2729,10 @@ spec:
                       - format: cidr
                       - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.\.(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$
                       - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))\.\.((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$
+                enableAddressSet:
+                  type: boolean
+                  default: false
+                  description: EnableAddressSet to work with policy-based routing and ACL
               required:
                 - subnet
                 - ips

charts/kube-ovn/templates/kube-ovn-crd.yaml

@@ -2682,6 +2682,9 @@ spec:
       - name: Subnet
         type: string
         jsonPath: .spec.subnet
+      - name: enableAddressSet
+        type: boolean
+        jsonPath: .spec.enableAddressSet
       - name: IPs
         type: string
         jsonPath: .spec.ips
@@ -2726,6 +2729,10 @@ spec:
                       - format: cidr
                       - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.\.(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$
                       - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))\.\.((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$
+                enableAddressSet:
+                  type: boolean
+                  default: false
+                  description: EnableAddressSet to work with policy-based routing and ACL
               required:
                 - subnet
                 - ips

dist/images/install.sh

@@ -2932,6 +2932,9 @@ spec:
       - name: Subnet
         type: string
         jsonPath: .spec.subnet
+      - name: enableAddressSet
+        type: boolean
+        jsonPath: .spec.enableAddressSet
       - name: IPs
         type: string
         jsonPath: .spec.ips
@@ -2976,6 +2979,10 @@ spec:
                       - format: cidr
                       - pattern: ^(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.\.(?:(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])\.){3}(?:[01]?\d{1,2}|2[0-4]\d|25[0-5])$
                       - pattern: ^((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))\.\.((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|:)))$
+                enableAddressSet:
+                  type: boolean
+                  default: false
+                  description: EnableAddressSet to work with policy-based routing and ACL
               required:
                 - subnet
                 - ips

makefiles/build.mk

@@ -10,6 +10,13 @@ else
 GO_BUILD_FLAGS = -trimpath -ldflags "-w -s $(GOLDFLAGS)"
 endif
 
+GO_MOD_VERSION := $(shell awk '/^go[[:space:]]+/ { print $$2; exit }' go.mod)
+ifeq ($(strip $(GO_MOD_VERSION)),)
+$(error failed to determine Go version from go.mod)
+endif
+GOTOOLCHAIN_VERSION := go$(GO_MOD_VERSION)
+MODERNIZE_ENV := GOTOOLCHAIN=$(GOTOOLCHAIN_VERSION)
+
 .PHONY: build-go
 build-go:
 	go mod tidy
@@ -147,11 +154,11 @@ lint:
     ifeq ($(CI),true)
 		@echo "Running in GitHub Actions"
 		golangci-lint run -v
-		go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest -test ./...
+		$(MODERNIZE_ENV) go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest -test ./...
     else
 		@echo "Running in local environment"
 		golangci-lint run -v --fix
-		go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest -test --fix ./...
+		$(MODERNIZE_ENV) go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest -test --fix ./...
     endif
 
 .PHONY: lint-windows

pkg/apis/kubeovn/v1/ippool.go

@@ -34,6 +34,8 @@ type IPPoolSpec struct {
 	Subnet     string   `json:"subnet,omitempty"`
 	Namespaces []string `json:"namespaces,omitempty"`
 	IPs        []string `json:"ips,omitempty"`
+	// EnableAddressSet to work with policy-based routing and ACL
+	EnableAddressSet bool `json:"enableAddressSet,omitempty"`
 }
 
 type IPPoolStatus struct {
@@ -110,7 +112,6 @@ func (s *IPPoolStatus) EnsureCondition(ctype ConditionType) {
 // EnsureStandardConditions - helper to inject standard conditions
 func (s *IPPoolStatus) EnsureStandardConditions() {
 	s.EnsureCondition(Ready)
-	s.EnsureCondition(Error)
 }
 
 // SetCondition updates or creates a new condition
@@ -138,11 +139,6 @@ func (s *IPPoolStatus) SetError(reason, message string) {
 	s.SetCondition(Error, reason, message)
 }
 
-// ClearError - shortcut to set error condition
-func (s *IPPoolStatus) ClearError() {
-	s.ClearCondition(Error, "NoError", "No error seen")
-}
-
 // IsConditionTrue - if condition is true
 func (s IPPoolStatus) IsConditionTrue(ctype ConditionType) bool {
 	if c := s.GetCondition(ctype); c != nil {

pkg/controller/controller.go

@@ -56,6 +56,7 @@ const (
 	u2oKey                        = "u2o"
 	adminNetworkPolicyKey         = "anp"
 	baselineAdminNetworkPolicyKey = "banp"
+	ippoolKey                     = "ippool"
 )
 
 // Controller is kube-ovn main controller that watch ns/pod/node/svc/ep and operate ovn

pkg/controller/init.go

@@ -983,6 +983,10 @@ func (c *Controller) syncFinalizers() error {
 		klog.Errorf("failed to sync ip finalizer: %v", err)
 		return err
 	}
+	if err := c.syncIPPoolFinalizer(cl); err != nil {
+		klog.Errorf("failed to sync ippool finalizer: %v", err)
+		return err
+	}
 	if err := c.syncOvnDnatFinalizer(cl); err != nil {
 		klog.Errorf("failed to sync ovn dnat finalizer: %v", err)
 		return err

pkg/controller/ippool.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"context"
+	"fmt"
 	"reflect"
 	"slices"
 
@@ -12,6 +13,8 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
 	"github.com/kubeovn/kube-ovn/pkg/util"
@@ -47,8 +50,14 @@ func (c *Controller) enqueueDeleteIPPool(obj any) {
 func (c *Controller) enqueueUpdateIPPool(oldObj, newObj any) {
 	oldIPPool := oldObj.(*kubeovnv1.IPPool)
 	newIPPool := newObj.(*kubeovnv1.IPPool)
+	if !newIPPool.DeletionTimestamp.IsZero() {
+		klog.V(3).Infof("enqueue delete ippool %s due to deletion timestamp", cache.MetaObjectToName(newIPPool).String())
+		c.deleteIPPoolQueue.Add(newIPPool.DeepCopy())
+		return
+	}
 	if !slices.Equal(oldIPPool.Spec.Namespaces, newIPPool.Spec.Namespaces) ||
-		!slices.Equal(oldIPPool.Spec.IPs, newIPPool.Spec.IPs) {
+		!slices.Equal(oldIPPool.Spec.IPs, newIPPool.Spec.IPs) ||
+		oldIPPool.Spec.EnableAddressSet != newIPPool.Spec.EnableAddressSet {
 		key := cache.MetaObjectToName(newIPPool).String()
 		klog.V(3).Infof("enqueue update ippool %s", key)
 		c.addOrUpdateIPPoolQueue.Add(key)
@@ -70,7 +79,22 @@ func (c *Controller) handleAddOrUpdateIPPool(key string) error {
 	klog.Infof("handle add/update ippool %s", cachedIPPool.Name)
 
 	ippool := cachedIPPool.DeepCopy()
+	if err = c.handleAddIPPoolFinalizer(ippool); err != nil {
+		klog.Errorf("failed to add finalizer for ippool %s: %v", ippool.Name, err)
+		return err
+	}
+	if !ippool.DeletionTimestamp.IsZero() {
+		klog.Infof("ippool %s is being deleted, skip add/update handling", ippool.Name)
+		return nil
+	}
 	ippool.Status.EnsureStandardConditions()
+	if err = c.reconcileIPPoolAddressSet(ippool); err != nil {
+		klog.Errorf("failed to reconcile address set for ippool %s: %v", ippool.Name, err)
+		if patchErr := c.patchIPPoolStatusCondition(ippool, "ReconcileAddressSetFailed", err.Error()); patchErr != nil {
+			klog.Error(patchErr)
+		}
+		return err
+	}
 	if err = c.ipam.AddOrUpdateIPPool(ippool.Spec.Subnet, ippool.Name, ippool.Spec.IPs); err != nil {
 		klog.Errorf("failed to add/update ippool %s with IPs %v in subnet %s: %v", ippool.Name, ippool.Spec.IPs, ippool.Spec.Subnet, err)
 		if patchErr := c.patchIPPoolStatusCondition(ippool, "UpdateIPAMFailed", err.Error()); patchErr != nil {
@@ -79,15 +103,7 @@ func (c *Controller) handleAddOrUpdateIPPool(key string) error {
 		return err
 	}
 
-	v4a, v4u, v6a, v6u, v4as, v4us, v6as, v6us := c.ipam.IPPoolStatistics(ippool.Spec.Subnet, ippool.Name)
-	ippool.Status.V4AvailableIPs = v4a
-	ippool.Status.V4UsingIPs = v4u
-	ippool.Status.V6AvailableIPs = v6a
-	ippool.Status.V6UsingIPs = v6u
-	ippool.Status.V4AvailableIPRange = v4as
-	ippool.Status.V4UsingIPRange = v4us
-	ippool.Status.V6AvailableIPRange = v6as
-	ippool.Status.V6UsingIPRange = v6us
+	c.updateIPPoolStatistics(ippool)
 
 	if err = c.patchIPPoolStatusCondition(ippool, "UpdateIPAMSucceeded", ""); err != nil {
 		klog.Error(err)
@@ -107,6 +123,10 @@ func (c *Controller) handleDeleteIPPool(ippool *kubeovnv1.IPPool) error {
 
 	klog.Infof("handle delete ippool %s", ippool.Name)
 	c.ipam.RemoveIPPool(ippool.Spec.Subnet, ippool.Name)
+	if err := c.OVNNbClient.DeleteAddressSet(util.IPPoolAddressSetName(ippool.Name)); err != nil {
+		klog.Errorf("failed to delete address set for ippool %s: %v", ippool.Name, err)
+		return err
+	}
 
 	namespaces, err := c.namespacesLister.List(labels.Everything())
 	if err != nil {
@@ -115,14 +135,16 @@ func (c *Controller) handleDeleteIPPool(ippool *kubeovnv1.IPPool) error {
 	}
 
 	for _, ns := range namespaces {
-		if len(ns.Annotations) == 0 {
-			continue
-		}
 		if ns.Annotations[util.IPPoolAnnotation] == ippool.Name {
 			c.enqueueAddNamespace(ns)
 		}
 	}
 
+	if err := c.handleDelIPPoolFinalizer(ippool); err != nil {
+		klog.Errorf("failed to remove finalizer for ippool %s: %v", ippool.Name, err)
+		return err
+	}
+
 	return nil
 }
 
@@ -140,15 +162,7 @@ func (c *Controller) handleUpdateIPPoolStatus(key string) error {
 	}
 
 	ippool := cachedIPPool.DeepCopy()
-	v4a, v4u, v6a, v6u, v4as, v4us, v6as, v6us := c.ipam.IPPoolStatistics(ippool.Spec.Subnet, ippool.Name)
-	ippool.Status.V4AvailableIPs = v4a
-	ippool.Status.V4UsingIPs = v4u
-	ippool.Status.V6AvailableIPs = v6a
-	ippool.Status.V6UsingIPs = v6u
-	ippool.Status.V4AvailableIPRange = v4as
-	ippool.Status.V4UsingIPRange = v4us
-	ippool.Status.V6AvailableIPRange = v6as
-	ippool.Status.V6UsingIPRange = v6us
+	c.updateIPPoolStatistics(ippool)
 	if reflect.DeepEqual(ippool.Status, cachedIPPool.Status) {
 		return nil
 	}
@@ -163,7 +177,6 @@ func (c Controller) patchIPPoolStatusCondition(ippool *kubeovnv1.IPPool, reason,
 		c.recorder.Eventf(ippool, corev1.EventTypeWarning, reason, errMsg)
 	} else {
 		ippool.Status.Ready(reason, "")
-		c.recorder.Eventf(ippool, corev1.EventTypeNormal, reason, errMsg)
 	}
 
 	return c.patchIPPoolStatus(ippool)
@@ -182,3 +195,109 @@ func (c Controller) patchIPPoolStatus(ippool *kubeovnv1.IPPool) error {
 
 	return nil
 }
+
+func (c *Controller) syncIPPoolFinalizer(cl client.Client) error {
+	ippools := &kubeovnv1.IPPoolList{}
+	return migrateFinalizers(cl, ippools, func(i int) (client.Object, client.Object) {
+		if i < 0 || i >= len(ippools.Items) {
+			return nil, nil
+		}
+		return ippools.Items[i].DeepCopy(), ippools.Items[i].DeepCopy()
+	})
+}
+
+func (c *Controller) handleAddIPPoolFinalizer(ippool *kubeovnv1.IPPool) error {
+	if ippool == nil || !ippool.DeletionTimestamp.IsZero() {
+		return nil
+	}
+	if controllerutil.ContainsFinalizer(ippool, util.KubeOVNControllerFinalizer) {
+		return nil
+	}
+
+	newIPPool := ippool.DeepCopy()
+	controllerutil.AddFinalizer(newIPPool, util.KubeOVNControllerFinalizer)
+	patch, err := util.GenerateMergePatchPayload(ippool, newIPPool)
+	if err != nil {
+		klog.Errorf("failed to generate patch payload for ippool %s: %v", ippool.Name, err)
+		return err
+	}
+	if _, err = c.config.KubeOvnClient.KubeovnV1().IPPools().Patch(context.Background(), ippool.Name,
+		types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		klog.Errorf("failed to add finalizer for ippool %s: %v", ippool.Name, err)
+		return err
+	}
+	return nil
+}
+
+func (c *Controller) handleDelIPPoolFinalizer(ippool *kubeovnv1.IPPool) error {
+	if ippool == nil || len(ippool.GetFinalizers()) == 0 {
+		return nil
+	}
+
+	newIPPool := ippool.DeepCopy()
+	controllerutil.RemoveFinalizer(newIPPool, util.DepreciatedFinalizerName)
+	controllerutil.RemoveFinalizer(newIPPool, util.KubeOVNControllerFinalizer)
+	patch, err := util.GenerateMergePatchPayload(ippool, newIPPool)
+	if err != nil {
+		klog.Errorf("failed to generate patch payload for ippool %s: %v", ippool.Name, err)
+		return err
+	}
+	if _, err = c.config.KubeOvnClient.KubeovnV1().IPPools().Patch(context.Background(), ippool.Name,
+		types.MergePatchType, patch, metav1.PatchOptions{}, ""); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		klog.Errorf("failed to remove finalizer from ippool %s: %v", ippool.Name, err)
+		return err
+	}
+	return nil
+}
+
+func (c *Controller) updateIPPoolStatistics(ippool *kubeovnv1.IPPool) {
+	v4a, v4u, v6a, v6u, v4as, v4us, v6as, v6us := c.ipam.IPPoolStatistics(ippool.Spec.Subnet, ippool.Name)
+	ippool.Status.V4AvailableIPs = v4a
+	ippool.Status.V4UsingIPs = v4u
+	ippool.Status.V6AvailableIPs = v6a
+	ippool.Status.V6UsingIPs = v6u
+	ippool.Status.V4AvailableIPRange = v4as
+	ippool.Status.V4UsingIPRange = v4us
+	ippool.Status.V6AvailableIPRange = v6as
+	ippool.Status.V6UsingIPRange = v6us
+}
+
+func (c *Controller) reconcileIPPoolAddressSet(ippool *kubeovnv1.IPPool) error {
+	asName := util.IPPoolAddressSetName(ippool.Name)
+
+	if !ippool.Spec.EnableAddressSet {
+		if err := c.OVNNbClient.DeleteAddressSet(asName); err != nil {
+			err = fmt.Errorf("failed to delete address set %s: %w", asName, err)
+			klog.Error(err)
+			return err
+		}
+		return nil
+	}
+
+	addresses, err := util.ExpandIPPoolAddressesForOVN(ippool.Spec.IPs)
+	if err != nil {
+		err = fmt.Errorf("failed to build address set entries for ippool %s: %w", ippool.Name, err)
+		klog.Error(err)
+		return err
+	}
+
+	if err := c.OVNNbClient.CreateAddressSet(asName, map[string]string{ippoolKey: ippool.Name}); err != nil {
+		err = fmt.Errorf("failed to create address set for ippool %s: %w", ippool.Name, err)
+		klog.Error(err)
+		return err
+	}
+
+	if err := c.OVNNbClient.AddressSetUpdateAddress(asName, addresses...); err != nil {
+		err = fmt.Errorf("failed to update address set for ippool %s: %w", ippool.Name, err)
+		klog.Error(err)
+		return err
+	}
+
+	return nil
+}