Add Cilium as an CNI option in tests

Signed-off-by: peppi-lotta <[email protected]>

Author: peppi-lotta

go.mod

@@ -45,7 +45,7 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.3.0 // indirect
 	github.com/go-openapi/jsonpointer v0.21.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
 	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
 	github.com/gobuffalo/flect v1.0.3 // indirect

go.sum

@@ -27,7 +27,6 @@ github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+
 github.com/coredns/corefile-migration v1.0.28 h1:O8YafUREqUcGbRtcJfOmWU6ifcw2HX76I1QvI5xZpsw=
 github.com/coredns/corefile-migration v1.0.28/go.mod h1:56DPqONc3njpVPsdilEnfijCwNGC3/kTJLl7i7SPavY=
 github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@@ -59,12 +58,10 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
 github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
 github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ=
 github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg=
-github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs=
 github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
 github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
-github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE=
-github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
-github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
+github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
 github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
 github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
 github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI=
@@ -111,11 +108,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=

test/e2e/config/e2e_conf.yaml

@@ -243,7 +243,7 @@ providers:
       targetName: "cluster-template-opensuse-leap.yaml"
 
 variables:
-  CNI: "/tmp/calico.yaml"
+  CNI: "/tmp/cni.yaml"
   KUBETEST_CONFIGURATION: "./data/kubetest/conformance.yaml"
   KUBERNETES_VERSION: "${KUBERNETES_VERSION:-v1.34.1}"
   KUBERNETES_VERSION_MINOR: "${KUBERNETES_VERSION%.*}"
@@ -274,8 +274,11 @@ variables:
   UPGRADED_BMO_IMAGE_TAG: "${UPGRADED_BMO_IMAGE_TAG:-main}"
 
   PROVIDER_ID_FORMAT: "metal3://{{ ds.meta_data.providerid }}"
+  CNI_NAME: "${CNI_NAME:-calico}"
   # Pin Calico version
   CALICO_VERSION: "${CALICO_VERSION:-v3.30.3}"
+  # Pin Cilium version 
+  CILIUM_VERSION: "${CILIUM_VERSION:-v1.18.0}"
   # Pin CertManager for upgrade tests
   CERT_MANAGER_RELEASE: v1.17.1
   # Default vars for the template, those values could be overridden by the env-vars.

test/e2e/e2e_suite_test.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/blang/semver/v4"
 	"github.com/jinzhu/copier"
 	bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
 	infrav1 "github.com/metal3-io/cluster-api-provider-metal3/api/v1beta1"
@@ -18,6 +19,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"gopkg.in/yaml.v3"
+	"helm.sh/helm/v3/pkg/cli"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/klog/v2"
 	clusterv1beta1 "sigs.k8s.io/cluster-api/api/core/v1beta1"
@@ -191,13 +193,22 @@ func CreateClusterctlLocalRepository(config *clusterctl.E2EConfig, repositoryFol
 
 	// Ensuring a CNI file is defined in the config and register a FileTransformation to inject the referenced file as in place of the CNI_RESOURCES envSubst variable.
 	Expect(config.Variables).To(HaveKey(capi_e2e.CNIPath), "Missing %s variable in the config", capi_e2e.CNIPath)
+	cniName := config.MustGetVariable("CNI_NAME")
 	cniPath := config.MustGetVariable(capi_e2e.CNIPath)
-	switch osType {
-	case osTypeLeap:
-		updateCalico(config, cniPath, "eth1")
+	cniInterface := "enp2s0"
+	if osType == osTypeLeap {
+		cniInterface = "eth1"
+	}
+
+	switch cniName {
+	case "cilium":
+		updateCilium(config, cniPath)
+	case "calico":
+		updateCalico(config, cniPath, cniInterface)
 	default:
-		updateCalico(config, cniPath, "enp2s0")
+		Expect(cniName).To(Or(Equal("calico"), Equal("cilium")), "Invalid CNI type %q, only 'cilium' and 'calico' are supported", cniName)
 	}
+
 	Expect(cniPath).To(BeAnExistingFile(), "The %s variable should resolve to an existing file", capi_e2e.CNIPath)
 	createRepositoryInput.RegisterClusterResourceSetConfigMapTransformation(cniPath, capi_e2e.CNIResources)
 
@@ -300,7 +311,7 @@ func updateCalico(config *clusterctl.E2EConfig, calicoYaml, calicoInterface stri
 		err = copier.CopyWithOption(addItem, calicoNodeContainerEnvs.Content[0], copier.Option{IgnoreEmpty: true, DeepCopy: true})
 		Expect(err).ToNot(HaveOccurred())
 		addItem.Content[1].SetString("IP_AUTODETECTION_METHOD")
-		addItem.Content[3].SetString("interface=" + calicoInterface)
+		addItem.Content[3].SetString("interface=" + cniInterface)
 		addItem.HeadComment = "Start section modified by CAPM3 e2e test framework"
 		addItem.FootComment = "End section modified by CAPM3 e2e test framework"
 		calicoNodeContainerEnvs.Content = append(calicoNodeContainerEnvs.Content, addItem)
@@ -312,6 +323,53 @@ func updateCalico(config *clusterctl.E2EConfig, calicoYaml, calicoInterface stri
 	Expect(err).ToNot(HaveOccurred(), "Cannot print out the update to the file")
 }
 
+// updateCilium generates and writes a Cilium CNI manifest to the CNI path specified in e2e config.
+// It retrieves the Cilium version from e2e configuration, downloads the corresponding Helm chart, generates a manifest from the chart template, and writes the manifest to the CNI path.
+func updateCilium(config *clusterctl.E2EConfig, cniPath string) {
+	ctx = context.Background()
+	ciliumVersion := e2eConfig.MustGetVariable("CILIUM_VERSION")
+	if ciliumVersion[0] == 'v' {
+		ciliumVersion = ciliumVersion[1:]
+	}
+	settings := cli.New()
+	settings.SetNamespace("kube-system")
+	helmDriver := os.Getenv("HELM_DRIVER")
+	opts := HelmOpts{
+		Logger:         log.Default(),
+		Settings:       settings,
+		ReleaseName:    "cilium",
+		ChartRef:       fmt.Sprintf("https://helm.cilium.io/cilium-%s.tgz", ciliumVersion),
+		ChartLocation:  fmt.Sprintf("/tmp/cilium-%s.tgz", ciliumVersion),
+		ReleaseVersion: semver.MustParse(ciliumVersion),
+		Driver:         helmDriver,
+	}
+
+	manifestOverwriteValues := map[string]interface{}{
+		"operator": map[string]interface{}{
+			"replicas": 1,
+			"updateStrategy": map[string]interface{}{
+				"rollingUpdate": map[string]interface{}{
+					"maxUnavailable": "100%",
+				},
+			},
+		},
+	}
+
+	manifest, err := generateTemplateFromHelmChart(ctx, opts, manifestOverwriteValues, e2eConfig)
+	Expect(err).ToNot(HaveOccurred(), "failed to generate template: %v", err)
+
+	// Replace ${BIN_PATH} with /opt/cni/bin. This is done to prevent
+	// framework.RegisterClusterResourceSetConfigMapTransformation from throwing
+	// an error due to unresolvable "envsubst" variable.
+	manifest = strings.ReplaceAll(manifest, "${BIN_PATH}", "/opt/cni/bin")
+
+	containerRegistry := config.MustGetVariable("CONTAINER_REGISTRY")
+	manifest = strings.ReplaceAll(manifest, "quay.io", containerRegistry)
+
+	err = os.WriteFile(cniPath, []byte(manifest), 0600)
+	Expect(err).ToNot(HaveOccurred(), "Failed to write Cilium manifest to file: %v", err)
+}
+
 // createBMHsInNamespace is a hook function that can be called after creating
 // a namespace, it creates the needed bmhs in the namespace hosting the cluster.
 func createBMHsInNamespace(clusterProxy framework.ClusterProxy, clusterNamespace string) {

test/e2e/helm_helper.go

@@ -0,0 +1,132 @@
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/blang/semver/v4"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/chart/loader"
+	"helm.sh/helm/v3/pkg/chartutil"
+	"helm.sh/helm/v3/pkg/cli"
+	"helm.sh/helm/v3/pkg/downloader"
+	"helm.sh/helm/v3/pkg/getter"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+)
+
+type HelmOpts struct {
+	Logger         *log.Logger
+	Settings       *cli.EnvSettings
+	ReleaseName    string
+	ChartRef       string
+	ChartLocation  string // Added to specify the chart location
+	ReleaseVersion semver.Version
+	Driver         string
+}
+
+func generateTemplateFromHelmChart(ctx context.Context, opts HelmOpts, manifestOverwriteValues map[string]interface{}, e2econfig *clusterctl.E2EConfig) (string, error) {
+	actionConfig, err := initActionConfig(opts)
+	if err != nil {
+		return "", fmt.Errorf("failed to init action config: %w", err)
+	}
+
+	kubeversion, err := chartutil.ParseKubeVersion(e2econfig.MustGetVariable("KUBERNETES_VERSION"))
+	if err != nil {
+		return "", fmt.Errorf("failed to parse kube version: %w", err)
+	}
+
+	pullClient := action.NewPullWithOpts(
+		action.WithConfig(actionConfig))
+	pullClient.DestDir = "/tmp/"
+	pullClient.Settings = opts.Settings
+	pullClient.Version = opts.ReleaseVersion.String()
+
+	_, err = pullClient.Run(opts.ChartRef)
+	if err != nil {
+		return "", fmt.Errorf("failed to pull chart: %w", err)
+	}
+
+	installClient := action.NewInstall(actionConfig)
+	installClient.DryRun = true
+	installClient.ClientOnly = true
+	installClient.Replace = true
+	installClient.ReleaseName = opts.ReleaseName
+	installClient.Namespace = opts.Settings.Namespace()
+	installClient.Version = opts.ReleaseVersion.String()
+	installClient.KubeVersion = kubeversion
+
+	chartPath, err := installClient.ChartPathOptions.LocateChart(opts.ChartLocation, opts.Settings)
+	if err != nil {
+		return "", err
+	}
+
+	providers := getter.All(opts.Settings)
+
+	chart, err := loader.Load(chartPath)
+	if err != nil {
+		return "", err
+	}
+
+	// Check chart dependencies
+	if chartDependencies := chart.Metadata.Dependencies; chartDependencies != nil {
+		if err = action.CheckDependencies(chart, chartDependencies); err != nil {
+			err = fmt.Errorf("failed to check chart dependencies: %w", err)
+			if !installClient.DependencyUpdate {
+				return "", err
+			}
+
+			manager := &downloader.Manager{
+				Out:              opts.Logger.Writer(),
+				ChartPath:        chartPath,
+				Keyring:          installClient.ChartPathOptions.Keyring,
+				SkipUpdate:       false,
+				Getters:          providers,
+				RepositoryConfig: opts.Settings.RepositoryConfig,
+				RepositoryCache:  opts.Settings.RepositoryCache,
+				Debug:            opts.Settings.Debug,
+				RegistryClient:   installClient.GetRegistryClient(),
+			}
+			if err = manager.Update(); err != nil {
+				return "", err
+			}
+			// Reload the chart with the updated Chart.lock file.
+			if chart, err = loader.Load(chartPath); err != nil {
+				return "", fmt.Errorf("failed to reload chart after repo update: %w", err)
+			}
+		}
+	}
+
+	release, err := installClient.RunWithContext(ctx, chart, manifestOverwriteValues)
+	if err != nil {
+		return "", fmt.Errorf("failed to run install: %w", err)
+	}
+
+	return release.Manifest, nil
+}
+
+func initActionConfig(opts HelmOpts) (*action.Configuration, error) {
+	return initActionConfigList(opts, false)
+}
+
+func initActionConfigList(opts HelmOpts, allNamespaces bool) (*action.Configuration, error) {
+	actionConfig := new(action.Configuration)
+	namespace := func() string {
+		// For list action, you can pass an empty string instead of settings.Namespace() to list
+		// all namespaces
+		if allNamespaces {
+			return ""
+		}
+		return opts.Settings.Namespace()
+	}()
+
+	if err := actionConfig.Init(
+		opts.Settings.RESTClientGetter(),
+		namespace,
+		opts.Driver,
+		opts.Logger.Printf); err != nil {
+		return nil, err
+	}
+
+	return actionConfig, nil
+}