From d7c3afedce251739d48631ebbeb8fc74fdaacc78 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Fri, 26 Jan 2024 21:07:16 +0000 Subject: [PATCH 1/2] Add Tolerations spec to chart configuration Signed-off-by: Daniel Miranda --- pkg/apis/helm.cattle.io/v1/types.go | 2 + pkg/controllers/chart/chart.go | 57 ++++--- test/suite/helm_test.go | 253 ++++++++++++++++++++++++++++ 3 files changed, 286 insertions(+), 26 deletions(-) diff --git a/pkg/apis/helm.cattle.io/v1/types.go b/pkg/apis/helm.cattle.io/v1/types.go index 3172ae9a..f48ad789 100644 --- a/pkg/apis/helm.cattle.io/v1/types.go +++ b/pkg/apis/helm.cattle.io/v1/types.go @@ -42,6 +42,8 @@ type HelmChartSpec struct { PodSecurityContext *corev1.PodSecurityContext `json:"podSecurityContext,omitempty"` SecurityContext *corev1.SecurityContext `json:"securityContext,omitempty"` + + Tolerations []corev1.Toleration `json:"tolerations,omitempty"` } type HelmChartStatus struct { diff --git a/pkg/controllers/chart/chart.go b/pkg/controllers/chart/chart.go index 291652d1..10cb21b8 100644 --- a/pkg/controllers/chart/chart.go +++ b/pkg/controllers/chart/chart.go @@ -74,6 +74,35 @@ var ( } ) +var ( + bootstrapTolerations = []corev1.Toleration{ + { + Key: corev1.TaintNodeNotReady, + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: TaintExternalCloudProvider, + Operator: corev1.TolerationOpEqual, + Value: "true", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "CriticalAddonsOnly", + Operator: corev1.TolerationOpExists, + }, + { + Key: LabelNodeRolePrefix + LabelEtcdSuffix, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoExecute, + }, + { + Key: LabelNodeRolePrefix + LabelControlPlaneSuffix, + Operator: corev1.TolerationOpExists, + Effect: corev1.TaintEffectNoSchedule, + }, + } +) + type Controller struct { systemNamespace string jobClusterRole string @@ -525,32 +554,7 @@ func job(chart *v1.HelmChart, apiServerPort string) (*batch.Job, *corev1.Secret, if chart.Spec.Bootstrap { job.Spec.Template.Spec.NodeSelector[LabelNodeRolePrefix+LabelControlPlaneSuffix] = "true" job.Spec.Template.Spec.HostNetwork = true - job.Spec.Template.Spec.Tolerations = []corev1.Toleration{ - { - Key: corev1.TaintNodeNotReady, - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: TaintExternalCloudProvider, - Operator: corev1.TolerationOpEqual, - Value: "true", - Effect: corev1.TaintEffectNoSchedule, - }, - { - Key: "CriticalAddonsOnly", - Operator: corev1.TolerationOpExists, - }, - { - Key: LabelNodeRolePrefix + LabelEtcdSuffix, - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoExecute, - }, - { - Key: LabelNodeRolePrefix + LabelControlPlaneSuffix, - Operator: corev1.TolerationOpExists, - Effect: corev1.TaintEffectNoSchedule, - }, - } + job.Spec.Template.Spec.Tolerations = bootstrapTolerations job.Spec.Template.Spec.Containers[0].Env = append(job.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{ { Name: "KUBERNETES_SERVICE_HOST", @@ -563,6 +567,7 @@ func job(chart *v1.HelmChart, apiServerPort string) (*batch.Job, *corev1.Secret, Value: "true"}, }...) } + job.Spec.Template.Spec.Tolerations = append(job.Spec.Template.Spec.Tolerations, chart.Spec.Tolerations...) setProxyEnv(job) setAuthSecret(job, chart) diff --git a/test/suite/helm_test.go b/test/suite/helm_test.go index 340d858f..e5b58c84 100644 --- a/test/suite/helm_test.go +++ b/test/suite/helm_test.go @@ -18,6 +18,33 @@ import ( "k8s.io/utils/pointer" ) +var bootstrapTolerations = &[]corev1.Toleration{ + { + Key: "node.kubernetes.io/not-ready", + Effect: "NoSchedule", + }, + { + Key: "node.cloudprovider.kubernetes.io/uninitialized", + Operator: "Equal", + Value: "true", + Effect: "NoSchedule", + }, + { + Key: "CriticalAddonsOnly", + Operator: "Exists", + }, + { + Key: "node-role.kubernetes.io/etcd", + Operator: "Exists", + Effect: "NoExecute", + }, + { + Key: "node-role.kubernetes.io/control-plane", + Operator: "Exists", + Effect: "NoSchedule", + }, +} + var _ = Describe("Helm Tests", Ordered, func() { framework, _ := framework.New() @@ -755,4 +782,230 @@ var _ = Describe("Helm Tests", Ordered, func() { }, 120*time.Second, 5*time.Second).Should(BeTrue()) }) }) + + Context("When no tolerations are specified (non-boostrap)", func() { + var ( + err error + chart *v1.HelmChart + job *batchv1.Job + ) + BeforeEach(func() { + chart = framework.NewHelmChart("traefik-example-no-tolerations", + "stable/traefik", + "1.86.1", + "v3", + map[string]intstr.IntOrString{ + "rbac.enabled": { + Type: intstr.String, + StrVal: "true", + }, + "ssl.enabled": { + Type: intstr.String, + StrVal: "true", + }, + }) + chart, err = framework.CreateHelmChart(chart, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + labelSelector := labels.SelectorFromSet(labels.Set{ + "owner": "helm", + "name": chart.Name, + }) + _, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1) + Expect(err).ToNot(HaveOccurred()) + + chart, err = framework.GetHelmChart(chart.Name, chart.Namespace) + Expect(err).ToNot(HaveOccurred()) + job, err = framework.GetJob(chart) + Expect(err).ToNot(HaveOccurred()) + }) + FIt("Should have no tolerations set", func() { + Expect(job.Spec.Template.Spec.Tolerations).To(BeEmpty()) + }) + AfterEach(func() { + err = framework.DeleteHelmChart(chart.Name, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + _, err := framework.GetHelmChart(chart.Name, framework.Namespace) + return err != nil && apierrors.IsNotFound(err) + }, 120*time.Second, 5*time.Second).Should(BeTrue()) + }) + }) + + Context("When custom tolerations are specified (non-boostrap)", func() { + var ( + err error + chart *v1.HelmChart + job *batchv1.Job + tolerations = &[]corev1.Toleration{ + { + Key: "test-1", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "test-2", + Effect: corev1.TaintEffectNoExecute, + }, + } + ) + BeforeEach(func() { + chart = framework.NewHelmChart("traefik-example-custom-tolerations", + "stable/traefik", + "1.86.1", + "v3", + map[string]intstr.IntOrString{ + "rbac.enabled": { + Type: intstr.String, + StrVal: "true", + }, + "ssl.enabled": { + Type: intstr.String, + StrVal: "true", + }, + }) + chart.Spec.Tolerations = *tolerations + chart, err = framework.CreateHelmChart(chart, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + labelSelector := labels.SelectorFromSet(labels.Set{ + "owner": "helm", + "name": chart.Name, + }) + _, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1) + Expect(err).ToNot(HaveOccurred()) + + chart, err = framework.GetHelmChart(chart.Name, chart.Namespace) + Expect(err).ToNot(HaveOccurred()) + job, err = framework.GetJob(chart) + Expect(err).ToNot(HaveOccurred()) + }) + FIt("Should have tolerations set", func() { + Expect(job.Spec.Template.Spec.Tolerations).To(Equal(*tolerations)) + }) + AfterEach(func() { + err = framework.DeleteHelmChart(chart.Name, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + _, err := framework.GetHelmChart(chart.Name, framework.Namespace) + return err != nil && apierrors.IsNotFound(err) + }, 120*time.Second, 5*time.Second).Should(BeTrue()) + }) + }) + + Context("When no tolerations are specified (boostrap)", func() { + var ( + err error + chart *v1.HelmChart + job *batchv1.Job + ) + BeforeEach(func() { + chart = framework.NewHelmChart("traefik-example-bootstrap-default-tolerations", + "stable/traefik", + "1.86.1", + "v3", + map[string]intstr.IntOrString{ + "rbac.enabled": { + Type: intstr.String, + StrVal: "true", + }, + "ssl.enabled": { + Type: intstr.String, + StrVal: "true", + }, + }) + chart.Spec.Bootstrap = true + chart, err = framework.CreateHelmChart(chart, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + labelSelector := labels.SelectorFromSet(labels.Set{ + "owner": "helm", + "name": chart.Name, + }) + _, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1) + Expect(err).ToNot(HaveOccurred()) + + chart, err = framework.GetHelmChart(chart.Name, chart.Namespace) + Expect(err).ToNot(HaveOccurred()) + job, err = framework.GetJob(chart) + Expect(err).ToNot(HaveOccurred()) + }) + FIt("Should have default tolerations set", func() { + Expect(job.Spec.Template.Spec.Tolerations).To(Equal(*bootstrapTolerations)) + }) + AfterEach(func() { + err = framework.DeleteHelmChart(chart.Name, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + _, err := framework.GetHelmChart(chart.Name, framework.Namespace) + return err != nil && apierrors.IsNotFound(err) + }, 120*time.Second, 5*time.Second).Should(BeTrue()) + }) + }) + + Context("When custom tolerations are specified (boostrap)", func() { + var ( + err error + chart *v1.HelmChart + job *batchv1.Job + specTolerations = &[]corev1.Toleration{ + { + Key: "test-1", + Effect: corev1.TaintEffectNoSchedule, + }, + { + Key: "test-2", + Effect: corev1.TaintEffectNoExecute, + }, + } + finalTolerations = append(*bootstrapTolerations, *specTolerations...) + ) + BeforeEach(func() { + chart = framework.NewHelmChart("traefik-example-bootstrap-custom-tolerations", + "stable/traefik", + "1.86.1", + "v3", + map[string]intstr.IntOrString{ + "rbac.enabled": { + Type: intstr.String, + StrVal: "true", + }, + "ssl.enabled": { + Type: intstr.String, + StrVal: "true", + }, + }) + chart.Spec.Tolerations = *specTolerations + chart.Spec.Bootstrap = true + chart, err = framework.CreateHelmChart(chart, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + labelSelector := labels.SelectorFromSet(labels.Set{ + "owner": "helm", + "name": chart.Name, + }) + _, err = framework.WaitForRelease(chart, labelSelector, 120*time.Second, 1) + Expect(err).ToNot(HaveOccurred()) + + chart, err = framework.GetHelmChart(chart.Name, chart.Namespace) + Expect(err).ToNot(HaveOccurred()) + job, err = framework.GetJob(chart) + Expect(err).ToNot(HaveOccurred()) + }) + FIt("Should have tolerations set", func() { + Expect(job.Spec.Template.Spec.Tolerations).To(Equal(finalTolerations)) + }) + AfterEach(func() { + err = framework.DeleteHelmChart(chart.Name, framework.Namespace) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func() bool { + _, err := framework.GetHelmChart(chart.Name, framework.Namespace) + return err != nil && apierrors.IsNotFound(err) + }, 120*time.Second, 5*time.Second).Should(BeTrue()) + }) + }) + }) From 2f990d829758b40bf162804b16115f7ad1e3c314 Mon Sep 17 00:00:00 2001 From: Daniel Miranda Date: Sat, 27 Jan 2024 10:41:47 +0000 Subject: [PATCH 2/2] Fix leftover FIt tests --- test/suite/helm_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/suite/helm_test.go b/test/suite/helm_test.go index e5b58c84..895a62a5 100644 --- a/test/suite/helm_test.go +++ b/test/suite/helm_test.go @@ -819,7 +819,7 @@ var _ = Describe("Helm Tests", Ordered, func() { job, err = framework.GetJob(chart) Expect(err).ToNot(HaveOccurred()) }) - FIt("Should have no tolerations set", func() { + It("Should have no tolerations set", func() { Expect(job.Spec.Template.Spec.Tolerations).To(BeEmpty()) }) AfterEach(func() { @@ -880,7 +880,7 @@ var _ = Describe("Helm Tests", Ordered, func() { job, err = framework.GetJob(chart) Expect(err).ToNot(HaveOccurred()) }) - FIt("Should have tolerations set", func() { + It("Should have tolerations set", func() { Expect(job.Spec.Template.Spec.Tolerations).To(Equal(*tolerations)) }) AfterEach(func() { @@ -931,7 +931,7 @@ var _ = Describe("Helm Tests", Ordered, func() { job, err = framework.GetJob(chart) Expect(err).ToNot(HaveOccurred()) }) - FIt("Should have default tolerations set", func() { + It("Should have default tolerations set", func() { Expect(job.Spec.Template.Spec.Tolerations).To(Equal(*bootstrapTolerations)) }) AfterEach(func() { @@ -994,7 +994,7 @@ var _ = Describe("Helm Tests", Ordered, func() { job, err = framework.GetJob(chart) Expect(err).ToNot(HaveOccurred()) }) - FIt("Should have tolerations set", func() { + It("Should have tolerations set", func() { Expect(job.Spec.Template.Spec.Tolerations).To(Equal(finalTolerations)) }) AfterEach(func() {