Skip to content

Commit e49b54d

Browse files
kamarabbas99kamarabbas99
committed
Add e2e tests for CPU startup boost
1 parent 18d8efd commit e49b54d

File tree

4 files changed

+377
-5
lines changed

4 files changed

+377
-5
lines changed

‎vertical-pod-autoscaler/e2e/v1/admission_controller.go‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1098,6 +1098,122 @@ var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
10981098
})
10991099
})
11001100

1101+
var _ = AdmissionControllerE2eDescribe("Admission-controller", func() {
1102+
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
1103+
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
1104+
1105+
ginkgo.BeforeEach(func() {
1106+
waitForVpaWebhookRegistration(f)
1107+
})
1108+
1109+
f.It("boosts CPU by factor on pod creation", framework.WithFeatureGate(features.CPUStartupBoost), func() {
1110+
initialCPU := ParseQuantityOrDie("100m")
1111+
expectedCPU := ParseQuantityOrDie("200m")
1112+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
1113+
1114+
ginkgo.By("Setting up a VPA with a startup boost policy (factor)")
1115+
containerName := utils.GetHamsterContainerNameByIndex(0)
1116+
factor := int32(2)
1117+
vpaCRD := test.VerticalPodAutoscaler().
1118+
WithName("hamster-vpa").
1119+
WithNamespace(f.Namespace.Name).
1120+
WithTargetRef(utils.HamsterTargetRef).
1121+
WithContainer(containerName).
1122+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
1123+
AppendRecommendation(
1124+
test.Recommendation().
1125+
WithContainer(containerName).
1126+
WithTarget("100m", "100Mi").
1127+
GetContainerResources(),
1128+
).
1129+
Get()
1130+
utils.InstallVPA(f, vpaCRD)
1131+
1132+
ginkgo.By("Starting the deployment and verifying the pod is boosted")
1133+
podList := utils.StartDeploymentPods(f, d)
1134+
pod := podList.Items[0]
1135+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1136+
gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1137+
})
1138+
1139+
f.It("boosts CPU by quantity on pod creation", framework.WithFeatureGate(features.CPUStartupBoost), func() {
1140+
initialCPU := ParseQuantityOrDie("100m")
1141+
boostCPUQuantity := ParseQuantityOrDie("500m")
1142+
expectedCPU := ParseQuantityOrDie("600m")
1143+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
1144+
1145+
ginkgo.By("Setting up a VPA with a startup boost policy (quantity)")
1146+
containerName := utils.GetHamsterContainerNameByIndex(0)
1147+
vpaCRD := test.VerticalPodAutoscaler().
1148+
WithName("hamster-vpa").
1149+
WithNamespace(f.Namespace.Name).
1150+
WithTargetRef(utils.HamsterTargetRef).
1151+
WithContainer(containerName).
1152+
WithCPUStartupBoost(vpa_types.QuantityStartupBoostType, nil, &boostCPUQuantity, "15s").
1153+
AppendRecommendation(
1154+
test.Recommendation().
1155+
WithContainer(containerName).
1156+
WithTarget("100m", "100Mi").
1157+
GetContainerResources(),
1158+
).
1159+
Get()
1160+
utils.InstallVPA(f, vpaCRD)
1161+
1162+
ginkgo.By("Starting the deployment and verifying the pod is boosted")
1163+
podList := utils.StartDeploymentPods(f, d)
1164+
pod := podList.Items[0]
1165+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1166+
gomega.Expect(pod.Spec.Containers[0].Resources.Limits.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1167+
})
1168+
1169+
f.It("boosts CPU on pod creation when VPA update mode is Off", framework.WithFeatureGate(features.CPUStartupBoost), func() {
1170+
initialCPU := ParseQuantityOrDie("100m")
1171+
expectedCPU := ParseQuantityOrDie("200m")
1172+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
1173+
1174+
ginkgo.By("Setting up a VPA with updateMode Off and a startup boost policy")
1175+
containerName := utils.GetHamsterContainerNameByIndex(0)
1176+
factor := int32(2)
1177+
vpaCRD := test.VerticalPodAutoscaler().
1178+
WithName("hamster-vpa").
1179+
WithNamespace(f.Namespace.Name).
1180+
WithTargetRef(utils.HamsterTargetRef).
1181+
WithContainer(containerName).
1182+
WithUpdateMode(vpa_types.UpdateModeOff). // VPA is off, but boost should still work
1183+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
1184+
Get()
1185+
utils.InstallVPA(f, vpaCRD)
1186+
1187+
ginkgo.By("Starting the deployment and verifying the pod is boosted")
1188+
podList := utils.StartDeploymentPods(f, d)
1189+
pod := podList.Items[0]
1190+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(expectedCPU)).To(gomega.Equal(0))
1191+
})
1192+
1193+
f.It("doesn't boost CPU on pod creation when scaling mode is Off", framework.WithFeatureGate(features.CPUStartupBoost), func() {
1194+
initialCPU := ParseQuantityOrDie("100m")
1195+
d := NewHamsterDeploymentWithResources(f, initialCPU, ParseQuantityOrDie("100Mi"))
1196+
1197+
ginkgo.By("Setting up a VPA with a startup boost policy and scaling mode Off")
1198+
containerName := utils.GetHamsterContainerNameByIndex(0)
1199+
factor := int32(2)
1200+
vpaCRD := test.VerticalPodAutoscaler().
1201+
WithName("hamster-vpa").
1202+
WithNamespace(f.Namespace.Name).
1203+
WithTargetRef(utils.HamsterTargetRef).
1204+
WithContainer(containerName).
1205+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "15s").
1206+
WithScalingMode(containerName, vpa_types.ContainerScalingModeOff).
1207+
Get()
1208+
utils.InstallVPA(f, vpaCRD)
1209+
1210+
ginkgo.By("Starting the deployment and verifying the pod is NOT boosted")
1211+
podList := utils.StartDeploymentPods(f, d)
1212+
pod := podList.Items[0]
1213+
gomega.Expect(pod.Spec.Containers[0].Resources.Requests.Cpu().Cmp(initialCPU)).To(gomega.Equal(0))
1214+
})
1215+
})
1216+
11011217
func waitForVpaWebhookRegistration(f *framework.Framework) {
11021218
ginkgo.By("Waiting for VPA webhook registration")
11031219
gomega.Eventually(func() bool {

‎vertical-pod-autoscaler/e2e/v1/common.go‎

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"encoding/json"
2222
"fmt"
23+
"strings"
2324
"time"
2425

2526
ginkgo "github.com/onsi/ginkgo/v2"
@@ -36,6 +37,7 @@ import (
3637
"k8s.io/autoscaler/vertical-pod-autoscaler/e2e/utils"
3738
vpa_types "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
3839
vpa_clientset "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/client/clientset/versioned"
40+
"k8s.io/autoscaler/vertical-pod-autoscaler/pkg/features"
3941
clientset "k8s.io/client-go/kubernetes"
4042
"k8s.io/kubernetes/test/e2e/framework"
4143
framework_deployment "k8s.io/kubernetes/test/e2e/framework/deployment"
@@ -244,14 +246,30 @@ func InstallRawVPA(f *framework.Framework, obj interface{}) error {
244246

245247
// AnnotatePod adds annotation for an existing pod.
246248
func AnnotatePod(f *framework.Framework, podName, annotationName, annotationValue string) {
247-
bytes, err := json.Marshal([]utils.PatchRecord{{
249+
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(context.TODO(), podName, metav1.GetOptions{})
250+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to get pod.")
251+
252+
patches := []utils.PatchRecord{}
253+
if pod.Annotations == nil {
254+
patches = append(patches, utils.PatchRecord{
255+
Op: "add",
256+
Path: "/metadata/annotations",
257+
Value: make(map[string]string),
258+
})
259+
}
260+
261+
patches = append(patches, utils.PatchRecord{
248262
Op: "add",
249-
Path: fmt.Sprintf("/metadata/annotations/%v", annotationName),
263+
Path: fmt.Sprintf("/metadata/annotations/%s", annotationName),
250264
Value: annotationValue,
251-
}})
252-
pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{})
265+
})
266+
267+
bytes, err := json.Marshal(patches)
268+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
269+
270+
patchedPod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Patch(context.TODO(), podName, types.JSONPatchType, bytes, metav1.PatchOptions{})
253271
gomega.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to patch pod.")
254-
gomega.Expect(pod.Annotations[annotationName]).To(gomega.Equal(annotationValue))
272+
gomega.Expect(patchedPod.Annotations[annotationName]).To(gomega.Equal(annotationValue))
255273
}
256274

257275
// ParseQuantityOrDie parses quantity from string and dies with an error if
@@ -466,3 +484,45 @@ func WaitForPodsUpdatedWithoutEviction(f *framework.Framework, initialPods *apiv
466484
framework.Logf("finished waiting for at least one pod to be updated without eviction")
467485
return err
468486
}
487+
488+
// checkInPlaceOrRecreateTestsEnabled check for enabled feature gates in the cluster used for the
489+
// InPlaceOrRecreate VPA feature.
490+
// Use this in a "beforeEach" call before any suites that use InPlaceOrRecreate featuregate.
491+
func checkInPlaceOrRecreateTestsEnabled(f *framework.Framework, checkAdmission, checkUpdater bool) {
492+
ginkgo.By("Checking InPlacePodVerticalScaling cluster feature gate is on")
493+
494+
if checkUpdater {
495+
ginkgo.By("Checking InPlaceOrRecreate VPA feature gate is enabled for updater")
496+
497+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-updater", metav1.GetOptions{})
498+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
499+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
500+
vpaUpdaterPod := deploy.Spec.Template.Spec.Containers[0]
501+
gomega.Expect(vpaUpdaterPod.Name).To(gomega.Equal("updater"))
502+
if !anyContainsSubstring(vpaUpdaterPod.Args, fmt.Sprintf("%s=true", string(features.InPlaceOrRecreate))) {
503+
ginkgo.Skip("Skipping suite: InPlaceOrRecreate feature gate is not enabled for the VPA updater")
504+
}
505+
}
506+
507+
if checkAdmission {
508+
ginkgo.By("Checking InPlaceOrRecreate VPA feature gate is enabled for admission controller")
509+
510+
deploy, err := f.ClientSet.AppsV1().Deployments(VpaNamespace).Get(context.TODO(), "vpa-admission-controller", metav1.GetOptions{})
511+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
512+
gomega.Expect(deploy.Spec.Template.Spec.Containers).To(gomega.HaveLen(1))
513+
vpaAdmissionPod := deploy.Spec.Template.Spec.Containers[0]
514+
gomega.Expect(vpaAdmissionPod.Name).To(gomega.Equal("admission-controller"))
515+
if !anyContainsSubstring(vpaAdmissionPod.Args, fmt.Sprintf("%s=true", string(features.InPlaceOrRecreate))) {
516+
ginkgo.Skip("Skipping suite: InPlaceOrRecreate feature gate is not enabled for VPA admission controller")
517+
}
518+
}
519+
}
520+
521+
func anyContainsSubstring(arr []string, substr string) bool {
522+
for _, s := range arr {
523+
if strings.Contains(s, substr) {
524+
return true
525+
}
526+
}
527+
return false
528+
}

‎vertical-pod-autoscaler/e2e/v1/full_vpa.go‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,121 @@ var _ = FullVpaE2eDescribe("Pods under VPA with non-recognized recommender expli
356356
})
357357
})
358358

359+
var _ = FullVpaE2eDescribe("Pods under VPA with CPUStartupBoost", func() {
360+
var (
361+
rc *ResourceConsumer
362+
)
363+
replicas := 3
364+
365+
ginkgo.AfterEach(func() {
366+
rc.CleanUp()
367+
})
368+
369+
f := framework.NewDefaultFramework("vertical-pod-autoscaling")
370+
f.NamespacePodSecurityEnforceLevel = podsecurity.LevelBaseline
371+
372+
ginkgo.Describe("have CPU startup boost recommendation applied", func() {
373+
ginkgo.BeforeEach(func() {
374+
waitForVpaWebhookRegistration(f)
375+
})
376+
377+
f.It("to all containers of a pod", framework.WithFeatureGate(features.CPUStartupBoost), func() {
378+
ns := f.Namespace.Name
379+
ginkgo.By("Setting up a VPA CRD with CPUStartupBoost")
380+
targetRef := &autoscaling.CrossVersionObjectReference{
381+
APIVersion: "apps/v1",
382+
Kind: "Deployment",
383+
Name: "hamster",
384+
}
385+
386+
containerName := utils.GetHamsterContainerNameByIndex(0)
387+
factor := int32(100)
388+
vpaCRD := test.VerticalPodAutoscaler().
389+
WithName("hamster-vpa").
390+
WithNamespace(f.Namespace.Name).
391+
WithTargetRef(targetRef).
392+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
393+
WithContainer(containerName).
394+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s").
395+
Get()
396+
utils.InstallVPA(f, vpaCRD)
397+
398+
ginkgo.By("Setting up a hamster deployment")
399+
rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment,
400+
replicas,
401+
1, /*initCPUTotal*/
402+
10, /*initMemoryTotal*/
403+
1, /*initCustomMetric*/
404+
initialCPU, /*cpuRequest*/
405+
initialMemory, /*memRequest*/
406+
f.ClientSet,
407+
f.ScalesGetter)
408+
409+
// Pods should be created with boosted CPU (10m * 100 = 1000m)
410+
err := waitForResourceRequestInRangeInPods(
411+
f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
412+
ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m"))
413+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
414+
415+
// Pods should be scaled back down in-place after they become Ready and
416+
// StartupBoost.CPU.Duration has elapsed
417+
err = waitForResourceRequestInRangeInPods(
418+
f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
419+
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
420+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
421+
})
422+
423+
f.It("to a subset of containers in a pod", framework.WithFeatureGate(features.CPUStartupBoost), func() {
424+
ns := f.Namespace.Name
425+
426+
ginkgo.By("Setting up a VPA CRD with CPUStartupBoost")
427+
targetRef := &autoscaling.CrossVersionObjectReference{
428+
APIVersion: "apps/v1",
429+
Kind: "Deployment",
430+
Name: "hamster",
431+
}
432+
433+
containerName := utils.GetHamsterContainerNameByIndex(0)
434+
factor := int32(100)
435+
vpaCRD := test.VerticalPodAutoscaler().
436+
WithName("hamster-vpa").
437+
WithNamespace(f.Namespace.Name).
438+
WithTargetRef(targetRef).
439+
WithUpdateMode(vpa_types.UpdateModeInPlaceOrRecreate).
440+
WithContainer(containerName).
441+
WithCPUStartupBoost(vpa_types.FactorStartupBoostType, &factor, nil, "10s").
442+
Get()
443+
444+
utils.InstallVPA(f, vpaCRD)
445+
446+
ginkgo.By("Setting up a hamster deployment")
447+
rc = NewDynamicResourceConsumer("hamster", ns, KindDeployment,
448+
replicas,
449+
1, /*initCPUTotal*/
450+
10, /*initMemoryTotal*/
451+
1, /*initCustomMetric*/
452+
initialCPU, /*cpuRequest*/
453+
initialMemory, /*memRequest*/
454+
f.ClientSet,
455+
f.ScalesGetter)
456+
457+
// Pods should be created with boosted CPU (10m * 100 = 1000m)
458+
err := waitForResourceRequestInRangeInPods(
459+
f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
460+
ParseQuantityOrDie("900m"), ParseQuantityOrDie("1100m"))
461+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
462+
463+
// Pods should be scaled back down in-place after they become Ready and
464+
// StartupBoost.CPU.Duration has elapsed
465+
err = waitForResourceRequestInRangeInPods(
466+
f, utils.PollTimeout, metav1.ListOptions{LabelSelector: "name=hamster"}, apiv1.ResourceCPU,
467+
ParseQuantityOrDie(minimalCPULowerBound), ParseQuantityOrDie(minimalCPUUpperBound))
468+
gomega.Expect(err).NotTo(gomega.HaveOccurred())
469+
})
470+
})
471+
472+
})
473+
359474
var _ = FullVpaE2eDescribe("OOMing pods under VPA", func() {
360475
const replicas = 3
361476

0 commit comments

Comments
 (0)