feat(resource reservation): add JSON configuration for CPU and memory resources in GPU reservation pods

Author: lokielse

cmd/binder/app/app.go

@@ -5,6 +5,7 @@ package app
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"time"
 
@@ -104,12 +105,21 @@ func New(options *Options, config *rest.Config) (*App, error) {
 	kubeClient := draversionawareclient.NewDRAAwareClient(kubernetes.NewForConfigOrDie(config))
 	informerFactory := informers.NewSharedInformerFactory(kubeClient, 0)
 
+	// Deserialize pod resources if provided
+	var podResources *corev1.ResourceRequirements
+	if options.ResourceReservationPodResourcesJSON != "" {
+		podResources = &corev1.ResourceRequirements{}
+		if err := json.Unmarshal([]byte(options.ResourceReservationPodResourcesJSON), podResources); err != nil {
+			setupLog.Error(err, "failed to unmarshal resource reservation pod resources")
+			return nil, fmt.Errorf("failed to unmarshal resource reservation pod resources: %w", err)
+		}
+	}
+
 	rrs := resourcereservation.NewService(options.FakeGPUNodes, clientWithWatch, options.ResourceReservationPodImage,
 		time.Duration(options.ResourceReservationAllocationTimeout)*time.Second,
 		options.ResourceReservationNamespace, options.ResourceReservationServiceAccount,
 		options.ResourceReservationAppLabel, options.ScalingPodNamespace, options.RuntimeClassName,
-		options.ResourceReservationPodCPURequest, options.ResourceReservationPodMemoryRequest,
-		options.ResourceReservationPodCPULimit, options.ResourceReservationPodMemoryLimit)
+		podResources)
 
 	reconcilerParams := &controllers.ReconcilerParams{
 		MaxConcurrentReconciles:     options.MaxConcurrentReconciles,

cmd/binder/app/options.go

@@ -17,10 +17,7 @@ type Options struct {
 	ResourceReservationPodImage          string
 	ResourceReservationAppLabel          string
 	ResourceReservationAllocationTimeout int
-	ResourceReservationPodCPURequest     string
-	ResourceReservationPodMemoryRequest  string
-	ResourceReservationPodCPULimit       string
-	ResourceReservationPodMemoryLimit    string
+	ResourceReservationPodResourcesJSON  string
 	ScalingPodNamespace                  string
 	QPS                                  float64
 	Burst                                int
@@ -61,18 +58,9 @@ func InitOptions(fs *pflag.FlagSet) *Options {
 	fs.IntVar(&options.ResourceReservationAllocationTimeout,
 		"resource-reservation-allocation-timeout", 40,
 		"Resource reservation allocation timeout in seconds")
-	fs.StringVar(&options.ResourceReservationPodCPURequest,
-		"resource-reservation-pod-cpu-request", "",
-		"CPU request for GPU reservation pods (optional, empty means not set)")
-	fs.StringVar(&options.ResourceReservationPodMemoryRequest,
-		"resource-reservation-pod-memory-request", "",
-		"Memory request for GPU reservation pods (optional, empty means not set)")
-	fs.StringVar(&options.ResourceReservationPodCPULimit,
-		"resource-reservation-pod-cpu-limit", "",
-		"CPU limit for GPU reservation pods (optional, empty means not set)")
-	fs.StringVar(&options.ResourceReservationPodMemoryLimit,
-		"resource-reservation-pod-memory-limit", "",
-		"Memory limit for GPU reservation pods (optional, empty means not set)")
+	fs.StringVar(&options.ResourceReservationPodResourcesJSON,
+		"resource-reservation-pod-resources", "",
+		"JSON-serialized ResourceRequirements for GPU reservation pods (optional, empty means not set)")
 	fs.StringVar(&options.ScalingPodNamespace,
 		"scale-adjust-namespace", constants.DefaultScaleAdjustName,
 		"Scaling pods namespace")

pkg/apis/kai/v1/binder/binder.go

@@ -103,7 +103,8 @@ type ResourceReservation struct {
 	// RuntimeClassName specifies the runtime class used by the reservation pods. Needs to allow access to the GPU
 	RuntimeClassName *string `json:"runtimeClassName,omitempty"`
 
-	// PodResources specifies the CPU and memory resource requests and limits for GPU reservation pods
+	// PodResources specifies the CPU and memory resource requests and limits for GPU reservation pods.
+	// If not set, Kubernetes defaults will be used, which allows for better backward compatibility.
 	// +kubebuilder:validation:Optional
 	PodResources *common.Resources `json:"podResources,omitempty"`
 }
@@ -117,7 +118,4 @@ func (r *ResourceReservation) SetDefaultsWhereNeeded() {
 	r.ServiceAccountName = common.SetDefault(r.ServiceAccountName, ptr.To(constants.DefaultResourceReservationName))
 	r.AppLabel = common.SetDefault(r.AppLabel, ptr.To(constants.DefaultResourceReservationName))
 	r.RuntimeClassName = common.SetDefault(r.RuntimeClassName, ptr.To(constants.DefaultRuntimeClassName))
-
-	// PodResources is optional - if not set, Kubernetes defaults will be used
-	// This allows for better backward compatibility
 }

pkg/binder/binding/resourcereservation/resource_reservation.go

@@ -53,10 +53,7 @@ type service struct {
 	appLabelValue       string
 	scalingPodNamespace string
 	runtimeClassName    string
-	podCPURequest       string
-	podMemoryRequest    string
-	podCPULimit         string
-	podMemoryLimit      string
+	podResources        *v1.ResourceRequirements
 }
 
 func NewService(
@@ -69,10 +66,7 @@ func NewService(
 	appLabelValue string,
 	scalingPodNamespace string,
 	runtimeClassName string,
-	podCPURequest string,
-	podMemoryRequest string,
-	podCPULimit string,
-	podMemoryLimit string,
+	podResources *v1.ResourceRequirements,
 ) *service {
 	return &service{
 		fakeGPuNodes:        fakeGPuNodes,
@@ -85,10 +79,7 @@ func NewService(
 		appLabelValue:       appLabelValue,
 		scalingPodNamespace: scalingPodNamespace,
 		runtimeClassName:    runtimeClassName,
-		podCPURequest:       podCPURequest,
-		podMemoryRequest:    podMemoryRequest,
-		podCPULimit:         podCPULimit,
-		podMemoryLimit:      podMemoryLimit,
+		podResources:        podResources,
 	}
 }
 
@@ -395,8 +386,7 @@ func (rsc *service) createGPUReservationPod(ctx context.Context, nodeName, gpuGr
 
 	podName := fmt.Sprintf("%s-%s-%s", gpuReservationPodPrefix, nodeName, rand.String(reservationPodRandomCharacters))
 
-	// Build resource requirements with configured values
-	// Only set CPU/Memory if explicitly configured (better backward compatibility)
+	// Build resource requirements starting with GPU resources
 	resources := v1.ResourceRequirements{
 		Limits: v1.ResourceList{
 			constants.GpuResource: *resource.NewQuantity(numberOfGPUsToReserve, resource.DecimalSI),
@@ -406,20 +396,24 @@ func (rsc *service) createGPUReservationPod(ctx context.Context, nodeName, gpuGr
 		},
 	}
 
-	// Only set CPU/Memory limits if configured
-	if rsc.podCPULimit != "" {
-		resources.Limits[v1.ResourceCPU] = resource.MustParse(rsc.podCPULimit)
-	}
-	if rsc.podMemoryLimit != "" {
-		resources.Limits[v1.ResourceMemory] = resource.MustParse(rsc.podMemoryLimit)
-	}
-
-	// Only set CPU/Memory requests if configured
-	if rsc.podCPURequest != "" {
-		resources.Requests[v1.ResourceCPU] = resource.MustParse(rsc.podCPURequest)
-	}
-	if rsc.podMemoryRequest != "" {
-		resources.Requests[v1.ResourceMemory] = resource.MustParse(rsc.podMemoryRequest)
+	// Merge in configured CPU/Memory resources if provided, but skip GPU resources to prevent override
+	if rsc.podResources != nil {
+		if rsc.podResources.Limits != nil {
+			for resourceName, quantity := range rsc.podResources.Limits {
+				// Skip GPU resources - they are already set correctly
+				if resourceName != constants.GpuResource {
+					resources.Limits[resourceName] = quantity
+				}
+			}
+		}
+		if rsc.podResources.Requests != nil {
+			for resourceName, quantity := range rsc.podResources.Requests {
+				// Skip GPU resources - they are already set correctly
+				if resourceName != constants.GpuResource {
+					resources.Requests[resourceName] = quantity
+				}
+			}
+		}
 	}
 
 	pod, err := rsc.createResourceReservationPod(nodeName, gpuGroup, podName, resources)

pkg/binder/binding/resourcereservation/resource_reservation_test.go

@@ -47,7 +47,7 @@ func initializeTestService(
 ) *service {
 	service := NewService(false, client, "", 40*time.Millisecond,
 		resourceReservationNameSpace, resourceReservationServiceAccount, resourceReservationAppLabelValue, scalingPodsNamespace, constants.DefaultRuntimeClassName,
-		"", "", "", "") // Empty resource configs to use defaults
+		nil) // nil podResources to use defaults
 
 	return service
 }
@@ -952,10 +952,16 @@ var _ = Describe("ResourceReservationService", func() {
 				reservationPodImage: "test-image:latest",
 				kubeClient:          fake.NewClientBuilder().Build(),
 				runtimeClassName:    "nvidia",
-				podCPURequest:       "2m",
-				podMemoryRequest:    "20Mi",
-				podCPULimit:         "100m",
-				podMemoryLimit:      "200Mi",
+				podResources: &v1.ResourceRequirements{
+					Requests: v1.ResourceList{
+						v1.ResourceCPU:    resource.MustParse("2m"),
+						v1.ResourceMemory: resource.MustParse("20Mi"),
+					},
+					Limits: v1.ResourceList{
+						v1.ResourceCPU:    resource.MustParse("100m"),
+						v1.ResourceMemory: resource.MustParse("200Mi"),
+					},
+				},
 				scalingPodNamespace: scalingPodsNamespace,
 			}
 
@@ -986,10 +992,7 @@ var _ = Describe("ResourceReservationService", func() {
 				reservationPodImage: "test-image:latest",
 				kubeClient:          fake.NewClientBuilder().Build(),
 				runtimeClassName:    "nvidia",
-				podCPURequest:       "",
-				podMemoryRequest:    "",
-				podCPULimit:         "",
-				podMemoryLimit:      "",
+				podResources:        nil,
 				scalingPodNamespace: scalingPodsNamespace,
 			}
 
@@ -1025,10 +1028,14 @@ var _ = Describe("ResourceReservationService", func() {
 				reservationPodImage: "test-image:latest",
 				kubeClient:          fake.NewClientBuilder().Build(),
 				runtimeClassName:    "nvidia",
-				podCPURequest:       "5m",
-				podMemoryRequest:    "",
-				podCPULimit:         "50m",
-				podMemoryLimit:      "",
+				podResources: &v1.ResourceRequirements{
+					Requests: v1.ResourceList{
+						v1.ResourceCPU: resource.MustParse("5m"),
+					},
+					Limits: v1.ResourceList{
+						v1.ResourceCPU: resource.MustParse("50m"),
+					},
+				},
 				scalingPodNamespace: scalingPodsNamespace,
 			}
 
@@ -1048,6 +1055,46 @@ var _ = Describe("ResourceReservationService", func() {
 			Expect(memRequestExists).To(BeFalse())
 			Expect(memLimitExists).To(BeFalse())
 		})
+
+		It("should not allow GPU resources to be overridden by podResources", func() {
+			// This test ensures that even if podResources contains GPU configuration,
+			// it won't override the GPU resource value set by the service
+			rsc := &service{
+				namespace:           "kai-resource-reservation",
+				appLabelValue:       "kai-reservation",
+				serviceAccountName:  "kai-sa",
+				reservationPodImage: "test-image:latest",
+				kubeClient:          fake.NewClientBuilder().Build(),
+				runtimeClassName:    "nvidia",
+				podResources: &v1.ResourceRequirements{
+					Requests: v1.ResourceList{
+						v1.ResourceCPU:        resource.MustParse("10m"),
+						constants.GpuResource: resource.MustParse("999"), // This should be ignored
+					},
+					Limits: v1.ResourceList{
+						v1.ResourceCPU:        resource.MustParse("100m"),
+						constants.GpuResource: resource.MustParse("999"), // This should be ignored
+					},
+				},
+				scalingPodNamespace: scalingPodsNamespace,
+			}
+
+			pod, err := rsc.createGPUReservationPod(context.TODO(), "test-node", "test-gpu-group")
+			Expect(err).To(BeNil())
+			Expect(pod).NotTo(BeNil())
+
+			container := pod.Spec.Containers[0]
+
+			// Verify CPU resources from podResources are set
+			Expect(container.Resources.Requests[v1.ResourceCPU]).To(Equal(resource.MustParse("10m")))
+			Expect(container.Resources.Limits[v1.ResourceCPU]).To(Equal(resource.MustParse("100m")))
+
+			// Verify GPU resources are NOT overridden - should always be 1
+			gpuRequest := container.Resources.Requests[constants.GpuResource]
+			gpuLimit := container.Resources.Limits[constants.GpuResource]
+			Expect(gpuRequest.Value()).To(Equal(int64(1)), "GPU request should be 1, not overridden by podResources")
+			Expect(gpuLimit.Value()).To(Equal(int64(1)), "GPU limit should be 1, not overridden by podResources")
+		})
 	})
 })
 

pkg/operator/operands/binder/resources.go

@@ -5,6 +5,7 @@ package binder
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 
 	appsv1 "k8s.io/api/apps/v1"
@@ -233,19 +234,15 @@ func buildArgsList(kaiConfig *kaiv1.Config, config *kaiv1binder.Binder, fakeGPU
 		args = append(args, []string{fmt.Sprintf("--runtime-class-name=%s", *config.ResourceReservation.RuntimeClassName)}...)
 	}
 
-	// Add GPU reservation pod resource configurations
+	// Serialize and add GPU reservation pod resource configurations
 	if config.ResourceReservation.PodResources != nil {
-		if cpuRequest, found := config.ResourceReservation.PodResources.Requests[v1.ResourceCPU]; found {
-			args = append(args, []string{"--resource-reservation-pod-cpu-request", cpuRequest.String()}...)
+		resourceRequirements := v1.ResourceRequirements{
+			Requests: config.ResourceReservation.PodResources.Requests,
+			Limits:   config.ResourceReservation.PodResources.Limits,
 		}
-		if memoryRequest, found := config.ResourceReservation.PodResources.Requests[v1.ResourceMemory]; found {
-			args = append(args, []string{"--resource-reservation-pod-memory-request", memoryRequest.String()}...)
-		}
-		if cpuLimit, found := config.ResourceReservation.PodResources.Limits[v1.ResourceCPU]; found {
-			args = append(args, []string{"--resource-reservation-pod-cpu-limit", cpuLimit.String()}...)
-		}
-		if memoryLimit, found := config.ResourceReservation.PodResources.Limits[v1.ResourceMemory]; found {
-			args = append(args, []string{"--resource-reservation-pod-memory-limit", memoryLimit.String()}...)
+		resourcesJSON, err := json.Marshal(resourceRequirements)
+		if err == nil {
+			args = append(args, []string{"--resource-reservation-pod-resources", string(resourcesJSON)}...)
 		}
 	}