Add scheduler deployment

Author: omer-dayan

cmd/scheduler/app/options/options.go

@@ -20,10 +20,11 @@ const (
 	defaultProfilerApiPort             = "8182"
 	defaultVerbosityLevel              = 3
 	defaultMaxConsolidationPreemptees  = 16
-	DefaultDetailedFitError            = false
+	defaultDetailedFitError            = false
 	DefaultPyroscopeMutexProfilerRate  = 5
 	DefaultPyroscopeBlockProfilerRate  = 5
 	defaultNumOfStatusRecordingWorkers = 5
+	defaultNodePoolLabelKey            = "runai/node-pool"
 )
 
 // ServerOption is the main context object for the controller manager.
@@ -69,7 +70,7 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	// kai-scheduler will ignore pods with scheduler names other than specified with the option
 	fs.StringVar(&s.SchedulerName, "scheduler-name", defaultSchedulerName, "kai-scheduler will handle pods with the scheduler-name")
 	fs.BoolVar(&s.RestrictSchedulingNodes, "restrict-node-scheduling", false, "kai-scheduler will allocate jobs only to restricted nodes")
-	fs.StringVar(&s.NodePoolLabelKey, "nodepool-label-key", "", "The label key by which to filter scheduling nodepool")
+	fs.StringVar(&s.NodePoolLabelKey, "nodepool-label-key", defaultNodePoolLabelKey, "The label key by which to filter scheduling nodepool")
 	fs.StringVar(&s.NodePoolLabelValue, "partition-label-value", "", "The label value by which to filter scheduling partition")
 	fs.StringVar(&s.SchedulerConf, "scheduler-conf", "", "The absolute path of scheduler configuration file")
 	fs.DurationVar(&s.SchedulePeriod, "schedule-period", defaultSchedulerPeriod, "The period between each scheduling cycle")
@@ -88,7 +89,7 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	fs.IntVar(&s.MaxNumberConsolidationPreemptees, "max-consolidation-preemptees", defaultMaxConsolidationPreemptees, "Maximum number of consolidation preemptees. Defaults to 16")
 	fs.IntVar(&s.QPS, "qps", 50, "Queries per second to the K8s API server")
 	fs.IntVar(&s.Burst, "burst", 300, "Burst to the K8s API server")
-	fs.BoolVar(&s.DetailedFitErrors, "detailed-fit-errors", DefaultDetailedFitError, "Write detailed fit errors for every node on every podgroup")
+	fs.BoolVar(&s.DetailedFitErrors, "detailed-fit-errors", defaultDetailedFitError, "Write detailed fit errors for every node on every podgroup")
 	fs.BoolVar(&s.ScheduleCSIStorage, "schedule-csi-storage", false, "Enables advanced scheduling (preempt, reclaim) for csi storage objects")
 	fs.BoolVar(&s.UseSchedulingSignatures, "use-scheduling-signatures", true, "Use scheduling signatures to avoid duplicate scheduling attempts for identical jobs")
 	fs.BoolVar(&s.FullHierarchyFairness, "full-hierarchy-fairness", true, "Fairness across project and department levels")

cmd/scheduler/app/options/options_test.go

@@ -45,6 +45,7 @@ func TestAddFlags(t *testing.T) {
 		PyroscopeMutexProfilerRate:        DefaultPyroscopeMutexProfilerRate,
 		GlobalDefaultStalenessGracePeriod: defaultStalenessGracePeriod,
 		NumOfStatusRecordingWorkers:       defaultNumOfStatusRecordingWorkers,
+		NodePoolLabelKey:                  defaultNodePoolLabelKey,
 	}
 
 	if !reflect.DeepEqual(expected, s) {

deployments/kai-scheduler/templates/rbac/scheduler-binding.yaml

@@ -0,0 +1,15 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: scheduler
+subjects:
+  - kind: ServiceAccount
+    name: scheduler
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: scheduler
+  apiGroup: rbac.authorization.k8s.io

deployments/kai-scheduler/templates/rbac/scheduler.yaml

@@ -0,0 +1,113 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: scheduler
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - events
+      - persistentvolumeclaims
+      - pods
+      - pods/status
+      - pods/binding
+      - services
+      - persistentvolumes
+      - namespaces
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - scheduling.run.ai
+    resources:
+      - bindrequests
+      - podgroups
+      - podgroups/status
+      - queues
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - get
+      - list
+      - watch
+      - patch
+  - apiGroups:
+      - storage.k8s.io
+    resources:
+      - storageclasses
+      - csinodes
+      - csidrivers
+      - csistoragecapacities
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - policy
+    resources:
+      - poddisruptionbudgets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - scheduling.k8s.io
+    resources:
+      - priorityclasses
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - resource.k8s.io
+    resources:
+      - deviceclasses
+      - resourceslices
+      - resourceclaims
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - configmaps
+      - events
+      - persistentvolumeclaims
+      - pods
+      - pods/status
+      - pods/binding
+      - pods/finalizers
+      - services
+      - services/finalizers
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update
+  - apiGroups:
+      - scheduling.run.ai
+    resources:
+      - bindrequests
+      - podgroups
+      - podgroups/status
+    verbs:
+      - create
+      - delete
+      - get
+      - list
+      - patch
+      - update

deployments/kai-scheduler/templates/services/scheduler-configmap.yaml

@@ -0,0 +1,32 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: v1
+data:
+  config.yaml: |
+    actions: allocate, consolidation, reclaim, preempt, stalegangeviction
+    tiers:
+      - plugins:
+          - name: predicates
+          - name: proportion
+          - name: priority
+          - name: nodeavailability
+          - name: resourcetype
+          - name: podaffinity
+          - name: elastic
+          - name: kubeflow
+          - name: ray
+          - name: taskorder
+          - name: nominatednode
+          - name: dynamicresources
+          - name: gpupack
+          - name: gpusharingorder
+          - name: nodeplacement
+            arguments:
+              cpu: {{ .Values.scheduler.placementStrategy }}
+              gpu: {{ .Values.scheduler.placementStrategy }}
+kind: ConfigMap
+metadata:
+  labels:
+    app: scheduler
+  name: scheduler-config

deployments/kai-scheduler/templates/services/scheduler-service.yaml

@@ -0,0 +1,17 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: scheduler
+spec:
+  selector:
+    app: scheduler
+  ports:
+    - name: http-metrics
+      protocol: TCP
+      port: {{ .Values.scheduler.ports.metricsPort }}
+      targetPort: {{ .Values.scheduler.ports.metricsPort }}
+  sessionAffinity: None
+  type: ClusterIP

deployments/kai-scheduler/templates/services/scheduler-serviceaccount.yaml

@@ -0,0 +1,7 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scheduler

deployments/kai-scheduler/templates/services/scheduler.yaml

@@ -0,0 +1,52 @@
+# Copyright 2025 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: scheduler
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: scheduler
+  template:
+    metadata:
+      labels:
+        app: scheduler
+    spec:
+      serviceAccountName: scheduler
+      containers:
+        - name: scheduler
+          image: "{{ .Values.global.registry }}/{{ .Values.scheduler.image.name }}:{{ .Chart.Version }}"
+          imagePullPolicy: {{ .Values.scheduler.image.pullPolicy }}
+          args:
+            - --scheduler-conf=/etc/config/config.yaml
+          {{- if .Values.scheduler.additionalArgs }}
+            {{- toYaml .Values.scheduler.additionalArgs | nindent 12 }}
+          {{- end }}
+          {{- if .Values.scheduler.resources }}
+          resources:
+            {{- toYaml .Values.scheduler.resources | nindent 12 }}
+          {{- end }}
+          {{- if .Values.global.securityContext }}
+          securityContext:
+            {{- toYaml .Values.global.securityContext | nindent 12 }}
+          {{- end }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.scheduler.ports.metricsPort }}
+          volumeMounts:
+            - name: config
+              subPath: "config.yaml"
+              mountPath: "/etc/config/config.yaml"
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml .Values.global.imagePullSecrets | nindent 8 }}
+      {{- end }}
+      volumes:
+        - name: config
+          configMap:
+            name: scheduler-config

deployments/kai-scheduler/values.yaml

@@ -43,6 +43,22 @@ binder:
       cpu: "250m"
       memory: "128Mi"
 
+scheduler:
+  image:
+    name: scheduler
+    pullPolicy: IfNotPresent
+  additionalArgs: []
+  placementStrategy: binpack
+  ports:
+    metricsPort: 8080
+  resources:
+    limits:
+      cpu: "500m"
+      memory: "256Mi"
+    requests:
+      cpu: "250m"
+      memory: "128Mi"
+
 webhookmanager:
   image:
     name: webhookmanager