feat(operator): Add default podAntiAffinity and service-level Affinity support (#619)

Signed-off-by: Omer Yahud <[email protected]>

Author: omeryahud

CHANGELOG.md

@@ -12,6 +12,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 - Option to configure reservation pods runtime class.
 - Added a tool to run time-aware fairness simulations over multiple cycles (see [Time-Aware Fairness Simulator](cmd/time-aware-simulator/README.md))
 - Added enforcement of the `nvidia` runtime class for GPU pods, with the option to enforce a custom runtime class, or disable enforcement entirely.
+- Added a preferred podAntiAffinity term by default for all services, can be set to required instead by setting `global.requireDefaultPodAffinityTerm`
+- Added support for service-level affinities
 
 ### Fixed
 - Fixed a bug where the scheduler would not re-try updating podgroup status after failure

deployments/kai-scheduler/.helmignore

@@ -35,3 +35,4 @@ stable-index/*
 .circleci/
 .github/
 tests/*
+

deployments/kai-scheduler/crds/kai.scheduler_configs.yaml

@@ -27,6 +27,9 @@ spec:
     affinity:
       {{- toYaml .Values.global.affinity | nindent 6 }}
     {{- end }}
+    {{- if .Values.global.requireDefaultPodAntiAffinityTerm }}
+    requireDefaultPodAntiAffinityTerm: true
+    {{- end }}
     {{- if .Values.global.tolerations }}
     tolerations:
       {{- toYaml .Values.global.tolerations | nindent 6 }}
@@ -52,6 +55,10 @@ spec:
       resources:
         {{- toYaml .Values.binder.resources | nindent 8 }}
       {{- end }}
+      {{- if .Values.binder.affinity }}
+      affinity:
+        {{- toYaml .Values.binder.affinity | nindent 8 }}
+      {{- end }}
     metricsPort: {{ .Values.binder.ports.metricsPort }}
     resourceReservation:
       {{- if .Values.binder.runtimeClassName }}
@@ -72,6 +79,10 @@ spec:
       resources:
         {{- toYaml .Values.podgroupcontroller.resources | nindent 8 }}
       {{- end }}
+      {{- if .Values.podgroupcontroller.affinity }}
+      affinity:
+        {{- toYaml .Values.podgroupcontroller.affinity | nindent 8 }}
+      {{- end }}
 
   queueController:
     service:
@@ -85,6 +96,10 @@ spec:
       resources:
         {{- toYaml .Values.queuecontroller.resources | nindent 8 }}
       {{- end }}
+      {{- if .Values.queuecontroller.affinity }}
+      affinity:
+        {{- toYaml .Values.queuecontroller.affinity | nindent 8 }}
+      {{- end }}
 
   admission:
     service:
@@ -98,6 +113,10 @@ spec:
       resources:
         {{- toYaml .Values.admission.resources | nindent 8 }}
       {{- end }}
+      {{- if .Values.admission.affinity }}
+      affinity:
+        {{- toYaml .Values.admission.affinity | nindent 8 }}
+      {{- end }}
     gpuSharing: {{ .Values.global.gpuSharing | default false }}
     queueLabelSelector: false
     webhook:
@@ -119,6 +138,10 @@ spec:
       resources:
         {{- toYaml .Values.nodescaleadjuster.resources | nindent 8 }}
       {{- end }}
+      {{- if .Values.nodescaleadjuster.affinity }}
+      affinity:
+        {{- toYaml .Values.nodescaleadjuster.affinity | nindent 8 }}
+      {{- end }}
     args:
       nodeScaleNamespace: {{ .Values.nodescaleadjuster.scalingPodNamespace }} 
       scalingPodImage:
@@ -138,6 +161,10 @@ spec:
       resources:
         {{- toYaml .Values.scheduler.resources | nindent 8 }}
       {{- end }}
+      {{- if .Values.scheduler.affinity }}
+      affinity:
+        {{- toYaml .Values.scheduler.affinity | nindent 8 }}
+      {{- end }}
     {{- if and .Values.scheduler.ports .Values.scheduler.ports.metricsPort }}
     schedulerService:
       port: {{ .Values.scheduler.ports.metricsPort }}