propagate scheduler service namespace for leader election in HA (#236)

* propagate scheduler service namespace for leader election in HA

* Added namespace argument to scheduler deployment

* Update cmd/scheduler/app/options/options.go

Co-authored-by: Roman Baron <[email protected]>

* Update cmd/scheduler/app/options/options.go

Co-authored-by: Roman Baron <[email protected]>

* Fixed options test in scheduler service

---------

Co-authored-by: Roman Baron <[email protected]>
Co-authored-by: Roman Baron <[email protected]>

Author: iris-shain-runai

cmd/scheduler/app/options/options.go

@@ -16,6 +16,7 @@ const (
 	defaultSchedulerName               = "kai-scheduler"
 	defaultResourceReservationAppLabel = "kai-resource-reservation"
 	defaultMetricsNamespace            = "kai"
+	defaultNamespace                   = "kai-scheduler"
 	defaultSchedulerPeriod             = time.Second
 	defaultStalenessGracePeriod        = 60 * time.Second
 	defaultListenAddress               = ":8080"
@@ -65,6 +66,7 @@ type ServerOption struct {
 	CPUWorkerNodeLabelKey             string
 	GPUWorkerNodeLabelKey             string
 	MIGWorkerNodeLabelKey             string
+	Namspace                          string
 
 	QPS   int
 	Burst int
@@ -80,6 +82,8 @@ func NewServerOption() *ServerOption {
 func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	// kai-scheduler will ignore pods with scheduler names other than specified with the option
 	fs.StringVar(&s.SchedulerName, "scheduler-name", defaultSchedulerName, "The scheduler name in pod spec that handled by this scheduler")
+	fs.StringVar(&s.Namspace, "namespace", defaultNamespace, "Scheduler service namespace")
+	fs.StringVar(&s.MetricsNamespace, "metrics-namespace", defaultMetricsNamespace, "Metrics namespace")
 	fs.StringVar(&s.ResourceReservationAppLabel, "resource-reservation-app-label", defaultResourceReservationAppLabel, "App label value of resource reservation pods")
 	fs.BoolVar(&s.RestrictSchedulingNodes, "restrict-node-scheduling", false, "kai-scheduler will allocate jobs only to restricted nodes")
 	fs.StringVar(&s.NodePoolLabelKey, "nodepool-label-key", defaultNodePoolLabelKey, "The label key by which to filter scheduling nodepool")
@@ -90,7 +94,6 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 		"Start a leader election client and gain leadership before "+
 			"executing the main loop. Enable this when running replicated kai-scheduler for high availability")
 	fs.BoolVar(&s.PrintVersion, "version", true, "Show version")
-	fs.StringVar(&s.MetricsNamespace, "metrics-namespace", defaultMetricsNamespace, "Metrics namespace")
 	fs.StringVar(&s.ListenAddress, "listen-address", defaultListenAddress, "The address to listen on for HTTP requests")
 	fs.BoolVar(&s.EnableProfiler, "enable-profiler", false, "Enable profiler")
 	fs.StringVar(&s.ProfilerApiPort, "profiler-port", defaultProfilerApiPort, "The port to listen for profiler api requests")

cmd/scheduler/app/options/options_test.go

@@ -28,10 +28,11 @@ func TestAddFlags(t *testing.T) {
 	// This is a snapshot of expected options parsed by args.
 	expected := &ServerOption{
 		SchedulerName:                     defaultSchedulerName,
+		Namspace:                          defaultNamespace,
+		MetricsNamespace:                  defaultMetricsNamespace,
 		ResourceReservationAppLabel:       defaultResourceReservationAppLabel,
 		SchedulePeriod:                    5 * time.Minute,
 		PrintVersion:                      true,
-		MetricsNamespace:                  defaultMetricsNamespace,
 		ListenAddress:                     defaultListenAddress,
 		ProfilerApiPort:                   defaultProfilerApiPort,
 		Verbosity:                         defaultVerbosityLevel,

cmd/scheduler/app/server.go

@@ -41,8 +41,6 @@ const (
 	leaseDuration = 15 * time.Second
 	renewDeadline = 10 * time.Second
 	retryPeriod   = 5 * time.Second
-
-	lockObjectNamespace = ""
 )
 
 var logFlushFreq = pflag.Duration("log-flush-frequency", 5*time.Second, "Maximum number of seconds between log flushes")
@@ -174,7 +172,7 @@ func Run(opt *options.ServerOption, config *restclient.Config, mux *http.ServeMu
 
 	// Prepare event clients.
 	broadcaster := record.NewBroadcaster()
-	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: leaderElectionClient.CoreV1().Events(lockObjectNamespace)})
+	broadcaster.StartRecordingToSink(&corev1.EventSinkImpl{Interface: leaderElectionClient.CoreV1().Events(opt.Namspace)})
 	eventRecorder := broadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: opt.SchedulerName})
 
 	hostname, err := os.Hostname()
@@ -185,7 +183,7 @@ func Run(opt *options.ServerOption, config *restclient.Config, mux *http.ServeMu
 	id := hostname + "_" + string(uuid.NewUUID())
 
 	rl, err := resourcelock.New(resourcelock.LeasesResourceLock,
-		lockObjectNamespace,
+		opt.Namspace,
 		opt.SchedulerName,
 		leaderElectionClient.CoreV1(),
 		leaderElectionClient.CoordinationV1(),

deployments/kai-scheduler/templates/services/scheduler.yaml

@@ -24,6 +24,7 @@ spec:
           imagePullPolicy: {{ .Values.scheduler.image.pullPolicy }}
           args:
             - "--scheduler-conf=/etc/config/config.yaml"
+            - "--namespace={{ .Release.Namespace }}"
             - "--resource-reservation-app-label={{ .Values.global.resourceReservation.appLabel }}"
           {{- if .Values.scheduler.additionalArgs }}
             {{- toYaml .Values.scheduler.additionalArgs | nindent 12 }}