Making scheduler metrics subsystem name configurable (#86)

Author: romanbaron

cmd/scheduler/app/options/options.go

@@ -14,6 +14,7 @@ import (
 
 const (
 	defaultSchedulerName               = "kai-scheduler"
+	defaultMetricsSubSystemName        = "kai"
 	defaultSchedulerPeriod             = time.Second
 	defaultStalenessGracePeriod        = 60 * time.Second
 	defaultListenAddress               = ":8080"
@@ -34,6 +35,7 @@ type ServerOption struct {
 	SchedulePeriod                    time.Duration
 	EnableLeaderElection              bool
 	PrintVersion                      bool
+	MetricsSubSystemName              string
 	RestrictSchedulingNodes           bool
 	NodePoolLabelKey                  string
 	NodePoolLabelValue                string
@@ -69,7 +71,7 @@ func NewServerOption() *ServerOption {
 // AddFlags adds flags for a specific CMServer to the specified FlagSet
 func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 	// kai-scheduler will ignore pods with scheduler names other than specified with the option
-	fs.StringVar(&s.SchedulerName, "scheduler-name", defaultSchedulerName, "kai-scheduler will handle pods with the scheduler-name")
+	fs.StringVar(&s.SchedulerName, "scheduler-name", defaultSchedulerName, "The scheduler name in pod spec that handled by this scheduler")
 	fs.BoolVar(&s.RestrictSchedulingNodes, "restrict-node-scheduling", false, "kai-scheduler will allocate jobs only to restricted nodes")
 	fs.StringVar(&s.NodePoolLabelKey, "nodepool-label-key", defaultNodePoolLabelKey, "The label key by which to filter scheduling nodepool")
 	fs.StringVar(&s.NodePoolLabelValue, "partition-label-value", "", "The label value by which to filter scheduling partition")
@@ -79,6 +81,7 @@ func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
 		"Start a leader election client and gain leadership before "+
 			"executing the main loop. Enable this when running replicated kai-scheduler for high availability")
 	fs.BoolVar(&s.PrintVersion, "version", true, "Show version")
+	fs.StringVar(&s.MetricsSubSystemName, "metrics-subsystem-name", defaultMetricsSubSystemName, "The name of the metrics subsystem")
 	fs.StringVar(&s.ListenAddress, "listen-address", defaultListenAddress, "The address to listen on for HTTP requests")
 	fs.BoolVar(&s.EnableProfiler, "enable-profiler", false, "Enable profiler")
 	fs.StringVar(&s.ProfilerApiPort, "profiler-port", defaultProfilerApiPort, "The port to listen for profiler api requests")

cmd/scheduler/app/options/options_test.go

@@ -30,6 +30,7 @@ func TestAddFlags(t *testing.T) {
 		SchedulerName:                     defaultSchedulerName,
 		SchedulePeriod:                    5 * time.Minute,
 		PrintVersion:                      true,
+		MetricsSubSystemName:              defaultMetricsSubSystemName,
 		ListenAddress:                     defaultListenAddress,
 		ProfilerApiPort:                   defaultProfilerApiPort,
 		Verbosity:                         defaultVerbosityLevel,

cmd/scheduler/app/server.go

@@ -30,9 +30,10 @@ import (
 
 	"github.com/NVIDIA/KAI-scheduler/cmd/scheduler/app/options"
 	"github.com/NVIDIA/KAI-scheduler/cmd/scheduler/profiling"
-	scheduler "github.com/NVIDIA/KAI-scheduler/pkg/scheduler"
+	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler"
 	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/conf"
 	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/log"
+	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/metrics"
 	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/version"
 )
 
@@ -131,6 +132,7 @@ func Run(opt *options.ServerOption, config *restclient.Config, mux *http.ServeMu
 	if opt.PrintVersion {
 		version.PrintVersion()
 	}
+	metrics.SetSubSystemName(opt.MetricsSubSystemName)
 
 	scheduler, err := scheduler.NewScheduler(config,
 		opt.SchedulerConf,

pkg/binder/binding/resourcereservation/resource_reservation_test.go

@@ -82,13 +82,12 @@ var _ = Describe("ResourceReservationService", func() {
 	)
 	Context("ReserveGpuDevice", func() {
 		for testName, testData := range map[string]struct {
-			reservationPod                 *v1.Pod
-			groupName                      string
-			clientInterceptFuncs           interceptor.Funcs
-			getGPUIndexFromMetricsAttempts int
-			numReservationPods             int
-			expectedGPUIndex               string
-			expectedErrorContains          string
+			reservationPod        *v1.Pod
+			groupName             string
+			clientInterceptFuncs  interceptor.Funcs
+			numReservationPods    int
+			expectedGPUIndex      string
+			expectedErrorContains string
 		}{
 			"reservation pod exists": {
 				groupName: existingGroup,

pkg/scheduler/metrics/metrics.go

@@ -11,8 +11,6 @@ import (
 )
 
 const (
-	RunaiNamespace = "runai"
-
 	// OnSessionOpen label
 	OnSessionOpen = "OnSessionOpen"
 
@@ -22,48 +20,49 @@ const (
 
 var (
 	currentAction string
+	subsystemName string
 
 	e2eSchedulingLatency = promauto.NewGauge(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "e2e_scheduling_latency_milliseconds",
 			Help:      "E2e scheduling latency in milliseconds (scheduling algorithm + binding), as a gauge",
 		},
 	)
 
 	openSessionLatency = promauto.NewGauge(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "open_session_latency_milliseconds",
 			Help:      "Open session latency in milliseconds, including all plugins, as a gauge",
 		},
 	)
 
 	closeSessionLatency = promauto.NewGauge(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "close_session_latency_milliseconds",
 			Help:      "Close session latency in milliseconds, including all plugins, as a gauge",
 		},
 	)
 
 	pluginSchedulingLatency = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "plugin_scheduling_latency_milliseconds",
 			Help:      "Plugin scheduling latency in milliseconds, as a gauge",
 		}, []string{"plugin", "OnSession"})
 
 	actionSchedulingLatency = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "action_scheduling_latency_milliseconds",
 			Help:      "Action scheduling latency in milliseconds, as a gauge",
 		}, []string{"action"})
 
 	taskSchedulingLatency = promauto.NewHistogram(
 		prometheus.HistogramOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "task_scheduling_latency_milliseconds",
 			Help:      "Task scheduling latency in milliseconds",
 			Buckets:   prometheus.ExponentialBuckets(5, 2, 10),
@@ -72,7 +71,7 @@ var (
 
 	taskBindLatency = promauto.NewHistogram(
 		prometheus.HistogramOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "task_bind_latency_milliseconds",
 			Help:      "Task bind latency histogram in milliseconds",
 			Buckets:   prometheus.ExponentialBuckets(5, 2, 10),
@@ -81,55 +80,55 @@ var (
 
 	podgroupsScheduledByAction = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "podgroups_scheduled_by_action",
 			Help:      "Count of podgroups scheduled per action",
 		}, []string{"action"})
 
 	podgroupsConsideredByAction = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "podgroups_acted_on_by_action",
 			Help:      "Count of podgroups tried per action",
 		}, []string{"action"})
 
 	scenariosSimulatedByAction = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "scenarios_simulation_by_action",
 			Help:      "Count of scenarios simulated per action",
 		}, []string{"action"})
 
 	scenariosFilteredByAction = promauto.NewCounterVec(
 		prometheus.CounterOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "scenarios_filtered_by_action",
 			Help:      "Count of scenarios filtered per action",
 		}, []string{"action"})
 
 	preemptionAttempts = promauto.NewCounter(
 		prometheus.CounterOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "total_preemption_attempts",
 			Help:      "Total preemption attempts in the cluster till now",
 		},
 	)
 
 	queueFairShareCPU = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "queue_fair_share_cpu_cores",
 			Help:      "CPU Fair share of queue, as a gauge. Value is in Cores",
 		}, []string{"queue_name"})
 	queueFairShareMemory = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "queue_fair_share_memory_gb",
 			Help:      "Memory Fair share of queue, as a gauge. Value is in GB",
 		}, []string{"queue_name"})
 	queueFairShareGPU = promauto.NewGaugeVec(
 		prometheus.GaugeOpts{
-			Subsystem: RunaiNamespace,
+			Subsystem: subsystemName,
 			Name:      "queue_fair_share_gpu",
 			Help:      "GPU Fair share of queue, as a gauge. Values in GPU devices",
 		}, []string{"queue_name"})
@@ -168,6 +167,10 @@ func UpdateTaskScheduleDuration(duration time.Duration) {
 	taskSchedulingLatency.Observe(float64(duration.Milliseconds()))
 }
 
+func SetSubSystemName(name string) {
+	subsystemName = name
+}
+
 func SetCurrentAction(action string) {
 	currentAction = action
 }