Add query latency metric

Author: itsomri

pkg/scheduler/cache/usagedb/usagedb.go

@@ -11,6 +11,7 @@ import (
 	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/api/queue_info"
 	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/cache/usagedb/api"
 	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/log"
+	"github.com/NVIDIA/KAI-scheduler/pkg/scheduler/metrics"
 )
 
 var defaultFetchInterval = 1 * time.Minute
@@ -135,14 +136,13 @@ func (l *UsageLister) WaitForCacheSync(stopCh <-chan struct{}) bool {
 }
 
 func (l *UsageLister) fetchAndUpdateUsage() {
-	// TODO: Add metrics for fetch times
+	now := time.Now()
 	usage, err := l.client.GetResourceUsage()
 	if err != nil {
 		log.InfraLogger.V(1).Errorf("failed to fetch usage data: %v", err)
 		return
 	}
-
-	now := time.Now()
+	metrics.UpdateUsageQueryLatency(time.Since(now))
 
 	l.lastUsageDataMutex.Lock()
 	defer l.lastUsageDataMutex.Unlock()

pkg/scheduler/metrics/metrics.go

@@ -54,6 +54,7 @@ var (
 	queueCPUUsage               *prometheus.GaugeVec
 	queueMemoryUsage            *prometheus.GaugeVec
 	queueGPUUsage               *prometheus.GaugeVec
+	usageQueryLatency           *prometheus.HistogramVec
 )
 
 func init() {
@@ -190,6 +191,14 @@ func InitMetrics(namespace string) {
 			Name:      "queue_gpu_usage_devices",
 			Help:      "GPU usage of queue, as a gauge. Value is proportional to gpu*hours usage with time decay applied",
 		}, []string{"queue_name"})
+
+	usageQueryLatency = promauto.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: namespace,
+			Name:      "usage_query_latency_milliseconds",
+			Help:      "Usage database query latency histogram in milliseconds",
+			Buckets:   prometheus.ExponentialBuckets(5, 2, 10),
+		}, []string{})
 }
 
 // UpdateOpenSessionDuration updates latency for open session, including all plugins
@@ -277,6 +286,10 @@ func ResetQueueUsage() {
 	queueGPUUsage.Reset()
 }
 
+func UpdateUsageQueryLatency(latency time.Duration) {
+	usageQueryLatency.WithLabelValues().Observe(float64(latency.Milliseconds()))
+}
+
 // RegisterPreemptionAttempts records number of attempts for preemption
 func RegisterPreemptionAttempts() {
 	preemptionAttempts.Inc()