Natasha/queue metrics fixes (#344)

* added mustregister to metrics

* added queuecontroller servicemonitor for metrics

* little fix

* removed servicemonitor..

Author: natasharomm

cmd/queuecontroller/app/app.go

@@ -5,6 +5,7 @@ package app
 
 import (
 	"flag"
+	"fmt"
 
 	"go.uber.org/zap/zapcore"
 
@@ -50,6 +51,7 @@ func Run() error {
 	initLogger()
 
 	metrics.InitMetrics(opts.MetricsNamespace, opts.QueueLabelToMetricLabel.Get(), opts.QueueLabelToDefaultMetricValue.Get())
+	setupLog.Info(fmt.Sprintf("Queue metrics initialized and registered with namespace: %s", opts.MetricsNamespace))
 
 	var err error
 	options := ctrl.Options{

deployments/kai-scheduler/templates/services/queuecontroller-service.yaml

@@ -13,5 +13,9 @@ spec:
     targetPort: 9443
     protocol: TCP
     name: webhook
+  - port: 8080
+    targetPort: 8080
+    protocol: TCP
+    name: metrics
   selector:
-    app: queuecontroller 
+    app: queuecontroller

pkg/queuecontroller/metrics/metrics.go

@@ -4,11 +4,13 @@
 package metrics
 
 import (
-	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/client_golang/prometheus/promauto" // auto-registry collectors in default registry
 	"sort"
 
 	v2 "github.com/NVIDIA/KAI-scheduler/pkg/apis/scheduling/v2"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto" // auto-registry collectors in default registry
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 )
 
 const (
@@ -92,6 +94,8 @@ func InitMetrics(namespace string, queueLabelToMetricLabelMap, queueLabelToDefau
 			Help:      "Queue quota memory",
 		}, queueMetricsLabels,
 	)
+
+	metrics.Registry.MustRegister(queueInfo, queueDeservedGPUs, queueQuotaCPU, queueQuotaMemory)
 }
 
 func SetQueueMetrics(queue *v2.Queue) {