feat(operator): add a metric for taint scheduling

Author: ayuskauskas

docs/metrics/grafana_values.yaml

@@ -1,7 +1,27 @@
+# 
+# LICENSE START
+#
+#    Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# LICENSE END
+# 
+
 datasources:
   datasources.yaml:
     apiVersion: 1
     datasources:
     - name: Prometheus
       type: prometheus
-      url: http://prometheus-server.default.svc.cluster.local
+      url: http://prometheus-server.default.svc.cluster.local

docs/metrics/prometheus_values.yaml

@@ -1,3 +1,23 @@
+# 
+# LICENSE START
+#
+#    Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+# LICENSE END
+# 
+
 extraScrapeConfigs: |
   - job_name: 'skyhook'
     metrics_path: /metrics
@@ -7,4 +27,3 @@ extraScrapeConfigs: |
     static_configs:
       - targets:
         - skyhook-operator-controller-manager-metrics-service.skyhook.svc.cluster.local:8443
-

k8s-tests/chainsaw/skyhook/taint-scheduling/chainsaw-test.yaml

@@ -49,10 +49,16 @@ spec:
         file: skyhook.yaml
     - assert:
         file: assert.yaml
+    - script:
+        content: |
+          ../metrics_test.py skyhook_node_taint_tolerance_issue_count 1 -t skyhook_name=taint-scheduling
     - patch:
         file: update-skyhook.yaml
     - assert:
         file: assert-update.yaml
+    - script:
+        content: |
+          ../metrics_test.py skyhook_node_taint_tolerance_issue_count 0 -t skyhook_name=taint-scheduling
     finally:
     - script:
         content: |

operator/internal/controller/cluster_state_v2.go

@@ -448,6 +448,7 @@ func (np *NodePicker) SelectNodes(s SkyhookNodes) []wrapper.SkyhookNode {
 			nodesWithTaintTolerationIssue = append(nodesWithTaintTolerationIssue, node.GetNode().Name)
 		}
 	}
+	skyhook_node_taint_tolerance_issue_count.WithLabelValues(s.GetSkyhook().Name).Set(float64(len(nodesWithTaintTolerationIssue)))
 
 	// if we have nodes that are not tolerable, we need to add a condition to the skyhook
 	if len(nodesWithTaintTolerationIssue) > 0 {

operator/internal/controller/metrics.go

@@ -131,6 +131,14 @@ var (
 		},
 		[]string{"skyhook_name", "package_name", "package_version", "stage"},
 	)
+
+	skyhook_node_taint_tolerance_issue_count = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "skyhook_node_taint_tolerance_issue_count",
+			Help: "Number of nodes in the cluster that have taint tolerance issues",
+		},
+		[]string{"skyhook_name"},
+	)
 )
 
 func zeroOutSkyhookMetrics(skyhook SkyhookNodes) {
@@ -142,6 +150,7 @@ func zeroOutSkyhookMetrics(skyhook SkyhookNodes) {
 	skyhook_node_complete_count.DeleteLabelValues(skyhook.GetSkyhook().Name)
 	skyhook_node_error_count.DeleteLabelValues(skyhook.GetSkyhook().Name)
 	skyhook_node_blocked_count.DeleteLabelValues(skyhook.GetSkyhook().Name)
+	skyhook_node_taint_tolerance_issue_count.DeleteLabelValues(skyhook.GetSkyhook().Name)
 	for _, _package := range skyhook.GetSkyhook().Spec.Packages {
 		zeroOutSkyhookPackageMetrics(skyhook.GetSkyhook().Name, _package.Name, _package.Version)
 	}
@@ -172,5 +181,6 @@ func init() {
 		skyhook_package_complete_count,
 		skyhook_package_stage_count,
 		skyhook_package_restarts_count,
+		skyhook_node_taint_tolerance_issue_count,
 	)
 }