Add postgresql as alternative datastore

Signed-off-by: Davanum Srinivas <[email protected]>

Author: dims

Tiltfile.postgresql

@@ -0,0 +1,157 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load('ext://helm_resource', 'helm_resource', 'helm_repo')
+load('ext://namespace', 'namespace_create', 'namespace_inject')
+
+update_settings(k8s_upsert_timeout_secs=600)
+
+num_gpu_nodes = int(os.getenv('NUM_GPU_NODES', '10'))
+
+# Install cert-manager for TLS certificates
+helm_repo('jetstack', 'https://charts.jetstack.io')
+helm_resource(
+    'cert-manager',
+    chart='jetstack/cert-manager',
+    namespace='cert-manager',
+    flags=[
+        '--create-namespace',
+        '--set=installCRDs=true',
+    ],
+)
+
+# Install Prometheus operator for monitoring
+helm_repo('prometheus-community', 'https://prometheus-community.github.io/helm-charts')
+helm_resource(
+    'prometheus-operator',
+    chart='prometheus-community/kube-prometheus-stack',
+    namespace='monitoring',
+    flags=[
+        '--create-namespace',
+        '--set=prometheus.enabled=false',
+        '--set=alertmanager.enabled=false',
+        '--set=grafana.enabled=false',
+        '--set=kubeStateMetrics.enabled=false',
+        '--set=nodeExporter.enabled=false',
+        '--set=prometheusOperator.enabled=true',
+    ],
+)
+
+# Install KWOK for fake GPU nodes
+helm_repo('sigs-kwok', 'https://kwok.sigs.k8s.io/charts/')
+helm_resource(
+    'kwok',
+    chart='sigs-kwok/kwok',
+    namespace='kube-system',
+    flags=[
+        '--set=hostNetwork=true'
+    ]
+)
+helm_resource(
+    'kwok-stage-fast',
+    chart='sigs-kwok/stage-fast',
+    resource_deps=['kwok'],
+    pod_readiness='ignore'
+)
+
+# Create namespaces
+namespace_create('gpu-operator')
+namespace_create('nvsentinel')
+
+# Create fake GPU nodes using KWOK
+kwok_node_template = str(read_file('./tilt/kwok-node-template.yaml'))
+for i in range(num_gpu_nodes):
+    node_yaml = kwok_node_template.replace('PLACEHOLDER', str(i))
+    k8s_yaml(blob(node_yaml))
+
+# Apply additional Kubernetes resources
+k8s_yaml('./tilt/nvidia-driver-daemonset.yaml')
+k8s_yaml('./tilt/nvidia-dcgm-daemonset.yaml')
+k8s_yaml('./tilt/janitor.dgxc.nvidia.com_rebootnodes.yaml')
+
+# Include component-specific Tiltfiles
+include('./fault-quarantine-module/Tiltfile')
+include('./fault-remediation-module/Tiltfile')
+include('./node-drainer-module/Tiltfile')
+include('./platform-connectors/Tiltfile')
+include('./health-events-analyzer/Tiltfile')
+include('./health-monitors/csp-health-monitor/Tiltfile')
+include('./labeler-module/Tiltfile')
+include('./tilt/simple-health-client/Tiltfile')
+include('./health-monitors/gpu-health-monitor/Tiltfile')
+include('./health-monitors/syslog-health-monitor/Tiltfile')
+
+# Deploy NVSentinel with PostgreSQL configuration using certificate authentication
+yaml = helm(
+    './distros/kubernetes/nvsentinel',
+    name='nvsentinel',
+    namespace='nvsentinel',
+    values=[
+        './distros/kubernetes/nvsentinel/values.yaml',
+        './distros/kubernetes/nvsentinel/values-tilt-postgresql.yaml'
+    ],
+)
+k8s_yaml(yaml)
+
+# Configure cert-manager resources for PostgreSQL
+k8s_resource(
+    new_name='cert-manager-resources',
+    objects=[
+        'postgresql-root-ca:certificate',
+        'postgresql-ca-issuer:issuer',
+        'selfsigned-ca-issuer:issuer',
+        'postgresql-server-cert:certificate',
+        'postgresql-client-cert:certificate'
+    ],
+    resource_deps=['cert-manager'],
+)
+
+# Configure Prometheus monitoring resources
+k8s_resource(
+    new_name='prometheus-resources',
+    objects=['nvsentinel-pod-monitor:podmonitor'],
+    resource_deps=['prometheus-operator'],
+)
+k8s_resource(
+    'prometheus-operator',
+    port_forwards='9090:9090',
+)
+
+# Configure KWOK fake nodes
+kwok_node_names = ['kwok-node-' + str(i) + ':node' for i in range(num_gpu_nodes)]
+k8s_resource(
+    new_name='kwok-fake-nodes',
+    objects=kwok_node_names,
+    resource_deps=['kwok', 'nvsentinel-platform-connector', 'nvsentinel-fault-quarantine', 'nvsentinel-fault-remediation',
+        'nvsentinel-labeler', 'nvsentinel-node-drainer', 'nvsentinel-postgresql', 'simple-health-client'
+    ],
+)
+k8s_resource(
+    'kwok-stage-fast',
+    pod_readiness='ignore',
+    resource_deps=['kwok']
+)
+
+# Configure GPU health monitor with special pod readiness
+k8s_resource(
+    workload='nvsentinel-gpu-health-monitor-dcgm-3.x',
+    pod_readiness='ignore'
+)
+
+# PostgreSQL-specific resources
+k8s_resource(
+    workload='nvsentinel-postgresql',
+    port_forwards=['5432:5432'],  # Port-forward PostgreSQL for debugging
+    resource_deps=['cert-manager-resources'],
+)

distros/kubernetes/nvsentinel/Chart.lock

@@ -5,6 +5,9 @@ dependencies:
 - name: mongodb-store
   repository: ""
   version: 0.1.0
+- name: postgresql
+  repository: https://charts.bitnami.com/bitnami
+  version: 15.5.38
 - name: fault-quarantine
   repository: ""
   version: 0.1.0
@@ -29,5 +32,5 @@ dependencies:
 - name: labeler
   repository: ""
   version: 0.1.0
-digest: sha256:c10f6e7fdb0b99a47f38e210c25e610da182c94fe32d89753a34352d12c0bb22
-generated: "2025-10-15T10:37:19.739789+05:30"
+digest: sha256:7a790375389163a770b2796f9b771d0ac5744e66df098336bfa67113bd740318
+generated: "2025-10-21T14:58:37.80844-04:00"

distros/kubernetes/nvsentinel/Chart.yaml

@@ -25,6 +25,10 @@ dependencies:
   - name: mongodb-store
     version: "0.1.0"
     condition: platformConnector.mongodbStore.enabled
+  - name: postgresql
+    version: "15.5.38"
+    repository: https://charts.bitnami.com/bitnami
+    condition: postgresql.enabled
   - name: fault-quarantine
     version: "0.1.0"
     condition: global.faultQuarantineModule.enabled

distros/kubernetes/nvsentinel/charts/csp-health-monitor/templates/deployment.yaml

@@ -74,7 +74,7 @@ spec:
             readOnly: true
           envFrom:
             - configMapRef:
-                name: mongodb-config
+                name: {{- if eq .Values.global.datastore.provider "postgresql" }} nvsentinel-datastore-config{{- else }} mongodb-config{{- end }}
                 optional: true
 
         - name: maintenance-notifier
@@ -104,7 +104,7 @@ spec:
             mountPath: /run/nvsentinel
           envFrom:
             - configMapRef:
-                name: mongodb-config
+                name: {{- if eq .Values.global.datastore.provider "postgresql" }} nvsentinel-datastore-config{{- else }} mongodb-config{{- end }}
                 optional: true
       restartPolicy: Always
       {{- with (.Values.global.systemNodeSelector | default .Values.nodeSelector) }}

distros/kubernetes/nvsentinel/charts/fault-quarantine/templates/deployment.yaml

@@ -68,7 +68,7 @@ spec:
               value: {{ .Values.clientCertMountPath }}
           envFrom:
             - configMapRef:
-                name: mongodb-config
+                name: {{- if eq .Values.global.datastore.provider "postgresql" }} nvsentinel-datastore-config{{- else }} mongodb-config{{- end }}
                 optional: true
       volumes:
       - name: config-volume

distros/kubernetes/nvsentinel/charts/fault-remediation/templates/deployment.yaml

@@ -75,7 +75,7 @@ spec:
             value: "{{ .Values.logCollector.enableGcpSosCollection }}"
           envFrom:
             - configMapRef:
-                name: mongodb-config
+                name: {{- if eq .Values.global.datastore.provider "postgresql" }} nvsentinel-datastore-config{{- else }} mongodb-config{{- end }}
                 optional: true
       volumes:
       - name: mongo-app-client-cert

distros/kubernetes/nvsentinel/charts/health-events-analyzer/templates/deployment.yaml

@@ -62,7 +62,7 @@ spec:
               value: {{ .Values.clientCertMountPath }}
           envFrom:
             - configMapRef:
-                name: mongodb-config
+                name: {{- if eq .Values.global.datastore.provider "postgresql" }} nvsentinel-datastore-config{{- else }} mongodb-config{{- end }}
                 optional: true
       volumes:
       - name: config-volume

distros/kubernetes/nvsentinel/charts/mongodb-store/charts/mongodb/values.yaml

@@ -130,7 +130,7 @@ diagnosticMode:
 ##
 image:
   registry: docker.io
-  repository: bitnami/mongodb
+  repository: bitnamilegacy/mongodb
   tag: 8.0.3-debian-12-r0
   digest: ""
   ## Specify a imagePullPolicy
@@ -256,7 +256,7 @@ tls:
   ##
   image:
     registry: docker.io
-    repository: bitnami/nginx
+    repository: bitnamilegacy/nginx
     tag: 1.27.2-debian-12-r2
     digest: ""
     pullPolicy: IfNotPresent
@@ -844,7 +844,7 @@ externalAccess:
     ##
     image:
       registry: docker.io
-      repository: bitnami/kubectl
+      repository: bitnamilegacy/kubectl
       tag: 1.31.2-debian-12-r3
       digest: ""
       ## Specify a imagePullPolicy
@@ -893,7 +893,7 @@ externalAccess:
     ##
     image:
       registry: docker.io
-      repository: bitnami/os-shell
+      repository: bitnamilegacy/os-shell
       tag: 12-debian-12-r32
       digest: ""
       ## Specify a imagePullPolicy
@@ -1488,7 +1488,7 @@ volumePermissions:
   ##
   image:
     registry: docker.io
-    repository: bitnami/os-shell
+    repository: bitnamilegacy/os-shell
     tag: 12-debian-12-r32
     digest: ""
     ## Specify a imagePullPolicy
@@ -2288,7 +2288,7 @@ metrics:
   ##
   image:
     registry: docker.io
-    repository: bitnami/mongodb-exporter
+    repository: bitnamilegacy/mongodb-exporter
     tag: 0.41.2-debian-12-r1
     digest: ""
     pullPolicy: IfNotPresent

distros/kubernetes/nvsentinel/charts/node-drainer/templates/deployment.yaml

@@ -61,7 +61,7 @@ spec:
             value: {{ .Values.clientCertMountPath }}
           envFrom:
             - configMapRef:
-                name: mongodb-config
+                name: {{- if eq .Values.global.datastore.provider "postgresql" }} nvsentinel-datastore-config{{- else }} mongodb-config{{- end }}
                 optional: true
       volumes:
       - name: config-volume

distros/kubernetes/nvsentinel/templates/certmanager-postgresql.yaml

@@ -0,0 +1,89 @@
+# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+{{- if or (eq .Values.global.clusterType "standalone") (eq .Values.global.clusterType "mgmt") }}
+{{- if eq .Values.global.datastore.provider "postgresql" }}
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: postgresql-ca-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  ca:
+    secretName: postgresql-root-ca-secret
+---
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: selfsigned-ca-issuer
+  namespace: {{ .Release.Namespace }}
+spec:
+  selfSigned: {}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: postgresql-root-ca
+  namespace: {{ .Release.Namespace }}
+spec:
+  isCA: true
+  commonName: postgresql-root-ca
+  secretName: postgresql-root-ca-secret
+  duration: 87600h   # 10 years
+  renewBefore: 720h  # 30 days before expiration
+  privateKey:
+    algorithm: RSA
+    size: 4096
+  issuerRef:
+    name: selfsigned-ca-issuer
+    kind: Issuer
+---
+# PostgreSQL Server Certificate
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: postgresql-server-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: postgresql-server-cert
+  duration: 8760h   # 1 year
+  renewBefore: 360h # 15 days before expiration
+  commonName: postgresql
+  dnsNames:
+    - postgresql
+    - {{ printf "%s-postgresql.%s.svc.cluster.local" .Release.Name .Release.Namespace }}
+    - {{ printf "%s-postgresql.%s.svc" .Release.Name .Release.Namespace }}
+    - {{ printf "postgresql.%s.svc.cluster.local" .Release.Namespace }}
+    - {{ printf "postgresql.%s.svc" .Release.Namespace }}
+  issuerRef:
+    name: postgresql-ca-issuer
+    kind: Issuer
+---
+# PostgreSQL Client Certificate
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: postgresql-client-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  secretName: postgresql-client-cert
+  duration: 8760h   # 1 year
+  renewBefore: 360h # 15 days before expiration
+  commonName: postgresql
+  issuerRef:
+    name: postgresql-ca-issuer
+    kind: Issuer
+{{- end }}
+{{- end }}