feat: use monitoring script that lists memory usage along with used models

Author: nikochiko

nvidia-monitor.Dockerfile

@@ -1,8 +1,7 @@
 FROM nvcr.io/nvidia/cuda:12.4.1-runtime-ubuntu22.04
 
-RUN wget -qO nerdctl.tar.gz "https://github.com/containerd/nerdctl/releases/download/v1.7.6/nerdctl-1.7.6-linux-amd64.tar.gz" \
-    && tar xvzf /usr/local/bin nerdctl.tar.gz \
-    && rm nerdctl.tar.gz
+WORKDIR /root
+COPY scripts/monitor.sh monitor.sh
 
-# for i in $(nerdctl  -a /host/run/containerd/containerd.sock -n k8s.io container ls --format "{{.ID}}"); do nerdctl  -a /host/run/containerd/containerd.sock -n k8s.io inspect -f '{{.State.Pid}} {{index .Config.Labels "io.kubernetes.pod.name"}}' $i; done | grep gooey-gpu
+# ./monitor.sh
 # nvidia-smi

scripts/monitor.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+ENV_VAR_FILTER="MODEL_IDS"
+
+function print3() {
+  printf "%-8s %-80s %12s\n" "$1" "${2%,}" "$3"
+}
+
+print3 "PID" "Model IDs" "GPU Mem"
+
+nvidia-smi --query-compute-apps=pid,used_memory --format=csv,noheader,nounits \
+    | while IFS=',' read -r pid mem; do
+        pid="${pid//[[:space:]]/}"
+        mem="$(echo $mem|tr -d '[:space:],')"
+        cmd_env=$(ps eww -p "$pid" -o command=)
+        val=$(printf '%s\n' "$cmd_env" | tr ' ' '\n' | grep "$ENV_VAR_FILTER=" | cut -d= -f2- | tr '\n' ',')
+        if [ -z "$val" ]; then
+          val=$(printf '%s\n' "$cmd_env" | sed -nE 's/.*celery@([^:]+):MainProcess.*/\1/p')
+        fi
+        mem_gib=$(awk "BEGIN{printf \"%.2f\", $mem/1024}")
+        print3 "$pid" "${val%,}" "${mem_gib}GiB"
+      done