Enable kubectl execution in sidecar (#24155)

Enable kubectl execution in sidecar

Author: FengPan-Frank

dockers/docker-telemetry-sidecar/systemd_scripts/telemetry.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Telemetry container
+Requires=database.service
+After=database.service swss.service syncd.service
+BindsTo=sonic.target
+After=sonic.target
+StartLimitIntervalSec=1200
+StartLimitBurst=3
+
+[Service]
+Type=simple
+User=root
+ExecStartPre=/usr/local/bin/telemetry.sh start   # start: now returns in non-blocking (fire-and-forget pod deletion)
+ExecStart=/usr/local/bin/telemetry.sh wait       # wait: long-lived loop that observes pod status
+ExecStop=/usr/local/bin/telemetry.sh stop        # stop will not be working after kubesonic since pod will be auto-deployed via kubernetes
+RestartSec=30
+TimeoutStartSec=30s
+TimeoutStopSec=30s
+Restart=always

dockers/docker-telemetry-sidecar/systemd_scripts/telemetry.sh

@@ -1,85 +1,141 @@
 #!/bin/bash
+# 1. Runs as root via systemd service, so direct access to kubelet.conf is available; sudo is not required
+# 2. Use kubectl to get pods and delete pods with retry
+# 3. start/stop/restart are NON-BLOCKING
+# 4. Only target pods matching POD_SELECTOR (default: raw_container_name=telemetry)
+
 set -euo pipefail
 
-SERVICE="telemetry"
-NS="${NS:-sonic}"                               # k8s namespace
-LABEL="raw_container_name=${SERVICE}"           # selector used by DaemonSet
-KUBECTL_BIN="${KUBECTL_BIN:-kubectl}"
-NODE_NAME="${NODE_NAME:-$(hostname)}"
-DEV="${2:-}"                                    # accepted for compatibility; unused (single-ASIC)
+NS="sonic"
+KUBECTL_BIN="/usr/bin/kubectl"
+KCF=(--kubeconfig=/etc/kubernetes/kubelet.conf)
+REQ_TIMEOUT="5s"
+MAX_ATTEMPTS=10
+BACKOFF_START=1
+BACKOFF_MAX=8
 
-log() { /usr/bin/logger -t "${SERVICE}#system" "$*"; }
+# Label selector for telemetry pods; can be overridden via env
+# Example override: POD_SELECTOR="app=telemetry" telemetry.sh start
+POD_SELECTOR="${POD_SELECTOR:-raw_container_name=telemetry}"
 
-require_kubectl() {
-  if ! command -v "${KUBECTL_BIN}" >/dev/null 2>&1; then
-    echo "ERROR: kubectl not found (KUBECTL_BIN=${KUBECTL_BIN})." >&2
-    exit 127
-  fi
-  # Try a sensible default if KUBECONFIG isn’t set
-  if [[ -z "${KUBECONFIG:-}" && -r /etc/kubernetes/kubelet.conf ]]; then
-    export KUBECONFIG=/etc/kubernetes/kubelet.conf
-  fi
+NODE_NAME="$(hostname | tr '[:upper:]' '[:lower:]')"
+log() { /usr/bin/logger -t "k8s-podctl#system" "$*"; }
+
+kubectl_retry() {
+  local attempt=1 backoff=${BACKOFF_START} out rc
+  while true; do
+    out="$("${KUBECTL_BIN}" "${KCF[@]}" --request-timeout="${REQ_TIMEOUT}" "$@" 2>&1)"; rc=$?
+    if (( rc == 0 )); then
+      printf '%s' "$out"
+      return 0
+    fi
+    if (( attempt >= MAX_ATTEMPTS )); then
+      echo "$out" >&2
+      return "$rc"
+    fi
+    log "kubectl retry ${attempt}/${MAX_ATTEMPTS} for: $*"
+    sleep "${backoff}"
+    (( backoff = backoff < BACKOFF_MAX ? backoff*2 : BACKOFF_MAX ))
+    (( attempt++ ))
+  done
 }
 
 pods_on_node() {
-  # Prints: "<name> <phase>" per line for this node
-  "${KUBECTL_BIN}" -n "${NS}" get pods \
-    -l "${LABEL}" \
+  kubectl_retry -n "${NS}" get pods \
     --field-selector "spec.nodeName=${NODE_NAME}" \
-    -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{"\n"}{end}' 2>/dev/null || true
+    -l "${POD_SELECTOR}" \
+    -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.phase}{"\n"}{end}' || true
+}
+
+pod_names_on_node() {
+  kubectl_retry -n "${NS}" get pods \
+    --field-selector "spec.nodeName=${NODE_NAME}" \
+    -l "${POD_SELECTOR}" \
+    -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' || true
+}
+
+delete_pod_with_retry() {
+  local name="$1"
+  local out rc
+  out=$(kubectl_retry -n "${NS}" delete pod "${name}" --force --grace-period=0 --wait=false 2>&1)
+  rc=$?
+  if (( rc != 0 )); then
+    log "ERROR delete pod '${name}' failed rc=${rc}: ${out}"
+  else
+    log "Deleted pod '${name}'"
+  fi
+  return "$rc"
 }
 
 kill_pods() {
-  require_kubectl
-  local found=0
-  while read -r name phase; do
-    [[ -z "${name}" ]] && continue
-    found=1
-    log "Deleting ${SERVICE} pod ${name} (phase=${phase}) on node ${NODE_NAME}"
-    # Force/instant delete to emulate “kill”; DaemonSet will recreate
-    "${KUBECTL_BIN}" -n "${NS}" delete pod "${name}" --grace-period=0 --force >/dev/null 2>&1 || true
-  done < <(pods_on_node)
-  if [[ "${found}" -eq 0 ]]; then
-    log "No ${SERVICE} pods found on node ${NODE_NAME} (namespace=${NS}, label=${LABEL})."
+  mapfile -t names < <(pod_names_on_node)
+  if (( ${#names[@]} == 0 )); then
+    log "No pods found on ${NODE_NAME} (ns=${NS}, selector=${POD_SELECTOR})."
+    return 0
+  fi
+
+  log "Deleting pods on ${NODE_NAME} (ns=${NS}, selector=${POD_SELECTOR}): ${names[*]}"
+
+  local rc_any=0
+  for p in "${names[@]}"; do
+    [[ -z "$p" ]] && continue
+    if ! delete_pod_with_retry "$p"; then
+      rc_any=1
+    fi
+  done
+
+  if (( rc_any != 0 )); then
+    log "ERROR one or more pod deletions failed on ${NODE_NAME} (selector=${POD_SELECTOR})"
+  else
+    log "All targeted pods deleted on ${NODE_NAME} (selector=${POD_SELECTOR})"
   fi
+  return "$rc_any"
+}
+
+cmd_start() {
+  if command -v systemd-cat >/dev/null 2>&1; then
+    # background + pipe to journald with distinct priorities
+    ( kill_pods ) \
+      > >(systemd-cat -t telemetry-start -p info) \
+      2> >(systemd-cat -t telemetry-start -p err)
+  else
+    # background + pipe to syslog via logger in case systemd-journald is masked/disabled
+    ( kill_pods ) \
+      > >(logger -t "telemetry-start" -p user.info) \
+      2> >(logger -t "telemetry-start" -p user.err)
+  fi &
+  disown
+  exit 0
 }
 
-cmd_start()   { kill_pods; }     # start == kill (DS restarts)
 cmd_stop()    { kill_pods; }
 cmd_restart() { kill_pods; }
 
 cmd_status() {
-  require_kubectl
-  local out; out="$(pods_on_node)"
-  if [[ -z "${out}" ]]; then
-    echo "${SERVICE}: NOT RUNNING (no pod on node ${NODE_NAME})"
+  local out=""; out="$(pods_on_node)"
+  if [[ -z "$out" ]]; then
+    echo "NOT RUNNING (no pod on node ${NODE_NAME} with selector '${POD_SELECTOR}')"
     exit 3
   fi
-  echo "${out}" | while read -r name phase; do
-    [[ -z "${name}" ]] && continue
-    echo "${SERVICE} pod ${name}: ${phase}"
-  done
-  # Exit 0 if at least one Running, 1 otherwise
-  if echo "${out}" | awk '$2=="Running"{found=1} END{exit found?0:1}'; then
+  while read -r name phase; do
+    [[ -z "$name" ]] && continue
+    echo "pod ${name}: ${phase}"
+  done <<<"$out"
+  if awk '$2=="Running"{found=1} END{exit found?0:1}' <<<"$out"; then
     exit 0
   else
     exit 1
   fi
 }
 
 cmd_wait() {
-  require_kubectl
-  log "Waiting on ${SERVICE} pods (ns=${NS}, label=${LABEL}) on node ${NODE_NAME}..."
-  # Keep the systemd service 'active' as long as at least one pod exists for this node.
+  log "Waiting on pods (ns=${NS}, selector=${POD_SELECTOR}) on node ${NODE_NAME}…"
   while true; do
-    local out; out="$(pods_on_node)"
-    if [[ -z "${out}" ]]; then
-      # no pod presently; keep waiting (DaemonSet may bring it up)
-      sleep 5
-      continue
+    local out=""; out="$(pods_on_node)"
+    if [[ -z "$out" ]]; then
+      sleep 5; continue
     fi
-    # If at least one is Running, sleep longer; otherwise poll faster
-    if echo "${out}" | awk '$2=="Running"{found=1} END{exit found?0:1}'; then
+    if awk '$2=="Running"{found=1} END{exit found?0:1}' <<<"$out"; then
       sleep 60
     else
       sleep 5
@@ -94,7 +150,7 @@ case "${1:-}" in
   wait)    cmd_wait ;;
   status)  cmd_status ;;
   *)
-    echo "Usage: $0 {start|stop|restart|wait|status} [asic-id(optional, ignored)]" >&2
+    echo "Usage: $0 {start|stop|restart|wait|status}" >&2
     exit 2
     ;;
 esac

dockers/docker-telemetry-sidecar/systemd_scripts/tests/test_systemd_stub.py

@@ -78,7 +78,7 @@ def fake_run_nsenter(args, *, text=True, input_bytes=None):
             target = args[-1]
             host_fs.pop(target, None)
             return 0, "" if text else b"", "" if text else b""
-        # sudo …
+        # sudo … (allow anything)
         if args[:1] == ["sudo"]:
             return 0, "" if text else b"", "" if text else b""
         return 1, "" if text else b"", "unsupported" if text else b"unsupported"
@@ -87,6 +87,7 @@ def fake_run_nsenter(args, *, text=True, input_bytes=None):
 
     # Fake container FS
     container_fs = {}
+
     def fake_read_file_bytes_local(path: str):
         return container_fs.get(path, None)
 
@@ -213,3 +214,31 @@ def test_env_controls_telemetry_src_default(monkeypatch):
     ss = importlib.import_module("systemd_stub")
     assert ss.IS_V1_ENABLED is False
     assert ss._TELEMETRY_SRC.endswith("telemetry.sh")
+
+
+def test_telemetry_service_syncs_to_host_when_different(ss):
+    ss, container_fs, host_fs, commands = ss
+
+    # Prepare container unit content and host old content
+    container_fs[ss.CONTAINER_TELEMETRY_SERVICE] = b"UNIT-NEW"
+    host_fs[ss.HOST_TELEMETRY_SERVICE] = b"UNIT-OLD"
+
+    # Only include the telemetry service item to make the assertion clear
+    ss.SYNC_ITEMS[:] = [
+        ss.SyncItem(ss.CONTAINER_TELEMETRY_SERVICE, ss.HOST_TELEMETRY_SERVICE, 0o644)
+    ]
+
+    # Add post actions for telemetry.service
+    ss.POST_COPY_ACTIONS[ss.HOST_TELEMETRY_SERVICE] = [
+        ["sudo", "systemctl", "daemon-reload"],
+        ["sudo", "systemctl", "restart", "telemetry"],
+    ]
+
+    ok = ss.ensure_sync()
+    assert ok is True
+    assert host_fs[ss.HOST_TELEMETRY_SERVICE] == b"UNIT-NEW"
+
+    # Verify systemctl actions were invoked
+    post_cmds = [args for _, args in commands if args and args[0] == "sudo"]
+    assert ("sudo", "systemctl", "daemon-reload") in post_cmds
+    assert ("sudo", "systemctl", "restart", "telemetry") in post_cmds