[nvidia-ctk-installer] do not revert cri-o config on shutdown

This commit updates the behavior of the nvidia-ctk-installer for cri-o.
On shutdown, we no longer delete the drop-in config file as long as
none of the nvidia runtime handlers are set as the default runtime.
This change was made to workaround an issue observed when uninstalling
the gpu-operator -- management containers launched with the nvidia
runtime handler would get stuck in the terminating state with the below
error message:

```
failed to find runtime handler nvidia from runtime list map[crun:... runc:...], failed to "KillPodSandbox" for ...
```

There appears to be a race condition where the nvidia-ctk-installer removes the drop-in file
and restarts cri-o. After the cri-o restart, if there are still pods / containers to terminate
that were started with the nvidia runtime, then cri-o fails to terminate them. The behavior
of cri-o, and its in-memory runtime handler cache, appears to differ from that of containerd as
we have never encountered such an issue with containerd.

This commit can be considered a stop-gap solution until more robust solution is developed.

Signed-off-by: Christopher Desiniotis <[email protected]>

Author: cdesiniotis

cmd/nvidia-ctk-installer/container/runtime/crio/config_test.go

@@ -88,7 +88,6 @@ func TestCrioConfigLifecycle(t *testing.T) {
 			},
 			assertCleanupPostConditions: func(t *testing.T, co *container.Options, _ *Options) error {
 				require.NoFileExists(t, co.TopLevelConfigPath)
-				require.NoFileExists(t, co.DropInConfig)
 				return nil
 			},
 		},
@@ -182,8 +181,6 @@ signature_policy = "/etc/crio/policy.json"
 			assertCleanupPostConditions: func(t *testing.T, co *container.Options, o *Options) error {
 				require.FileExists(t, co.TopLevelConfigPath)
 
-				require.NoFileExists(t, co.DropInConfig)
-
 				actualTopLevel, err := os.ReadFile(co.TopLevelConfigPath)
 				require.NoError(t, err)
 
@@ -480,8 +477,6 @@ plugin_dirs = [
 `
 				require.Equal(t, expected, string(actual))
 
-				require.NoFileExists(t, co.DropInConfig)
-
 				return nil
 			},
 		},

cmd/nvidia-ctk-installer/container/runtime/crio/crio.go

@@ -180,6 +180,11 @@ func cleanupHook(co *Options) error {
 
 // cleanupConfig removes the NVIDIA container runtime from the cri-o config
 func cleanupConfig(o *container.Options) error {
+	if !o.SetAsDefault {
+		log.Infof("")
+		return nil
+	}
+
 	log.Infof("Reverting config file modifications")
 
 	cfg, err := getRuntimeConfig(o)