make compute-domain kubelet dir configurable

Signed-off-by: Swati Gupta <[email protected]>

Author: guptaNswati

cmd/compute-domain-kubelet-plugin/computedomain.go

@@ -43,7 +43,6 @@ const (
 	informerResyncPeriod = 10 * time.Minute
 	cleanupInterval      = 10 * time.Minute
 
-	ComputeDomainDaemonSettingsRoot       = DriverPluginPath + "/domains"
 	ComputeDomainDaemonConfigTemplatePath = "/templates/compute-domain-daemon-config.tmpl.cfg"
 )
 
@@ -67,6 +66,10 @@ type ComputeDomainDaemonSettings struct {
 	nodesConfigPath string
 }
 
+func (c Config) ComputeDomainDaemonSettingsRoot() string {
+	return filepath.Join(c.DriverPluginPath(), "domains")
+}
+
 func NewComputeDomainManager(config *Config, configFilesRoot, cliqueID string) *ComputeDomainManager {
 	factory := nvinformers.NewSharedInformerFactory(config.clientsets.Nvidia, informerResyncPeriod)
 	informer := factory.Resource().V1beta1().ComputeDomains().Informer()

cmd/compute-domain-kubelet-plugin/device_state.go

@@ -90,13 +90,13 @@ func NewDeviceState(ctx context.Context, config *Config) (*DeviceState, error) {
 		return nil, fmt.Errorf("error getting cliqueID: %w", err)
 	}
 
-	computeDomainManager := NewComputeDomainManager(config, ComputeDomainDaemonSettingsRoot, cliqueID)
+	computeDomainManager := NewComputeDomainManager(config, config.ComputeDomainDaemonSettingsRoot(), cliqueID)
 
 	if err := cdi.CreateStandardDeviceSpecFile(allocatable); err != nil {
 		return nil, fmt.Errorf("unable to create base CDI spec file: %v", err)
 	}
 
-	checkpointManager, err := checkpointmanager.NewCheckpointManager(DriverPluginPath)
+	checkpointManager, err := checkpointmanager.NewCheckpointManager(config.DriverPluginPath())
 	if err != nil {
 		return nil, fmt.Errorf("unable to create checkpoint manager: %v", err)
 	}

cmd/compute-domain-kubelet-plugin/driver.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"path/filepath"
 	"sync"
 	"time"
 
@@ -42,10 +43,6 @@ const (
 	// that deadline, retryable errors are retried (with backoff) via the
 	// workqueue abstraction.
 	ErrorRetryMaxTimeout = 45 * time.Second
-	// DriverPrepUprepFlockPath is the path to a lock file used to make sure
-	// that calls to nodePrepareResource() / nodeUnprepareResource() never
-	// interleave, node-globally.
-	DriverPrepUprepFlockPath = DriverPluginPath + "/pu.lock"
 )
 
 // permanentError defines an error indicating that it is permanent.
@@ -64,6 +61,13 @@ type driver struct {
 	pulock       *flock.Flock
 }
 
+// DriverPrepUprepFlockPath is the path to a lock file used to make sure
+// that calls to nodePrepareResource() / nodeUnprepareResource() never
+// interleave, node-globally.
+func (c Config) DriverPrepUprepFlockPath() string {
+	return filepath.Join(c.DriverPluginPath(), "pu.lock")
+}
+
 func NewDriver(ctx context.Context, config *Config) (*driver, error) {
 	state, err := NewDeviceState(ctx, config)
 	if err != nil {
@@ -73,7 +77,7 @@ func NewDriver(ctx context.Context, config *Config) (*driver, error) {
 	driver := &driver{
 		client: config.clientsets.Core,
 		state:  state,
-		pulock: flock.NewFlock(DriverPrepUprepFlockPath),
+		pulock: flock.NewFlock(config.DriverPrepUprepFlockPath()),
 	}
 
 	helper, err := kubeletplugin.Start(
@@ -89,6 +93,8 @@ func NewDriver(ctx context.Context, config *Config) (*driver, error) {
 		// prepare() must be incoming). Concurrency management for incoming
 		// requests is done with this driver's work queue abstraction.
 		kubeletplugin.Serialize(false),
+		kubeletplugin.RegistrarDirectoryPath(config.flags.kubeletRegistrarDirectoryPath),
+		kubeletplugin.PluginDataDirectoryPath(config.DriverPluginPath()),
 	)
 	if err != nil {
 		return nil, err

cmd/compute-domain-kubelet-plugin/main.go

@@ -26,6 +26,7 @@ import (
 
 	"github.com/urfave/cli/v2"
 
+	"k8s.io/dynamic-resource-allocation/kubeletplugin"
 	"k8s.io/klog/v2"
 
 	"github.com/NVIDIA/k8s-dra-driver-gpu/internal/info"
@@ -34,7 +35,6 @@ import (
 
 const (
 	DriverName                         = "compute-domain.nvidia.com"
-	DriverPluginPath                   = "/var/lib/kubelet/plugins/" + DriverName
 	DriverPluginCheckpointFileBasename = "checkpoint.json"
 )
 
@@ -43,19 +43,25 @@ type Flags struct {
 	loggingConfig     *flags.LoggingConfig
 	featureGateConfig *flags.FeatureGateConfig
 
-	nodeName            string
-	namespace           string
-	cdiRoot             string
-	containerDriverRoot string
-	hostDriverRoot      string
-	nvidiaCDIHookPath   string
+	nodeName                      string
+	namespace                     string
+	cdiRoot                       string
+	containerDriverRoot           string
+	hostDriverRoot                string
+	nvidiaCDIHookPath             string
+	kubeletRegistrarDirectoryPath string
+	kubeletPluginsDirectoryPath   string
 }
 
 type Config struct {
 	flags      *Flags
 	clientsets flags.ClientSets
 }
 
+func (c Config) DriverPluginPath() string {
+	return filepath.Join(c.flags.kubeletPluginsDirectoryPath, DriverName)
+}
+
 func main() {
 	if err := newApp().Run(os.Args); err != nil {
 		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
@@ -111,6 +117,20 @@ func newApp() *cli.App {
 			Destination: &flags.nvidiaCDIHookPath,
 			EnvVars:     []string{"NVIDIA_CDI_HOOK_PATH"},
 		},
+		&cli.StringFlag{
+			Name:        "kubelet-registrar-directory-path",
+			Usage:       "Absolute path to the directory where kubelet stores plugin registrations.",
+			Value:       kubeletplugin.KubeletRegistryDir,
+			Destination: &flags.kubeletRegistrarDirectoryPath,
+			EnvVars:     []string{"KUBELET_REGISTRAR_DIRECTORY_PATH"},
+		},
+		&cli.StringFlag{
+			Name:        "kubelet-plugins-directory-path",
+			Usage:       "Absolute path to the directory where kubelet stores plugin data.",
+			Value:       kubeletplugin.KubeletPluginsDir,
+			Destination: &flags.kubeletPluginsDirectoryPath,
+			EnvVars:     []string{"KUBELET_PLUGINS_DIRECTORY_PATH"},
+		},
 	}
 	cliFlags = append(cliFlags, flags.kubeClientConfig.Flags()...)
 	cliFlags = append(cliFlags, flags.featureGateConfig.Flags()...)
@@ -158,13 +178,13 @@ func newApp() *cli.App {
 // StartPlugin initializes and runs the compute domain kubelet plugin.
 func StartPlugin(ctx context.Context, config *Config) error {
 	// Create the plugin directory
-	err := os.MkdirAll(DriverPluginPath, 0750)
+	err := os.MkdirAll(config.DriverPluginPath(), 0750)
 	if err != nil {
 		return err
 	}
 
 	// Setup nvidia-cdi-hook binary
-	if err := config.flags.setNvidiaCDIHookPath(); err != nil {
+	if err := config.setNvidiaCDIHookPath(); err != nil {
 		return fmt.Errorf("error setting up nvidia-cdi-hook: %w", err)
 	}
 
@@ -215,13 +235,13 @@ func StartPlugin(ctx context.Context, config *Config) error {
 // to this path. The /usr/bin/nvidia-cdi-hook is present in the current
 // container image because it is copied from the toolkit image into this
 // container at build time.
-func (f *Flags) setNvidiaCDIHookPath() error {
-	if f.nvidiaCDIHookPath != "" {
+func (c Config) setNvidiaCDIHookPath() error {
+	if c.flags.nvidiaCDIHookPath != "" {
 		return nil
 	}
 
 	sourcePath := "/usr/bin/nvidia-cdi-hook"
-	targetPath := filepath.Join(DriverPluginPath, "nvidia-cdi-hook")
+	targetPath := filepath.Join(c.DriverPluginPath(), "nvidia-cdi-hook")
 
 	input, err := os.ReadFile(sourcePath)
 	if err != nil {
@@ -232,7 +252,7 @@ func (f *Flags) setNvidiaCDIHookPath() error {
 		return fmt.Errorf("error copying nvidia-cdi-hook: %w", err)
 	}
 
-	f.nvidiaCDIHookPath = targetPath
+	c.flags.nvidiaCDIHookPath = targetPath
 
 	return nil
 }