Add MIG Reconfigure API for vgpu-device-manager

This change adds a mig/reconfigure package that implements the
functionality added to the vgpu-device-manager.

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

pkg/mig/reconfigure/api.go

@@ -0,0 +1,11 @@
+package reconfigure
+
+const (
+	MIGConfigStateLabel  = "nvidia.com/mig.config.state"
+	VGPUConfigStateLabel = "nvidia.com/vgpu.config.state"
+)
+
+// A Reconfigurer applies applies applies the specified config.
+type Reconfigurer interface {
+	Reconfigure(string, string) error
+}

pkg/mig/reconfigure/find.go

@@ -0,0 +1,91 @@
+/*
+# Copyright 2024 NVIDIA CORPORATION
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+// TODO: This should be pulled in from nvidia-container-toolkit or go-nvlib.
+
+package reconfigure
+
+import (
+	"fmt"
+	"path/filepath"
+)
+
+type root string
+
+// getDriverLibraryPath returns path to `libnvidia-ml.so.1` in the driver root.
+// The folder for this file is also expected to be the location of other driver files.
+func (r root) getDriverLibraryPath() (string, error) {
+	librarySearchPaths := []string{
+		"/usr/lib64",
+		"/usr/lib/x86_64-linux-gnu",
+		"/usr/lib/aarch64-linux-gnu",
+		"/lib64",
+		"/lib/x86_64-linux-gnu",
+		"/lib/aarch64-linux-gnu",
+	}
+
+	libraryPath, err := r.findFile("libnvidia-ml.so.1", librarySearchPaths...)
+	if err != nil {
+		return "", err
+	}
+
+	return libraryPath, nil
+}
+
+// getNvidiaSMIPath returns path to the `nvidia-smi` executable in the driver root.
+func (r root) getNvidiaSMIPath() (string, error) {
+	binarySearchPaths := []string{
+		"/usr/bin",
+		"/usr/sbin",
+		"/bin",
+		"/sbin",
+	}
+
+	binaryPath, err := r.findFile("nvidia-smi", binarySearchPaths...)
+	if err != nil {
+		return "", err
+	}
+
+	return binaryPath, nil
+}
+
+// findFile searches the root for a specified file.
+// A number of folders can be specified to search in addition to the root itself.
+// If the file represents a symlink, this is resolved and the final path is returned.
+func (r root) findFile(name string, searchIn ...string) (string, error) {
+
+	for _, d := range append([]string{"/"}, searchIn...) {
+		l := filepath.Join(string(r), d, name)
+		candidate, err := resolveLink(l)
+		if err != nil {
+			continue
+		}
+		return candidate, nil
+	}
+
+	return "", fmt.Errorf("error locating %q", name)
+}
+
+// resolveLink finds the target of a symlink or the file itself in the
+// case of a regular file.
+// This is equivalent to running `readlink -f ${l}`.
+func resolveLink(l string) (string, error) {
+	resolved, err := filepath.EvalSymlinks(l)
+	if err != nil {
+		return "", fmt.Errorf("error resolving link '%s': %w", l, err)
+	}
+	return resolved, nil
+}

pkg/mig/reconfigure/options.go

@@ -0,0 +1,134 @@
+package reconfigure
+
+import (
+	"k8s.io/client-go/kubernetes"
+)
+
+// An Option represents a functional option passed to the constructor.
+type Option func(*options)
+
+// reconfigureMIGOptions contains configuration options for reconfiguring MIG
+// settings on a Kubernetes node. This struct is used to manage the various
+// parameters required for applying MIG configurations through mig-parted, including node identification, configuration files, reboot behavior, and host
+// system service management.
+type reconfigureMIGOptions struct {
+	// NodeName is the kubernetes node to change the MIG configuration on.
+	// Its validation follows the RFC 1123 standard for DNS subdomain names.
+	// Source: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names
+	// NodeName string `validate:"required,hostname_rfc1123"`
+
+	// MIGPartedConfigFile is the mig-parted configuration file path.
+	// Deprecated: Pass the config file as an argument.
+	MIGPartedConfigFile string `validate:"required,filepath"`
+
+	// SelectedMIGConfig is the selected mig-parted configuration to apply to the
+	// node.
+	// Deprecated: Pass the selected config as an argument.
+	SelectedMIGConfig string
+
+	// DriverLibrayPath is the path to libnvidia-ml.so.1 in the container.
+	DriverLibraryPath string `validate:"required,filepath"`
+
+	// WithReboot reboots the node if changing the MIG mode fails for any reason.
+	WithReboot bool
+
+	// WithShutdownHostGPUClients shutdowns/restarts any required host GPU clients
+	// across a MIG configuration.
+	WithShutdownHostGPUClients bool
+
+	// HostRootMount is the container path where host root directory is mounted.
+	HostRootMount string `validate:"dirpath"`
+
+	// HostMIGManagerStateFile is the path where the systemd mig-manager state
+	// file is located.
+	HostMIGManagerStateFile string `validate:"filepath"`
+
+	// HostGPUClientServices is a comma separated list of host systemd services to
+	// shutdown/restart across a MIG reconfiguration.
+	HostGPUClientServices []string `validate:"dive,systemd_service_name"`
+
+	// HostKubeletService is the name of the host's 'kubelet' systemd service
+	// which may need to be shutdown/restarted across a MIG mode reconfiguration.
+	HostKubeletService string `validate:"systemd_service_name"`
+
+	configStateLabel string
+}
+
+type manager struct {
+	clientset *kubernetes.Clientset
+	nodeName  string
+}
+
+type options struct {
+	manager
+
+	driverRoot root
+
+	reconfigureMIGOptions
+}
+
+func WithClientset(clientset *kubernetes.Clientset) Option {
+	return func(o *options) {
+		o.clientset = clientset
+	}
+}
+
+func WithNodeName(nodeName string) Option {
+	return func(o *options) {
+		o.nodeName = nodeName
+	}
+}
+
+func WithDriverRoot[T string | root](driverRoot T) Option {
+	return func(o *options) {
+		o.driverRoot = root(driverRoot)
+	}
+}
+
+func WithDriverLibraryPath(driverLibraryPath string) Option {
+	return func(o *options) {
+		o.DriverLibraryPath = driverLibraryPath
+	}
+}
+
+func WithShutdownHostGPUClients(shutdownHostGPUClients bool) Option {
+	return func(o *options) {
+		o.WithShutdownHostGPUClients = shutdownHostGPUClients
+	}
+}
+
+func WithHostGPUClientServices(hostGPUClientServices ...string) Option {
+	return func(o *options) {
+		o.HostGPUClientServices = append([]string{}, hostGPUClientServices...)
+	}
+}
+
+func WithHostKubeletService(hostKubeletService string) Option {
+	return func(o *options) {
+		o.HostKubeletService = hostKubeletService
+	}
+}
+
+func WithHostMIGManagerStateFile(hostMIGManagerStateFile string) Option {
+	return func(o *options) {
+		o.HostMIGManagerStateFile = hostMIGManagerStateFile
+	}
+}
+
+func WithHostRootMount(hostRootMount string) Option {
+	return func(o *options) {
+		o.HostRootMount = hostRootMount
+	}
+}
+
+func WithAllowReboot(allowReboot bool) Option {
+	return func(o *options) {
+		o.WithReboot = allowReboot
+	}
+}
+
+func WithConfigStateLabel(configStateLabel string) Option {
+	return func(o *options) {
+		o.configStateLabel = configStateLabel
+	}
+}

pkg/mig/reconfigure/reconfigure.go

@@ -0,0 +1,69 @@
+package reconfigure
+
+import (
+	"context"
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func New(opts ...Option) (Reconfigurer, error) {
+	o := &options{}
+
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	if o.clientset == nil {
+		return nil, fmt.Errorf("a k8s clientset is required")
+	}
+	if o.nodeName == "" {
+		return nil, fmt.Errorf("a node name is required")
+	}
+	if o.configStateLabel == "" {
+		return nil, fmt.Errorf("a config state label must be specified")
+	}
+
+	// TODO: Add validation.
+
+	return o, nil
+}
+
+func (o *options) Reconfigure(migPartedConfigFile string, selectedMIGConfig string) error {
+	// TODO: These should be passed as arguments.
+	o.reconfigureMIGOptions.MIGPartedConfigFile = migPartedConfigFile
+	o.reconfigureMIGOptions.SelectedMIGConfig = selectedMIGConfig
+
+	return o.reconfigureMIG(&o.reconfigureMIGOptions)
+}
+
+func (m *manager) getNodeLabelValue(label string) (string, error) {
+	node, err := m.clientset.CoreV1().Nodes().Get(context.TODO(), m.nodeName, metav1.GetOptions{})
+	if err != nil {
+		return "", fmt.Errorf("unable to get node object: %v", err)
+	}
+
+	value, ok := node.Labels[label]
+	if !ok {
+		return "", nil
+	}
+
+	return value, nil
+}
+
+func (m *manager) setNodeLabelValue(label, value string) error {
+	node, err := m.clientset.CoreV1().Nodes().Get(context.TODO(), m.nodeName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("unable to get node object: %v", err)
+	}
+
+	labels := node.GetLabels()
+	labels[label] = value
+	node.SetLabels(labels)
+	_, err = m.clientset.CoreV1().Nodes().Update(context.TODO(), node, metav1.UpdateOptions{})
+	if err != nil {
+		return fmt.Errorf("unable to update node object: %v", err)
+	}
+
+	return nil
+}