Ignore errors getting device memory using NVML

On certain systems, the NVML nvmlDeviceGetMemoryInformation API
is not supported and returns an error. In these cases we ignore
these errors and log a warning instead. This means that:

* For the GPU Device Plugin, memory limits will be enforced for
  MPS partioning.
* For GFD, no nvidia.com/gpu.memory label will be generated.

Signed-off-by: Evan Lezar <[email protected]>

Author: elezar

internal/lm/resource.go

@@ -21,6 +21,8 @@ import (
 	"regexp"
 	"strings"
 
+	"k8s.io/klog/v2"
+
 	spec "github.com/NVIDIA/k8s-device-plugin/api/config/v1"
 	"github.com/NVIDIA/k8s-device-plugin/internal/resource"
 )
@@ -46,7 +48,7 @@ func NewGPUResourceLabeler(config *spec.Config, device resource.Device, count in
 
 	totalMemoryMiB, err := device.GetTotalMemoryMiB()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get memory info for device: %v", err)
+		klog.Warningf("Ignoring error getting memory info for device: %v", err)
 	}
 
 	resourceLabeler := newResourceLabeler(fullGPUResourceName, config)

internal/rm/devices.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"strings"
 
+	"k8s.io/klog/v2"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 )
 
@@ -73,7 +74,7 @@ func BuildDevice(index string, d deviceInfo) (*Device, error) {
 
 	totalMemory, err := d.GetTotalMemory()
 	if err != nil {
-		return nil, fmt.Errorf("error getting device memory: %w", err)
+		klog.Warningf("Ignoring error getting device memory: %v", err)
 	}
 
 	computeCapability, err := d.GetComputeCapability()