soft check for VFs before attempting unbind

Signed-off-by: Varun Ramachandra Sekar <[email protected]>

Author: varunrsekar

cmd/gpu-kubelet-plugin/device_state.go

@@ -98,7 +98,7 @@ func NewDeviceState(ctx context.Context, config *Config) (*DeviceState, error) {
 
 	var vfioPciManager *VfioPciManager
 	if featuregates.Enabled(featuregates.PassthroughSupport) {
-		manager := NewVfioPciManager(string(containerDriverRoot))
+		manager := NewVfioPciManager(string(containerDriverRoot), nvdevlib)
 		if err := manager.Prechecks(); err == nil {
 			vfioPciManager = manager
 		} else {

cmd/gpu-kubelet-plugin/vfio-device.go

@@ -47,12 +47,14 @@ const (
 type VfioPciManager struct {
 	driverRoot string
 	driver     string
+	nvlib      *deviceLib
 }
 
-func NewVfioPciManager(driverRoot string) *VfioPciManager {
+func NewVfioPciManager(driverRoot string, nvlib *deviceLib) *VfioPciManager {
 	vm := &VfioPciManager{
 		driverRoot: driverRoot,
 		driver:     vfioPciDriver,
+		nvlib:      nvlib,
 	}
 	if !vm.isVfioPCIModuleLoaded() {
 		err := vm.loadVfioPciModule()
@@ -149,6 +151,19 @@ func (vm *VfioPciManager) WaitForGPUFree(info *VfioDeviceInfo) error {
 	}
 }
 
+// Verify there are no VFs on the GPU.
+func (vm *VfioPciManager) verifyDisabledVFs(pcieBusID string) error {
+	gpu, err := vm.nvlib.nvpci.GetGPUByPciBusID(pcieBusID)
+	if err != nil {
+		return err
+	}
+	numVFs := gpu.SriovInfo.PhysicalFunction.NumVFs
+	if numVFs > 0 {
+		return fmt.Errorf("gpu has %d VFs, cannot unbind", numVFs)
+	}
+	return nil
+}
+
 // Configure binds the GPU to the vfio-pci driver.
 func (vm *VfioPciManager) Configure(info *VfioDeviceInfo) error {
 	perGpuLock.Get(info.pcieBusID).Lock()
@@ -165,6 +180,10 @@ func (vm *VfioPciManager) Configure(info *VfioDeviceInfo) error {
 	if err != nil {
 		return err
 	}
+	err = vm.verifyDisabledVFs(info.pcieBusID)
+	if err != nil {
+		return err
+	}
 	err = vm.changeDriver(info.pcieBusID, vm.driver)
 	if err != nil {
 		return err