Skip to content

Commit cc34bd7

Browse files
winsopcwinsopc
committed
VFIO device support
Add both PF and VF VFIO mode support, which is needed by kubevirt support. Signed-off-by: Zhen(Winson) Wang <[email protected]>
1 parent 6921b4c commit cc34bd7

File tree

8 files changed

+461
-110
lines changed

8 files changed

+461
-110
lines changed

‎cmd/ib-sriov-cni/main.go‎

Lines changed: 145 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,24 @@ func unlockCNIExecution(lock *flock.Flock) {
9595
_ = lock.Unlock()
9696
}
9797

98+
func handleVfioPciDetection(netConf *localtypes.NetConf) error {
99+
if netConf.DeviceID == "" {
100+
return nil
101+
}
102+
103+
// If vfioPciMode is explicitly set to true, use it; otherwise auto-detect
104+
if !netConf.VfioPciMode {
105+
isVfioPci, err := utils.IsVfioPciDevice(netConf.DeviceID)
106+
if err != nil {
107+
return fmt.Errorf("failed to check vfio-pci driver binding for device %s: %v", netConf.DeviceID, err)
108+
}
109+
if isVfioPci {
110+
netConf.VfioPciMode = true
111+
}
112+
}
113+
return nil
114+
}
115+
98116
// Get network config, updated with GUID, device info and network namespace.
99117
func getNetConfNetns(args *skel.CmdArgs) (*localtypes.NetConf, ns.NetNS, error) {
100118
netConf, err := config.LoadConf(args.StdinData)
@@ -116,6 +134,21 @@ func getNetConfNetns(args *skel.CmdArgs) (*localtypes.NetConf, ns.NetNS, error)
116134
"infiniband SRIOV-CNI failed, Unexpected error. GUID must be provided by ib-kubernetes")
117135
}
118136

137+
// Handle vfio-pci detection
138+
if err := handleVfioPciDetection(netConf); err != nil {
139+
return nil, nil, err
140+
}
141+
142+
// If vfioPciMode is true (either set manually or auto-detected), skip SR-IOV setup
143+
if netConf.VfioPciMode {
144+
// Skip normal SR-IOV setup for vfio-pci devices, just get netns and return
145+
netns, err := ns.GetNS(args.Netns)
146+
if err != nil {
147+
return nil, nil, fmt.Errorf("failed to open netns %q: %v", netns, err)
148+
}
149+
return netConf, netns, nil
150+
}
151+
119152
if netConf.RdmaIso {
120153
err = utils.EnsureRdmaSystemMode()
121154
if err != nil {
@@ -215,59 +248,65 @@ func runIPAMPlugin(stdinData []byte, netConf *localtypes.NetConf) (_ *current.Re
215248
return newResult, nil
216249
}
217250

218-
func cmdAdd(args *skel.CmdArgs) (retErr error) {
219-
netConf, netns, err := getNetConfNetns(args)
251+
// handleVfioPciMode handles VFIO PCI devices with VF GUID setting if needed
252+
func handleVfioPciMode(args *skel.CmdArgs, netConf *localtypes.NetConf, netns ns.NetNS) error {
253+
// Check if the device is a VF (Virtual Function) or PF (Physical Function)
254+
isVF, err := utils.IsVirtualFunction(netConf.DeviceID)
220255
if err != nil {
221-
return err
256+
return fmt.Errorf("failed to determine if device %s is VF or PF: %v", netConf.DeviceID, err)
222257
}
223-
defer netns.Close()
224258

225-
sm := sriov.NewSriovManager()
259+
// Only handle GUID setting for VF devices, keep previous code path for PF devices
260+
if isVF && netConf.GUID != "" {
261+
// Load device info needed for GUID setting (VFIO VF version - no network interface)
262+
err = config.LoadDeviceInfoVfioVF(netConf)
263+
if err != nil {
264+
return fmt.Errorf("failed to get device specific information for vfio-pci VF device: %v", err)
265+
}
226266

227-
// Lock CNI operation to serialize the operation
228-
lock, err := lockCNIExecution()
229-
if err != nil {
230-
return err
267+
sm := sriov.NewSriovManager()
268+
if err := sm.ApplyVFConfig(netConf); err != nil {
269+
return fmt.Errorf("failed to configure VF GUID for vfio-pci device: %v", err)
270+
}
231271
}
232-
defer unlockCNIExecution(lock)
233272

234-
err = doVFConfig(sm, netConf, netns, args)
235-
if err != nil {
236-
return err
273+
result := &current.Result{}
274+
result.Interfaces = []*current.Interface{{
275+
Name: args.IfName,
276+
Sandbox: netns.Path(),
277+
}}
278+
279+
// Cache NetConf for CmdDel (minimal config for vfio-pci mode)
280+
if err = utils.SaveNetConf(args.ContainerID, config.DefaultCNIDir, args.IfName, netConf); err != nil {
281+
return fmt.Errorf("error saving NetConf %q", err)
237282
}
238-
defer func() {
239-
if retErr != nil {
240-
nsErr := netns.Do(func(_ ns.NetNS) error {
241-
_, innerErr := netlink.LinkByName(args.IfName)
242-
return innerErr
243-
})
244-
if nsErr == nil {
245-
_ = sm.ReleaseVF(netConf, args.IfName, args.ContainerID, netns)
246-
}
247-
if netConf.RdmaIso {
248-
_ = utils.MoveRdmaDevFromNs(netConf.RdmaNetState.ContainerRdmaDevName, netns)
249-
}
250-
}
251-
}()
252283

284+
return types.PrintResult(result, netConf.CNIVersion)
285+
}
286+
287+
// setupIPAMAndResult handles IPAM configuration and creates the result
288+
func setupIPAMAndResult(args *skel.CmdArgs, netConf *localtypes.NetConf, netns ns.NetNS) (*current.Result, func(error), error) {
253289
result := &current.Result{}
254290
result.Interfaces = []*current.Interface{{
255291
Name: args.IfName,
256292
Sandbox: netns.Path(),
257293
}}
258294

295+
// Default cleanup function (no-op)
296+
cleanup := func(error) {}
297+
259298
if netConf.IPAM.Type != "" {
260-
var newResult *current.Result
261-
newResult, err = runIPAMPlugin(args.StdinData, netConf)
299+
newResult, err := runIPAMPlugin(args.StdinData, netConf)
262300
if err != nil {
263-
return err
301+
return nil, cleanup, err
264302
}
265-
// If runIPAMPlugin failed, than ExecDel was called. Defer if no error
266-
defer func() {
303+
304+
// Return cleanup function for IPAM
305+
cleanup = func(retErr error) {
267306
if retErr != nil {
268307
_ = ipam.ExecDel(netConf.IPAM.Type, args.StdinData)
269308
}
270-
}()
309+
}
271310

272311
newResult.Interfaces = result.Interfaces
273312

@@ -280,12 +319,66 @@ func cmdAdd(args *skel.CmdArgs) (retErr error) {
280319
return ipam.ConfigureIface(args.IfName, newResult)
281320
})
282321
if err != nil {
283-
return err
322+
return nil, cleanup, err
284323
}
285324

286325
result = newResult
287326
}
288327

328+
return result, cleanup, nil
329+
}
330+
331+
func cmdAdd(args *skel.CmdArgs) (retErr error) {
332+
netConf, netns, err := getNetConfNetns(args)
333+
if err != nil {
334+
return err
335+
}
336+
defer netns.Close()
337+
338+
// If device is bound to vfio-pci, handle VF GUID setting if needed, then skip SR-IOV configuration
339+
if netConf.VfioPciMode {
340+
return handleVfioPciMode(args, netConf, netns)
341+
}
342+
343+
sm := sriov.NewSriovManager()
344+
345+
// Lock CNI operation to serialize the operation
346+
lock, err := lockCNIExecution()
347+
if err != nil {
348+
return err
349+
}
350+
defer unlockCNIExecution(lock)
351+
352+
err = doVFConfig(sm, netConf, netns, args)
353+
if err != nil {
354+
return err
355+
}
356+
defer func() {
357+
if retErr != nil {
358+
// Skip cleanup for VFIO devices as they don't have network interfaces to manage
359+
if !netConf.VfioPciMode {
360+
nsErr := netns.Do(func(_ ns.NetNS) error {
361+
_, innerErr := netlink.LinkByName(args.IfName)
362+
return innerErr
363+
})
364+
if nsErr == nil {
365+
_ = sm.ReleaseVF(netConf, args.IfName, args.ContainerID, netns)
366+
}
367+
}
368+
if netConf.RdmaIso {
369+
_ = utils.MoveRdmaDevFromNs(netConf.RdmaNetState.ContainerRdmaDevName, netns)
370+
}
371+
}
372+
}()
373+
374+
result, ipamCleanup, err := setupIPAMAndResult(args, netConf, netns)
375+
if err != nil {
376+
return err
377+
}
378+
defer func() {
379+
ipamCleanup(retErr)
380+
}()
381+
289382
// Cache NetConf for CmdDel
290383
if err = utils.SaveNetConf(args.ContainerID, config.DefaultCNIDir, args.IfName, netConf); err != nil {
291384
return fmt.Errorf("error saving NetConf %q", err)
@@ -294,6 +387,16 @@ func cmdAdd(args *skel.CmdArgs) (retErr error) {
294387
return types.PrintResult(result, netConf.CNIVersion)
295388
}
296389

390+
func handleIPAMCleanup(netConf *localtypes.NetConf, stdinData []byte) error {
391+
if netConf.IPAM.Type == "" {
392+
return nil
393+
}
394+
if netConf.IPAM.Type == ipamDHCP {
395+
return fmt.Errorf("ipam type dhcp is not supported")
396+
}
397+
return ipam.ExecDel(netConf.IPAM.Type, stdinData)
398+
}
399+
297400
func cmdDel(args *skel.CmdArgs) (retErr error) {
298401
// https://github.com/kubernetes/kubernetes/pull/35240
299402
if args.Netns == "" {
@@ -317,16 +420,16 @@ func cmdDel(args *skel.CmdArgs) (retErr error) {
317420
}()
318421
}
319422

423+
// If device was in vfio-pci mode, skip all SR-IOV and IPAM cleanup
424+
if netConf.VfioPciMode {
425+
return nil
426+
}
427+
320428
sm := sriov.NewSriovManager()
321429

322-
if netConf.IPAM.Type != "" {
323-
if netConf.IPAM.Type == ipamDHCP {
324-
return fmt.Errorf("ipam type dhcp is not supported")
325-
}
326-
err = ipam.ExecDel(netConf.IPAM.Type, args.StdinData)
327-
if err != nil {
328-
return err
329-
}
430+
err = handleIPAMCleanup(netConf, args.StdinData)
431+
if err != nil {
432+
return err
330433
}
331434

332435
netns, err := ns.GetNS(args.Netns)

‎pkg/config/config.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,27 @@ func LoadDeviceInfo(netConf *types.NetConf) error {
6060
return nil
6161
}
6262

63+
// LoadDeviceInfoVfioVF loads device information for VFIO VF devices (no network interface)
64+
func LoadDeviceInfoVfioVF(netConf *types.NetConf) error {
65+
// DeviceID takes precedence; if we are given a VF pciaddr then work from there
66+
if netConf.DeviceID != "" {
67+
// Get rest of the VF information
68+
pfName, vfID, err := getVfInfo(netConf.DeviceID)
69+
if err != nil {
70+
return fmt.Errorf("load config: failed to get VF information: %q", err)
71+
}
72+
netConf.VFID = vfID
73+
netConf.Master = pfName
74+
75+
// For VFIO VF, we don't have network interface, so set HostIFNames to empty
76+
netConf.HostIFNames = ""
77+
} else {
78+
return fmt.Errorf("load config: vf pci addr is required")
79+
}
80+
81+
return nil
82+
}
83+
6384
func getVfInfo(vfPci string) (string, int, error) {
6485
var vfID int
6586

0 commit comments

Comments
 (0)