diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 8991543f..dc55c8e9 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -24,9 +24,9 @@ on: workflow_dispatch: env: - GO_VERSION: "1.23" - K8S_VERSION: "v1.32.0" - KIND_VERSION: "v0.25.0" + GO_VERSION: "1.24" + K8S_VERSION: "v1.32.2" + KIND_VERSION: "v0.27.0" IMAGE_NAME: ghcr.io/google/dranet KIND_CLUSTER_NAME: kind diff --git a/go.mod b/go.mod index df793a24..18d86dee 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.32.3 k8s.io/utils v0.0.0-20250321185631-1f6e0b77f77e + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -93,5 +94,4 @@ require ( sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/pkg/apis/types.go b/pkg/apis/types.go new file mode 100644 index 00000000..2973b69e --- /dev/null +++ b/pkg/apis/types.go @@ -0,0 +1,76 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +// TODO Generate code and keep in sync golang types on schema +type NetworkConfig struct { + Name string `json:"name"` // new name inside the namespace + IPs []string `json:"ips"` + Routes []Route `json:"routes"` + MTU int `json:"mtu"` + Mode Mode `json:"mode"` + Macvlan *MacvlanConfig `json:"macvlan,omitempty"` + Macvtap *MacvlanConfig `json:"macvtap,omitempty"` + IPvlan *IPvlanConfig `json:"ipvlan,omitempty"` +} + +// Route represents a route configuration. +type Route struct { + Destination string `json:"destination"` + Gateway string `json:"gateway"` +} + +// Mode represents the network mode. +type Mode string + +// Enumerated Mode values. +const ( + ModeMacvlan Mode = "macvlan" + ModeMacvtap Mode = "macvtap" + ModeIPvlan Mode = "ipvlan" + ModeDedicated Mode = "dedicated" +) + +// MacvlanConfig represents the Macvlan configuration. +type MacvlanConfig struct { + Mode MacvlanMode `json:"macvlanMode"` +} + +// MacvlanMode represents the macvlan mode. +type MacvlanMode string + +// Enumerated Macvlan mode values. +const ( + MacvlanModeBridge MacvlanMode = "bridge" + MacvlanModePrivate MacvlanMode = "private" + MacvlanModeVepa MacvlanMode = "vepa" + MacvlanModePassthru MacvlanMode = "passthru" +) + +// IPvlanConfig represents the IPvlan configuration. +type IPvlanConfig struct { + Mode IPvlanMode `json:"ipvlanMode"` +} + +// IPvlanMode represents the ipvlan mode. +type IPvlanMode string + +// Enumerated IPvlan mode values. +const ( + IPvlanModeL2 IPvlanMode = "l2" + IPvlanModeL3 IPvlanMode = "l3" +) diff --git a/pkg/apis/validation.go b/pkg/apis/validation.go new file mode 100644 index 00000000..10634057 --- /dev/null +++ b/pkg/apis/validation.go @@ -0,0 +1,78 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + "errors" + "fmt" + "net/netip" + + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/yaml" +) + +// ValidateConfig validates the data in a runtime.RawExtension against the OpenAPI schema. +func ValidateConfig(raw *runtime.RawExtension) (*NetworkConfig, error) { + if raw == nil || raw.Raw == nil { + return nil, nil + } + // Check if raw.Raw is empty + if len(raw.Raw) == 0 { + return nil, nil + } + var errorsList []error + var config NetworkConfig + if err := yaml.Unmarshal(raw.Raw, &config, yaml.DisallowUnknownFields); err != nil { + return nil, fmt.Errorf("failed to unmarshal YAML data: %w", err) + } + + switch config.Mode { + case ModeMacvtap: + if config.Macvtap == nil { + return nil, fmt.Errorf("vlan config is missing") + } + case ModeMacvlan: + if config.Macvlan == nil { + errorsList = append(errorsList, fmt.Errorf("macvlan config is missing")) + } + case ModeIPvlan: + if config.IPvlan == nil { + errorsList = append(errorsList, fmt.Errorf("ipvlan config is missing")) + } + default: + // No mode specified or ModeDedicated + } + + for _, ip := range config.IPs { + if _, err := netip.ParsePrefix(ip); err != nil { + errorsList = append(errorsList, fmt.Errorf("invalid IP in CIDR format %s", ip)) + } + } + + for _, route := range config.Routes { + if route.Destination == "" || route.Gateway == "" { + errorsList = append(errorsList, fmt.Errorf("invalid route %v", route)) + } + if _, err := netip.ParsePrefix(route.Destination); err != nil { + errorsList = append(errorsList, fmt.Errorf("invalid CIDR %s", route.Destination)) + } + if _, err := netip.ParseAddr(route.Gateway); err != nil { + errorsList = append(errorsList, fmt.Errorf("invalid IP address %s", route.Gateway)) + } + } + return &config, errors.Join(errorsList...) +} diff --git a/pkg/apis/validation_test.go b/pkg/apis/validation_test.go new file mode 100644 index 00000000..34f98299 --- /dev/null +++ b/pkg/apis/validation_test.go @@ -0,0 +1,173 @@ +/* +Copyright 2025 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package apis + +import ( + "testing" + + "k8s.io/apimachinery/pkg/runtime" +) + +func TestValidateConfig(t *testing.T) { + testCases := []struct { + name string + config string + wantErr bool + }{ + { + name: "valid config", + config: ` +ips: +- 192.168.1.10/24 +routes: +- destination: 10.0.0.0/8 + gateway: 192.168.1.1 +mtu: 1500 +`, + wantErr: false, + }, + { + name: "invalid ip", + config: ` +ips: +- a.b.c.d/24 +routes: +- destination: 10.0.0.0/8 + gateway: 192.168.1.1 +mtu: 1500 +`, + wantErr: true, + }, + { + name: "invalid route destination", + config: ` +ips: +- 192.168.1.10/24 +routes: +- destination: a.b.c.d/8 + gateway: 192.168.1.1 +mtu: 1500 +`, + wantErr: true, + }, + { + name: "Empty config", + config: ``, + wantErr: false, + }, + { + name: "invalid route", + config: ` +ips: +- 192.168.1.10/24 +routes: +- destination: 10.0.0.0/8 +mtu: 1500 +`, + wantErr: true, + }, + { + name: "invalid route", + config: ` +ips: +- 192.168.1.10/24 +routes: +- gateway: 192.168.1.1 +mtu: 1500 +`, + wantErr: true, + }, + { + name: "invalid route gateway", + config: ` +ips: +- 192.168.1.10/24 +routes: +- destination: 10.0.0.0/8 + gateway: a.b.c.d +mtu: 1500 +`, + wantErr: true, + }, + { + name: "invalid yaml", + config: ` +ips: +- 192.168.1.10/24 +routes: +- destination: 10.0.0.0/8 + gateway: 192.168.1.1 +mtu: 1500 +foo: +- bar +`, + wantErr: true, + }, + { + name: "valid config with name", + config: ` +ips: +- 192.168.1.10/24 +routes: +- destination: 10.0.0.0/8 + gateway: 192.168.1.1 +name: eth1 +mtu: 1500 +`, + wantErr: false, + }, + { + name: "valid config with ipv6", + config: ` +ips: +- 2001:db8::1/64 +routes: +- destination: 2001:db8:1::/64 + gateway: 2001:db8::2 +name: eth1 +mtu: 1500 +`, + wantErr: false, + }, + { + name: "invalid config with ipv6", + config: ` +ips: +- 2001:db8::1/64 +routes: +- destination: 2001:db8:1::/64 + gateway: 2001:db8::z +name: eth1 +mtu: 1500 +`, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + raw := &runtime.RawExtension{} + raw.Raw = []byte(tc.config) + + _, err := ValidateConfig(raw) + if (err != nil) != tc.wantErr { + t.Errorf("ValidateConfig() error = %v, wantErr %v", err, tc.wantErr) + return + } + }) + } +} diff --git a/pkg/driver/driver.go b/pkg/driver/driver.go index 69f7244b..20f61a9d 100644 --- a/pkg/driver/driver.go +++ b/pkg/driver/driver.go @@ -24,10 +24,12 @@ import ( "slices" "time" - "github.com/google/cel-go/cel" + "github.com/google/dranet/pkg/apis" "github.com/google/dranet/pkg/filter" "github.com/google/dranet/pkg/inventory" + "github.com/google/cel-go/cel" + "github.com/Mellanox/rdmamap" "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/stub" @@ -238,6 +240,7 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) } // Process the configurations of the ResourceClaim + var netconf *apis.NetworkConfig for _, config := range claim.Status.Allocation.Devices.Config { if config.Opaque == nil { continue @@ -245,27 +248,39 @@ func (np *NetworkDriver) RunPodSandbox(ctx context.Context, pod *api.PodSandbox) if len(config.Requests) > 0 && !slices.Contains(config.Requests, result.Request) { continue } - klog.V(4).Infof("podStartHook Configuration %s", string(config.Opaque.Parameters.String())) - // TODO get config options here, it can add ips or commands - // to add routes, run dhcp, rename the interface ... whatever + // TODO: handle the case with multiple configurations (is that possible, should we merge them?) + netconf, err = apis.ValidateConfig(&config.Opaque.Parameters) + if err != nil { + return err + } + if netconf != nil { + klog.V(4).Infof("Configuration %#v", netconf) + break + } } klog.Infof("RunPodSandbox allocation.Devices.Result: %#v", result) - // TODO signal this via DRA - if rdmaDev, _ := rdmamap.GetRdmaDeviceForNetdevice(result.Device); rdmaDev != "" { - err := nsAttachRdmadev(rdmaDev, ns) + if netconf == nil || netconf.Mode == apis.ModeDedicated { + // TODO signal this via DRA + if rdmaDev, _ := rdmamap.GetRdmaDeviceForNetdevice(result.Device); rdmaDev != "" { + err := nsAttachRdmadev(rdmaDev, ns) + if err != nil { + klog.Infof("RunPodSandbox error getting RDMA device %s to namespace %s: %v", result.Device, ns, err) + continue + } + } + + // TODO config options to rename the device and pass parameters + // use https://github.com/opencontainers/runtime-spec/pull/1271 + err := nsAttachNetdev(result.Device, ns, result.Device) if err != nil { - klog.Infof("RunPodSandbox error getting RDMA device %s to namespace %s: %v", result.Device, ns, err) - continue + klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err) + return err } - } + } else if netconf.Mode == apis.ModeMacvlan { + } else if netconf.Mode == apis.ModeMacvtap { + } else if netconf.Mode == apis.ModeIPvlan { - // TODO config options to rename the device and pass parameters - // use https://github.com/opencontainers/runtime-spec/pull/1271 - err := nsAttachNetdev(result.Device, ns, result.Device) - if err != nil { - klog.Infof("RunPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err) - return err } } } @@ -304,22 +319,32 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox continue } + // Process the configurations of the ResourceClaim + var netconf *apis.NetworkConfig for _, config := range claim.Status.Allocation.Devices.Config { if config.Opaque == nil { continue } - klog.V(4).Infof("podStopHook Configuration %s", string(config.Opaque.Parameters.String())) - // TODO get config options here, it can add ips or commands - // to add routes, run dhcp, rename the interface ... whatever + if len(config.Requests) > 0 && !slices.Contains(config.Requests, result.Request) { + continue + } + // TODO: handle the case with multiple configurations (is that possible, should we merge them?) + netconf, err = apis.ValidateConfig(&config.Opaque.Parameters) + if err != nil { + return err + } + if netconf != nil { + klog.V(4).Infof("Configuration %#v", netconf) + break + } } - klog.V(4).Infof("podStopHook Device %s", result.Device) - // TODO config options to rename the device and pass parameters - // use https://github.com/opencontainers/runtime-spec/pull/1271 - err := nsDetachNetdev(ns, result.Device) - if err != nil { - klog.Infof("StopPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err) - continue + if netconf == nil || netconf.Mode == apis.ModeDedicated { + err := nsDetachNetdev(ns, result.Device) + if err != nil { + klog.Infof("StopPodSandbox error moving device %s to namespace %s: %v", result.Device, ns, err) + continue + } } } } @@ -328,6 +353,8 @@ func (np *NetworkDriver) StopPodSandbox(ctx context.Context, pod *api.PodSandbox func (np *NetworkDriver) RemovePodSandbox(_ context.Context, pod *api.PodSandbox) error { klog.V(2).Infof("RemovePodSandbox pod %s/%s: ips=%v", pod.GetNamespace(), pod.GetName(), pod.GetIps()) + defer np.netdb.RemovePodNetns(podKey(pod)) + // get the pod network namespace ns := getNetworkNamespace(pod) if ns == "" { @@ -426,6 +453,10 @@ func (np *NetworkDriver) nodePrepareResource(ctx context.Context, claimReq *drap len(config.Requests) > 0 && !slices.Contains(config.Requests, requestName) { continue } + _, err := apis.ValidateConfig(&config.Opaque.Parameters) + if err != nil { + return nil, err + } } device := drapb.Device{ PoolName: result.Pool, diff --git a/pkg/driver/subinterfaces.go b/pkg/driver/subinterfaces.go new file mode 100644 index 00000000..8faba5ae --- /dev/null +++ b/pkg/driver/subinterfaces.go @@ -0,0 +1,145 @@ +/* +Copyright 2024 Google LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package driver + +import ( + "fmt" + + "github.com/google/dranet/pkg/apis" + "github.com/vishvananda/netlink" + "github.com/vishvananda/netns" +) + +var ( + macvlanModeMap = map[apis.MacvlanMode]netlink.MacvlanMode{ + MacvlanModeBridge: netlink.MACVLAN_MODE_BRIDGE, + MacvlanModePrivate: netlink.MACVLAN_MODE_PRIVATE, + MacvlanModeVepa: netlink.MACVLAN_MODE_VEPA, + } + + macvtapModeMap = map[apis.MacvtapMode]netlink.MacvtapMode{ + MacvlanModeBridge: netlink.MACVLAN_MODE_BRIDGE, + MacvlanModePrivate: netlink.MACVLAN_MODE_PRIVATE, + MacvlanModeVepa: netlink.MACVLAN_MODE_VEPA, + } +) + +func addMacVlan(containerNsPAth string, devName string, cfg apis.NetworkConfig) error { + containerNs, err := netns.GetFromPath(containerNsPAth) + if err != nil { + return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, devName, err) + } + defer containerNs.Close() + + parentLink, err := netlink.LinkByName(devName) + if err != nil { + return fmt.Errorf("could not find parent interface %s : %w", devName, err) + } + newName := cfg.Name + if newName == "" { + newName = devName + } + + mode, ok := macvlanModeMap[cfg.Macvlan.Mode] + if !ok { + return fmt.Errorf("unknown mode") + } + + macvlan := &netlink.Macvlan{ + LinkAttrs: netlink.LinkAttrs{ + Name: newName, + ParentIndex: parentLink.Attrs().Index, + NetNsID: int(containerNs), + }, + Mode: mode, + } + if err := netlink.LinkAdd(macvlan); err != nil { + // If a user creates a macvlan and ipvlan on same parent, only one slave iface can be active at a time. + return fmt.Errorf("failed to create the %s macvlan interface: %v", macvlan.Name, err) + } + + return nil +} + +func addMacVtap(containerNsPAth string, devName string, cfg apis.NetworkConfig) error { + containerNs, err := netns.GetFromPath(containerNsPAth) + if err != nil { + return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, devName, err) + } + defer containerNs.Close() + + parentLink, err := netlink.LinkByName(devName) + if err != nil { + return fmt.Errorf("could not find parent interface %s : %w", devName, err) + } + + newName := cfg.Name + if newName == "" { + newName = devName + } + + mode, ok := macvlanModeMap[cfg.Macvlan.Mode] + if !ok { + return fmt.Errorf("unknown mode") + } + + macvtap := &netlink.Macvtap{ + Macvlan: netlink.Macvlan{ + LinkAttrs: netlink.LinkAttrs{ + Name: "dranet-" + devName, + ParentIndex: parentLink.Attrs().Index, + NetNsID: int(containerNs), + }, + Mode: mode, + }, + } + if err := netlink.LinkAdd(macvtap); err != nil { + // If a user creates a macvlan and ipvlan on same parent, only one slave iface can be active at a time. + return fmt.Errorf("failed to create the %s macvtap interface: %v", macvtap.Name, err) + } + + return nil +} + +func addIPVlan(containerNsPAth string, devName string, cfg apis.NetworkConfig) error { + containerNs, err := netns.GetFromPath(containerNsPAth) + if err != nil { + return fmt.Errorf("could not get network namespace from path %s for network device %s : %w", containerNsPAth, devName, err) + } + defer containerNs.Close() + + parentLink, err := netlink.LinkByName(devName) + if err != nil { + return fmt.Errorf("could not find parent interface %s : %w", devName, err) + } + + ipvlan := &netlink.IPVlan{ + LinkAttrs: netlink.LinkAttrs{ + Name: "dranet-" + devName, + ParentIndex: parentLink.Attrs().Index, + NetNsID: int(containerNs), + }, + Mode: mode, + } + + if err := netlink.LinkAdd(ipvlan); err != nil { + // If a user creates a macvlan and ipvlan on same parent, only one slave iface can be active at a time. + return fmt.Errorf("failed to create the %s ipvlan interface: %v", ipvlan.Name, err) + } + + return nil +}