[PoC] Restore vpp interfaces

Signed-off-by: Artem Glazychev <[email protected]>

Author: glazychev-art

pkg/networkservice/chains/forwarder/options.go

@@ -36,6 +36,7 @@ type forwarderOptions struct {
 	authorizeServer networkservice.NetworkServiceServer
 	clientURL       *url.URL
 	dialTimeout     time.Duration
+	dumpCleanTimeou time.Duration
 	domain2Device   map[string]string
 	statsOpts       []stats.Option
 	cleanupOpts     []cleanup.Option
@@ -77,6 +78,13 @@ func WithDialTimeout(dialTimeout time.Duration) Option {
 	}
 }
 
+// WithDumpCleanupTimeout sets
+func WithDumpCleanupTimeout(timeout time.Duration) Option {
+	return func(o *forwarderOptions) {
+		o.dumpCleanTimeou = timeout
+	}
+}
+
 // WithVlanDomain2Device sets vlan option
 func WithVlanDomain2Device(domain2Device map[string]string) Option {
 	return func(o *forwarderOptions) {

pkg/networkservice/chains/forwarder/server.go

@@ -26,6 +26,8 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/networkservicemesh/sdk-vpp/pkg/tools/dumptool"
+
 	"git.fd.io/govpp.git/api"
 	"github.com/google/uuid"
 
@@ -81,11 +83,18 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, vppConn
 		authorizeServer: authorize.NewServer(authorize.Any()),
 		clientURL:       &url.URL{Scheme: "unix", Host: "connect.to.socket"},
 		dialTimeout:     time.Millisecond * 200,
+		dumpCleanTimeou: time.Minute,
 		domain2Device:   make(map[string]string),
 	}
 	for _, opt := range options {
 		opt(opts)
 	}
+
+	dumpOption := &dumptool.DumpOption{
+		PodName: opts.name,
+		Timeout: opts.dumpCleanTimeou,
+	}
+
 	nseClient := registryclient.NewNetworkServiceEndpointRegistryClient(ctx, opts.clientURL,
 		registryclient.WithNSEAdditionalFunctionality(
 			registryrecvfd.NewNetworkServiceEndpointRegistryClient(),
@@ -111,9 +120,10 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, vppConn
 		mechanisms.NewServer(map[string]networkservice.NetworkServiceServer{
 			memif.MECHANISM: memif.NewServer(ctx, vppConn,
 				memif.WithDirectMemif(),
-				memif.WithChangeNetNS()),
-			kernel.MECHANISM:    kernel.NewServer(vppConn),
-			vxlan.MECHANISM:     vxlan.NewServer(vppConn, tunnelIP, opts.vxlanOpts...),
+				memif.WithChangeNetNS(),
+				memif.WithDump(dumpOption)),
+			kernel.MECHANISM:    kernel.NewServer(vppConn, kernel.WithDump(dumpOption)),
+			vxlan.MECHANISM:     vxlan.NewServer(vppConn, tunnelIP, append(opts.vxlanOpts, vxlan.WithDump(dumpOption))...),
 			wireguard.MECHANISM: wireguard.NewServer(vppConn, tunnelIP),
 		}),
 		pinhole.NewServer(vppConn),
@@ -134,9 +144,10 @@ func NewServer(ctx context.Context, tokenGenerator token.GeneratorFunc, vppConn
 					// mechanisms
 					memif.NewClient(vppConn,
 						memif.WithChangeNetNS(),
+						memif.WithDump(dumpOption),
 					),
-					kernel.NewClient(vppConn),
-					vxlan.NewClient(vppConn, tunnelIP, opts.vxlanOpts...),
+					kernel.NewClient(vppConn, kernel.WithDump(dumpOption)),
+					vxlan.NewClient(vppConn, tunnelIP, append(opts.vxlanOpts, vxlan.WithDump(dumpOption))...),
 					wireguard.NewClient(vppConn, tunnelIP),
 					vlan.NewClient(vppConn, opts.domain2Device),
 					filtermechanisms.NewClient(),

pkg/networkservice/mechanisms/kernel/client.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020-2021 Cisco and/or its affiliates.
+// Copyright (c) 2020-2022 Cisco and/or its affiliates.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -30,9 +30,14 @@ import (
 )
 
 // NewClient - returns a new Client chain element implementing the kernel mechanism with vpp
-func NewClient(vppConn api.Connection) networkservice.NetworkServiceClient {
+func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceClient {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
 	if _, err := os.Stat(vnetFilename); err == nil {
-		return kerneltap.NewClient(vppConn)
+		return kerneltap.NewClient(vppConn, kerneltap.WithDump(o.dumpOpt))
 	}
 	return kernelvethpair.NewClient(vppConn)
 }

pkg/networkservice/mechanisms/kernel/kerneltap/client.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020-2021 Cisco and/or its affiliates.
+// Copyright (c) 2020-2022 Cisco and/or its affiliates.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -28,6 +28,7 @@ import (
 
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
 	"github.com/networkservicemesh/api/pkg/api/networkservice/mechanisms/cls"
+	"github.com/networkservicemesh/sdk-vpp/pkg/tools/dumptool"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/core/next"
 	"github.com/networkservicemesh/sdk/pkg/networkservice/utils/metadata"
 	"github.com/networkservicemesh/sdk/pkg/tools/log"
@@ -36,12 +37,30 @@ import (
 
 type kernelTapClient struct {
 	vppConn api.Connection
+	dumpMap *dumptool.Map
 }
 
 // NewClient - return a new Client chain element implementing the kernel mechanism with vpp using tapv2
-func NewClient(vppConn api.Connection) networkservice.NetworkServiceClient {
+func NewClient(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceClient {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	ctx := context.Background()
+	dumpMap := dumptool.NewMap(ctx, 0)
+	if o.dumpOpt != nil {
+		var err error
+		dumpMap, err = dump(ctx, vppConn, o.dumpOpt.PodName, o.dumpOpt.Timeout, true)
+		if err != nil {
+			log.FromContext(ctx).Errorf("failed to Dump: %v", err)
+			/* TODO: set empty dumpMap here? */
+		}
+	}
+
 	return &kernelTapClient{
 		vppConn: vppConn,
+		dumpMap: dumpMap,
 	}
 }
 
@@ -60,7 +79,7 @@ func (k *kernelTapClient) Request(ctx context.Context, request *networkservice.N
 		return nil, err
 	}
 
-	if err := create(ctx, conn, k.vppConn, metadata.IsClient(k)); err != nil {
+	if err := create(ctx, conn, k.vppConn, k.dumpMap, metadata.IsClient(k)); err != nil {
 		closeCtx, cancelClose := postponeCtxFunc()
 		defer cancelClose()
 
@@ -75,7 +94,7 @@ func (k *kernelTapClient) Request(ctx context.Context, request *networkservice.N
 }
 
 func (k *kernelTapClient) Close(ctx context.Context, conn *networkservice.Connection, opts ...grpc.CallOption) (*empty.Empty, error) {
-	err := del(ctx, conn, k.vppConn, metadata.IsClient(k))
+	err := del(ctx, conn, k.vppConn, k.dumpMap, metadata.IsClient(k))
 	if err != nil {
 		log.FromContext(ctx).Error(err)
 	}

pkg/networkservice/mechanisms/kernel/kerneltap/common.go

@@ -22,6 +22,8 @@ import (
 	"context"
 	"time"
 
+	"github.com/networkservicemesh/sdk-vpp/pkg/tools/dumptool"
+
 	"git.fd.io/govpp.git/api"
 	interfaces "github.com/edwarnicke/govpp/binapi/interface"
 	"github.com/edwarnicke/govpp/binapi/interface_types"
@@ -38,8 +40,11 @@ import (
 	"github.com/networkservicemesh/sdk-vpp/pkg/tools/mechutils"
 )
 
-func create(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, isClient bool) error {
+func create(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, dumpMap *dumptool.Map, isClient bool) error {
 	if mechanism := kernel.ToMechanism(conn.GetMechanism()); mechanism != nil {
+		if val, loaded := dumpMap.LoadAndDelete(conn.GetId()); loaded {
+			ifindex.Store(ctx, isClient, val.(interface_types.InterfaceIndex))
+		}
 		// Construct the netlink handle for the target namespace for this kernel interface
 		handle, err := kernellink.GetNetlinkHandle(mechanism.GetNetNSURL())
 		if err != nil {
@@ -53,7 +58,7 @@ func create(ctx context.Context, conn *networkservice.Connection, vppConn api.Co
 			}
 		}
 		// Delete the kernel interface if there is one in the target namespace
-		_ = del(ctx, conn, vppConn, isClient)
+		_ = del(ctx, conn, vppConn, dumpMap, isClient)
 
 		nsFilename, err := mechutils.ToNSFilename(mechanism)
 		if err != nil {
@@ -141,8 +146,11 @@ func create(ctx context.Context, conn *networkservice.Connection, vppConn api.Co
 	return nil
 }
 
-func del(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, isClient bool) error {
+func del(ctx context.Context, conn *networkservice.Connection, vppConn api.Connection, dumpMap *dumptool.Map, isClient bool) error {
 	if mechanism := kernel.ToMechanism(conn.GetMechanism()); mechanism != nil {
+		if val, loaded := dumpMap.LoadAndDelete(conn.GetId()); loaded {
+			ifindex.Store(ctx, isClient, val.(interface_types.InterfaceIndex))
+		}
 		swIfIndex, ok := ifindex.LoadAndDelete(ctx, isClient)
 		if !ok {
 			return nil
@@ -162,3 +170,21 @@ func del(ctx context.Context, conn *networkservice.Connection, vppConn api.Conne
 	}
 	return nil
 }
+
+func dump(ctx context.Context, vppConn api.Connection, podName string, timeout time.Duration, isClient bool) (*dumptool.Map, error) {
+	return dumptool.DumpInterfaces(ctx, vppConn, podName, timeout, isClient,
+		/* Function on dump */
+		func(details *interfaces.SwInterfaceDetails) (interface{}, error) {
+			if details.InterfaceDevType == dumptool.DevTypeTap {
+				return details.SwIfIndex, nil
+			}
+			return nil, errors.New("Doesn't match the tap interface")
+		},
+		/* Function on delete */
+		func(ifindex interface{}) error {
+			_, err := tapv2.NewServiceClient(vppConn).TapDeleteV2(ctx, &tapv2.TapDeleteV2{
+				SwIfIndex: ifindex.(interface_types.InterfaceIndex),
+			})
+			return err
+		})
+}

pkg/networkservice/mechanisms/kernel/kerneltap/options.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2022 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package kerneltap
+
+import "github.com/networkservicemesh/sdk-vpp/pkg/tools/dumptool"
+
+type options struct {
+	dumpOpt *dumptool.DumpOption
+}
+
+// Option is an option pattern for kernel
+type Option func(o *options)
+
+// WithDump - sets dump parameters
+func WithDump(dump *dumptool.DumpOption) Option {
+	return func(o *options) {
+		o.dumpOpt = dump
+	}
+}

pkg/networkservice/mechanisms/kernel/kerneltap/server.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020-2021 Cisco and/or its affiliates.
+// Copyright (c) 2020-2022 Cisco and/or its affiliates.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -23,6 +23,7 @@ import (
 
 	"git.fd.io/govpp.git/api"
 	"github.com/golang/protobuf/ptypes/empty"
+	"github.com/networkservicemesh/sdk-vpp/pkg/tools/dumptool"
 	"github.com/pkg/errors"
 
 	"github.com/networkservicemesh/api/pkg/api/networkservice"
@@ -34,12 +35,30 @@ import (
 
 type kernelTapServer struct {
 	vppConn api.Connection
+	dumpMap *dumptool.Map
 }
 
 // NewServer - return a new Server chain element implementing the kernel mechanism with vpp using tapv2
-func NewServer(vppConn api.Connection) networkservice.NetworkServiceServer {
+func NewServer(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceServer {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	ctx := context.Background()
+	dumpMap := dumptool.NewMap(ctx, 0)
+	if o.dumpOpt != nil {
+		var err error
+		dumpMap, err = dump(ctx, vppConn, o.dumpOpt.PodName, o.dumpOpt.Timeout, false)
+		if err != nil {
+			log.FromContext(ctx).Errorf("failed to Dump: %v", err)
+			/* TODO: set empty dumpMap here? */
+		}
+	}
+
 	return &kernelTapServer{
 		vppConn: vppConn,
+		dumpMap: dumpMap,
 	}
 }
 
@@ -51,7 +70,7 @@ func (k *kernelTapServer) Request(ctx context.Context, request *networkservice.N
 		return nil, err
 	}
 
-	if err := create(ctx, conn, k.vppConn, metadata.IsClient(k)); err != nil {
+	if err := create(ctx, conn, k.vppConn, k.dumpMap, metadata.IsClient(k)); err != nil {
 		closeCtx, cancelClose := postponeCtxFunc()
 		defer cancelClose()
 
@@ -66,7 +85,7 @@ func (k *kernelTapServer) Request(ctx context.Context, request *networkservice.N
 }
 
 func (k *kernelTapServer) Close(ctx context.Context, conn *networkservice.Connection) (*empty.Empty, error) {
-	err := del(ctx, conn, k.vppConn, metadata.IsClient(k))
+	err := del(ctx, conn, k.vppConn, k.dumpMap, metadata.IsClient(k))
 	if err != nil {
 		log.FromContext(ctx).Error(err)
 	}

pkg/networkservice/mechanisms/kernel/options.go

@@ -0,0 +1,35 @@
+// Copyright (c) 2022 Cisco and/or its affiliates.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at:
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build linux
+
+package kernel
+
+import "github.com/networkservicemesh/sdk-vpp/pkg/tools/dumptool"
+
+type options struct {
+	dumpOpt *dumptool.DumpOption
+}
+
+// Option is an option pattern for kernel
+type Option func(o *options)
+
+// WithDump - sets dump parameters
+func WithDump(dump *dumptool.DumpOption) Option {
+	return func(o *options) {
+		o.dumpOpt = dump
+	}
+}

pkg/networkservice/mechanisms/kernel/server.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020-2021 Cisco and/or its affiliates.
+// Copyright (c) 2020-2022 Cisco and/or its affiliates.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -30,9 +30,14 @@ import (
 )
 
 // NewServer return a NetworkServiceServer chain element that correctly handles the kernel Mechanism
-func NewServer(vppConn api.Connection) networkservice.NetworkServiceServer {
+func NewServer(vppConn api.Connection, opts ...Option) networkservice.NetworkServiceServer {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
 	if _, err := os.Stat(vnetFilename); err == nil {
-		return kerneltap.NewServer(vppConn)
+		return kerneltap.NewServer(vppConn, kerneltap.WithDump(o.dumpOpt))
 	}
 	return kernelvethpair.NewServer(vppConn)
 }