Workaround for K8s dual stack bug

K8s doesn't have a functioning way to get the IPv6 Pod IP via the
downward API.

This hacks around it if env variable

NSM_TUNNEL_IP_TO_V6 = true

The first GlobalUnicast IPv6 address from the same kernel interface as
the IPv4 Tunnel IP is used instead.

Signed-off-by: Ed Warnicke <[email protected]>

Author: edwarnicke

.golangci.yml

@@ -171,6 +171,10 @@ issues:
       linters:
         - funlen
       text: "Function 'main'"
+    - path: main.go
+      linters:
+        - gocyclo
+      text: "cyclomatic complexity 17 of func `main` is high"
     - path: internal/vppinit/vppinit.go
       linters:
         - funlen

internal/config/config.go

@@ -45,6 +45,7 @@ type Config struct {
 	OpenTelemetryEndpoint string            `default:"otel-collector.observability.svc.cluster.local:4317" desc:"OpenTelemetry Collector Endpoint"`
 
 	TunnelIP     net.IP       `desc:"IP to use for tunnels" split_words:"true"`
+	TunnelIPToV6 bool         `desc:"If NSM_TUNNEL_IP set to IPv4 address, use IPv6 address from same interface" split_words:"true"`
 	VxlanPort    uint16       `default:"0" desc:"VXLAN port to use" split_words:"true"`
 	VppAPISocket string       `default:"/var/run/vpp/external/vpp-api.sock" desc:"filename of socket to connect to existing VPP instance.  If empty a VPP instance is run in forwarder" split_words:"true"`
 	VppInit      vppinit.Func `default:"NONE" desc:"type of VPP initialization. Must be NONE or AF_PACKET" split_words:"true"`

internal/vppinit/vppinit.go

@@ -337,3 +337,39 @@ func linkByIP(ctx context.Context, ipaddress net.IP) (netlink.Link, error) {
 	}
 	return nil, nil
 }
+
+// TunnelIPtoIPv6 - Converts TunnelIP to IPv6
+//
+// In DualStack K8s, it is not currently possible to get the IPv6 PodIP
+// via the Downward API.  For this reason, we need a mechanism to support
+// taking the IPv4 Pod address received via the Downward API and replacing
+// it with the IPv6 IP on the same interface.  This function does so
+// by taking the first GlobalUnicast IPv6 address from the same interface and
+// returning it.
+func TunnelIPtoIPv6(ctx context.Context, tunnelIP net.IP) (net.IP, error) {
+	if tunnelIP == nil || tunnelIP.To4() == nil {
+		return tunnelIP, nil
+	}
+
+	link, err := linkByIP(ctx, tunnelIP)
+	if err != nil {
+		return nil, err
+	}
+	now := time.Now()
+	addrs, err := netlink.AddrList(link, netlink.FAMILY_ALL)
+	if err != nil {
+		return nil, errors.Wrapf(err, "could not retrieve addresses for link %s", link.Attrs().Name)
+	}
+
+	log.FromContext(ctx).
+		WithField("duration", time.Since(now)).
+		WithField("link", link.Attrs().Name).
+		WithField("netlink", "AddrList").Debug("completed")
+
+	for _, addr := range addrs {
+		if addr.IP != nil && addr.IP.To4() == nil && addr.IP.IsGlobalUnicast() {
+			tunnelIP = addr.IP
+		}
+	}
+	return tunnelIP, nil
+}

main.go

@@ -190,6 +190,13 @@ func main() {
 		log.FromContext(ctx).Warn("SR-IOV is not enabled")
 	}
 
+	if cfg.TunnelIP != nil && cfg.TunnelIP.To4() != nil && cfg.TunnelIPToV6 {
+		cfg.TunnelIP, err = vppinit.TunnelIPtoIPv6(ctx, cfg.TunnelIP)
+		if err != nil {
+			logrus.Fatalf("error converting IPv4 TunnelIP to IPv6 TunnelIP: %+v", err)
+		}
+	}
+
 	deviceMap := setupDeviceMap(ctx, cfg)
 	err = vppinit.InitLinks(ctx, vppConn, deviceMap, cfg.TunnelIP)
 	if err != nil {