fix(cilium): use dsr load-balancer mode

This should maybe fix issues with pinging VIPs https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode

Signed-off-by: Vegard Hagen <[email protected]>

Author: vehagn

k8s/infra/network/cilium/bgp-advertise-lb-services.yaml

@@ -6,10 +6,11 @@ metadata:
     bgp.cilium.io/advertise: loadbalancer-services
 spec:
   advertisements:
-    - advertisementType: "Service"
+    - advertisementType: Service
       service:
         addresses:
           - LoadBalancerIP
+          - ExternalIP
       selector:
         matchLabels:
           bgp.cilium.io/advertise-service: default

k8s/infra/network/cilium/bgp-cluster-config.yaml

@@ -6,12 +6,13 @@ spec:
   nodeSelector:
     matchLabels:
       node-role.kubernetes.io/control-plane: ""
+      kubernetes.io/hostname: "ctrl-01"
   bgpInstances:
     - name: "65200"
       localASN: 65200
       peers:
         - name: "ucg-max-65100"
           peerASN: 65100
-          peerAddress: 172.20.10.1
+          peerAddress: 192.168.1.1
           peerConfigRef:
             name: ucg-max

k8s/infra/network/cilium/kustomization.yaml

@@ -2,7 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - announce.yaml
+#  - announce.yaml
   - bgp-ip-pool.yaml
   - bgp-advertise-lb-services.yaml
   - bgp-peer-ucg-max-config.yaml

k8s/infra/network/cilium/values.yaml

@@ -22,6 +22,8 @@ cgroup:
 # https://docs.cilium.io/en/stable/operations/performance/tuning/#ebpf-host-routing
 bpf:
   hostLegacyRouting: true
+  # due to loadBalancer.mode: "dsr".
+  masquerade: true
 
 # https://docs.cilium.io/en/stable/network/concepts/ipam/
 ipam:
@@ -59,6 +61,9 @@ resources:
 #debug:
 #  enabled: true
 
+l2announcements:
+  enabled: false
+
 # https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/
 bgpControlPlane:
   enabled: true
@@ -71,6 +76,26 @@ externalIPs:
 loadBalancer:
   # https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#maglev-consistent-hashing
   algorithm: maglev
+  # https://docs.cilium.io/en/stable/network/kubernetes/kubeproxy-free/#dsr-mode
+  mode: dsr
+  l7:
+    backend: envoy
+
+# The default "vxlan" is incompatible with loadBalancer.mode: "dsr".
+routingMode: native
+#tunnelProtocol: ""
+
+ipMasqAgent:
+  enabled: true
+
+#
+#enableIPv4Masquerade: true
+
+# invalid daemon configuration: native routing cidr must be configured with option
+# --ipv4-native-routing-cidr
+#  in combination with --enable-ipv4=true
+# --enable-ipv4-masquerade=true
+#  --enable-ip-masq-agent=false --routing-mode=native --ipam=kubernetes"
 
 gatewayAPI:
   enabled: true

k8s/infra/vpn/coturn/svc.yaml

@@ -10,6 +10,8 @@ metadata:
     lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
+  externalTrafficPolicy: Local
+  internalTrafficPolicy: Local
   selector:
     app.kubernetes.io/name: coturn
   ports: