feat(bgp): enable bgp in cilium and disable l2 announcements

vehagn · vehagn · commit 25aac986cdd0 · 2025-10-27T09:33:02.000+01:00
Also change IPs off all services to a different subnet

Signed-off-by: Vegard Hagen &lt;vegard@stonegarden.dev&gt;
diff --git a/k8s/apps/dev/whoami/svc.yaml b/k8s/apps/dev/whoami/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: whoami
   namespace: whoami
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.223
+    io.cilium/lb-ipam-ips: 172.20.10.200
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/apps/external/proxmox/svc.yaml b/k8s/apps/external/proxmox/svc.yaml
@@ -8,4 +8,5 @@ spec:
     - name: https
       protocol: TCP
       port: 443
+      appProtocol: kubernetes.io/h2c
       targetPort: 8006
diff --git a/k8s/apps/external/truenas/svc.yaml b/k8s/apps/external/truenas/svc.yaml
@@ -8,3 +8,4 @@ spec:
     - name: https
       protocol: TCP
       port: 443
+      appProtocol: kubernetes.io/h2c
diff --git a/k8s/apps/media/plex/kustomization.yaml b/k8s/apps/media/plex/kustomization.yaml
@@ -8,7 +8,7 @@ configMapGenerator:
     namespace: plex
     literals:
       - TZ="Europe/Oslo"
-      - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443,http://192.168.1.228:32400
+      - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443
 
 resources:
   - ns.yaml
diff --git a/k8s/apps/utils/torrent/svc-torrent.yaml b/k8s/apps/utils/torrent/svc-torrent.yaml
@@ -4,7 +4,10 @@ metadata:
   name: torrent-torrent
   namespace: torrent
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.225
+    io.cilium/lb-ipam-ips: 172.20.10.250
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/infra/auth/authelia/cnpg-db.yaml b/k8s/infra/auth/authelia/cnpg-db.yaml
@@ -27,7 +27,7 @@ spec:
   #            metadata:
   #              name: authelia-postgres-db
   #              annotations:
-  #                io.cilium/lb-ipam-ips: 192.168.1.240
+  #                io.cilium/lb-ipam-ips: 172.20.10.241
   #            spec:
   #              type: LoadBalancer
   storage:
diff --git a/k8s/infra/auth/lldap/cnpg-db.yaml b/k8s/infra/auth/lldap/cnpg-db.yaml
@@ -27,7 +27,7 @@ spec:
   #            metadata:
   #              name: lldap-postgres-db
   #              annotations:
-  #                io.cilium/lb-ipam-ips: 192.168.1.241
+  #                io.cilium/lb-ipam-ips: 172.20.10.240
   #            spec:
   #              type: LoadBalancer
   storage:
diff --git a/k8s/infra/auth/lldap/svc.yaml b/k8s/infra/auth/lldap/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: lldap
   namespace: lldap
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.242
+    io.cilium/lb-ipam-ips: 172.20.10.120
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default
 spec:
   type: LoadBalancer
   # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
diff --git a/k8s/infra/network/cilium/bgp-advertisement.yaml b/k8s/infra/network/cilium/bgp-advertisement.yaml
@@ -0,0 +1,15 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPAdvertisement
+metadata:
+  name: loadbalancer-services
+  labels:
+    bgp.cilium.io/advertise: loadbalancer-services
+spec:
+  advertisements:
+    - advertisementType: "Service"
+      service:
+        addresses:
+          - LoadBalancerIP
+      selector:
+        matchLabels:
+          bgp.cilium.io/advertise-service: default
diff --git a/k8s/infra/network/cilium/bgp-cluster-config.yaml b/k8s/infra/network/cilium/bgp-cluster-config.yaml
@@ -0,0 +1,17 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPClusterConfig
+metadata:
+  name: cilium-unifi
+spec:
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/control-plane: ""
+  bgpInstances:
+    - name: "65200"
+      localASN: 65200
+      peers:
+        - name: "ucg-max-65100"
+          peerASN: 65100
+          peerAddress: 172.20.10.1
+          peerConfigRef:
+            name: cilium-peer
diff --git a/k8s/infra/network/cilium/bgp-peer-config.yaml b/k8s/infra/network/cilium/bgp-peer-config.yaml
@@ -0,0 +1,14 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPPeerConfig
+metadata:
+  name: cilium-peer
+spec:
+  gracefulRestart:
+    enabled: true
+    restartTimeSeconds: 15
+  families:
+    - afi: ipv4
+      safi: unicast
+      advertisements:
+        matchLabels:
+          bgp.cilium.io/advertise: loadbalancer-services
diff --git a/k8s/infra/network/cilium/default-ip-pool.yaml b/k8s/infra/network/cilium/default-ip-pool.yaml
@@ -0,0 +1,10 @@
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: default-ip-pool
+spec:
+  blocks:
+    - cidr: "172.20.10.0/24"
+  serviceSelector:
+    matchLabels:
+      lb-ipam.cilium.io/ip-pool: default
diff --git a/k8s/infra/network/cilium/ip-pool.yaml b/k8s/infra/network/cilium/ip-pool.yaml
diff --git a/k8s/infra/network/cilium/kustomization.yaml b/k8s/infra/network/cilium/kustomization.yaml
@@ -3,7 +3,10 @@ kind: Kustomization
 
 resources:
   - announce.yaml
-  - ip-pool.yaml
+  - default-ip-pool.yaml
+  - bgp-advertisement.yaml
+  - bgp-peer-config.yaml
+  - bgp-cluster-config.yaml
   - dashboards/cilium.yaml
   - dashboards/cilium-operator.yaml
 
diff --git a/k8s/infra/network/cilium/values.yaml b/k8s/infra/network/cilium/values.yaml
@@ -23,6 +23,11 @@ cgroup:
 bpf:
   hostLegacyRouting: true
 
+## ???
+# routingMode: "host"
+# ipv4NativeRoutingCIDR=10.0.0.0/8
+# k8s.requireIPv4PodCIDR=true
+
 # https://docs.cilium.io/en/stable/network/concepts/ipam/
 ipam:
   mode: kubernetes
@@ -59,12 +64,8 @@ resources:
 #debug:
 #  enabled: true
 
-# Increase rate limit when doing L2 announcements
-k8sClientRateLimit:
-  qps: 20
-  burst: 100
-
-l2announcements:
+# https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/
+bgpControlPlane:
   enabled: true
 
 externalIPs:
diff --git a/k8s/infra/network/dns/adguard/svc.yaml b/k8s/infra/network/dns/adguard/svc.yaml
@@ -4,9 +4,14 @@ metadata:
   name: adguard
   namespace: dns
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.253
+    io.cilium/lb-ipam-ips: 172.20.10.153
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default
 spec:
   type: LoadBalancer
+  # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
+  clusterIP: 10.96.0.12
   ports:
     - name: http
       protocol: TCP
diff --git a/k8s/infra/network/dns/unbound/config/a-records.conf b/k8s/infra/network/dns/unbound/config/a-records.conf
@@ -1,5 +1,10 @@
 # A Record
      #local-data: "somecomputer.local. A 192.168.1.1"
+     local-data: "*.stonegarden.dev A 172.20.10.100"
+     local-data: "stonegarden.dev A 172.20.10.100"
+     local-data: "coturn.stonegarden.dev A 172.20.10.140"
 
 # PTR Record
      #local-data-ptr: "192.168.1.1 somecomputer.local."
+     local-data-ptr: "172.20.10.100 stonegarden.dev."
+     local-data-ptr: "172.20.10.140 coturn.stonegarden.dev."
diff --git a/k8s/infra/network/dns/unbound/deployment.yaml b/k8s/infra/network/dns/unbound/deployment.yaml
@@ -33,7 +33,6 @@ spec:
               cpu: 10m
               memory: 64Mi
             limits:
-              cpu: 500m
               memory: 128Mi
           startupProbe:
             exec:
diff --git a/k8s/infra/network/dns/unbound/svc.yaml b/k8s/infra/network/dns/unbound/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: unbound
   namespace: dns
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.252
+    io.cilium/lb-ipam-ips: 172.20.10.150
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default
 spec:
   type: LoadBalancer
   # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
diff --git a/k8s/infra/network/gateway/gw-external.yaml b/k8s/infra/network/gateway/gw-external.yaml
@@ -5,9 +5,13 @@ metadata:
   namespace: gateway
 spec:
   gatewayClassName: cilium
+  infrastructure:
+    labels:
+      bgp.cilium.io/advertise-service: default
+      lb-ipam.cilium.io/ip-pool: default
   addresses:
     - type: IPAddress
-      value: 192.168.1.222
+      value: 172.20.10.110
   listeners:
     - protocol: HTTPS
       port: 443
diff --git a/k8s/infra/network/gateway/gw-internal.yaml b/k8s/infra/network/gateway/gw-internal.yaml
@@ -5,9 +5,13 @@ metadata:
   namespace: gateway
 spec:
   gatewayClassName: cilium
+  infrastructure:
+    labels:
+      bgp.cilium.io/advertise-service: default
+      lb-ipam.cilium.io/ip-pool: default
   addresses:
     - type: IPAddress
-      value: 192.168.1.220
+      value: 172.20.10.100
   listeners:
     - protocol: TLS
       port: 443
diff --git a/k8s/infra/vpn/coturn/svc.yaml b/k8s/infra/vpn/coturn/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: coturn
   namespace: coturn
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.226
+    io.cilium/lb-ipam-ips: 172.20.10.140
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/infra/vpn/netbird/management/deployment.yaml b/k8s/infra/vpn/netbird/management/deployment.yaml
@@ -19,9 +19,10 @@ spec:
       nodeSelector:
         topology.kubernetes.io/zone: abel
       dnsConfig:
-        # Internal AdGuard Home DNS
         nameservers:
-          - 192.168.1.253
+          - 172.20.10.153 # AdGuard Home
+          - 10.96.0.12 # AdGuard Home
+          - 10.96.0.11 # Unbound
       dnsPolicy: None
       securityContext:
         seccompProfile:
