feat(bgp): enable bgp in cilium and disable l2 announcements

vehagn · vehagn · commit d6daa93ca48c · 2025-10-12T22:13:33.000+02:00
Also change IPs off all services to a different subnet

Signed-off-by: Vegard Hagen &lt;vegard@stonegarden.dev&gt;
diff --git a/k8s/apps/dev/whoami/svc.yaml b/k8s/apps/dev/whoami/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: whoami
   namespace: whoami
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.223
+    io.cilium/lb-ipam-ips: 172.20.10.200
+  labels:
+    ip-pool: default
+    advertise: bgp
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/apps/media/plex/kustomization.yaml b/k8s/apps/media/plex/kustomization.yaml
@@ -8,7 +8,7 @@ configMapGenerator:
     namespace: plex
     literals:
       - TZ="Europe/Oslo"
-      - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443,http://192.168.1.228:32400
+      - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443
 
 resources:
   - ns.yaml
diff --git a/k8s/apps/utils/torrent/svc-torrent.yaml b/k8s/apps/utils/torrent/svc-torrent.yaml
@@ -4,7 +4,10 @@ metadata:
   name: torrent-torrent
   namespace: torrent
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.225
+    io.cilium/lb-ipam-ips: 172.20.10.250
+  labels:
+    ip-pool: default
+    advertise: bgp
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/infra/auth/authelia/cnpg-db.yaml b/k8s/infra/auth/authelia/cnpg-db.yaml
@@ -27,7 +27,7 @@ spec:
   #            metadata:
   #              name: authelia-postgres-db
   #              annotations:
-  #                io.cilium/lb-ipam-ips: 192.168.1.240
+  #                io.cilium/lb-ipam-ips: 172.20.10.241
   #            spec:
   #              type: LoadBalancer
   storage:
diff --git a/k8s/infra/auth/lldap/cnpg-db.yaml b/k8s/infra/auth/lldap/cnpg-db.yaml
@@ -27,7 +27,7 @@ spec:
   #            metadata:
   #              name: lldap-postgres-db
   #              annotations:
-  #                io.cilium/lb-ipam-ips: 192.168.1.241
+  #                io.cilium/lb-ipam-ips: 172.20.10.240
   #            spec:
   #              type: LoadBalancer
   storage:
diff --git a/k8s/infra/auth/lldap/svc.yaml b/k8s/infra/auth/lldap/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: lldap
   namespace: lldap
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.242
+    io.cilium/lb-ipam-ips: 172.20.10.120
+  labels:
+    ip-pool: default
+    advertise: bgp
 spec:
   type: LoadBalancer
   # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
diff --git a/k8s/infra/network/cilium/bgp-advertisement.yaml b/k8s/infra/network/cilium/bgp-advertisement.yaml
@@ -0,0 +1,15 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPAdvertisement
+metadata:
+  name: loadbalancer-services
+  labels:
+    advertise: loadbalancer-services
+spec:
+  advertisements:
+    - advertisementType: "Service"
+      service:
+        addresses:
+          - LoadBalancerIP
+      selector:
+        matchLabels:
+          advertise: bgp
diff --git a/k8s/infra/network/cilium/bgp-cluster-config.yaml b/k8s/infra/network/cilium/bgp-cluster-config.yaml
@@ -0,0 +1,17 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPClusterConfig
+metadata:
+  name: cilium-unifi
+spec:
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/control-plane: ""
+  bgpInstances:
+    - name: "65200"
+      localASN: 65200
+      peers:
+        - name: "ucg-max-65100"
+          peerASN: 65100
+          peerAddress: 172.20.10.1
+          peerConfigRef:
+            name: cilium-peer
diff --git a/k8s/infra/network/cilium/bgp-peer-config.yaml b/k8s/infra/network/cilium/bgp-peer-config.yaml
@@ -0,0 +1,14 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPPeerConfig
+metadata:
+  name: cilium-peer
+spec:
+  gracefulRestart:
+    enabled: true
+    restartTimeSeconds: 15
+  families:
+    - afi: ipv4
+      safi: unicast
+      advertisements:
+        matchLabels:
+          advertise: loadbalancer-services
diff --git a/k8s/infra/network/cilium/default-ip-pool.yaml b/k8s/infra/network/cilium/default-ip-pool.yaml
@@ -0,0 +1,10 @@
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: default-ip-pool
+spec:
+  blocks:
+    - cidr: "172.20.10.0/24"
+  serviceSelector:
+    matchLabels:
+      ip-pool: default
diff --git a/k8s/infra/network/cilium/ip-pool.yaml b/k8s/infra/network/cilium/ip-pool.yaml
diff --git a/k8s/infra/network/cilium/kustomization.yaml b/k8s/infra/network/cilium/kustomization.yaml
@@ -3,7 +3,10 @@ kind: Kustomization
 
 resources:
   - announce.yaml
-  - ip-pool.yaml
+  - default-ip-pool.yaml
+  - bgp-advertisement.yaml
+  - bgp-peer-config.yaml
+  - bgp-cluster-config.yaml
   - dashboards/cilium.yaml
   - dashboards/cilium-operator.yaml
 
diff --git a/k8s/infra/network/cilium/values.yaml b/k8s/infra/network/cilium/values.yaml
@@ -23,6 +23,11 @@ cgroup:
 bpf:
   hostLegacyRouting: true
 
+## ???
+# routingMode: "host"
+# ipv4NativeRoutingCIDR=10.0.0.0/8
+# k8s.requireIPv4PodCIDR=true
+
 # https://docs.cilium.io/en/stable/network/concepts/ipam/
 ipam:
   mode: kubernetes
@@ -59,12 +64,8 @@ resources:
 #debug:
 #  enabled: true
 
-# Increase rate limit when doing L2 announcements
-k8sClientRateLimit:
-  qps: 20
-  burst: 100
-
-l2announcements:
+# https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/
+bgpControlPlane:
   enabled: true
 
 externalIPs:
diff --git a/k8s/infra/network/dns/adguard/svc.yaml b/k8s/infra/network/dns/adguard/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: adguard
   namespace: dns
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.253
+    io.cilium/lb-ipam-ips: 172.20.10.153
+  labels:
+    ip-pool: default
+    advertise: bgp
 spec:
   type: LoadBalancer
   ports:
diff --git a/k8s/infra/network/dns/unbound/svc.yaml b/k8s/infra/network/dns/unbound/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: unbound
   namespace: dns
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.252
+    io.cilium/lb-ipam-ips: 172.20.10.150
+  labels:
+    ip-pool: default
+    advertise: bgp
 spec:
   type: LoadBalancer
   # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
diff --git a/k8s/infra/network/gateway/gw-external.yaml b/k8s/infra/network/gateway/gw-external.yaml
@@ -5,9 +5,15 @@ metadata:
   namespace: gateway
 spec:
   gatewayClassName: cilium
+  infrastructure:
+    labels:
+      ip-pool: default
+      advertise: bgp
   addresses:
     - type: IPAddress
       value: 192.168.1.222
+    - type: IPAddress
+      value: 172.20.10.110
   listeners:
     - protocol: HTTPS
       port: 443
diff --git a/k8s/infra/network/gateway/gw-internal.yaml b/k8s/infra/network/gateway/gw-internal.yaml
@@ -5,9 +5,15 @@ metadata:
   namespace: gateway
 spec:
   gatewayClassName: cilium
+  infrastructure:
+    labels:
+      ip-pool: default
+      advertise: bgp
   addresses:
     - type: IPAddress
       value: 192.168.1.220
+    - type: IPAddress
+      value: 172.20.10.100
   listeners:
     - protocol: TLS
       port: 443
diff --git a/k8s/infra/vpn/coturn/svc.yaml b/k8s/infra/vpn/coturn/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: coturn
   namespace: coturn
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.226
+    io.cilium/lb-ipam-ips: 172.20.10.140
+  labels:
+    ip-pool: default
+    advertise: bgp
 spec:
   type: LoadBalancer
   selector:
