feat(bgp): enable bgp in cilium and disable l2 announcements

vehagn · vehagn · commit 5709b4486022 · 2025-11-05T23:18:51.000+01:00
Also change IPs off all services to a different subnet

Signed-off-by: Vegard Hagen &lt;vegard@stonegarden.dev&gt;
diff --git a/k8s/apps/dev/whoami/svc.yaml b/k8s/apps/dev/whoami/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: whoami
   namespace: whoami
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.223
+    io.cilium/lb-ipam-ips: 172.20.10.200
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/apps/external/proxmox/svc.yaml b/k8s/apps/external/proxmox/svc.yaml
@@ -8,4 +8,5 @@ spec:
     - name: https
       protocol: TCP
       port: 443
+      appProtocol: kubernetes.io/h2c
       targetPort: 8006
diff --git a/k8s/apps/external/truenas/svc.yaml b/k8s/apps/external/truenas/svc.yaml
@@ -8,3 +8,4 @@ spec:
     - name: https
       protocol: TCP
       port: 443
+      appProtocol: kubernetes.io/h2c
diff --git a/k8s/apps/media/plex/kustomization.yaml b/k8s/apps/media/plex/kustomization.yaml
@@ -8,7 +8,7 @@ configMapGenerator:
     namespace: plex
     literals:
       - TZ="Europe/Oslo"
-      - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443,http://192.168.1.228:32400
+      - PLEX_ADVERTISE_URL=https://plex.stonegarden.dev:443
 
 resources:
   - ns.yaml
diff --git a/k8s/apps/utils/torrent/svc-torrent.yaml b/k8s/apps/utils/torrent/svc-torrent.yaml
@@ -4,7 +4,10 @@ metadata:
   name: torrent-torrent
   namespace: torrent
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.225
+    io.cilium/lb-ipam-ips: 172.20.10.250
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/apps/utils/unpoller/config/up.conf b/k8s/apps/utils/unpoller/config/up.conf
@@ -26,7 +26,7 @@
   dynamic = false
 
 [unifi.defaults]
-  url = "https://192.168.1.1"
+  url = "https://172.20.10.1"
 
   user = "unifipoller"
   pass = "file:///mnt/secrets/password"
diff --git a/k8s/infra/auth/authelia/cnpg-db.yaml b/k8s/infra/auth/authelia/cnpg-db.yaml
@@ -27,7 +27,7 @@ spec:
   #            metadata:
   #              name: authelia-postgres-db
   #              annotations:
-  #                io.cilium/lb-ipam-ips: 192.168.1.240
+  #                io.cilium/lb-ipam-ips: 172.20.10.241
   #            spec:
   #              type: LoadBalancer
   storage:
diff --git a/k8s/infra/auth/lldap/cnpg-db.yaml b/k8s/infra/auth/lldap/cnpg-db.yaml
@@ -27,7 +27,7 @@ spec:
   #            metadata:
   #              name: lldap-postgres-db
   #              annotations:
-  #                io.cilium/lb-ipam-ips: 192.168.1.241
+  #                io.cilium/lb-ipam-ips: 172.20.10.240
   #            spec:
   #              type: LoadBalancer
   storage:
diff --git a/k8s/infra/auth/lldap/svc.yaml b/k8s/infra/auth/lldap/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: lldap
   namespace: lldap
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.242
+    io.cilium/lb-ipam-ips: 172.20.10.120
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
   # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
diff --git a/k8s/infra/network/cilium/announce.yaml b/k8s/infra/network/cilium/announce.yaml
diff --git a/k8s/infra/network/cilium/bgp-advertise-lb-services.yaml b/k8s/infra/network/cilium/bgp-advertise-lb-services.yaml
@@ -0,0 +1,15 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPAdvertisement
+metadata:
+  name: loadbalancer-services
+  labels:
+    bgp.cilium.io/advertise: loadbalancer-services
+spec:
+  advertisements:
+    - advertisementType: "Service"
+      service:
+        addresses:
+          - LoadBalancerIP
+      selector:
+        matchLabels:
+          bgp.cilium.io/advertise-service: default
diff --git a/k8s/infra/network/cilium/bgp-cluster-config.yaml b/k8s/infra/network/cilium/bgp-cluster-config.yaml
@@ -0,0 +1,17 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPClusterConfig
+metadata:
+  name: cilium-unifi
+spec:
+  nodeSelector:
+    matchLabels:
+      node-role.kubernetes.io/control-plane: ""
+  bgpInstances:
+    - name: "65200"
+      localASN: 65200
+      peers:
+        - name: "ucg-max-65100"
+          peerASN: 65100
+          peerAddress: 172.20.10.1
+          peerConfigRef:
+            name: ucg-max
diff --git a/k8s/infra/network/cilium/bgp-ip-pool.yaml b/k8s/infra/network/cilium/bgp-ip-pool.yaml
@@ -0,0 +1,10 @@
+apiVersion: cilium.io/v2alpha1
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: default-bgp-ip-pool
+spec:
+  blocks:
+    - cidr: 172.20.10.0/24
+  serviceSelector:
+    matchLabels:
+      lb-ipam.cilium.io/ip-pool: default-bgp
diff --git a/k8s/infra/network/cilium/bgp-peer-ucg-max-config.yaml b/k8s/infra/network/cilium/bgp-peer-ucg-max-config.yaml
@@ -0,0 +1,17 @@
+apiVersion: cilium.io/v2
+kind: CiliumBGPPeerConfig
+metadata:
+  name: ucg-max
+spec:
+  authSecretRef: bgp-peer-ucg-max-password
+  timers:
+    connectRetryTimeSeconds: 10
+  gracefulRestart:
+    enabled: true
+    restartTimeSeconds: 15
+  families:
+    - afi: ipv4
+      safi: unicast
+      advertisements:
+        matchLabels:
+          bgp.cilium.io/advertise: loadbalancer-services
diff --git a/k8s/infra/network/cilium/bgp-peer-ucg-max-password.yaml b/k8s/infra/network/cilium/bgp-peer-ucg-max-password.yaml
@@ -0,0 +1,12 @@
+apiVersion: bitnami.com/v1alpha1
+kind: SealedSecret
+metadata:
+  name: bgp-peer-ucg-max-password
+  namespace: cilium-secrets
+spec:
+  encryptedData:
+    password: AgAfFRgvvxVt/KTOs7Ko646DwXNDTgXf+UuL21TjywwtVU7frAMfFx7U3+EfiBmXAbWsq2zAek0PGB8qhbaw9Q3Vnn5OgNed16dVOuYkV4QKqbarai/U2HXySts8pA6OR5ICE4WqJN/ayLsbN62jbMfjJMRlwIwXoRgW+6xLuspp81E/VUOQ0VBBKVlONzUWQWKgckpGjXLi7E1EEYQ6JVcKnDVUacM5adF0CjKkkQw/2MK4RnYhZXzmGTV1AnPe+ZK9HgjlmN5fqZ1cks4N1mfWPIGsRJX4CmWw4deqZ/nuB+3dhrDsdYCjxVNuOf9gTlyNwfBjETlYXuR9UCo/YuDzautmo6NlOy1nzl1O/nZbN4Rat+mbgFmn1sv5TwDUawcz4tNhKlrA+5iyVhmhNSYMqkNpkXRBJNRm+fK4Yspdo2yRl5jOX/dfkoWOgk9NijhfoDI2XGGpb7VIrD945i+9EBVe0UwMhnrhRWQZCTerx/1/kNajw77CUuD87KR+akeWVpxO1sG+tDwIceafeKbORYvl5S3TI24NkpS16Fdxl6PoFPVtiZlLqVxWRsnzK6Vb6cvWNGrlehHxURPrL9oB7+R+AM4W10/vqvX2NTNdIbzc2aT8qEq8gUlkPaRjcfeS9DKgnwM3OgF9UDyxjbRiHJ92dMOi3Gvu1F80crUZ0W6fdKhdsWQNqv84aS0JL0yaDLfbqNWf
+  template:
+    metadata:
+      name: bgp-peer-ucg-max-password
+      namespace: cilium-secrets
diff --git a/k8s/infra/network/cilium/ip-pool.yaml b/k8s/infra/network/cilium/ip-pool.yaml
diff --git a/k8s/infra/network/cilium/kustomization.yaml b/k8s/infra/network/cilium/kustomization.yaml
@@ -2,8 +2,11 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 
 resources:
-  - announce.yaml
-  - ip-pool.yaml
+  - bgp-ip-pool.yaml
+  - bgp-advertise-lb-services.yaml
+  - bgp-peer-ucg-max-config.yaml
+  - bgp-peer-ucg-max-password.yaml
+  - bgp-cluster-config.yaml
   - dashboards/cilium.yaml
   - dashboards/cilium-operator.yaml
 
diff --git a/k8s/infra/network/cilium/values.yaml b/k8s/infra/network/cilium/values.yaml
@@ -59,13 +59,11 @@ resources:
 #debug:
 #  enabled: true
 
-# Increase rate limit when doing L2 announcements
-k8sClientRateLimit:
-  qps: 20
-  burst: 100
-
-l2announcements:
+# https://docs.cilium.io/en/latest/network/bgp-control-plane/bgp-control-plane-v2/
+bgpControlPlane:
   enabled: true
+  secretsNamespace:
+    name: cilium-secrets
 
 externalIPs:
   enabled: true
diff --git a/k8s/infra/network/dns/adguard/config/AdGuardHome.yaml b/k8s/infra/network/dns/adguard/config/AdGuardHome.yaml
@@ -144,12 +144,12 @@ filtering:
   parental_block_host: family-block.dns.adguard.com
   safebrowsing_block_host: standard-block.dns.adguard.com
   rewrites:
+    - domain: coturn.stonegarden.dev
+      answer: 172.20.10.140
     - domain: '*.stonegarden.dev'
-      answer: 192.168.1.220
+      answer: 172.20.10.100
     - domain: stonegarden.dev
-      answer: 192.168.1.220
-    - domain: coturn.stonegarden.dev
-      answer: 192.168.1.226
+      answer: 172.20.10.100
   safe_fs_patterns:
     - /opt/adguardhome/work/data/userfilters/*
   safebrowsing_cache_size: 1048576
diff --git a/k8s/infra/network/dns/adguard/svc.yaml b/k8s/infra/network/dns/adguard/svc.yaml
@@ -4,9 +4,14 @@ metadata:
   name: adguard
   namespace: dns
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.253
+    io.cilium/lb-ipam-ips: 172.20.10.153
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
+  # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
+  clusterIP: 10.96.0.12
   ports:
     - name: http
       protocol: TCP
diff --git a/k8s/infra/network/dns/unbound/config/a-records.conf b/k8s/infra/network/dns/unbound/config/a-records.conf
@@ -1,5 +1,10 @@
 # A Record
      #local-data: "somecomputer.local. A 192.168.1.1"
+     local-data: "coturn.stonegarden.dev A 172.20.10.140"
+     local-data: "*.stonegarden.dev A 172.20.10.100"
+     local-data: "stonegarden.dev A 172.20.10.100"
 
 # PTR Record
      #local-data-ptr: "192.168.1.1 somecomputer.local."
+     local-data-ptr: "172.20.10.100 stonegarden.dev."
+     local-data-ptr: "172.20.10.140 coturn.stonegarden.dev."
diff --git a/k8s/infra/network/dns/unbound/deployment.yaml b/k8s/infra/network/dns/unbound/deployment.yaml
@@ -33,7 +33,6 @@ spec:
               cpu: 10m
               memory: 64Mi
             limits:
-              cpu: 500m
               memory: 128Mi
           startupProbe:
             exec:
diff --git a/k8s/infra/network/dns/unbound/svc.yaml b/k8s/infra/network/dns/unbound/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: unbound
   namespace: dns
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.252
+    io.cilium/lb-ipam-ips: 172.20.10.150
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
   # https://kubernetes.io/docs/concepts/services-networking/cluster-ip-allocation/
diff --git a/k8s/infra/network/gateway/gw-external.yaml b/k8s/infra/network/gateway/gw-external.yaml
@@ -5,9 +5,13 @@ metadata:
   namespace: gateway
 spec:
   gatewayClassName: cilium
+  infrastructure:
+    labels:
+      bgp.cilium.io/advertise-service: default
+      lb-ipam.cilium.io/ip-pool: default-bgp
   addresses:
     - type: IPAddress
-      value: 192.168.1.222
+      value: 172.20.10.110
   listeners:
     - protocol: HTTPS
       port: 443
diff --git a/k8s/infra/network/gateway/gw-internal.yaml b/k8s/infra/network/gateway/gw-internal.yaml
@@ -5,9 +5,13 @@ metadata:
   namespace: gateway
 spec:
   gatewayClassName: cilium
+  infrastructure:
+    labels:
+      bgp.cilium.io/advertise-service: default
+      lb-ipam.cilium.io/ip-pool: default-bgp
   addresses:
     - type: IPAddress
-      value: 192.168.1.220
+      value: 172.20.10.100
   listeners:
     - protocol: TLS
       port: 443
diff --git a/k8s/infra/vpn/coturn/svc.yaml b/k8s/infra/vpn/coturn/svc.yaml
@@ -4,7 +4,10 @@ metadata:
   name: coturn
   namespace: coturn
   annotations:
-    io.cilium/lb-ipam-ips: 192.168.1.226
+    io.cilium/lb-ipam-ips: 172.20.10.140
+  labels:
+    bgp.cilium.io/advertise-service: default
+    lb-ipam.cilium.io/ip-pool: default-bgp
 spec:
   type: LoadBalancer
   selector:
diff --git a/k8s/infra/vpn/netbird/management/deployment.yaml b/k8s/infra/vpn/netbird/management/deployment.yaml
@@ -19,9 +19,10 @@ spec:
       nodeSelector:
         topology.kubernetes.io/zone: abel
       dnsConfig:
-        # Internal AdGuard Home DNS
         nameservers:
-          - 192.168.1.253
+          - 172.20.10.153 # AdGuard Home
+          - 10.96.0.12 # AdGuard Home
+          - 10.96.0.11 # Unbound
       dnsPolicy: None
       securityContext:
         seccompProfile:
