Add NAT64 to enable IPv6 provision in v4 only host

nuhakala · nuhakala · commit 1abf0cebe1b2 · 2025-09-04T09:52:12.000Z
- Install Tayga as NAT64 service
- Install Bind9 and configure it as local DNS server with DNS64
- Add IPv6 specific addresses and logic to cluster, controlplane, and
  worker templates
- Update vars.md with new variables
- Update CNI configuration for IPv6

Signed-off-by: Nuutti Hakala &lt;nuutti.hakala@est.tech&gt;
diff --git a/tests/roles/run_tests/tasks/verify.yml b/tests/roles/run_tests/tasks/verify.yml
@@ -39,14 +39,60 @@
       replace: "{{ POD_CIDR }}"
     register: updated_manifest
 
-  - name: Add IP_AUTODETECTION_METHOD in calico config Ubuntu
-    blockinfile:
-      path: /tmp/calico.yaml
-      insertafter: "{{ POD_CIDR }}"
-      block: |
-          # for indentation
-                      - name: IP_AUTODETECTION_METHOD
-                        value: "cidr={{ EXTERNAL_SUBNET_V4_HOST }}/{{ EXTERNAL_SUBNET_V4_PREFIX }}"
+  - name: Calico IPv4 specific setup
+    block:
+      - name: Add IP_AUTODETECTION_METHOD in calico config Ubuntu
+        blockinfile:
+          path: /tmp/calico.yaml
+          insertafter: "{{ POD_CIDR }}"
+          block: |
+              # for indentation
+                          - name: IP_AUTODETECTION_METHOD
+                            value: "cidr={{ EXTERNAL_SUBNET_V4_HOST }}/{{ EXTERNAL_SUBNET_V4_PREFIX }}"
+
+    when: IP_STACK == "v4" or IP_STACK == "v4v6"
+
+  - name: Calico IPv6 specific setup
+    block: 
+
+      - name: Edit IPAM pool
+        replace:
+          path: /tmp/calico.yaml
+          regexp: '"type": "calico-ipam"'
+          replace: '"type": "calico-ipam", "assign_ipv4": "false", "assign_ipv6": "true"'
+
+      - name: Add IP_AUTODETECTION_METHOD and POD_CIDR in calico config Ubuntu
+        blockinfile:
+          path: /tmp/calico.yaml
+          insertafter: "{{ POD_CIDR }}"
+          block: |
+              # for indentation
+                          - name: IP_AUTODETECTION_METHOD
+                            value: "cidr=127.0.0.1/24"
+                          - name: IP6_AUTODETECTION_METHOD
+                            value: "cidr={{ EXTERNAL_SUBNET_V6_HOST }}/{{ EXTERNAL_SUBNET_V6_PREFIX }}"
+                          - name: IP6
+                            value: "autodetect"
+                          # CALICO_ROUTER_ID needed when v4 is disabled
+                          - name: CALICO_ROUTER_ID
+                            value: "hash"
+
+      - name: Disable IPv4 autodetection
+        replace:
+          path: /tmp/calico.yaml
+          after: "name: IP"
+          before: "name: CALICO_IPV4POOL_IPIP"
+          regexp: "autodetect"
+          replace: "none"
+
+      - name: set FELIX_IPV6SUPPORT to true
+        replace:
+          path: /tmp/calico.yaml
+          after: "name: FELIX_IPV6SUPPORT"
+          before: "name: FELIX_HEALTHENABLED"
+          regexp: "false"
+          replace: "true"
+    when: IP_STACK == "v6"
 
   - name: Apply Calico manifest
     kubernetes.core.k8s:
diff --git a/tests/roles/run_tests/templates/main/cluster-template-cluster.yaml b/tests/roles/run_tests/templates/main/cluster-template-cluster.yaml
@@ -25,7 +25,7 @@ metadata:
   namespace: ${ NAMESPACE }
 spec:
   controlPlaneEndpoint:
-    host: ${ CLUSTER_APIENDPOINT_HOST }
+    host: ${ CLUSTER_APIENDPOINT_IP }
     port: ${ CLUSTER_APIENDPOINT_PORT }
   noCloudProvider: true
 ---
diff --git a/tests/roles/run_tests/templates/main/cluster-template-controlplane-kubeadm-config-centos.yaml b/tests/roles/run_tests/templates/main/cluster-template-controlplane-kubeadm-config-centos.yaml
@@ -1,5 +1,7 @@
 # CentOS specific controlplane kubeadm config
 preKubeadmCommands:
+  - sysctl -w net.ipv6.conf.all.forwarding=1
+  - sysctl -w net.ipv6.conf.default.forwarding=1
   - systemctl restart NetworkManager.service
   - nmcli connection load /etc/NetworkManager/system-connections/{{ IRONIC_ENDPOINT_BRIDGE }}.nmconnection
   - nmcli connection up {{ IRONIC_ENDPOINT_BRIDGE }}
@@ -66,7 +68,11 @@ files:
           priority 101
           advert_int 1
           virtual_ipaddress {
-              {{ CLUSTER_APIENDPOINT_HOST }}
+{% if IP_STACK == "v6" %}
+{# According to VRRP, link local address should come first with v6 #}
+              {{ CLUSTER_APIENDPOINT_LINK_LOCAL }}
+{% endif %}
+              {{ CLUSTER_APIENDPOINT_IP }}
           }
           track_script {
               k8s_api_check
@@ -92,26 +98,38 @@ files:
       type=bridge
       autoconnect=yes
       autoconnect-priority=1
+
       [bridge]
       interface-name={{ IRONIC_ENDPOINT_BRIDGE }}
       stp=false
+
+{% if IP_STACK == "v6" %}
       [ipv4]
+      method=disabled
+
+      [ipv6]
       address1={{ "{{ ds.meta_data.provisioningIP }}" }}/{{ "{{ ds.meta_data.provisioningCIDR }}" }}
+      addr-gen-mode=eui64
       method=manual
+{% else %}
+      [ipv4]
+      address1={{ "{{ ds.meta_data.provisioningIP }}" }}/{{ "{{ ds.meta_data.provisioningCIDR }}" }}
+      method=manual
+
       [ipv6]
       addr-gen-mode=eui64
       method=ignore
+{% endif %}
   - path: /etc/yum.repos.d/kubernetes.repo
     owner: root:root
     permissions: '0644'
     content: |
       [kubernetes]
       name=Kubernetes
-      baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
+      baseurl=https://pkgs.k8s.io/core:/stable:/v1.34/rpm/
       enabled=1
       gpgcheck=1
-      repo_gpgcheck=0
-      gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+      gpgkey=https://pkgs.k8s.io/core:/stable:/v1.34/rpm/repodata/repomd.xml.key
   - path : /etc/containers/registries.conf
     content: |
       [registries.search]
diff --git a/tests/roles/run_tests/templates/main/cluster-template-workers-kubeadm-config-centos.yaml b/tests/roles/run_tests/templates/main/cluster-template-workers-kubeadm-config-centos.yaml
@@ -1,5 +1,7 @@
 # CentOS specific worker kubeadm config
 preKubeadmCommands:
+  - sysctl -w net.ipv6.conf.all.forwarding=1
+  - sysctl -w net.ipv6.conf.default.forwarding=1
   - rm /etc/cni/net.d/*
   - systemctl restart NetworkManager.service
   - nmcli connection load /etc/NetworkManager/system-connections/{{ IRONIC_ENDPOINT_BRIDGE }}.nmconnection
@@ -57,24 +59,33 @@ files:
       [bridge]
       stp=false
 
+{% if IP_STACK == "v6" %}
+      [ipv4]
+      method=disabled
+
+      [ipv6]
+      address1={{ "{{ ds.meta_data.provisioningIP }}" }}/{{ "{{ ds.meta_data.provisioningCIDR }}" }}
+      addr-gen-mode=eui64
+      method=manual
+{% else %}
       [ipv4]
       address1={{ "{{ ds.meta_data.provisioningIP }}" }}/{{ "{{ ds.meta_data.provisioningCIDR }}" }}
       method=manual
 
       [ipv6]
       addr-gen-mode=eui64
       method=ignore
+{% endif %}
   - path: /etc/yum.repos.d/kubernetes.repo
     owner: root:root
     permissions: '0644'
     content: |
       [kubernetes]
       name=Kubernetes
-      baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
+      baseurl=https://pkgs.k8s.io/core:/stable:/v1.34/rpm/
       enabled=1
       gpgcheck=1
-      repo_gpgcheck=0
-      gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+      gpgkey=https://pkgs.k8s.io/core:/stable:/v1.34/rpm/repodata/repomd.xml.key
   - path : /etc/containers/registries.conf
     owner: root:root
     permissions: '0644'
diff --git a/tests/roles/run_tests/templates/main/metal3datatemplate-template.yaml b/tests/roles/run_tests/templates/main/metal3datatemplate-template.yaml
@@ -66,5 +66,5 @@ spec:
         - 8.8.8.8
 {% endif %}
 {% if IP_STACK == 'v6' or IP_STACK == 'v4v6' %}
-        - 2001:4860:4860::8888
+        - {{ LOCAL_DNS_V6 }}
 {% endif %}
diff --git a/tests/roles/run_tests/vars/main.yml b/tests/roles/run_tests/vars/main.yml
@@ -10,6 +10,7 @@ CAPI_CONFIG_DIR: "{{ lookup('env', 'CAPI_CONFIG_DIR') | default('$HOME/.config/c
 CLUSTER_APIENDPOINT_HOST: "{{ lookup('env', 'CLUSTER_APIENDPOINT_HOST') }}"
 CLUSTER_APIENDPOINT_PORT: "{{ lookup('env', 'CLUSTER_APIENDPOINT_PORT') | default(6443, true) }}"
 CLUSTER_APIENDPOINT_IP: "{{ lookup('env', 'CLUSTER_APIENDPOINT_IP') }}"
+CLUSTER_APIENDPOINT_LINK_LOCAL: "{{ ansible_facts['ironicendpoint']['ipv6'][2]['address'] }}"
 CLUSTER_NAME: "{{ lookup('env', 'CLUSTER_NAME') | default('test1', true) }}"
 CONTROL_PLANE_MACHINE_COUNT: "{{ lookup('env', 'CONTROL_PLANE_MACHINE_COUNT') | default(1, true) }}"
 WORKER_MACHINE_COUNT: "{{ lookup('env', 'WORKER_MACHINE_COUNT') | default(1, true) }}"
@@ -48,7 +49,7 @@ DEPLOY_RAMDISK_URL: "{{ lookup('env', 'DEPLOY_RAMDISK_URL') }}"
 IRONIC_URL: "{{ lookup('env', 'IRONIC_URL') }}"
 IRONIC_INSPECTOR_URL: "{{ lookup('env', 'IRONIC_INSPECTOR_URL') }}"
 POD_CIDR: "{{ lookup('env', 'POD_CIDR') }}"
-SERVICE_CIDR: "10.96.0.0/12"
+SERVICE_CIDR: "{{ lookup('env', 'SERVICE_CIDR') | default('10.96.0.0/12', true) }}"
 CAPM3RELEASE: "{{ lookup('env', 'CAPM3RELEASE') | default('v1.1.2', true) }}"
 CAPIRELEASE: "{{ lookup('env', 'CAPIRELEASE') | default('v1.1.4', true) }}"
 IPAMRELEASE: "{{ lookup('env', 'IPAMRELEASE') | default('v1.1.2', true) }}"
@@ -61,6 +62,7 @@ CALICO_MINOR_RELEASE: "{{ lookup('env', 'CALICO_MINOR_RELEASE') | default('v3.25
 CALICO_PATCH_RELEASE: "{{ lookup('env', 'CALICO_PATCH_RELEASE') | default('v3.25.1', true) }}"
 DOCKER_HUB_PROXY: "{{ lookup('env', 'DOCKER_HUB_PROXY') }}"
 WORKING_DIR: "{{ lookup('env', 'WORKING_DIR') | default('/opt/metal3-dev-env', true) }}"
+LOCAL_DNS_V6: "{{ lookup('env', 'LOCAL_DNS_V6') | default('fd00:abcd::1', true) }}"
 
 # Environment variables for deployment. IMAGE_OS can be centos or ubuntu, change accordingly to your needs.
 IMAGE_OS: "{{ lookup('env', 'IMAGE_OS') | default('centos', true) }}"
diff --git a/vars.md b/vars.md
@@ -12,6 +12,11 @@ assured that they are persisted.
 | Name | Option | Allowed values | Default |
 | :------ | :------- | :--------------- | :-------- |
 | DOCKER_USE_IPV6_INTERNALLY | Choose whether Docker will use IPv6 internally. | "true", "false" | false |
+| ENABLE_NAT64 | Enable NAT64 functionality to emulate IPv6 capable host. | "true", "false" | "false
+| DNS_UPSTREAM | Specify upstream DNS server for locally run DNS. Only used when ENABLE_NAT64 = "true" | IPv4 or IPv6 address | 10.1.0.2
+| DNS64_PREFIX | Prefix for DNS64 server. Only used when ENABLE_NAT64 = "true" | Any valid prefix | 64:ff9b::/96
+| LOCAL_DNS_V4 | IPv4 address for local DNS server. Only used when ENABLE_NAT64 = "true". Using loopback is fine, because in the IPv6 setup VMs don't access DNS server over IPv4. | IPv4 address | 127.0.0.2
+| LOCAL_DNS_V6 | IPv6 address for local DNS server. Only used when ENABLE_NAT64 = "true". NOT RECOMMENDED TO USE LOOPBACK ::1, because then VMs cannot access the server. | IPv6 address | fd00:abcd::1
 | MAX_SURGE_VALUE | This variable defines if controlplane should scale-in or scale-out during upgrade. | 0 (scale-in) or 1 (scale-out) |1|
 | EPHEMERAL_CLUSTER | Tool for running management/ephemeral cluster. | minikube, kind, tilt | "kind" when using docker as the container runtime (the default on Ubuntu), "minikube" otherwise |
 | IP_STACK | Choose whether the "external" libvirt network will use IPv4, IPv6, or IPv4+IPv6. This network is the primary network interface for the virtual bare metal hosts. Note that this only sets up the underlying network, and fully provisioning IPv6 kubernetes clusters is not yet automated. If IPv6 is enabled, DHCPv6 will be available to the virtual bare metal hosts. | "v4", "v6", "v4v6" (dual-stack)) | v4 |
@@ -217,6 +222,10 @@ networking, the following additional steps are required:
 1. You need to replace the default IPv4 addresses with IPv6 addresses in the
    environment variables.
 
+Notice that if your host does not have support for native IPv6, you need to
+enable NAT64 and DNS64 to provision the VMs. If you enable NAT64 with the
+variable below, it will also enable DNS64.
+
 The following variables need to be set:
 
 ``` sh
@@ -229,4 +238,5 @@ export EXTERNAL_SUBNET_V6="fd55::/64"
 export BARE_METAL_PROVISIONER_SUBNET_IPV6_ONLY=true
 export DOCKER_USE_IPV6_INTERNALLY=true
 export POD_CIDR="fd00:6969::/64"
+export ENABLE_NAT64=true # use if host does not support native IPv6
 ```
diff --git a/vm-setup/roles/nat64/defaults/main.yml b/vm-setup/roles/nat64/defaults/main.yml
@@ -0,0 +1,8 @@
+nat64_action: setup
+ENABLE_NAT64: "{{ lookup('env', 'ENABLE_NAT64') | default('false', true) }}"
+TEMPLATES_PATH: "{{ metal3_dir }}/vm-setup/roles/nat64/files"
+DNS_UPSTREAM: "{{ lookup('env', 'DNS_UPSTREAM') | default('10.1.0.2', true) }}"
+DNS64_PREFIX: "{{ lookup('env', 'DNS64_PREFIX') | default('fd00:ffff:ffff::/96', true) }}"
+# Local DNS is only used in IPv6 only env
+LOCAL_DNS_V4: "{{ lookup('env', 'LOCAL_DNS_V4') | default('127.0.0.2', true) }}"
+LOCAL_DNS_V6: "{{ lookup('env', 'LOCAL_DNS_V6') | default('fd00:abcd::1', true) }}"
diff --git a/vm-setup/roles/nat64/tasks/main.yml b/vm-setup/roles/nat64/tasks/main.yml
@@ -0,0 +1,17 @@
+- name: Install IPv6 only required packages
+  package:
+    name:
+      - tayga
+      - bind9
+    state: present
+  when: nat64_action == "setup"
+
+- name: Setup NAT64
+  block:
+    - include_tasks: setup_nat.yml
+  when: nat64_action == "setup"
+
+- name: Teardown NAT64
+  block:
+    - include_tasks: teardown_nat.yml
+  when: nat64_action == "teardown"
diff --git a/vm-setup/roles/nat64/tasks/setup_nat.yml b/vm-setup/roles/nat64/tasks/setup_nat.yml
@@ -0,0 +1,76 @@
+# Libvirt needs dnsmasq to manage networks. Installing it will likely automatically
+# enable global dnsmasq. We don't want it.
+- name: Ensure dnsmasq is stopped
+  ansible.builtin.systemd_service:
+    state: stopped
+    name: dnsmasq
+  become: yes
+
+- name: Write Tayga configuration
+  template:
+    src: "../templates/tayga.conf"
+    dest: "/etc/tayga.conf"
+
+- name: ensure IP forwarding is on
+  shell: |
+    sysctl net.ipv4.ip_forward=1
+    sysctl net.ipv6.conf.all.forwarding=1
+  become: yes
+
+- name: Start NAT64 service tayga
+  ansible.builtin.systemd_service:
+    state: started
+    name: tayga
+  become: yes
+
+- name: Generate Bind9 options file
+  template:
+    src: "../templates/named.conf.options"
+    dest: /etc/bind/named.conf.options
+    owner: root
+    group: root
+    mode: '0644'
+
+# IPv6 only supported on ubuntu, so assume that systemd-resolved is
+# the default dns resolution
+- name: Disable systemd-resolved
+  ansible.builtin.systemd_service:
+    state: stopped
+    name: systemd-resolved
+  become: yes
+
+- name: Write /etc/resolv.conf to contain local DNS server
+  template:
+    src: "../templates/resolv.conf"
+    dest: /etc/resolv.conf
+    owner: root
+    group: root
+    mode: '0644'
+    backup: true
+
+# Note that if the address is regular 127.0.0.1, then this will add a duplicate 
+# address to the interface but with different subnet mask, the default
+# subnetmask is 8 for IPv4 and 
+- name: Add DNS-server IPv4-address to loopback interface
+  shell:
+    cmd: "ip addr add {{ LOCAL_DNS_V4 }}/32 dev lo"
+  become: yes
+  ignore_errors: true
+
+- name: Add DNS-server IPv6-address to loopback interface
+  shell:
+    cmd: "ip addr add {{ LOCAL_DNS_V6 }}/128 dev lo"
+  become: yes
+  ignore_errors: true
+
+- name: Restart Bind9 service
+  ansible.builtin.systemd_service:
+    state: restarted
+    daemon_reload: true
+    name: named.service
+  become: yes
+
+- name: Add local hostname to /etc/hosts
+  ansible.builtin.lineinfile:
+    path: /etc/hosts
+    line: 127.0.0.1 {{ ansible_hostname }}
diff --git a/vm-setup/roles/nat64/tasks/teardown_nat.yml b/vm-setup/roles/nat64/tasks/teardown_nat.yml
@@ -0,0 +1,23 @@
+- name: Stop NAT64 service tayga
+  ansible.builtin.systemd_service:
+    state: stopped
+    name: tayga
+  become: yes
+
+- name: Clean NAT64 interface
+  shell: |
+    ip link del nat64
+  become: yes
+  ignore_errors: true
+
+- name: Delete DNS-server IPv4-address from loopback interface
+  shell:
+    cmd: "ip addr del {{ LOCAL_DNS_V4 }}/32 dev lo"
+  become: yes
+  ignore_errors: true
+
+- name: Delete DNS-server IPv6-address from loopback interface
+  shell:
+    cmd: "ip addr del {{ LOCAL_DNS_V6 }}/128 dev lo"
+  become: yes
+  ignore_errors: true
diff --git a/vm-setup/roles/nat64/templates/named.conf.options b/vm-setup/roles/nat64/templates/named.conf.options
@@ -0,0 +1,20 @@
+options {
+        directory "/var/cache/bind";
+
+        forwarders {
+                {{ DNS_UPSTREAM }};
+        };
+
+        dnssec-validation auto;
+        listen-on { {{ LOCAL_DNS_V4 }}; };
+        listen-on-v6 { {{ LOCAL_DNS_V6 }}; };
+        dns64 {{ DNS64_PREFIX }} {
+                // Accept all requests
+                clients { any; };
+                // maps all ipv4 addresses
+                mapped { any; };
+                // excludes all native ipv6 addresses: even if domain has
+                // native ipv6 address bind will return mapped ipv4 address
+                exclude { any; };
+        };
+};
diff --git a/vm-setup/roles/nat64/templates/resolv.conf b/vm-setup/roles/nat64/templates/resolv.conf
@@ -0,0 +1,2 @@
+nameserver {{ LOCAL_DNS_V4 }}
+nameserver {{ LOCAL_DNS_V6 }}
diff --git a/vm-setup/roles/nat64/templates/tayga.conf b/vm-setup/roles/nat64/templates/tayga.conf
diff --git a/vm-setup/roles/packages_installation/defaults/main.yml b/vm-setup/roles/packages_installation/defaults/main.yml
diff --git a/vm-setup/roles/packages_installation/tasks/ubuntu_required_packages.yml b/vm-setup/roles/packages_installation/tasks/ubuntu_required_packages.yml
diff --git a/vm-setup/roles/packages_installation/templates/daemon.json b/vm-setup/roles/packages_installation/templates/daemon.json
diff --git a/vm-setup/setup-playbook.yml b/vm-setup/setup-playbook.yml
diff --git a/vm-setup/teardown-playbook.yml b/vm-setup/teardown-playbook.yml
