azure: Add support for multi zonal NAT gateways

Adding support to install multiple NAT gateways per subnet.
The zone in which they will be installed is not customizable
due to needing an upstream PR to fix CAPZ.

Author: rna-afk

data/data/install.openshift.io_installconfigs.yaml

@@ -5277,6 +5277,27 @@ spec:
                           type: string
                         type: array
                     type: object
+                  natGatewaySpec:
+                    description: |-
+                      NatGatewaySpec allows the user to specify the subnets and the nat gateway configuration for each subnet.
+                      Since only one nat gateway is allowed per subnet, users can create multiple subnets and create nat gateway
+                      for each subnet for zone resilience.
+                    items:
+                      description: NatGatewaySpec allows the user to specify the subnets
+                        and the nat gateway configuration for each subnet.
+                      properties:
+                        name:
+                          description: Name of the nat gateway to be created.
+                          type: string
+                        subnet:
+                          description: Subnet specifies the name of the subnet to
+                            be created.
+                          type: string
+                      required:
+                      - name
+                      - subnet
+                      type: object
+                    type: array
                   networkResourceGroupName:
                     description: NetworkResourceGroupName specifies the network resource
                       group that contains an existing VNet
@@ -5326,6 +5347,7 @@ spec:
                       VNet for the installer to use
                     type: string
                 required:
+                - natGatewaySpec
                 - region
                 type: object
               baremetal:

pkg/asset/manifests/azure/cluster.go

@@ -179,7 +179,20 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	azEnv := string(installConfig.Azure.CloudName)
 
-	computeSubnetSpec := capz.SubnetSpec{
+	subnetSpec := capz.Subnets{
+		{
+			SubnetClassSpec: capz.SubnetClassSpec{
+				Name: controlPlaneSubnet,
+				Role: capz.SubnetControlPlane,
+				CIDRBlocks: []string{
+					subnets[0].String(),
+				},
+			},
+			SecurityGroup: securityGroup,
+		},
+	}
+
+	computeSubnetSpec := []capz.SubnetSpec{{
 		ID: nodeSubnetID,
 		SubnetClassSpec: capz.SubnetClassSpec{
 			Name: computeSubnet,
@@ -189,14 +202,36 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 			},
 		},
 		SecurityGroup: securityGroup,
-	}
+	}}
 
 	if installConfig.Config.Azure.OutboundType == azure.NATGatewaySingleZoneOutboundType {
-		computeSubnetSpec.NatGateway = capz.NatGateway{
+		computeSubnetSpec[0].NatGateway = capz.NatGateway{
 			NatGatewayClassSpec: capz.NatGatewayClassSpec{Name: fmt.Sprintf("%s-natgw", clusterID.InfraID)},
 		}
+	} else if installConfig.Config.Azure.OutboundType == azure.NATGatewayMultiZoneOutboundType {
+		computeSubnetSpec = []capz.SubnetSpec{}
+		for index, spec := range installConfig.Config.Azure.NatGatewaySpec {
+			computeSubnetSpec = append(computeSubnetSpec, capz.SubnetSpec{
+				ID: "UNKNOWN",
+				SubnetClassSpec: capz.SubnetClassSpec{
+					Name: computeSubnet,
+					Role: capz.SubnetNode,
+					CIDRBlocks: []string{
+						spec.Subnet,
+					},
+				},
+				NatGateway: capz.NatGateway{
+					NatGatewayIP: capz.PublicIPSpec{
+						Name: fmt.Sprintf("%s-natgw-public-ip-%d", clusterID.InfraID, index),
+					},
+					NatGatewayClassSpec: capz.NatGatewayClassSpec{Name: spec.Name},
+				},
+				SecurityGroup: securityGroup,
+			})
+		}
 	}
 
+	subnetSpec = append(subnetSpec, computeSubnetSpec...)
 	azureCluster := &capz.AzureCluster{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      clusterID.InfraID,
@@ -236,19 +271,7 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 				},
 				APIServerLB:            &apiServerLB,
 				ControlPlaneOutboundLB: controlPlaneOutboundLB,
-				Subnets: capz.Subnets{
-					{
-						SubnetClassSpec: capz.SubnetClassSpec{
-							Name: controlPlaneSubnet,
-							Role: capz.SubnetControlPlane,
-							CIDRBlocks: []string{
-								subnets[0].String(),
-							},
-						},
-						SecurityGroup: securityGroup,
-					},
-					computeSubnetSpec,
-				},
+				Subnets:                subnetSpec,
 			},
 		},
 	}

pkg/explain/printer_test.go

@@ -322,6 +322,12 @@ If empty, the value is equal to "AzurePublicCloud".
 installing on Azure for machine pools which do not define their own
 platform configuration.
 
+    natGatewaySpec <[]object> -required-
+      NatGatewaySpec allows the user to specify the subnets and the nat gateway configuration for each subnet.
+Since only one nat gateway is allowed per subnet, users can create multiple subnets and create nat gateway
+for each subnet for zone resilience.
+      NatGatewaySpec allows the user to specify the subnets and the nat gateway configuration for each subnet.
+
     networkResourceGroupName <string>
       NetworkResourceGroupName specifies the network resource group that contains an existing VNet
 

pkg/types/azure/platform.go

@@ -23,6 +23,9 @@ const (
 	// see https://learn.microsoft.com/en-us/azure/virtual-network/nat-gateway/nat-gateway-resource
 	NATGatewaySingleZoneOutboundType OutboundType = "NATGatewaySingleZone"
 
+	// NATGatewayMultiZoneOutboundType uses NAT gateways in multiple zones in the compute node subnets for outbound access.
+	NATGatewayMultiZoneOutboundType OutboundType = "MultiZoneNatGateway"
+
 	// UserDefinedRoutingOutboundType uses user defined routing for egress from the cluster.
 	// see https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview
 	UserDefinedRoutingOutboundType OutboundType = "UserDefinedRouting"
@@ -83,6 +86,11 @@ type Platform struct {
 	// +optional
 	OutboundType OutboundType `json:"outboundType"`
 
+	// NatGatewaySpec allows the user to specify the subnets and the nat gateway configuration for each subnet.
+	// Since only one nat gateway is allowed per subnet, users can create multiple subnets and create nat gateway
+	// for each subnet for zone resilience.
+	NatGatewaySpec []NatGatewaySpec `json:"natGatewaySpec"`
+
 	// ResourceGroupName is the name of an already existing resource group where the cluster should be installed.
 	// This resource group should only be used for this specific cluster and the cluster components will assume
 	// ownership of all resources in the resource group. Destroying the cluster using installer will delete this
@@ -110,6 +118,14 @@ type Platform struct {
 	UserProvisionedDNS dns.UserProvisionedDNS `json:"userProvisionedDNS,omitempty"`
 }
 
+// NatGatewaySpec allows the user to specify the subnets and the nat gateway configuration for each subnet.
+type NatGatewaySpec struct {
+	// Name of the nat gateway to be created.
+	Name string `json:"name"`
+	// Subnet specifies the name of the subnet to be created.
+	Subnet string `json:"subnet"`
+}
+
 // KeyVault defines an Azure Key Vault.
 type KeyVault struct {
 	// ResourceGroup defines the Azure resource group used by the key

pkg/types/azure/validation/platform.go

@@ -239,6 +239,7 @@ func findDuplicateTagKeys(tagSet map[string]string) error {
 var (
 	validOutboundTypes = map[azure.OutboundType]struct{}{
 		azure.LoadbalancerOutboundType:         {},
+		azure.NATGatewayMultiZoneOutboundType:  {},
 		azure.NATGatewaySingleZoneOutboundType: {},
 		azure.UserDefinedRoutingOutboundType:   {},
 	}