azure: Add support for multi zonal NAT gateways

Adding support to install multiple NAT gateways per subnet.
The zone in which they will be installed is not customizable
due to needing an upstream PR to fix CAPZ.

Also, allowing the users to bring their own subnets.

Author: rna-afk

pkg/asset/manifests/azure/cluster.go

@@ -179,7 +179,20 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 
 	azEnv := string(installConfig.Azure.CloudName)
 
-	computeSubnetSpec := capz.SubnetSpec{
+	// Set default control plane subnets for default installs.
+	defaultControlPlaneSubnet := capz.Subnets{
+		{
+			SubnetClassSpec: capz.SubnetClassSpec{
+				Name: controlPlaneSubnet,
+				Role: capz.SubnetControlPlane,
+				CIDRBlocks: []string{
+					subnets[0].String(),
+				},
+			},
+			SecurityGroup: securityGroup,
+		},
+	}
+	defaultComputeSubnetSpec := capz.SubnetSpec{
 		ID: nodeSubnetID,
 		SubnetClassSpec: capz.SubnetClassSpec{
 			Name: computeSubnet,
@@ -191,10 +204,53 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 		SecurityGroup: securityGroup,
 	}
 
-	if installConfig.Config.Azure.OutboundType == azure.NATGatewaySingleZoneOutboundType {
-		computeSubnetSpec.NatGateway = capz.NatGateway{
-			NatGatewayClassSpec: capz.NatGatewayClassSpec{Name: fmt.Sprintf("%s-natgw", clusterID.InfraID)},
+	subnetSpec := []capz.SubnetSpec{}
+	hasControlPlaneSubnet := false
+	hasComputePlaneSubnet := false
+
+	// Add the user specified subnets to the spec.
+	// For single zone, alter the compute subnet to have a NATGateway and add default control plane subnet
+	// configuration.
+	for index, spec := range installConfig.Config.Azure.Subnets {
+		specGen := capz.SubnetSpec{
+			ID: "UNKNOWN",
+			SubnetClassSpec: capz.SubnetClassSpec{
+				Name: spec.Name,
+				Role: spec.Role,
+			},
+			SecurityGroup: securityGroup,
+		}
+		// The CIDR information is optional since it could be a byo subnet.
+		if len(spec.SubnetCIDR) != 0 {
+			specGen.CIDRBlocks = spec.SubnetCIDR
 		}
+		// If role is compute node and outbound type is single node, add a NAT gateway to the subnet.
+		// Only adding a NAT gateway to the first subnet.
+		if !hasComputePlaneSubnet && spec.Role == capz.SubnetNode && installConfig.Config.Azure.OutboundType == azure.NATGatewaySingleZoneOutboundType {
+			specGen.NatGateway = capz.NatGateway{
+				NatGatewayClassSpec: capz.NatGatewayClassSpec{Name: fmt.Sprintf("%s-natgw", clusterID.InfraID)},
+			}
+		} else if installConfig.Config.Azure.OutboundType == azure.NATGatewayMultiZoneOutboundType {
+			if spec.NatGatewayName != "" {
+				specGen.NatGateway = capz.NatGateway{
+					NatGatewayIP: capz.PublicIPSpec{
+						Name: fmt.Sprintf("%s-natgw-public-ip-%d", clusterID.InfraID, index),
+					},
+					NatGatewayClassSpec: capz.NatGatewayClassSpec{Name: spec.NatGatewayName},
+				}
+			}
+		}
+		hasControlPlaneSubnet = hasControlPlaneSubnet || spec.Role == capz.SubnetControlPlane
+		hasComputePlaneSubnet = hasComputePlaneSubnet || spec.Role == capz.SubnetNode
+		subnetSpec = append(subnetSpec, specGen)
+	}
+	// Make sure there's at least one subnet for compute and control plane.
+	// Ordinary installs will get the default setup.
+	if !hasComputePlaneSubnet {
+		subnetSpec = append(subnetSpec, defaultComputeSubnetSpec)
+	}
+	if !hasControlPlaneSubnet {
+		subnetSpec = append(subnetSpec, defaultControlPlaneSubnet...)
 	}
 
 	azureCluster := &capz.AzureCluster{
@@ -236,19 +292,7 @@ func GenerateClusterAssets(installConfig *installconfig.InstallConfig, clusterID
 				},
 				APIServerLB:            &apiServerLB,
 				ControlPlaneOutboundLB: controlPlaneOutboundLB,
-				Subnets: capz.Subnets{
-					{
-						SubnetClassSpec: capz.SubnetClassSpec{
-							Name: controlPlaneSubnet,
-							Role: capz.SubnetControlPlane,
-							CIDRBlocks: []string{
-								subnets[0].String(),
-							},
-						},
-						SecurityGroup: securityGroup,
-					},
-					computeSubnetSpec,
-				},
+				Subnets:                subnetSpec,
 			},
 		},
 	}

pkg/types/azure/platform.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	"github.com/openshift/installer/pkg/types/dns"
+	capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 )
 
 // aro is a setting to enable aro-only modifications
@@ -23,6 +24,9 @@ const (
 	// see https://learn.microsoft.com/en-us/azure/virtual-network/nat-gateway/nat-gateway-resource
 	NATGatewaySingleZoneOutboundType OutboundType = "NATGatewaySingleZone"
 
+	// NATGatewayMultiZoneOutboundType uses NAT gateways in multiple zones in the compute node subnets for outbound access.
+	NATGatewayMultiZoneOutboundType OutboundType = "MultiZoneNatGateway"
+
 	// UserDefinedRoutingOutboundType uses user defined routing for egress from the cluster.
 	// see https://docs.microsoft.com/en-us/azure/virtual-network/virtual-networks-udr-overview
 	UserDefinedRoutingOutboundType OutboundType = "UserDefinedRouting"
@@ -83,6 +87,11 @@ type Platform struct {
 	// +optional
 	OutboundType OutboundType `json:"outboundType"`
 
+	// Subnets is the list of subnets the user can bring into the cluster to be used.
+	//
+	// +optional
+	Subnets []SubnetSpec `json:"subnets,omitempty"`
+
 	// ResourceGroupName is the name of an already existing resource group where the cluster should be installed.
 	// This resource group should only be used for this specific cluster and the cluster components will assume
 	// ownership of all resources in the resource group. Destroying the cluster using installer will delete this
@@ -110,6 +119,20 @@ type Platform struct {
 	UserProvisionedDNS dns.UserProvisionedDNS `json:"userProvisionedDNS,omitempty"`
 }
 
+// SubnetSpec specifies the properties the subnet needs to be used in the cluster.
+type SubnetSpec struct {
+	// Name of the subnet.
+	Name string `json:"name"`
+	// Role specifies the actual role which the subnet should be used in.
+	Role capz.SubnetRole `json:"role"`
+	// SubnetCIDR specifies the CIDR for the subnet if it needs to be created.
+	// Should not be mentioned if the subnet is a bring your own subnet.
+	SubnetCIDR []string `json:"cidr"`
+	// NatGatewayName specifies the name of the NAT gateway to be created for this subnet.
+	// Can only be used if the outbound type is set to MultiZoneNatGateway.
+	NatGatewayName string `json:"natGateway"`
+}
+
 // KeyVault defines an Azure Key Vault.
 type KeyVault struct {
 	// ResourceGroup defines the Azure resource group used by the key