@@ -13,28 +13,45 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
1313See the License for the specific language governing permissions and
1414limitations under the License.
1515*/
16+
1617package infracluster
1718
1819import (
1920 "context"
2021 "errors"
2122 "fmt"
23+ "net"
24+ "slices"
25+ "strings"
2226
2327 "github.com/go-logr/logr"
2428 configv1 "github.com/openshift/api/config/v1"
29+ mapiv1beta1 "github.com/openshift/api/machine/v1beta1"
2530 apierrors "k8s.io/apimachinery/pkg/api/errors"
2631 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2732 "k8s.io/klog/v2"
2833 "k8s.io/utils/ptr"
2934 clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
3035 "sigs.k8s.io/controller-runtime/pkg/client"
3136
37+ "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/extensions/layer3/routers"
38+ "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/ports"
39+ "github.com/gophercloud/gophercloud/v2/openstack/networking/v2/subnets"
3240 openstackv1 "sigs.k8s.io/cluster-api-provider-openstack/api/v1beta1"
41+ openstackclients "sigs.k8s.io/cluster-api-provider-openstack/pkg/clients"
42+ openstackscope "sigs.k8s.io/cluster-api-provider-openstack/pkg/scope"
3343)
3444
3545var (
3646 errUnsupportedOpenStackLoadBalancerType = errors .New ("unsupported load balancer type for OpenStack" )
3747 errOpenStackNoAPIServerInternalIPs = errors .New ("no APIServerInternalIPs available" )
48+ errOpenStackNoDefaultRouter = errors .New ("unable to determine default router from control plane machines" )
49+ errOpenStackNoDefaultSubnet = errors .New ("unable to determine default subnet from control plane machines" )
50+ errOpenStackNoControlPlaneMachines = errors .New ("no control plane machines found" )
51+ )
52+
53+ var (
54+ scopeCacheMaxSize int //nolint:gochecknoglobals
3855)
3956
4057// ensureOpenStackCluster ensures the OpenStackCluster object exists.
@@ -113,7 +130,6 @@ func (r *InfraClusterController) ensureOpenStackCluster(ctx context.Context, log
113130 },
114131 // NOTE(stephenfin): We deliberately don't add subnet here: CAPO will use all subnets in network,
115132 // which should also cover dual stack deployments. Everything else is populated below.
116- // FIXME(stephenfin): Populate these.
117133 Network : nil ,
118134 Router : nil ,
119135 ExternalNetwork : nil ,
@@ -122,6 +138,40 @@ func (r *InfraClusterController) ensureOpenStackCluster(ctx context.Context, log
122138 },
123139 }
124140
141+ // FIXME(stephenfin): Where can I source caCertificates from? The legacy infracluster controller
142+ // had the same issue.
143+ caCertificates := []byte {} // PEM encoded CA certificates
144+ scopeFactory := openstackscope .NewFactory (scopeCacheMaxSize )
145+
146+ scope , err := scopeFactory .NewClientScopeFromObject (ctx , r .Client , caCertificates , log , target )
147+ if err != nil {
148+ return nil , fmt .Errorf ("creating OpenStack client: %w" , err )
149+ }
150+
151+ networkClient , err := scope .NewNetworkClient ()
152+ if err != nil {
153+ return nil , fmt .Errorf ("creating OpenStack compute service: %w" , err )
154+ }
155+
156+ defaultSubnet , err := getDefaultSubnetFromMachines (ctx , log , r .Client , networkClient , platformStatus )
157+ if err != nil {
158+ return nil , err
159+ }
160+ // NOTE(stephenfin): As noted previously, we deliberately *do not* add subnet here: CAPO will
161+ // use all subnets in network, which should also cover dual-stack deployments
162+ target .Spec .Network = & openstackv1.NetworkParam {ID : ptr .To (defaultSubnet .NetworkID )}
163+
164+ router , err := getDefaultRouterFromSubnet (ctx , networkClient , defaultSubnet )
165+ if err != nil {
166+ return nil , err
167+ }
168+
169+ target .Spec .Router = & openstackv1.RouterParam {ID : ptr .To (router .ID )}
170+ // NOTE(stephenfin): The only reason we set ExternalNetworkID in the cluster spec is to avoid
171+ // an error reconciling the external network if it isn't set. If CAPO ever no longer requires
172+ // this we can just not set it and remove much of the code above. We don't actually use it.
173+ target .Spec .ExternalNetwork = & openstackv1.NetworkParam {ID : ptr .To (router .GatewayInfo .NetworkID )}
174+
125175 if err := r .Create (ctx , target ); err != nil {
126176 return nil , fmt .Errorf ("failed to create InfraCluster: %w" , err )
127177 }
@@ -130,3 +180,140 @@ func (r *InfraClusterController) ensureOpenStackCluster(ctx context.Context, log
130180
131181 return target , nil
132182}
183+
184+ // getDefaultRouterFromSubnet attempts to infer the default router used for
185+ // the network by looking for ports with the given subnet and gateway IP
186+ // associated with them.
187+ func getDefaultRouterFromSubnet (_ context.Context , networkClient openstackclients.NetworkClient , subnet * subnets.Subnet ) (* routers.Router , error ) {
188+ // Find the port which owns the subnet's gateway IP
189+ ports , err := networkClient .ListPort (ports.ListOpts {
190+ NetworkID : subnet .NetworkID ,
191+ FixedIPs : []ports.FixedIPOpts {
192+ {
193+ IPAddress : subnet .GatewayIP ,
194+ // XXX: We should search on both subnet and IP
195+ // address here, but can't because of
196+ // https://github.com/gophercloud/gophercloud/issues/2807
197+ // SubnetID: subnet.ID,
198+ },
199+ },
200+ })
201+ if err != nil {
202+ return nil , fmt .Errorf ("listing ports: %w" , err )
203+ }
204+
205+ if len (ports ) == 0 {
206+ return nil , fmt .Errorf ("%w: no ports found for subnet %s" , errOpenStackNoDefaultRouter , subnet .ID )
207+ }
208+
209+ if len (ports ) > 1 {
210+ return nil , fmt .Errorf ("%w: multiple ports found for subnet %s" , errOpenStackNoDefaultRouter , subnet .ID )
211+ }
212+
213+ routerID := ports [0 ].DeviceID
214+
215+ router , err := networkClient .GetRouter (routerID )
216+ if err != nil {
217+ return nil , fmt .Errorf ("getting router %s: %w" , routerID , err )
218+ }
219+
220+ if router .GatewayInfo .NetworkID == "" {
221+ return nil , fmt .Errorf ("%w: router %s does not have an external gateway" , errOpenStackNoDefaultRouter , routerID )
222+ }
223+
224+ return router , nil
225+ }
226+
227+ // getDefaultSubnetFromMachines attempts to infer the default cluster subnet by
228+ // directly examining the control plane machines. Specifically it looks for a
229+ // subnet attached to a control plane machine whose CIDR contains the API
230+ // loadbalancer internal VIP.
231+ //
232+ // This heuristic is only valid when the API loadbalancer type is
233+ // LoadBalancerTypeOpenShiftManagedDefault.
234+ //
235+ //nolint:gocognit,funlen
236+ func getDefaultSubnetFromMachines (ctx context.Context , log logr.Logger , kubeclient client.Client , networkClient openstackclients.NetworkClient , platformStatus * configv1.OpenStackPlatformStatus ) (* subnets.Subnet , error ) {
237+ mapiMachines := mapiv1beta1.MachineList {}
238+ if err := kubeclient .List (
239+ ctx ,
240+ & mapiMachines ,
241+ client .InNamespace ("openshift-machine-api" ),
242+ client.MatchingLabels {"machine.openshift.io/cluster-api-machine-role" : "master" },
243+ ); err != nil {
244+ return nil , fmt .Errorf ("listing control plane machines: %w" , err )
245+ }
246+
247+ if len (mapiMachines .Items ) == 0 {
248+ return nil , errOpenStackNoControlPlaneMachines
249+ }
250+
251+ apiServerInternalIPs := make ([]net.IP , len (platformStatus .APIServerInternalIPs ))
252+ for i , ipStr := range platformStatus .APIServerInternalIPs {
253+ apiServerInternalIPs [i ] = net .ParseIP (ipStr )
254+ }
255+
256+ for _ , mapiMachine := range mapiMachines .Items {
257+ log := log .WithValues ("machine" , mapiMachine .Name )
258+
259+ providerID := mapiMachine .Spec .ProviderID
260+ if providerID == nil {
261+ log .V (3 ).Info ("Skipping machine: providerID is not set" )
262+ continue
263+ }
264+
265+ if ! strings .HasPrefix (* providerID , "openstack:///" ) {
266+ log .V (2 ).Info ("Skipping machine: providerID has unexpected format" , "providerID" , * providerID )
267+ continue
268+ }
269+
270+ instanceID := (* providerID )[len ("openstack:///" ):]
271+
272+ portOpts := ports.ListOpts {
273+ DeviceID : instanceID ,
274+ }
275+
276+ ports , err := networkClient .ListPort (portOpts )
277+ if err != nil {
278+ return nil , fmt .Errorf ("listing ports for instance %s: %w" , instanceID , err )
279+ }
280+
281+ if len (ports ) == 0 {
282+ return nil , fmt .Errorf ("%w: no ports found for instance %s" , errOpenStackNoDefaultSubnet , instanceID )
283+ }
284+
285+ for _ , port := range ports {
286+ log := log .WithValues ("port" , port .ID )
287+
288+ for _ , fixedIP := range port .FixedIPs {
289+ if fixedIP .SubnetID == "" {
290+ continue
291+ }
292+
293+ subnet , err := networkClient .GetSubnet (fixedIP .SubnetID )
294+ if err != nil {
295+ return nil , fmt .Errorf ("getting subnet %s: %w" , fixedIP .SubnetID , err )
296+ }
297+
298+ _ , cidr , err := net .ParseCIDR (subnet .CIDR )
299+ if err != nil {
300+ return nil , fmt .Errorf ("parsing subnet CIDR %s: %w" , subnet .CIDR , err )
301+ }
302+
303+ if slices .ContainsFunc (
304+ apiServerInternalIPs , func (ip net.IP ) bool { return cidr .Contains (ip ) },
305+ ) {
306+ return subnet , nil
307+ }
308+
309+ log .V (6 ).Info ("subnet does not match any APIServerInternalIPs" , "subnet" , subnet .CIDR )
310+ }
311+
312+ log .V (6 ).Info ("port does not match any APIServerInternalIPs" )
313+ }
314+
315+ log .V (6 ).Info ("machine does not match any APIServerInternalIPs" )
316+ }
317+
318+ return nil , fmt .Errorf ("%w: no matching subnets found" , errOpenStackNoDefaultSubnet )
319+ }
0 commit comments