Skip to content

Commit 74efdc0

Browse files
sbueringersbueringer
committed
Introduce reconciler rate-limiting and hook caching
1 parent bf546f7 commit 74efdc0

File tree

46 files changed

+794
-237
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+794
-237
lines changed

‎.golangci.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,8 @@ linters:
180180
alias: topologynames
181181
- pkg: sigs.k8s.io/cluster-api/internal/util/client
182182
alias: "clientutil"
183+
- pkg: sigs.k8s.io/cluster-api/internal/util/reconciler
184+
alias: "reconcilerutil"
183185
# CAPD
184186
- pkg: sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1alpha3
185187
alias: infrav1alpha3

‎bootstrap/kubeadm/config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
- "--leader-elect"
2323
- "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
2424
- "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
25-
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false}"
25+
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false}"
2626
- "--bootstrap-token-ttl=${KUBEADM_BOOTSTRAP_TOKEN_TTL:=15m}"
2727
image: controller:latest
2828
name: manager

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go‎

Lines changed: 6 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ import (
3737
"k8s.io/klog/v2"
3838
"k8s.io/utils/ptr"
3939
ctrl "sigs.k8s.io/controller-runtime"
40-
"sigs.k8s.io/controller-runtime/pkg/builder"
4140
"sigs.k8s.io/controller-runtime/pkg/client"
4241
"sigs.k8s.io/controller-runtime/pkg/controller"
4342
"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -53,6 +52,7 @@ import (
5352
bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
5453
"sigs.k8s.io/cluster-api/controllers/clustercache"
5554
"sigs.k8s.io/cluster-api/feature"
55+
reconcilerutil "sigs.k8s.io/cluster-api/internal/util/reconciler"
5656
"sigs.k8s.io/cluster-api/internal/util/taints"
5757
"sigs.k8s.io/cluster-api/util"
5858
"sigs.k8s.io/cluster-api/util/conditions"
@@ -116,36 +116,31 @@ func (r *KubeadmConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl
116116
}
117117

118118
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "kubeadmconfig")
119-
b := ctrl.NewControllerManagedBy(mgr).
119+
b := reconcilerutil.ControllerManagedBy(mgr, predicateLog).
120120
For(&bootstrapv1.KubeadmConfig{}).
121121
WithOptions(options).
122122
Watches(
123123
&clusterv1.Machine{},
124124
handler.EnqueueRequestsFromMapFunc(r.MachineToBootstrapMapFunc),
125-
builder.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog)),
126125
).WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue))
127126

128127
if feature.Gates.Enabled(feature.MachinePool) {
129128
b = b.Watches(
130129
&clusterv1.MachinePool{},
131130
handler.EnqueueRequestsFromMapFunc(r.MachinePoolToBootstrapMapFunc),
132-
builder.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog)),
133131
)
134132
}
135133

136134
b = b.Watches(
137135
&clusterv1.Cluster{},
138136
handler.EnqueueRequestsFromMapFunc(r.ClusterToKubeadmConfigs),
139-
builder.WithPredicates(
140-
predicates.All(mgr.GetScheme(), predicateLog,
141-
predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog),
142-
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
143-
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
144-
),
137+
predicates.All(mgr.GetScheme(), predicateLog,
138+
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
139+
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
145140
),
146141
).WatchesRawSource(r.ClusterCache.GetClusterSource("kubeadmconfig", r.ClusterToKubeadmConfigs))
147142

148-
if err := b.Complete(r); err != nil {
143+
if _, err := b.Build(r); err != nil {
149144
return errors.Wrap(err, "failed setting up with a controller manager")
150145
}
151146
return nil

‎config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ spec:
2323
- "--leader-elect"
2424
- "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
2525
- "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
26-
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false},MachineSetPreflightChecks=${EXP_MACHINE_SET_PREFLIGHT_CHECKS:=true},MachineWaitForVolumeDetachConsiderVolumeAttachments=${EXP_MACHINE_WAITFORVOLUMEDETACH_CONSIDER_VOLUMEATTACHMENTS:=true},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false},MachineTaintPropagation=${EXP_MACHINE_TAINT_PROPAGATION:=false}"
26+
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false},MachineSetPreflightChecks=${EXP_MACHINE_SET_PREFLIGHT_CHECKS:=true},MachineWaitForVolumeDetachConsiderVolumeAttachments=${EXP_MACHINE_WAITFORVOLUMEDETACH_CONSIDER_VOLUMEATTACHMENTS:=true},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false},MachineTaintPropagation=${EXP_MACHINE_TAINT_PROPAGATION:=false}"
2727
image: controller:latest
2828
name: manager
2929
env:

‎controllers/clustercache/cluster_cache.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import (
4444
"sigs.k8s.io/controller-runtime/pkg/source"
4545

4646
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
47+
reconcilerutil "sigs.k8s.io/cluster-api/internal/util/reconciler"
4748
"sigs.k8s.io/cluster-api/util/predicates"
4849
)
4950

@@ -328,7 +329,8 @@ func SetupWithManager(ctx context.Context, mgr manager.Manager, options Options,
328329
cacheCtxCancel: cacheCtxCancel,
329330
}
330331

331-
err := ctrl.NewControllerManagedBy(mgr).
332+
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "clustercache")
333+
err := reconcilerutil.ControllerManagedBy(mgr, predicateLog).
332334
Named("clustercache").
333335
For(&clusterv1.Cluster{}).
334336
WithOptions(controllerOptions).

‎controllers/crdmigrator/crd_migrator.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import (
4545
"sigs.k8s.io/controller-runtime/pkg/controller"
4646

4747
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
48+
reconcilerutil "sigs.k8s.io/cluster-api/internal/util/reconciler"
4849
"sigs.k8s.io/cluster-api/util/cache"
4950
"sigs.k8s.io/cluster-api/util/contract"
5051
"sigs.k8s.io/cluster-api/util/predicates"
@@ -121,7 +122,7 @@ func (r *CRDMigrator) SetupWithManager(ctx context.Context, mgr ctrl.Manager, co
121122
}
122123

123124
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "crdmigrator")
124-
err := ctrl.NewControllerManagedBy(mgr).
125+
err := reconcilerutil.ControllerManagedBy(mgr, predicateLog).
125126
For(&apiextensionsv1.CustomResourceDefinition{},
126127
// This controller uses a PartialObjectMetadata watch/informer to avoid an informer for CRDs
127128
// to reduce memory usage.

‎controlplane/kubeadm/config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
- "--leader-elect"
2323
- "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
2424
- "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
25-
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false}"
25+
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false}"
2626
image: controller:latest
2727
name: manager
2828
env:

‎controlplane/kubeadm/internal/controllers/controller.go‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ import (
3636
"k8s.io/klog/v2"
3737
"k8s.io/utils/ptr"
3838
ctrl "sigs.k8s.io/controller-runtime"
39-
"sigs.k8s.io/controller-runtime/pkg/builder"
4039
"sigs.k8s.io/controller-runtime/pkg/client"
4140
"sigs.k8s.io/controller-runtime/pkg/controller"
4241
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -50,6 +49,7 @@ import (
5049
runtimeclient "sigs.k8s.io/cluster-api/exp/runtime/client"
5150
"sigs.k8s.io/cluster-api/feature"
5251
"sigs.k8s.io/cluster-api/internal/util/inplace"
52+
reconcilerutil "sigs.k8s.io/cluster-api/internal/util/reconciler"
5353
"sigs.k8s.io/cluster-api/internal/util/ssa"
5454
"sigs.k8s.io/cluster-api/util"
5555
"sigs.k8s.io/cluster-api/util/cache"
@@ -85,7 +85,7 @@ type KubeadmControlPlaneReconciler struct {
8585
APIReader client.Reader
8686
SecretCachingClient client.Client
8787
RuntimeClient runtimeclient.Client
88-
controller controller.Controller
88+
controller reconcilerutil.Controller
8989
recorder record.EventRecorder
9090
ClusterCache clustercache.ClusterCache
9191

@@ -132,22 +132,19 @@ func (r *KubeadmControlPlaneReconciler) SetupWithManager(ctx context.Context, mg
132132
}
133133

134134
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "kubeadmcontrolplane")
135-
c, err := ctrl.NewControllerManagedBy(mgr).
135+
c, err := reconcilerutil.ControllerManagedBy(mgr, predicateLog).
136136
For(&controlplanev1.KubeadmControlPlane{}).
137-
Owns(&clusterv1.Machine{}, builder.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog))).
137+
Owns(&clusterv1.Machine{}).
138138
WithOptions(options).
139139
WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue)).
140140
Watches(
141141
&clusterv1.Cluster{},
142142
handler.EnqueueRequestsFromMapFunc(r.ClusterToKubeadmControlPlane),
143-
builder.WithPredicates(
144-
predicates.All(mgr.GetScheme(), predicateLog,
145-
predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog),
146-
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
147-
predicates.Any(mgr.GetScheme(), predicateLog,
148-
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
149-
predicates.ClusterTopologyVersionChanged(mgr.GetScheme(), predicateLog),
150-
),
143+
predicates.All(mgr.GetScheme(), predicateLog,
144+
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
145+
predicates.Any(mgr.GetScheme(), predicateLog,
146+
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
147+
predicates.ClusterTopologyVersionChanged(mgr.GetScheme(), predicateLog),
151148
),
152149
),
153150
).

‎controlplane/kubeadm/internal/controllers/scale.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"strings"
23+
"time"
2324

2425
"github.com/pkg/errors"
2526
corev1 "k8s.io/api/core/v1"
@@ -190,6 +191,7 @@ func (r *KubeadmControlPlaneReconciler) preflightChecks(ctx context.Context, con
190191
}
191192
log.Info(fmt.Sprintf("Waiting for a version upgrade to %s to be propagated", v))
192193
controlPlane.PreflightCheckResults.TopologyVersionMismatch = true
194+
r.controller.DeferNextReconcileForObject(controlPlane.KCP, time.Now().Add(5*time.Second))
193195
return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}
194196
}
195197
}
@@ -198,6 +200,7 @@ func (r *KubeadmControlPlaneReconciler) preflightChecks(ctx context.Context, con
198200
if controlPlane.HasDeletingMachine() {
199201
controlPlane.PreflightCheckResults.HasDeletingMachine = true
200202
log.Info("Waiting for machines to be deleted", "machines", strings.Join(controlPlane.Machines.Filter(collections.HasDeletionTimestamp).Names(), ", "))
203+
r.controller.DeferNextReconcileForObject(controlPlane.KCP, time.Now().Add(5*time.Second))
201204
return ctrl.Result{RequeueAfter: deleteRequeueAfter}
202205
}
203206

@@ -254,7 +257,7 @@ loopmachines:
254257
r.recorder.Eventf(controlPlane.KCP, corev1.EventTypeWarning, "ControlPlaneUnhealthy",
255258
"Waiting for control plane to pass preflight checks to continue reconciliation: %v", aggregatedError)
256259
log.Info("Waiting for control plane to pass preflight checks", "failures", aggregatedError.Error())
257-
260+
r.controller.DeferNextReconcileForObject(controlPlane.KCP, time.Now().Add(5*time.Second))
258261
return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}
259262
}
260263

‎controlplane/kubeadm/internal/controllers/scale_test.go‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal/desiredstate"
4040
"sigs.k8s.io/cluster-api/feature"
4141
"sigs.k8s.io/cluster-api/util"
42+
"sigs.k8s.io/cluster-api/util/cache"
4243
"sigs.k8s.io/cluster-api/util/collections"
4344
)
4445

@@ -153,6 +154,7 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) {
153154
Client: env,
154155
managementCluster: fmc,
155156
managementClusterUncached: fmc,
157+
reconcileCache: cache.New[cache.ReconcileEntry](cache.DefaultTTL),
156158
recorder: record.NewFakeRecorder(32),
157159
}
158160
controlPlane := &internal.ControlPlane{
@@ -223,6 +225,7 @@ func TestKubeadmControlPlaneReconciler_scaleUpControlPlane(t *testing.T) {
223225
SecretCachingClient: secretCachingClient,
224226
managementCluster: fmc,
225227
managementClusterUncached: fmc,
228+
reconcileCache: cache.New[cache.ReconcileEntry](cache.DefaultTTL),
226229
recorder: record.NewFakeRecorder(32),
227230
}
228231

@@ -357,6 +360,7 @@ func TestKubeadmControlPlaneReconciler_scaleDownControlPlane_NoError(t *testing.
357360
managementCluster: &fakeManagementCluster{
358361
Workload: &fakeWorkloadCluster{},
359362
},
363+
reconcileCache: cache.New[cache.ReconcileEntry](cache.DefaultTTL),
360364
}
361365

362366
cluster := &clusterv1.Cluster{}
@@ -781,7 +785,8 @@ func TestPreflightChecks(t *testing.T) {
781785
g := NewWithT(t)
782786

783787
r := &KubeadmControlPlaneReconciler{
784-
recorder: record.NewFakeRecorder(32),
788+
reconcileCache: cache.New[cache.ReconcileEntry](cache.DefaultTTL),
789+
recorder: record.NewFakeRecorder(32),
785790
}
786791
cluster := &clusterv1.Cluster{}
787792
if tt.cluster != nil {

0 commit comments

Comments
 (0)