Skip to content

Commit 277a694

Browse files
sbueringersbueringer
committed
Introduce reconciler rate-limiting and hook caching
1 parent bf546f7 commit 277a694

File tree

68 files changed

+1263
-459
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+1263
-459
lines changed

‎.golangci.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ linters:
2020
- durationcheck # multiplying two durations
2121
- errcheck # unchecked errors
2222
- errchkjson # invalid types passed to json encoder
23+
- forbidigo # allows to block usage of funcs
2324
- ginkgolinter # ginkgo and gomega
2425
- gocritic # bugs, performance, style (we could add custom ones to this one)
2526
- godot # checks that comments end in a period
@@ -52,6 +53,10 @@ linters:
5253
# TODO: It will be dropped when the Go version migration is done.
5354
- usetesting
5455
settings:
56+
forbidigo:
57+
forbid:
58+
- pattern: ctrl.NewControllerManagedBy
59+
msg: Use capicontrollerutil.NewControllerManagedBy instead
5560
ginkgolinter:
5661
forbid-focus-container: true
5762
gocritic:
@@ -180,6 +185,8 @@ linters:
180185
alias: topologynames
181186
- pkg: sigs.k8s.io/cluster-api/internal/util/client
182187
alias: "clientutil"
188+
- pkg: sigs.k8s.io/cluster-api/internal/util/controller
189+
alias: "capicontrollerutil"
183190
# CAPD
184191
- pkg: sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1alpha3
185192
alias: infrav1alpha3

‎bootstrap/kubeadm/config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
- "--leader-elect"
2323
- "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
2424
- "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
25-
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false}"
25+
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false}"
2626
- "--bootstrap-token-ttl=${KUBEADM_BOOTSTRAP_TOKEN_TTL:=15m}"
2727
image: controller:latest
2828
name: manager

‎bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go‎

Lines changed: 4 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ import (
3737
"k8s.io/klog/v2"
3838
"k8s.io/utils/ptr"
3939
ctrl "sigs.k8s.io/controller-runtime"
40-
"sigs.k8s.io/controller-runtime/pkg/builder"
4140
"sigs.k8s.io/controller-runtime/pkg/client"
4241
"sigs.k8s.io/controller-runtime/pkg/controller"
4342
"sigs.k8s.io/controller-runtime/pkg/handler"
@@ -53,6 +52,7 @@ import (
5352
bsutil "sigs.k8s.io/cluster-api/bootstrap/util"
5453
"sigs.k8s.io/cluster-api/controllers/clustercache"
5554
"sigs.k8s.io/cluster-api/feature"
55+
capicontrollerutil "sigs.k8s.io/cluster-api/internal/util/controller"
5656
"sigs.k8s.io/cluster-api/internal/util/taints"
5757
"sigs.k8s.io/cluster-api/util"
5858
"sigs.k8s.io/cluster-api/util/conditions"
@@ -116,33 +116,26 @@ func (r *KubeadmConfigReconciler) SetupWithManager(ctx context.Context, mgr ctrl
116116
}
117117

118118
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "kubeadmconfig")
119-
b := ctrl.NewControllerManagedBy(mgr).
119+
b := capicontrollerutil.NewControllerManagedBy(mgr, predicateLog).
120120
For(&bootstrapv1.KubeadmConfig{}).
121121
WithOptions(options).
122122
Watches(
123123
&clusterv1.Machine{},
124124
handler.EnqueueRequestsFromMapFunc(r.MachineToBootstrapMapFunc),
125-
builder.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog)),
126125
).WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue))
127126

128127
if feature.Gates.Enabled(feature.MachinePool) {
129128
b = b.Watches(
130129
&clusterv1.MachinePool{},
131130
handler.EnqueueRequestsFromMapFunc(r.MachinePoolToBootstrapMapFunc),
132-
builder.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog)),
133131
)
134132
}
135133

136134
b = b.Watches(
137135
&clusterv1.Cluster{},
138136
handler.EnqueueRequestsFromMapFunc(r.ClusterToKubeadmConfigs),
139-
builder.WithPredicates(
140-
predicates.All(mgr.GetScheme(), predicateLog,
141-
predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog),
142-
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
143-
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
144-
),
145-
),
137+
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
138+
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
146139
).WatchesRawSource(r.ClusterCache.GetClusterSource("kubeadmconfig", r.ClusterToKubeadmConfigs))
147140

148141
if err := b.Complete(r); err != nil {
@@ -268,12 +261,6 @@ func (r *KubeadmConfigReconciler) Reconcile(ctx context.Context, req ctrl.Reques
268261
if err := patchHelper.Patch(ctx, config, patchOpts...); err != nil {
269262
rerr = kerrors.NewAggregate([]error{rerr, err})
270263
}
271-
272-
// Note: controller-runtime logs a warning that non-empty result is ignored
273-
// if error is not nil, so setting result here to empty to avoid noisy warnings.
274-
if rerr != nil {
275-
retRes = ctrl.Result{}
276-
}
277264
}()
278265

279266
// Ignore deleted KubeadmConfigs.

‎config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ spec:
2323
- "--leader-elect"
2424
- "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
2525
- "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
26-
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false},MachineSetPreflightChecks=${EXP_MACHINE_SET_PREFLIGHT_CHECKS:=true},MachineWaitForVolumeDetachConsiderVolumeAttachments=${EXP_MACHINE_WAITFORVOLUMEDETACH_CONSIDER_VOLUMEATTACHMENTS:=true},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false},MachineTaintPropagation=${EXP_MACHINE_TAINT_PROPAGATION:=false}"
26+
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},RuntimeSDK=${EXP_RUNTIME_SDK:=false},MachineSetPreflightChecks=${EXP_MACHINE_SET_PREFLIGHT_CHECKS:=true},MachineWaitForVolumeDetachConsiderVolumeAttachments=${EXP_MACHINE_WAITFORVOLUMEDETACH_CONSIDER_VOLUMEATTACHMENTS:=true},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false},MachineTaintPropagation=${EXP_MACHINE_TAINT_PROPAGATION:=false}"
2727
image: controller:latest
2828
name: manager
2929
env:

‎controllers/clustercache/cluster_cache.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ import (
4444
"sigs.k8s.io/controller-runtime/pkg/source"
4545

4646
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
47+
capicontrollerutil "sigs.k8s.io/cluster-api/internal/util/controller"
4748
"sigs.k8s.io/cluster-api/util/predicates"
4849
)
4950

@@ -328,7 +329,8 @@ func SetupWithManager(ctx context.Context, mgr manager.Manager, options Options,
328329
cacheCtxCancel: cacheCtxCancel,
329330
}
330331

331-
err := ctrl.NewControllerManagedBy(mgr).
332+
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "clustercache")
333+
err := capicontrollerutil.NewControllerManagedBy(mgr, predicateLog).
332334
Named("clustercache").
333335
For(&clusterv1.Cluster{}).
334336
WithOptions(controllerOptions).

‎controllers/crdmigrator/crd_migrator.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ import (
4545
"sigs.k8s.io/controller-runtime/pkg/controller"
4646

4747
clusterv1 "sigs.k8s.io/cluster-api/api/core/v1beta2"
48+
capicontrollerutil "sigs.k8s.io/cluster-api/internal/util/controller"
4849
"sigs.k8s.io/cluster-api/util/cache"
4950
"sigs.k8s.io/cluster-api/util/contract"
5051
"sigs.k8s.io/cluster-api/util/predicates"
@@ -121,7 +122,7 @@ func (r *CRDMigrator) SetupWithManager(ctx context.Context, mgr ctrl.Manager, co
121122
}
122123

123124
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "crdmigrator")
124-
err := ctrl.NewControllerManagedBy(mgr).
125+
err := capicontrollerutil.NewControllerManagedBy(mgr, predicateLog).
125126
For(&apiextensionsv1.CustomResourceDefinition{},
126127
// This controller uses a PartialObjectMetadata watch/informer to avoid an informer for CRDs
127128
// to reduce memory usage.

‎controlplane/kubeadm/config/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ spec:
2222
- "--leader-elect"
2323
- "--diagnostics-address=${CAPI_DIAGNOSTICS_ADDRESS:=:8443}"
2424
- "--insecure-diagnostics=${CAPI_INSECURE_DIAGNOSTICS:=false}"
25-
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false}"
25+
- "--feature-gates=MachinePool=${EXP_MACHINE_POOL:=true},ClusterTopology=${CLUSTER_TOPOLOGY:=false},KubeadmBootstrapFormatIgnition=${EXP_KUBEADM_BOOTSTRAP_FORMAT_IGNITION:=false},PriorityQueue=${EXP_PRIORITY_QUEUE:=false},ReconcilerRateLimiting=${EXP_RECONCILER_RATE_LIMITING:=false},InPlaceUpdates=${EXP_IN_PLACE_UPDATES:=false}"
2626
image: controller:latest
2727
name: manager
2828
env:

‎controlplane/kubeadm/internal/controllers/controller.go‎

Lines changed: 8 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ import (
3636
"k8s.io/klog/v2"
3737
"k8s.io/utils/ptr"
3838
ctrl "sigs.k8s.io/controller-runtime"
39-
"sigs.k8s.io/controller-runtime/pkg/builder"
4039
"sigs.k8s.io/controller-runtime/pkg/client"
4140
"sigs.k8s.io/controller-runtime/pkg/controller"
4241
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -49,6 +48,7 @@ import (
4948
"sigs.k8s.io/cluster-api/controlplane/kubeadm/internal"
5049
runtimeclient "sigs.k8s.io/cluster-api/exp/runtime/client"
5150
"sigs.k8s.io/cluster-api/feature"
51+
capicontrollerutil "sigs.k8s.io/cluster-api/internal/util/controller"
5252
"sigs.k8s.io/cluster-api/internal/util/inplace"
5353
"sigs.k8s.io/cluster-api/internal/util/ssa"
5454
"sigs.k8s.io/cluster-api/util"
@@ -85,7 +85,7 @@ type KubeadmControlPlaneReconciler struct {
8585
APIReader client.Reader
8686
SecretCachingClient client.Client
8787
RuntimeClient runtimeclient.Client
88-
controller controller.Controller
88+
controller capicontrollerutil.Controller
8989
recorder record.EventRecorder
9090
ClusterCache clustercache.ClusterCache
9191

@@ -132,23 +132,18 @@ func (r *KubeadmControlPlaneReconciler) SetupWithManager(ctx context.Context, mg
132132
}
133133

134134
predicateLog := ctrl.LoggerFrom(ctx).WithValues("controller", "kubeadmcontrolplane")
135-
c, err := ctrl.NewControllerManagedBy(mgr).
135+
c, err := capicontrollerutil.NewControllerManagedBy(mgr, predicateLog).
136136
For(&controlplanev1.KubeadmControlPlane{}).
137-
Owns(&clusterv1.Machine{}, builder.WithPredicates(predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog))).
137+
Owns(&clusterv1.Machine{}).
138138
WithOptions(options).
139139
WithEventFilter(predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue)).
140140
Watches(
141141
&clusterv1.Cluster{},
142142
handler.EnqueueRequestsFromMapFunc(r.ClusterToKubeadmControlPlane),
143-
builder.WithPredicates(
144-
predicates.All(mgr.GetScheme(), predicateLog,
145-
predicates.ResourceIsChanged(mgr.GetScheme(), predicateLog),
146-
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
147-
predicates.Any(mgr.GetScheme(), predicateLog,
148-
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
149-
predicates.ClusterTopologyVersionChanged(mgr.GetScheme(), predicateLog),
150-
),
151-
),
143+
predicates.ResourceHasFilterLabel(mgr.GetScheme(), predicateLog, r.WatchFilterValue),
144+
predicates.Any(mgr.GetScheme(), predicateLog,
145+
predicates.ClusterPausedTransitionsOrInfrastructureProvisioned(mgr.GetScheme(), predicateLog),
146+
predicates.ClusterTopologyVersionChanged(mgr.GetScheme(), predicateLog),
152147
),
153148
).
154149
WatchesRawSource(r.ClusterCache.GetClusterSource("kubeadmcontrolplane", r.ClusterToKubeadmControlPlane,
@@ -283,12 +278,6 @@ func (r *KubeadmControlPlaneReconciler) Reconcile(ctx context.Context, req ctrl.
283278
res = ctrl.Result{RequeueAfter: 20 * time.Second}
284279
}
285280
}
286-
287-
// Note: controller-runtime logs a warning that non-empty result is ignored
288-
// if error is not nil, so setting result here to empty to avoid noisy warnings.
289-
if reterr != nil {
290-
res = ctrl.Result{}
291-
}
292281
}()
293282

294283
if !kcp.DeletionTimestamp.IsZero() {

‎controlplane/kubeadm/internal/controllers/scale.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"context"
2121
"fmt"
2222
"strings"
23+
"time"
2324

2425
"github.com/pkg/errors"
2526
corev1 "k8s.io/api/core/v1"
@@ -190,6 +191,7 @@ func (r *KubeadmControlPlaneReconciler) preflightChecks(ctx context.Context, con
190191
}
191192
log.Info(fmt.Sprintf("Waiting for a version upgrade to %s to be propagated", v))
192193
controlPlane.PreflightCheckResults.TopologyVersionMismatch = true
194+
r.controller.DeferNextReconcileForObject(controlPlane.KCP, time.Now().Add(5*time.Second))
193195
return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}
194196
}
195197
}
@@ -198,6 +200,7 @@ func (r *KubeadmControlPlaneReconciler) preflightChecks(ctx context.Context, con
198200
if controlPlane.HasDeletingMachine() {
199201
controlPlane.PreflightCheckResults.HasDeletingMachine = true
200202
log.Info("Waiting for machines to be deleted", "machines", strings.Join(controlPlane.Machines.Filter(collections.HasDeletionTimestamp).Names(), ", "))
203+
r.controller.DeferNextReconcileForObject(controlPlane.KCP, time.Now().Add(5*time.Second))
201204
return ctrl.Result{RequeueAfter: deleteRequeueAfter}
202205
}
203206

@@ -254,7 +257,7 @@ loopmachines:
254257
r.recorder.Eventf(controlPlane.KCP, corev1.EventTypeWarning, "ControlPlaneUnhealthy",
255258
"Waiting for control plane to pass preflight checks to continue reconciliation: %v", aggregatedError)
256259
log.Info("Waiting for control plane to pass preflight checks", "failures", aggregatedError.Error())
257-
260+
r.controller.DeferNextReconcileForObject(controlPlane.KCP, time.Now().Add(5*time.Second))
258261
return ctrl.Result{RequeueAfter: preflightFailedRequeueAfter}
259262
}
260263

0 commit comments

Comments
 (0)