Skip to content
This repository was archived by the owner on Jan 11, 2023. It is now read-only.

Commit 03f1595

Browse files
jackfrancisjackfrancis
committed
Actually allow cloudprovider rate limit / backoff disabling (#3891)
1 parent 781379b commit 03f1595

File tree

9 files changed

+335
-90
lines changed

9 files changed

+335
-90
lines changed

‎pkg/acsengine/const.go‎

Lines changed: 2 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -83,21 +83,9 @@ const (
8383
// DefaultKubernetesCtrlMgrUseSvcAccountCreds is "true", see --use-service-account-credentials at https://kubernetes.io/docs/admin/kube-controller-manager/
8484
DefaultKubernetesCtrlMgrUseSvcAccountCreds = "false"
8585
// DefaultKubernetesCloudProviderBackoff is false to disable cloudprovider backoff implementation for API calls
86-
DefaultKubernetesCloudProviderBackoff = false
87-
// DefaultKubernetesCloudProviderBackoffRetries is 6, takes effect if DefaultKubernetesCloudProviderBackoff is true
88-
DefaultKubernetesCloudProviderBackoffRetries = 6
89-
// DefaultKubernetesCloudProviderBackoffJitter is 1, takes effect if DefaultKubernetesCloudProviderBackoff is true
90-
DefaultKubernetesCloudProviderBackoffJitter = 1.0
91-
// DefaultKubernetesCloudProviderBackoffDuration is 5, takes effect if DefaultKubernetesCloudProviderBackoff is true
92-
DefaultKubernetesCloudProviderBackoffDuration = 5
93-
// DefaultKubernetesCloudProviderBackoffExponent is 1.5, takes effect if DefaultKubernetesCloudProviderBackoff is true
94-
DefaultKubernetesCloudProviderBackoffExponent = 1.5
86+
DefaultKubernetesCloudProviderBackoff = true
9587
// DefaultKubernetesCloudProviderRateLimit is false to disable cloudprovider rate limiting implementation for API calls
96-
DefaultKubernetesCloudProviderRateLimit = false
97-
// DefaultKubernetesCloudProviderRateLimitQPS is 3, takes effect if DefaultKubernetesCloudProviderRateLimit is true
98-
DefaultKubernetesCloudProviderRateLimitQPS = 3.0
99-
// DefaultKubernetesCloudProviderRateLimitBucket is 10, takes effect if DefaultKubernetesCloudProviderRateLimit is true
100-
DefaultKubernetesCloudProviderRateLimitBucket = 10
88+
DefaultKubernetesCloudProviderRateLimit = true
10189
// DefaultKubeHeapsterDeploymentAddonName is the name of the kube-heapster-deployment addon
10290
DefaultKubeHeapsterDeploymentAddonName = "kube-heapster-deployment"
10391
// DefaultKubeDNSDeploymentAddonName is the name of the kube-dns-deployment addon

‎pkg/acsengine/defaults.go‎

Lines changed: 11 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -278,8 +278,6 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) {
278278

279279
switch o.OrchestratorType {
280280
case api.Kubernetes:
281-
k8sVersion := o.OrchestratorVersion
282-
283281
if o.KubernetesConfig == nil {
284282
o.KubernetesConfig = &api.KubernetesConfig{}
285283
}
@@ -342,32 +340,21 @@ func setOrchestratorDefaults(cs *api.ContainerService, isUpdate bool) {
342340
if o.KubernetesConfig.ServiceCIDR == "" {
343341
o.KubernetesConfig.ServiceCIDR = DefaultKubernetesServiceCIDR
344342
}
345-
// Enforce sane cloudprovider backoff defaults, if CloudProviderBackoff is true in KubernetesConfig
346-
o.KubernetesConfig.CloudProviderBackoff = true
347-
if o.KubernetesConfig.CloudProviderBackoffDuration == 0 {
348-
o.KubernetesConfig.CloudProviderBackoffDuration = DefaultKubernetesCloudProviderBackoffDuration
349-
}
350-
if o.KubernetesConfig.CloudProviderBackoffExponent == 0 {
351-
o.KubernetesConfig.CloudProviderBackoffExponent = DefaultKubernetesCloudProviderBackoffExponent
343+
344+
if o.KubernetesConfig.CloudProviderBackoff == nil {
345+
o.KubernetesConfig.CloudProviderBackoff = helpers.PointerToBool(DefaultKubernetesCloudProviderBackoff)
352346
}
353-
if o.KubernetesConfig.CloudProviderBackoffJitter == 0 {
354-
o.KubernetesConfig.CloudProviderBackoffJitter = DefaultKubernetesCloudProviderBackoffJitter
347+
// Enforce sane cloudprovider backoff defaults, if CloudProviderBackoff is true in KubernetesConfig
348+
if helpers.IsTrueBoolPointer(o.KubernetesConfig.CloudProviderBackoff) {
349+
o.KubernetesConfig.SetCloudProviderBackoffDefaults()
355350
}
356-
if o.KubernetesConfig.CloudProviderBackoffRetries == 0 {
357-
o.KubernetesConfig.CloudProviderBackoffRetries = DefaultKubernetesCloudProviderBackoffRetries
351+
352+
if o.KubernetesConfig.CloudProviderRateLimit == nil {
353+
o.KubernetesConfig.CloudProviderRateLimit = helpers.PointerToBool(DefaultKubernetesCloudProviderRateLimit)
358354
}
359-
k8sSemVer, _ := semver.Make(k8sVersion)
360-
minVersion, _ := semver.Make("1.6.6")
361355
// Enforce sane cloudprovider rate limit defaults, if CloudProviderRateLimit is true in KubernetesConfig
362-
// For k8s version greater or equal to 1.6.6, we will set the default CloudProviderRate* settings
363-
o.KubernetesConfig.CloudProviderRateLimit = true
364-
if o.KubernetesConfig.CloudProviderRateLimit && k8sSemVer.GTE(minVersion) {
365-
if o.KubernetesConfig.CloudProviderRateLimitQPS == 0 {
366-
o.KubernetesConfig.CloudProviderRateLimitQPS = DefaultKubernetesCloudProviderRateLimitQPS
367-
}
368-
if o.KubernetesConfig.CloudProviderRateLimitBucket == 0 {
369-
o.KubernetesConfig.CloudProviderRateLimitBucket = DefaultKubernetesCloudProviderRateLimitBucket
370-
}
356+
if helpers.IsTrueBoolPointer(o.KubernetesConfig.CloudProviderRateLimit) {
357+
o.KubernetesConfig.SetCloudProviderRateLimitDefaults()
371358
}
372359

373360
if o.KubernetesConfig.PrivateCluster == nil {

‎pkg/acsengine/defaults_test.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,40 @@ func TestDefaultDisableRbac(t *testing.T) {
679679
}
680680
}
681681

682+
func TestDefaultCloudProvider(t *testing.T) {
683+
mockCS := getMockBaseContainerService("1.10.3")
684+
properties := mockCS.Properties
685+
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
686+
setOrchestratorDefaults(&mockCS, true)
687+
688+
if !helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff) {
689+
t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
690+
helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
691+
}
692+
693+
if !helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit) {
694+
t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
695+
helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
696+
}
697+
698+
mockCS = getMockBaseContainerService("1.10.3")
699+
properties = mockCS.Properties
700+
properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
701+
properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff = helpers.PointerToBool(false)
702+
properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit = helpers.PointerToBool(false)
703+
setOrchestratorDefaults(&mockCS, true)
704+
705+
if !helpers.IsFalseBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff) {
706+
t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
707+
helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
708+
}
709+
710+
if !helpers.IsFalseBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderRateLimit) {
711+
t.Fatalf("got unexpected CloudProviderBackoff expected true, got %t",
712+
helpers.IsTrueBoolPointer(properties.OrchestratorProfile.KubernetesConfig.CloudProviderBackoff))
713+
}
714+
}
715+
682716
func getMockAddon(name string) api.KubernetesAddon {
683717
return api.KubernetesAddon{
684718
Name: name,

‎pkg/acsengine/k8s_versions.go‎

Lines changed: 43 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"strconv"
55
"strings"
66

7+
"github.com/Azure/acs-engine/pkg/api"
78
"github.com/Azure/acs-engine/pkg/api/common"
89
)
910

@@ -31,12 +32,12 @@ var k8sComponentVersions = map[string]map[string]string{
3132
"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
3233
"podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout,
3334
"routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
34-
"backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
35-
"backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
36-
"backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
37-
"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
38-
"ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
39-
"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
35+
"backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
36+
"backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
37+
"backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
38+
"backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
39+
"ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
40+
"ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
4041
"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
4142
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
4243
},
@@ -63,12 +64,12 @@ var k8sComponentVersions = map[string]map[string]string{
6364
"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
6465
"podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout,
6566
"routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
66-
"backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
67-
"backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
68-
"backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
69-
"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
70-
"ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
71-
"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
67+
"backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
68+
"backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
69+
"backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
70+
"backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
71+
"ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
72+
"ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
7273
"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
7374
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
7475
},
@@ -95,12 +96,12 @@ var k8sComponentVersions = map[string]map[string]string{
9596
"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
9697
"podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout,
9798
"routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
98-
"backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
99-
"backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
100-
"backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
101-
"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
102-
"ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
103-
"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
99+
"backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
100+
"backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
101+
"backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
102+
"backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
103+
"ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
104+
"ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
104105
"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
105106
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
106107
},
@@ -126,12 +127,12 @@ var k8sComponentVersions = map[string]map[string]string{
126127
"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
127128
"podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout,
128129
"routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
129-
"backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
130-
"backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
131-
"backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
132-
"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
133-
"ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
134-
"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
130+
"backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
131+
"backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
132+
"backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
133+
"backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
134+
"ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
135+
"ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
135136
"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
136137
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
137138
},
@@ -155,12 +156,12 @@ var k8sComponentVersions = map[string]map[string]string{
155156
"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
156157
"podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout,
157158
"routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
158-
"backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
159-
"backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
160-
"backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
161-
"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
162-
"ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
163-
"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
159+
"backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
160+
"backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
161+
"backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
162+
"backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
163+
"ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
164+
"ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
164165
"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
165166
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
166167
},
@@ -184,12 +185,12 @@ var k8sComponentVersions = map[string]map[string]string{
184185
"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
185186
"podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout,
186187
"routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
187-
"backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
188-
"backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
189-
"backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
190-
"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
191-
"ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
192-
"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
188+
"backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
189+
"backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
190+
"backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
191+
"backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
192+
"ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
193+
"ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
193194
"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
194195
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
195196
},
@@ -212,12 +213,12 @@ var k8sComponentVersions = map[string]map[string]string{
212213
"nodegraceperiod": DefaultKubernetesCtrlMgrNodeMonitorGracePeriod,
213214
"podeviction": DefaultKubernetesCtrlMgrPodEvictionTimeout,
214215
"routeperiod": DefaultKubernetesCtrlMgrRouteReconciliationPeriod,
215-
"backoffretries": strconv.Itoa(DefaultKubernetesCloudProviderBackoffRetries),
216-
"backoffjitter": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
217-
"backoffduration": strconv.Itoa(DefaultKubernetesCloudProviderBackoffDuration),
218-
"backoffexponent": strconv.FormatFloat(DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
219-
"ratelimitqps": strconv.FormatFloat(DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
220-
"ratelimitbucket": strconv.Itoa(DefaultKubernetesCloudProviderRateLimitBucket),
216+
"backoffretries": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffRetries),
217+
"backoffjitter": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffJitter, 'f', -1, 64),
218+
"backoffduration": strconv.Itoa(api.DefaultKubernetesCloudProviderBackoffDuration),
219+
"backoffexponent": strconv.FormatFloat(api.DefaultKubernetesCloudProviderBackoffExponent, 'f', -1, 64),
220+
"ratelimitqps": strconv.FormatFloat(api.DefaultKubernetesCloudProviderRateLimitQPS, 'f', -1, 64),
221+
"ratelimitbucket": strconv.Itoa(api.DefaultKubernetesCloudProviderRateLimitBucket),
221222
"gchighthreshold": strconv.Itoa(DefaultKubernetesGCHighThreshold),
222223
"gclowthreshold": strconv.Itoa(DefaultKubernetesGCLowThreshold),
223224
},

0 commit comments

Comments
 (0)