Skip to content
This repository was archived by the owner on Jan 11, 2023. It is now read-only.

Commit 168af82

Browse files
jackfrancisjackfrancis
authored
1.12 uses coredns (#3987)
1 parent 2841bc9 commit 168af82

File tree

10 files changed

+286
-8
lines changed

10 files changed

+286
-8
lines changed

‎parts/k8s/addons/coredns.yaml‎

Lines changed: 183 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,183 @@
1+
# Warning: This is a file generated from the base underscore template file: coredns.yaml.base
2+
3+
apiVersion: v1
4+
kind: ServiceAccount
5+
metadata:
6+
name: coredns
7+
namespace: kube-system
8+
labels:
9+
kubernetes.io/cluster-service: "true"
10+
addonmanager.kubernetes.io/mode: Reconcile
11+
---
12+
apiVersion: rbac.authorization.k8s.io/v1
13+
kind: ClusterRole
14+
metadata:
15+
labels:
16+
kubernetes.io/bootstrapping: rbac-defaults
17+
addonmanager.kubernetes.io/mode: Reconcile
18+
name: system:coredns
19+
rules:
20+
- apiGroups:
21+
- ""
22+
resources:
23+
- endpoints
24+
- services
25+
- pods
26+
- namespaces
27+
verbs:
28+
- list
29+
- watch
30+
---
31+
apiVersion: rbac.authorization.k8s.io/v1
32+
kind: ClusterRoleBinding
33+
metadata:
34+
annotations:
35+
rbac.authorization.kubernetes.io/autoupdate: "true"
36+
labels:
37+
kubernetes.io/bootstrapping: rbac-defaults
38+
addonmanager.kubernetes.io/mode: EnsureExists
39+
name: system:coredns
40+
roleRef:
41+
apiGroup: rbac.authorization.k8s.io
42+
kind: ClusterRole
43+
name: system:coredns
44+
subjects:
45+
- kind: ServiceAccount
46+
name: coredns
47+
namespace: kube-system
48+
---
49+
apiVersion: v1
50+
kind: ConfigMap
51+
metadata:
52+
name: coredns
53+
namespace: kube-system
54+
labels:
55+
addonmanager.kubernetes.io/mode: EnsureExists
56+
data:
57+
Corefile: |
58+
.:53 {
59+
errors
60+
health
61+
kubernetes <kubernetesKubeletClusterDomain> in-addr.arpa ip6.arpa {
62+
pods insecure
63+
upstream
64+
fallthrough in-addr.arpa ip6.arpa
65+
}
66+
prometheus :9153
67+
proxy . /etc/resolv.conf
68+
cache 30
69+
loop
70+
reload
71+
loadbalance
72+
}
73+
---
74+
apiVersion: extensions/v1beta1
75+
kind: Deployment
76+
metadata:
77+
name: coredns
78+
namespace: kube-system
79+
labels:
80+
k8s-app: kube-dns
81+
kubernetes.io/cluster-service: "true"
82+
addonmanager.kubernetes.io/mode: Reconcile
83+
kubernetes.io/name: "CoreDNS"
84+
spec:
85+
# replicas: not specified here:
86+
# 1. In order to make Addon Manager do not reconcile this replicas parameter.
87+
# 2. Default is 1.
88+
# 3. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
89+
strategy:
90+
type: RollingUpdate
91+
rollingUpdate:
92+
maxUnavailable: 1
93+
selector:
94+
matchLabels:
95+
k8s-app: kube-dns
96+
template:
97+
metadata:
98+
labels:
99+
k8s-app: kube-dns
100+
annotations:
101+
seccomp.security.alpha.kubernetes.io/pod: 'docker/default'
102+
spec:
103+
serviceAccountName: coredns
104+
tolerations:
105+
- key: node-role.kubernetes.io/master
106+
effect: NoSchedule
107+
- key: "CriticalAddonsOnly"
108+
operator: "Exists"
109+
containers:
110+
- name: coredns
111+
image: <kubernetesCoreDNSSpec>
112+
imagePullPolicy: IfNotPresent
113+
resources:
114+
limits:
115+
memory: 170Mi
116+
requests:
117+
cpu: 100m
118+
memory: 70Mi
119+
args: [ "-conf", "/etc/coredns/Corefile" ]
120+
volumeMounts:
121+
- name: config-volume
122+
mountPath: /etc/coredns
123+
readOnly: true
124+
ports:
125+
- containerPort: 53
126+
name: dns
127+
protocol: UDP
128+
- containerPort: 53
129+
name: dns-tcp
130+
protocol: TCP
131+
- containerPort: 9153
132+
name: metrics
133+
protocol: TCP
134+
livenessProbe:
135+
httpGet:
136+
path: /health
137+
port: 8080
138+
scheme: HTTP
139+
initialDelaySeconds: 60
140+
timeoutSeconds: 5
141+
successThreshold: 1
142+
failureThreshold: 5
143+
securityContext:
144+
allowPrivilegeEscalation: false
145+
capabilities:
146+
add:
147+
- NET_BIND_SERVICE
148+
drop:
149+
- all
150+
readOnlyRootFilesystem: true
151+
dnsPolicy: Default
152+
volumes:
153+
- name: config-volume
154+
configMap:
155+
name: coredns
156+
items:
157+
- key: Corefile
158+
path: Corefile
159+
---
160+
apiVersion: v1
161+
kind: Service
162+
metadata:
163+
name: kube-dns
164+
namespace: kube-system
165+
annotations:
166+
prometheus.io/port: "9153"
167+
prometheus.io/scrape: "true"
168+
labels:
169+
k8s-app: kube-dns
170+
kubernetes.io/cluster-service: "true"
171+
addonmanager.kubernetes.io/mode: Reconcile
172+
kubernetes.io/name: "CoreDNS"
173+
spec:
174+
selector:
175+
k8s-app: kube-dns
176+
clusterIP: <kubeDNSServiceIP>
177+
ports:
178+
- name: dns
179+
port: 53
180+
protocol: UDP
181+
- name: dns-tcp
182+
port: 53
183+
protocol: TCP

‎parts/k8s/addons/dns-autoscaler.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
name: dns-autoscaler
5+
namespace: kube-system
6+
labels:
7+
k8s-app: dns-autoscaler
8+
kubernetes.io/cluster-service: "true"
9+
addonmanager.kubernetes.io/mode: Reconcile
10+
spec:
11+
selector:
12+
matchLabels:
13+
k8s-app: dns-autoscaler
14+
template:
15+
metadata:
16+
labels:
17+
k8s-app: dns-autoscaler
18+
spec:
19+
containers:
20+
- name: autoscaler
21+
image: k8s.gcr.io/cluster-proportional-autoscaler-amd64:1.1.1
22+
resources:
23+
requests:
24+
cpu: "20m"
25+
memory: "10Mi"
26+
command:
27+
- /cluster-proportional-autoscaler
28+
- --namespace=kube-system
29+
- --configmap=dns-autoscaler
30+
- --target=Deployment/coredns
31+
# When cluster is using large nodes(with more cores), "coresPerReplica" should dominate.
32+
# If using small nodes, "nodesPerReplica" should dominate.
33+
- --default-params={"linear":{"coresPerReplica":256,"nodesPerReplica":16,"min":1}}
34+
- --logtostderr=true
35+
- --v=2

‎parts/k8s/kubernetesmastercustomdata.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,7 +253,11 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
253253
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
254254
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
255255
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
256+
{{if IsKubernetesVersionGe "1.12.0"}}
257+
sed -i "s|<kubernetesCoreDNSSpec>|{{WrapAsParameter "kubernetesCoreDNSSpec"}}|g; s|<kubernetesKubeletClusterDomain>|{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s|<kubeDNSServiceIP>|{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/coredns.yaml"
258+
{{else}}
256259
sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsParameter "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsParameter "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsParameter "kubernetesExecHealthzSpec"}}|g; s|<kubernetesDNSSidecarSpec>|{{WrapAsParameter "kubernetesDNSSidecarSpec"}}|g; s|<kubernetesKubeletClusterDomain>|{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s|<kubeDNSServiceIP>|{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
260+
{{end}}
257261
sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
258262

259263
{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}}

‎parts/k8s/kubernetesmastercustomdatavmss.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,7 +255,11 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
255255
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-controller-manager.yaml"
256256
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
257257
sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsParameter "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsParameter "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
258+
{{if IsKubernetesVersionGe "1.12.0"}}
259+
sed -i "s|<kubernetesCoreDNSSpec>|{{WrapAsParameter "kubernetesCoreDNSSpec"}}|g; s|<kubernetesKubeletClusterDomain>|{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s|<kubeDNSServiceIP>|{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/coredns.yaml"
260+
{{else}}
258261
sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsParameter "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsParameter "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsParameter "kubernetesExecHealthzSpec"}}|g; s|<kubernetesDNSSidecarSpec>|{{WrapAsParameter "kubernetesDNSSidecarSpec"}}|g; s|<kubernetesKubeletClusterDomain>|{{WrapAsParameter "kubernetesKubeletClusterDomain"}}|g; s|<kubeDNSServiceIP>|{{WrapAsParameter "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
262+
{{end}}
259263
sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsParameter "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsParameter "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
260264

261265
{{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}}

‎parts/k8s/kubernetesparams.t‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,12 @@
631631
},
632632
"type": "string"
633633
},
634+
"kubernetesCoreDNSSpec": {
635+
"metadata": {
636+
"description": "The container spec for coredns"
637+
},
638+
"type": "string"
639+
},
634640
"kubernetesDNSMasqSpec": {
635641
"metadata": {
636642
"description": "The container spec for kube-dnsmasq-amd64."

‎pkg/acsengine/artifacts.go‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,28 @@ func kubernetesAddonSettingsInit(profile *api.Properties) []kubernetesAddonSetti
3535
kubernetesFeatureSetting{
3636
"kubernetesmasteraddons-kube-dns-deployment.yaml",
3737
"kube-dns-deployment.yaml",
38-
true,
38+
!common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.12.0"),
3939
},
4040
profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultKubeDNSDeploymentAddonName),
4141
},
42+
{
43+
kubernetesFeatureSetting{
44+
"coredns.yaml",
45+
"coredns.yaml",
46+
common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.12.0"),
47+
},
48+
profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultCoreDNSAddonName),
49+
},
50+
{
51+
kubernetesFeatureSetting{
52+
"dns-autoscaler.yaml",
53+
"dns-autoscaler.yaml",
54+
// TODO enable this when it has been smoke tested
55+
//common.IsKubernetesVersionGe(profile.OrchestratorProfile.OrchestratorVersion, "1.12.0"),
56+
false,
57+
},
58+
profile.OrchestratorProfile.KubernetesConfig.GetAddonScript(DefaultDNSAutoscalerAddonName),
59+
},
4260
{
4361
kubernetesFeatureSetting{
4462
"kubernetesmasteraddons-kube-proxy-daemonset.yaml",

‎pkg/acsengine/const.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,10 @@ const (
8888
DefaultKubeHeapsterDeploymentAddonName = "kube-heapster-deployment"
8989
// DefaultKubeDNSDeploymentAddonName is the name of the kube-dns-deployment addon
9090
DefaultKubeDNSDeploymentAddonName = "kube-dns-deployment"
91+
// DefaultCoreDNSAddonName is the name of the coredns addon
92+
DefaultCoreDNSAddonName = "coredns"
93+
// DefaultDNSAutoscalerAddonName is the name of the coredns addon
94+
DefaultDNSAutoscalerAddonName = "dns-autoscaler"
9195
// DefaultKubeProxyAddonName is the name of the kube-proxy config addon
9296
DefaultKubeProxyAddonName = "kube-proxy-daemonset"
9397
// DefaultAzureStorageClassesAddonName is the name of the azure storage classes addon

‎pkg/acsengine/params_k8s.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ func assignKubernetesParameters(properties *api.Properties, parametersMap params
246246
}
247247
}
248248
}
249+
addValue(parametersMap, "kubernetesCoreDNSSpec", "coredns/coredns:1.2.2")
249250
addValue(parametersMap, "kubernetesKubeDNSSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["dns"])
250251
addValue(parametersMap, "kubernetesPodInfraContainerSpec", cloudSpecConfig.KubernetesSpecConfig.KubernetesImageBase+KubeConfigs[k8sVersion]["pause"])
251252
addValue(parametersMap, "cloudproviderConfig", api.CloudProviderConfig{

‎test/e2e/kubernetes/kubernetes_test.go‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ var _ = BeforeSuite(func() {
7070
masterSSHPort = "22"
7171
}
7272
masterSSHPrivateKeyFilepath = cfg.GetSSHKeyPath()
73+
// TODO
74+
// If no user-configurable stability iteration value is passed in, run stability tests once
75+
/*if cfg.StabilityIterations == 0 {
76+
cfg.StabilityIterations = 1
77+
}*/
7378
})
7479

7580
var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", func() {
@@ -115,13 +120,18 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
115120

116121
It("should have stable internal container networking", func() {
117122
name := fmt.Sprintf("alpine-%s", cfg.Name)
118-
command := fmt.Sprintf("nc -vz kubernetes 443")
123+
var command string
124+
if common.IsKubernetesVersionGe(eng.ExpandedDefinition.Properties.OrchestratorProfile.OrchestratorVersion, "1.12.0") {
125+
command = fmt.Sprintf("nc -vz kubernetes 443 && nc -vz kubernetes.default.svc 443 && nc -vz kubernetes.default.svc.cluster.local 443")
126+
} else {
127+
command = fmt.Sprintf("nc -vz kubernetes 443")
128+
}
119129
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
120130
Expect(err).NotTo(HaveOccurred())
121131
Expect(successes).To(Equal(cfg.StabilityIterations))
122132
})
123133

124-
It("should have functional DNS", func() {
134+
It("should be able to launch a long-running container networking DNS liveness pod", func() {
125135
if !eng.HasNetworkPolicy("calico") {
126136
var err error
127137
var p *pod.Pod
@@ -138,7 +148,9 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
138148
Expect(err).NotTo(HaveOccurred())
139149
Expect(running).To(Equal(true))
140150
}
151+
})
141152

153+
It("should have functional host OS DNS", func() {
142154
kubeConfig, err := GetConfig()
143155
Expect(err).NotTo(HaveOccurred())
144156
master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
@@ -222,7 +234,9 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
222234
if err != nil {
223235
log.Printf("Error while querying DNS: %s\n", err)
224236
}
237+
})
225238

239+
It("should have functional container networking DNS", func() {
226240
By("Ensuring that we have functional DNS resolution from a container")
227241
j, err := job.CreateJobFromFile(filepath.Join(WorkloadDir, "validate-dns.yaml"), "validate-dns", "default")
228242
Expect(err).NotTo(HaveOccurred())
@@ -235,16 +249,25 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
235249
Expect(err).NotTo(HaveOccurred())
236250
Expect(ready).To(Equal(true))
237251

238-
By("Ensuring that we have stable DNS resolution from a container")
252+
By("Ensuring that we have stable external DNS resolution from a container")
239253
name := fmt.Sprintf("alpine-%s", cfg.Name)
240254
command := fmt.Sprintf("nc -vz bbc.co.uk 80 || nc -vz google.com 443 || nc -vz microsoft.com 80")
241255
successes, err := pod.RunCommandMultipleTimes(pod.RunLinuxPod, "alpine", name, command, cfg.StabilityIterations)
242256
Expect(err).NotTo(HaveOccurred())
243257
Expect(successes).To(Equal(cfg.StabilityIterations))
244258
})
245259

246-
It("should have kube-dns running", func() {
247-
running, err := pod.WaitOnReady("kube-dns", "kube-system", 3, 30*time.Second, cfg.Timeout)
260+
It("should have DNS pod running", func() {
261+
var err error
262+
var running bool
263+
if common.IsKubernetesVersionGe(eng.ExpandedDefinition.Properties.OrchestratorProfile.OrchestratorVersion, "1.12.0") {
264+
By("Ensuring that coredns is running")
265+
running, err = pod.WaitOnReady("coredns", "kube-system", 3, 30*time.Second, cfg.Timeout)
266+
267+
} else {
268+
By("Ensuring that kube-dns is running")
269+
running, err = pod.WaitOnReady("kube-dns", "kube-system", 3, 30*time.Second, cfg.Timeout)
270+
}
248271
Expect(err).NotTo(HaveOccurred())
249272
Expect(running).To(Equal(true))
250273
})

0 commit comments

Comments
 (0)