Skip to content

Commit 61ae556

Browse files
mselim00mselim00
authored
feat(eksapi): accept and configure nodeadm feature gates (#641)
1 parent fa74d48 commit 61ae556

File tree

6 files changed

+148
-27
lines changed

6 files changed

+148
-27
lines changed

‎internal/deployers/eksapi/deployer.go‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -67,25 +67,26 @@ type deployerOptions struct {
6767
EmitMetrics bool `flag:"emit-metrics" desc:"Record and emit metrics to CloudWatch"`
6868
ExpectedAMI string `flag:"expected-ami" desc:"Expected AMI of nodes. Up will fail if the actual nodes are not utilizing the expected AMI. Defaults to --ami if defined."`
6969
// TODO: remove this once it's no longer used in downstream jobs
70-
GenerateSSHKey bool `flag:"generate-ssh-key" desc:"Generate an SSH key to use for tests. The generated key should not be used in production, as it will not have a passphrase."`
71-
InstanceTypes []string `flag:"instance-types" desc:"Node instance types. Cannot be used with --instance-type-archs"`
72-
InstanceTypeArchs []string `flag:"instance-type-archs" desc:"Use default node instance types for specific architectures. Cannot be used with --instance-types"`
73-
IPFamily string `flag:"ip-family" desc:"IP family for the cluster (ipv4 or ipv6)"`
74-
KubeconfigPath string `flag:"kubeconfig" desc:"Path to kubeconfig"`
75-
KubernetesVersion string `flag:"kubernetes-version" desc:"cluster Kubernetes version"`
76-
LogBucket string `flag:"log-bucket" desc:"S3 bucket for storing logs for each run. If empty, logs will not be stored."`
77-
NodeCreationTimeout time.Duration `flag:"node-creation-timeout" desc:"Time to wait for nodes to be created/launched. This should consider instance availability."`
78-
NodeReadyTimeout time.Duration `flag:"node-ready-timeout" desc:"Time to wait for all nodes to become ready"`
79-
Nodes int `flag:"nodes" desc:"number of nodes to launch in cluster"`
80-
NodeNameStrategy string `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"`
81-
Region string `flag:"region" desc:"AWS region for EKS cluster"`
82-
SkipNodeReadinessChecks bool `flag:"skip-node-readiness-checks" desc:"Skip performing readiness checks on created nodes"`
83-
StaticClusterName string `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"`
84-
TuneVPCCNI bool `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"`
85-
UnmanagedNodes bool `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"`
86-
UpClusterHeaders []string `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."`
87-
UserDataFormat string `flag:"user-data-format" desc:"Format of the node instance user data"`
88-
ZoneType string `flag:"zone-type" desc:"Type of zone to use for infrastructure (availability-zone, local-zone, etc). Defaults to availability-zone"`
70+
GenerateSSHKey bool `flag:"generate-ssh-key" desc:"Generate an SSH key to use for tests. The generated key should not be used in production, as it will not have a passphrase."`
71+
InstanceTypes []string `flag:"instance-types" desc:"Node instance types. Cannot be used with --instance-type-archs"`
72+
InstanceTypeArchs []string `flag:"instance-type-archs" desc:"Use default node instance types for specific architectures. Cannot be used with --instance-types"`
73+
IPFamily string `flag:"ip-family" desc:"IP family for the cluster (ipv4 or ipv6)"`
74+
KubeconfigPath string `flag:"kubeconfig" desc:"Path to kubeconfig"`
75+
KubernetesVersion string `flag:"kubernetes-version" desc:"cluster Kubernetes version"`
76+
LogBucket string `flag:"log-bucket" desc:"S3 bucket for storing logs for each run. If empty, logs will not be stored."`
77+
NodeadmFeatureGates []string `flag:"nodeadm-feature-gates" desc:"Feature gates to enable for nodeadm (key=value pairs)"`
78+
NodeCreationTimeout time.Duration `flag:"node-creation-timeout" desc:"Time to wait for nodes to be created/launched. This should consider instance availability."`
79+
NodeReadyTimeout time.Duration `flag:"node-ready-timeout" desc:"Time to wait for all nodes to become ready"`
80+
Nodes int `flag:"nodes" desc:"number of nodes to launch in cluster"`
81+
NodeNameStrategy string `flag:"node-name-strategy" desc:"Specifies the naming strategy for node. Allowed values: ['SessionName', 'EC2PrivateDNSName'], default to EC2PrivateDNSName"`
82+
Region string `flag:"region" desc:"AWS region for EKS cluster"`
83+
SkipNodeReadinessChecks bool `flag:"skip-node-readiness-checks" desc:"Skip performing readiness checks on created nodes"`
84+
StaticClusterName string `flag:"static-cluster-name" desc:"Optional when re-use existing cluster and node group by querying the kubeconfig and run test"`
85+
TuneVPCCNI bool `flag:"tune-vpc-cni" desc:"Apply tuning parameters to the VPC CNI DaemonSet"`
86+
UnmanagedNodes bool `flag:"unmanaged-nodes" desc:"Use an AutoScalingGroup instead of an EKS-managed nodegroup. Requires --ami"`
87+
UpClusterHeaders []string `flag:"up-cluster-header" desc:"Additional header to add to eks:CreateCluster requests. Specified in the same format as curl's -H flag."`
88+
UserDataFormat string `flag:"user-data-format" desc:"Format of the node instance user data"`
89+
ZoneType string `flag:"zone-type" desc:"Type of zone to use for infrastructure (availability-zone, local-zone, etc). Defaults to availability-zone"`
8990
}
9091

9192
// NewDeployer implements deployer.New for EKS using the EKS (and other AWS) API(s) directly (no cloudformation)

‎internal/deployers/eksapi/node.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,7 @@ func (m *nodeManager) createUnmanagedNodegroup(infra *Infrastructure, cluster *C
349349
var capacityReservationId string
350350
stackName := m.getUnmanagedNodegroupStackName()
351351
klog.Infof("creating unmanaged nodegroup stack %s...", stackName)
352-
userData, userDataIsMimePart, err := generateUserData(opts.UserDataFormat, cluster, opts)
352+
userData, userDataIsMimePart, err := generateUserData(cluster, opts)
353353
if err != nil {
354354
return err
355355
}

‎internal/deployers/eksapi/templates/templates.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ type UserDataTemplateData struct {
6767
CIDR string
6868
APIServerEndpoint string
6969
KubeletFeatureGates map[string]bool
70+
NodeadmFeatureGates map[string]bool
7071
}
7172

7273
var (

‎internal/deployers/eksapi/templates/userdata_nodeadm.yaml.mimepart.template‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,12 @@ MIME-Version: 1.0
55
apiVersion: node.eks.aws/v1alpha1
66
kind: NodeConfig
77
spec:
8+
{{- if .NodeadmFeatureGates}}
9+
featureGates:
10+
{{- range $gate, $value := .NodeadmFeatureGates }}
11+
{{$gate}}: {{$value}}
12+
{{- end }}
13+
{{- end }}
814
cluster:
915
name: {{.Name}}
1016
apiServerEndpoint: {{.APIServerEndpoint}}

‎internal/deployers/eksapi/userdata.go‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,17 @@ package eksapi
33
import (
44
"bytes"
55
"fmt"
6+
"strconv"
7+
"strings"
68
"text/template"
79

810
"github.com/aws/aws-k8s-tester/internal/deployers/eksapi/templates"
911
)
1012

11-
func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (string, bool, error) {
13+
func generateUserData(cluster *Cluster, opts *deployerOptions) (string, bool, error) {
1214
userDataIsMimePart := true
1315
var t *template.Template
14-
switch format {
16+
switch opts.UserDataFormat {
1517
case "bootstrap.sh":
1618
t = templates.UserDataBootstrapSh
1719
case "nodeadm":
@@ -21,7 +23,7 @@ func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (s
2123
t = templates.UserDataBottlerocket
2224
userDataIsMimePart = false
2325
default:
24-
return "", false, fmt.Errorf("uknown user data format: '%s'", format)
26+
return "", false, fmt.Errorf("unknown user data format: '%s'", opts.UserDataFormat)
2527
}
2628

2729
kubeletFeatureGates := map[string]bool{}
@@ -30,15 +32,37 @@ func generateUserData(format string, cluster *Cluster, opts *deployerOptions) (s
3032
kubeletFeatureGates["DynamicResourceAllocation"] = true
3133
}
3234

35+
nodeadmFeatureGates, err := extractFeatureGates(opts.NodeadmFeatureGates)
36+
if err != nil {
37+
return "", false, err
38+
}
39+
3340
var buf bytes.Buffer
3441
if err := t.Execute(&buf, templates.UserDataTemplateData{
3542
APIServerEndpoint: cluster.endpoint,
3643
CertificateAuthority: cluster.certificateAuthorityData,
3744
CIDR: cluster.cidr,
3845
Name: cluster.name,
3946
KubeletFeatureGates: kubeletFeatureGates,
47+
NodeadmFeatureGates: nodeadmFeatureGates,
4048
}); err != nil {
4149
return "", false, err
4250
}
4351
return buf.String(), userDataIsMimePart, nil
4452
}
53+
54+
func extractFeatureGates(featureGatePairs []string) (map[string]bool, error) {
55+
featureGateMap := make(map[string]bool)
56+
for _, keyValuePair := range featureGatePairs {
57+
components := strings.Split(keyValuePair, "=")
58+
if len(components) != 2 {
59+
return featureGateMap, fmt.Errorf("expected key=value pairs but %s has %d components", keyValuePair, len(components))
60+
}
61+
boolValue, err := strconv.ParseBool(components[1])
62+
if err != nil {
63+
return featureGateMap, fmt.Errorf("expected bool value in %s: %v", keyValuePair, err)
64+
}
65+
featureGateMap[components[0]] = boolValue
66+
}
67+
return featureGateMap, nil
68+
}

‎internal/deployers/eksapi/userdata_test.go‎

Lines changed: 93 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,41 @@ spec:
3535
certificateAuthority: certificateAuthority
3636
cidr: 10.100.0.0/16
3737
`
38+
39+
const nodeadmUserDataKubeletDRA = `Content-Type: application/node.eks.aws
40+
MIME-Version: 1.0
41+
42+
---
43+
apiVersion: node.eks.aws/v1alpha1
44+
kind: NodeConfig
45+
spec:
46+
cluster:
47+
name: cluster
48+
apiServerEndpoint: https://example.com
49+
certificateAuthority: certificateAuthority
50+
cidr: 10.100.0.0/16
51+
kubelet:
52+
config:
53+
featureGates:
54+
DynamicResourceAllocation: true
55+
`
56+
57+
const nodeadmUserDataFeatureGate = `Content-Type: application/node.eks.aws
58+
MIME-Version: 1.0
59+
60+
---
61+
apiVersion: node.eks.aws/v1alpha1
62+
kind: NodeConfig
63+
spec:
64+
featureGates:
65+
foo: true
66+
cluster:
67+
name: cluster
68+
apiServerEndpoint: https://example.com
69+
certificateAuthority: certificateAuthority
70+
cidr: 10.100.0.0/16
71+
`
72+
3873
const bottlerocketUserData = `[settings.kubernetes]
3974
"cluster-name" = "cluster"
4075
"api-server" = "https://example.com"
@@ -47,9 +82,12 @@ device-ownership-from-security-context = true
4782

4883
func Test_generateUserData(t *testing.T) {
4984
cases := []struct {
50-
format string
51-
expected string
52-
expectedIsMimePart bool
85+
format string
86+
expected string
87+
expectedIsMimePart bool
88+
kubernetesVersion string
89+
NodeadmFeatureGates []string
90+
wantErr bool
5391
}{
5492
{
5593
format: "bootstrap.sh",
@@ -66,10 +104,28 @@ func Test_generateUserData(t *testing.T) {
66104
expected: bottlerocketUserData,
67105
expectedIsMimePart: false,
68106
},
107+
{
108+
format: "nodeadm",
109+
expected: nodeadmUserDataKubeletDRA,
110+
kubernetesVersion: "1.33",
111+
expectedIsMimePart: true,
112+
},
113+
{
114+
format: "nodeadm",
115+
expected: nodeadmUserDataFeatureGate,
116+
kubernetesVersion: "1.30",
117+
NodeadmFeatureGates: []string{"foo=true"},
118+
expectedIsMimePart: true,
119+
},
69120
}
70121
for _, c := range cases {
71122
t.Run(c.format, func(t *testing.T) {
72-
actual, isMimePart, err := generateUserData(c.format, &cluster, &deployerOptions{})
123+
deployerOpts := &deployerOptions{
124+
KubernetesVersion: c.kubernetesVersion,
125+
NodeadmFeatureGates: c.NodeadmFeatureGates,
126+
UserDataFormat: c.format,
127+
}
128+
actual, isMimePart, err := generateUserData(&cluster, deployerOpts)
73129
if err != nil {
74130
t.Log(err)
75131
t.Error(err)
@@ -79,3 +135,36 @@ func Test_generateUserData(t *testing.T) {
79135
})
80136
}
81137
}
138+
139+
func Test_extractFeatureGates(t *testing.T) {
140+
testCases := []struct {
141+
input []string
142+
expected map[string]bool
143+
expectErr bool
144+
}{
145+
{
146+
input: []string{"foo=true", "bar=false"},
147+
expected: map[string]bool{
148+
"foo": true,
149+
"bar": false,
150+
},
151+
},
152+
{
153+
input: []string{"foo:true"},
154+
expectErr: true,
155+
},
156+
{
157+
input: []string{"foo=bar"},
158+
expectErr: true,
159+
},
160+
}
161+
for _, testCase := range testCases {
162+
output, err := extractFeatureGates(testCase.input)
163+
if testCase.expectErr {
164+
assert.Error(t, err)
165+
} else {
166+
assert.NoError(t, err)
167+
assert.Equal(t, testCase.expected, output)
168+
}
169+
}
170+
}

0 commit comments

Comments
 (0)