Skip to content

Commit df43a07

Browse files
pregnorpregnor
authored
merge(#3648): deployed EBS CSI driver
Install AWS EBS CSI driver by default as EKS addon or PKE AWS 1.23+ Helm chart #3648
2 parents 7cb7513 + 61aff24 commit df43a07

File tree

10 files changed

+176
-25
lines changed

10 files changed

+176
-25
lines changed

‎cmd/worker/aws.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ func registerAwsWorkflows(
3636
awsSecretStore awsworkflow.SecretStore,
3737
) {
3838
createClusterWorkflow := pkeworkflow.NewCreateClusterWorkflow(
39+
config.Distribution.PKE.Amazon.DefaultEBSCSIDriverChartVersion,
3940
config.Distribution.PKE.Amazon.DefaultNodeVolumeSize,
4041
config.Distribution.PKE.Amazon.GlobalRegion,
4142
)

‎cmd/worker/main.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,15 @@ func main() {
387387
restoreBackupActivity := velero.NewRestoreBackupActivity(clusterManager, unifiedHelmReleaser, global.DB(), config.Cluster.DisasterRecovery)
388388
worker.RegisterActivityWithOptions(restoreBackupActivity.Execute, activity.RegisterOptions{Name: clustersetup.RestoreBackupActivityName})
389389

390+
deployAWSEBSCSIDriverActivity := clustersetup.NewDeployAWSEBSCSIDriverActivity(
391+
config.Cluster.Labels,
392+
unifiedHelmReleaser,
393+
)
394+
worker.RegisterActivityWithOptions(
395+
deployAWSEBSCSIDriverActivity.Execute,
396+
activity.RegisterOptions{Name: clustersetup.DeployAWSEBSCSIDriverActivityName},
397+
)
398+
390399
deployIngressControllerActivity := clustersetup.NewDeployIngressControllerActivity(
391400
config.Cluster.Labels,
392401
unifiedHelmReleaser,

‎config/config.yaml.dist‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,7 @@ dex:
386386
#helm:
387387
# home: "./var/cache"
388388
# repositories:
389+
# aws-ebs-csi-driver: "https://kubernetes-sigs.github.io/aws-ebs-csi-driver"
389390
# stable: "https://charts.helm.sh/stable"
390391
# banzaicloud-stable: "https://kubernetes-charts.banzaicloud.com"
391392
# bitnami: "https://charts.bitnami.com/bitnami"
@@ -421,6 +422,7 @@ distribution:
421422
amazon:
422423
enabled: true
423424
# globalRegion: us-east-1
425+
# defaultEBSCSIDriverChartVersion: "2.12.1"
424426
# defaultImages: {}
425427
# defaultNetworkProvider: "cilium"
426428
# defaultNodeVolumeSize: 0 # GiB, 0/fallback: max(50, AMISize)

‎internal/cluster/clustersetup/BUILD.plz‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ go_library(
1919
"//src/auth",
2020
"//src/dns",
2121
"//third_party/go:emperror.dev__errors",
22+
"//third_party/go:github.com__Masterminds__semver__v3",
2223
"//third_party/go:github.com__aws__aws-sdk-go__aws",
2324
"//third_party/go:github.com__ghodss__yaml",
2425
"//third_party/go:go.uber.org__cadence",
@@ -50,6 +51,7 @@ go_test(
5051
"//src/auth",
5152
"//src/dns",
5253
"//third_party/go:emperror.dev__errors",
54+
"//third_party/go:github.com__Masterminds__semver__v3",
5355
"//third_party/go:github.com__aws__aws-sdk-go__aws",
5456
"//third_party/go:github.com__ghodss__yaml",
5557
"//third_party/go:github.com__stretchr__testify__mock",
@@ -90,6 +92,7 @@ go_test(
9092
"//src/auth",
9193
"//src/dns",
9294
"//third_party/go:emperror.dev__errors",
95+
"//third_party/go:github.com__Masterminds__semver__v3",
9396
"//third_party/go:github.com__aws__aws-sdk-go__aws",
9497
"//third_party/go:github.com__ghodss__yaml",
9598
"//third_party/go:github.com__stretchr__testify__mock",

‎internal/cluster/clustersetup/activity_deploy_aws_ebs_csi_driver.go‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright © 2021 Banzai Cloud
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package clustersetup
16+
17+
import (
18+
"context"
19+
20+
"emperror.dev/errors"
21+
"github.com/Masterminds/semver/v3"
22+
"go.uber.org/cadence/activity"
23+
24+
"github.com/banzaicloud/pipeline/internal/cluster/clusterconfig"
25+
"github.com/banzaicloud/pipeline/internal/global"
26+
)
27+
28+
const DeployAWSEBSCSIDriverActivityName = "deploy-aws-ebs-csi-driver"
29+
30+
type DeployAWSEBSCSIDriverActivity struct {
31+
config clusterconfig.LabelConfig
32+
helmService HelmService
33+
}
34+
35+
type DeployAWSEBSCSIDriverActivityInput struct {
36+
ClusterID uint
37+
KubernetesVersion string
38+
ChartVersion string
39+
}
40+
41+
// NewDeployAWSEBSCSIDriverActivity returns a new DeployAWSEBSCSIDriverActivity.
42+
func NewDeployAWSEBSCSIDriverActivity(
43+
config clusterconfig.LabelConfig,
44+
helmService HelmService,
45+
) DeployAWSEBSCSIDriverActivity {
46+
return DeployAWSEBSCSIDriverActivity{
47+
config: config,
48+
helmService: helmService,
49+
}
50+
}
51+
52+
func (a DeployAWSEBSCSIDriverActivity) Execute(ctx context.Context, input DeployAWSEBSCSIDriverActivityInput) error {
53+
logger := activity.GetLogger(ctx).Sugar().With(
54+
"clusterID", input.ClusterID,
55+
"KubernetesVersion", input.KubernetesVersion,
56+
"chartVersion", input.ChartVersion,
57+
)
58+
59+
ebsCSIDriverConstraint, err := semver.NewConstraint(">= 1.23")
60+
if err != nil {
61+
return errors.WrapIf(err, "creating semver constraint for EBS CSI driver failed")
62+
}
63+
64+
k8sVersion, err := semver.NewVersion(input.KubernetesVersion)
65+
if err != nil {
66+
return errors.WrapIf(err, "creating semver from Kubernetes version failed")
67+
}
68+
69+
if !ebsCSIDriverConstraint.Check(k8sVersion) {
70+
logger.Infof("kubernetesVersion failed ebsCSIDriverConstraint check", "k8sVersion", k8sVersion)
71+
72+
return nil
73+
}
74+
75+
return a.helmService.ApplyDeployment(
76+
ctx,
77+
input.ClusterID,
78+
global.Config.Cluster.Namespace,
79+
"aws-ebs-csi-driver/aws-ebs-csi-driver",
80+
"aws-ebs-csi-driver",
81+
nil,
82+
input.ChartVersion,
83+
)
84+
}

‎internal/cluster/distribution/eks/eksprovider/workflow/bootstrap_workflow.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,12 @@ func (a *BootstrapWorkflow) Execute(ctx workflow.Context, input BootstrapWorkflo
108108
KubernetesVersion: input.KubernetesVersion,
109109
AddonName: "kube-proxy",
110110
}),
111+
workflow.ExecuteActivity(ctx, CreateAddonActivityName,
112+
CreateAddonActivityInput{
113+
EKSActivityInput: commonActivityInput,
114+
KubernetesVersion: input.KubernetesVersion,
115+
AddonName: "aws-ebs-csi-driver",
116+
}),
111117
)
112118
}
113119

‎internal/cmd/config.go‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -515,11 +515,12 @@ type DistributionConfig struct {
515515

516516
PKE struct {
517517
Amazon struct {
518-
Enabled bool
519-
GlobalRegion string
520-
DefaultImages map[string]string
521-
DefaultNetworkProvider string
522-
DefaultNodeVolumeSize int
518+
Enabled bool
519+
GlobalRegion string
520+
DefaultEBSCSIDriverChartVersion string
521+
DefaultImages map[string]string
522+
DefaultNetworkProvider string
523+
DefaultNodeVolumeSize int
523524
}
524525
Azure struct {
525526
Enabled bool
@@ -866,6 +867,7 @@ traefik:
866867

867868
// Helm configuration
868869
v.SetDefault("helm::home", "./var/cache")
870+
v.SetDefault("helm::repositories::aws-ebs-csi-driver", "https://kubernetes-sigs.github.io/aws-ebs-csi-driver")
869871
v.SetDefault("helm::repositories::stable", "https://charts.helm.sh/stable")
870872
v.SetDefault("helm::repositories::banzaicloud-stable", "https://kubernetes-charts.banzaicloud.com")
871873
v.SetDefault("helm::repositories::bitnami", "https://charts.bitnami.com/bitnami")
@@ -885,6 +887,7 @@ traefik:
885887

886888
v.SetDefault("distribution::pke::amazon::enabled", true)
887889
v.SetDefault("distribution::pke::amazon::globalRegion", "us-east-1")
890+
v.SetDefault("distribution::pke::amazon::defaultEBSCSIDriverChartVersion", "2.12.1")
888891
v.SetDefault("distribution::pke::amazon::defaultImages", map[string]string{})
889892
v.SetDefault("distribution::pke::amazon::defaultNetworkProvider", "cilium")
890893
v.SetDefault("distribution::pke::amazon::defaultNodeVolumeSize", 0)

‎internal/providers/pke/pkeworkflow/BUILD.plz‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ go_library(
1010
"//internal/cluster",
1111
"//internal/cluster/auth",
1212
"//internal/cluster/clustersecret",
13+
"//internal/cluster/clustersetup",
1314
"//internal/cluster/distribution/eks/eksprovider/workflow",
1415
"//internal/cluster/distribution/pke/pkeaws",
1516
"//internal/providers/amazon",
@@ -55,6 +56,7 @@ go_test(
5556
"//internal/cluster",
5657
"//internal/cluster/auth",
5758
"//internal/cluster/clustersecret",
59+
"//internal/cluster/clustersetup",
5860
"//internal/cluster/distribution/eks/eksprovider/workflow",
5961
"//internal/cluster/distribution/pke/pkeaws",
6062
"//internal/providers/amazon",

‎internal/providers/pke/pkeworkflow/create_cluster.go‎

Lines changed: 54 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import (
2222
"go.uber.org/cadence"
2323
"go.uber.org/cadence/workflow"
2424

25+
"github.com/banzaicloud/pipeline/internal/cluster/clustersetup"
2526
"github.com/banzaicloud/pipeline/pkg/sdk/brn"
2627
"github.com/banzaicloud/pipeline/pkg/sdk/cadence/lib/pipeline/processlog"
2728
)
@@ -47,19 +48,26 @@ type CreateClusterWorkflowInput struct {
4748
PipelineExternalURLInsecure bool
4849
OIDCEnabled bool
4950
VPCID string
51+
KubernetesVersion string
5052
}
5153

5254
type CreateClusterWorkflow struct {
53-
DefaultNodeVolumeSize int
54-
GlobalRegion string
55-
processLogger processlog.ProcessLogger
55+
DefaultEBSCSIDriverChartVersion string
56+
DefaultNodeVolumeSize int
57+
GlobalRegion string
58+
processLogger processlog.ProcessLogger
5659
}
5760

58-
func NewCreateClusterWorkflow(defaultNodeVolumeSize int, globalRegion string) CreateClusterWorkflow {
61+
func NewCreateClusterWorkflow(
62+
defaultEBSCSIDriverChartVersion string,
63+
defaultNodeVolumeSize int,
64+
globalRegion string,
65+
) CreateClusterWorkflow {
5966
return CreateClusterWorkflow{
60-
DefaultNodeVolumeSize: defaultNodeVolumeSize,
61-
GlobalRegion: globalRegion,
62-
processLogger: processlog.New(),
67+
DefaultEBSCSIDriverChartVersion: defaultEBSCSIDriverChartVersion,
68+
DefaultNodeVolumeSize: defaultNodeVolumeSize,
69+
GlobalRegion: globalRegion,
70+
processLogger: processlog.New(),
6371
}
6472
}
6573

@@ -406,9 +414,9 @@ func (w CreateClusterWorkflow) Execute(ctx workflow.Context, input CreateCluster
406414

407415
// Create nodes
408416
{
409-
futures := make([]workflow.Future, len(nodePools))
417+
futures := make([]workflow.Future, 0, len(nodePools))
410418

411-
for i, np := range nodePools {
419+
for _, np := range nodePools {
412420
if !np.Master {
413421
subnetIDs := np.Subnets
414422
if len(np.Subnets) == 0 {
@@ -430,19 +438,51 @@ func (w CreateClusterWorkflow) Execute(ctx workflow.Context, input CreateCluster
430438
SSHKeyName: keyOut.KeyName,
431439
}
432440

433-
futures[i] = workflow.ExecuteActivity(ctx, CreateWorkerPoolActivityName, createWorkerPoolActivityInput)
441+
futures = append(
442+
futures,
443+
workflow.ExecuteActivity(ctx, CreateWorkerPoolActivityName, createWorkerPoolActivityInput),
444+
)
434445
}
435446
}
436447

437-
errs := make([]error, len(futures))
438-
for i, future := range futures {
448+
errs := make([]error, 0, len(futures))
449+
for index, future := range futures {
439450
if future != nil {
440-
errs[i] = errors.Wrapf(future.Get(ctx, nil), "couldn't create nodepool %q", nodePools[i].Name)
451+
if err := future.Get(ctx, nil); err != nil {
452+
errs = append(errs, errors.Wrapf(err, "couldn't create nodepool %q", nodePools[index].Name))
453+
}
441454
}
442455
}
443456

444-
return errors.Combine(errs...)
457+
if len(errs) > 0 {
458+
return errors.Combine(errs...)
459+
}
460+
}
461+
462+
// Note: install EBS CSI driver for PVCs from K8s 1.23.
463+
// Source: https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi.html
464+
{
465+
activityInput := clustersetup.DeployAWSEBSCSIDriverActivityInput{
466+
ClusterID: input.ClusterID,
467+
KubernetesVersion: input.KubernetesVersion,
468+
ChartVersion: w.DefaultEBSCSIDriverChartVersion,
469+
}
470+
471+
err := workflow.ExecuteActivity(
472+
ctx,
473+
clustersetup.DeployAWSEBSCSIDriverActivityName,
474+
activityInput,
475+
).Get(ctx, nil)
476+
if err != nil {
477+
return errors.WrapIfWithDetails(
478+
err,
479+
"installing EBS CSI driver Helm chart failed",
480+
"activityInput", activityInput,
481+
)
482+
}
445483
}
484+
485+
return nil
446486
}
447487

448488
type decodableError struct {

‎src/cluster/manager_common_creator.go‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -74,11 +74,10 @@ func NewClusterCreator(request *pkgCluster.CreateClusterRequest, cluster CommonC
7474

7575
if strings.HasPrefix(cluster.GetDistribution(), pkgCluster.PKE) && cluster.GetCloud() == pkgCluster.Amazon {
7676
return &pkeCreator{
77-
workflowClient: workflowClient,
78-
79-
commonCreator: *common,
80-
81-
oidcEnabled: request.Properties.CreateClusterPKE.Kubernetes.OIDC.Enabled,
77+
workflowClient: workflowClient,
78+
commonCreator: *common,
79+
oidcEnabled: request.Properties.CreateClusterPKE.Kubernetes.OIDC.Enabled,
80+
kubernetesVersion: request.Properties.CreateClusterPKE.Kubernetes.Version,
8281
}
8382
}
8483

@@ -94,7 +93,8 @@ type pkeCreator struct {
9493

9594
commonCreator
9695

97-
oidcEnabled bool
96+
oidcEnabled bool
97+
kubernetesVersion string
9898
}
9999

100100
// Create implements the clusterCreator interface.
@@ -119,6 +119,7 @@ func (c *pkeCreator) Create(ctx context.Context) error {
119119
PipelineExternalURL: externalBaseURL,
120120
PipelineExternalURLInsecure: externalBaseURLInsecure,
121121
OIDCEnabled: c.oidcEnabled,
122+
KubernetesVersion: c.kubernetesVersion,
122123
}
123124

124125
providerConfig := c.request.Properties.CreateClusterPKE.Network.ProviderConfig

0 commit comments

Comments
 (0)