Merge pull request #1577 from 0ekk/passvault

Support ansible vault password

Author: ErikJiang

api/constants/constants.go

@@ -16,6 +16,8 @@ const (
 	KeySprayRelease = "kubean.io/sprayRelease"
 	KeySprayCommit  = "kubean.io/sprayCommit"
 
+	AnnotationHostsConfVaultPasswordRef = "kubean.io/vault-password-ref"
+
 	KubeanConfigMapName                  = "kubean-config"
 	DefaultClusterOperationsBackEndLimit = 30
 	MaxClusterOperationsBackEndLimit     = 200

pkg/controllers/clusterops/controller.go

@@ -469,6 +469,27 @@ func (c *Controller) NewKubesprayJob(clusterOps *clusteroperationv1alpha1.Cluste
 				},
 			})
 	}
+	if vaultRef := c.getVaultSecret(clusterOps); vaultRef != nil {
+		if len(job.Spec.Template.Spec.Containers) > 0 && job.Spec.Template.Spec.Containers[0].Name == SprayJobPodName {
+			job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts,
+				corev1.VolumeMount{
+					Name:      "vault-password",
+					MountPath: "/auth/vault-password",
+					SubPath:   "vault-password",
+					ReadOnly:  true,
+				})
+		}
+		job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes,
+			corev1.Volume{
+				Name: "vault-password",
+				VolumeSource: corev1.VolumeSource{
+					Secret: &corev1.SecretVolumeSource{
+						SecretName:  vaultRef.Name,
+						DefaultMode: &PrivatekeyMode,
+					},
+				},
+			})
+	}
 	if clusterOps.Spec.ActiveDeadlineSeconds != nil && *clusterOps.Spec.ActiveDeadlineSeconds > 0 {
 		job.Spec.ActiveDeadlineSeconds = clusterOps.Spec.ActiveDeadlineSeconds
 	}
@@ -606,7 +627,7 @@ func (c *Controller) CreateEntryPointShellConfigMap(clusterOps *clusteroperation
 	if !clusterOps.Spec.EntrypointSHRef.IsEmpty() {
 		return false, nil
 	}
-	entryPointData := entrypoint.NewEntryPoint()
+	entryPointData := entrypoint.NewEntryPoint(c.getVaultSecret(clusterOps) != nil)
 	isPrivateKey := !clusterOps.Spec.SSHAuthRef.IsEmpty()
 	builtinActionSource := clusteroperationv1alpha1.BuiltinActionSource
 	for _, action := range clusterOps.Spec.PreHook {
@@ -766,8 +787,9 @@ func (c *Controller) CopyConfigMap(clusterOps *clusteroperationv1alpha1.ClusterO
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      newName,
-			Namespace: namespace,
+			Name:        newName,
+			Namespace:   namespace,
+			Annotations: oldConfigMap.Annotations,
 		},
 		Data: oldConfigMap.Data,
 	}
@@ -977,3 +999,25 @@ func (c *Controller) CheckClusterDataRef(cluster *clusterv1alpha1.Cluster, clust
 	}
 	return nil
 }
+
+func (c *Controller) getVaultSecret(clusterOps *clusteroperationv1alpha1.ClusterOperation) *apis.SecretRef {
+	if clusterOps.Spec.HostsConfRef.IsEmpty() {
+		return nil
+	}
+	hostsConf, err := c.ClientSet.CoreV1().ConfigMaps(clusterOps.Spec.HostsConfRef.NameSpace).Get(context.Background(), clusterOps.Spec.HostsConfRef.Name, metav1.GetOptions{})
+	if err != nil {
+		return nil
+	}
+	vaultRef, ok := hostsConf.Annotations[constants.AnnotationHostsConfVaultPasswordRef]
+	if !ok || vaultRef == "" {
+		return nil
+	}
+	if !c.CheckSecretExist(util.GetCurrentNSOrDefault(), vaultRef) {
+		klog.Warningf("vault password ref %s not found in namespace %s", vaultRef, util.GetCurrentNSOrDefault())
+		return nil
+	}
+	return &apis.SecretRef{
+		NameSpace: util.GetCurrentNSOrDefault(),
+		Name:      vaultRef,
+	}
+}

pkg/controllers/clusterops/controller_test.go

@@ -2923,3 +2923,102 @@ func (MockManager) GetLogger() logr.Logger { return logr.Logger{} }
 func (MockManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec {
 	return v1alpha1.ControllerConfigurationSpec{}
 }
+
+func TestGetVaultSecret(t *testing.T) {
+	controller := Controller{
+		Client:    newFakeClient(),
+		ClientSet: clientsetfake.NewSimpleClientset(),
+	}
+	clusterOps := &clusteroperationv1alpha1.ClusterOperation{}
+
+	tests := []struct {
+		name  string
+		setup func()
+		want  *apis.SecretRef
+	}{
+		{
+			name: "No HostsConfRef",
+			setup: func() {
+				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{}
+			},
+			want: nil,
+		},
+		{
+			name: "ConfigMap Not Found",
+			setup: func() {
+				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "nonexistent"}
+			},
+			want: nil,
+		},
+		{
+			name: "No Vault Annotation",
+			setup: func() {
+				cm := &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "default",
+						Name:      "hostsconf",
+					},
+				}
+				controller.ClientSet.CoreV1().ConfigMaps("default").Create(context.Background(), cm, metav1.CreateOptions{})
+				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "hostsconf"}
+			},
+			want: nil,
+		},
+		{
+			name: "Vault Secret Not Found",
+			setup: func() {
+				cm := &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace:   "default",
+						Name:        "hostsconf",
+						Annotations: map[string]string{constants.AnnotationHostsConfVaultPasswordRef: "vault-secret"},
+					},
+				}
+				controller.ClientSet.CoreV1().ConfigMaps("default").Create(context.Background(), cm, metav1.CreateOptions{})
+				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "hostsconf"}
+			},
+			want: nil,
+		},
+		{
+			name: "Successful Retrieval",
+			setup: func() {
+				cm := &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace:   "default",
+						Name:        "hostsconf1",
+						Annotations: map[string]string{constants.AnnotationHostsConfVaultPasswordRef: "vault-secret"},
+					},
+					Data: map[string]string{
+						"vault-password": "vault-password",
+					},
+				}
+				_, err := controller.ClientSet.CoreV1().ConfigMaps("default").Create(context.Background(), cm, metav1.CreateOptions{})
+				if err != nil {
+					t.Fatalf("failed to create config map: %v", err)
+				}
+				secret := &corev1.Secret{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: "default",
+						Name:      "vault-secret",
+					},
+				}
+				_, err = controller.ClientSet.CoreV1().Secrets("default").Create(context.Background(), secret, metav1.CreateOptions{})
+				if err != nil {
+					t.Fatalf("failed to create secret: %v", err)
+				}
+				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "hostsconf1"}
+			},
+			want: &apis.SecretRef{NameSpace: "default", Name: "vault-secret"},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			test.setup()
+			got := controller.getVaultSecret(clusterOps)
+			if !reflect.DeepEqual(got, test.want) {
+				t.Fatalf("got %v, want %v", got, test.want)
+			}
+		})
+	}
+}

pkg/util/entrypoint/entrypoint.go

@@ -48,6 +48,9 @@ const (
 //go:embed entrypoint.sh.template
 var entrypointTemplate string
 
+//go:embed inventory_decrypt.sh
+var inventoryDecryptScript string
+
 type void struct{}
 
 var member void
@@ -90,14 +93,20 @@ func (argsError ArgsError) Error() string {
 }
 
 type EntryPoint struct {
-	PreHookCMDs  []string
-	SprayCMD     string
-	PostHookCMDs []string
-	Actions      *Actions
+	PrerequisitesCMDs []string
+	PreHookCMDs       []string
+	SprayCMD          string
+	PostHookCMDs      []string
+	Actions           *Actions
+
+	isVaultEncryped bool
 }
 
-func NewEntryPoint() *EntryPoint {
-	ep := &EntryPoint{}
+func NewEntryPoint(isVaultEncryped bool) *EntryPoint {
+	ep := &EntryPoint{
+		isVaultEncryped: isVaultEncryped,
+	}
+	ep.PrerequisiteRunPart()
 	ep.Actions = NewActions()
 	return ep
 }
@@ -108,7 +117,11 @@ func (ep *EntryPoint) buildPlaybookCmd(action, extraArgs string, isPrivateKey, b
 			return "", ArgsError{fmt.Sprintf("unknown playbook type, the currently supported ranges include: %s", ep.Actions.Playbooks.List)}
 		}
 	}
-	playbookCmd := "ansible-playbook -i /conf/hosts.yml -b --become-user root -e \"@/conf/group_vars.yml\""
+	inventory := "/conf/hosts.yml"
+	if ep.isVaultEncryped {
+		inventory = "/dev/fd/200"
+	}
+	playbookCmd := fmt.Sprintf("ansible-playbook -i %s -b --become-user root -e \"@/conf/group_vars.yml\"", inventory)
 	if isPrivateKey {
 		playbookCmd = fmt.Sprintf("%s --private-key /auth/ssh-privatekey", playbookCmd)
 	}
@@ -145,6 +158,12 @@ func (ep *EntryPoint) hookRunPart(actionType, action, extraArgs string, isPrivat
 	return hookRunCmd, nil
 }
 
+func (ep *EntryPoint) PrerequisiteRunPart() {
+	if ep.isVaultEncryped {
+		ep.PrerequisitesCMDs = append(ep.PrerequisitesCMDs, inventoryDecryptScript)
+	}
+}
+
 func (ep *EntryPoint) PreHookRunPart(actionType, action, extraArgs string, isPrivateKey, builtinAction bool) error {
 	prehook, err := ep.hookRunPart(actionType, action, extraArgs, isPrivateKey, builtinAction)
 	if err != nil {

pkg/util/entrypoint/entrypoint.sh.template

@@ -4,6 +4,11 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
+# prerequisite
+{{ range $prerequisiteCMD := .PrerequisitesCMDs }}
+{{- $prerequisiteCMD }}
+{{ end }}
+
 # preinstall
 {{ range $preCMD := .PreHookCMDs }}
 {{- $preCMD }}