Merge pull request #1636 from 0ekk/add-asc

Support asymmetric encryption for ssh password

Author: 0ekk

api/constants/constants.go

@@ -16,9 +16,8 @@ const (
 	KeySprayRelease = "kubean.io/sprayRelease"
 	KeySprayCommit  = "kubean.io/sprayCommit"
 
-	AnnotationHostsConfVaultPasswordRef = "kubean.io/vault-password-ref"
-
 	KubeanConfigMapName                  = "kubean-config"
+	KubeanPubKeyConfigMapName            = "kubean-pubkey"
 	DefaultClusterOperationsBackEndLimit = 30
 	MaxClusterOperationsBackEndLimit     = 200
 )

artifacts/ssh_password_encyptor.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+set -e
+
+PUBLICKEY_PATH=${PUBLICKEY_PATH:-}
+KUBECONFIG=${KUBECONFIG:-"$HOME/.kube/config"}
+
+for cmd in base64 openssl; do
+  if ! which "$cmd" &>/dev/null; then
+    echo "Need $cmd"
+    exit 1
+  fi
+done
+
+if [ -z "$PUBLICKEY_PATH" ]; then
+  PUBLICKEY_PATH=$(mktemp)
+  trap "rm -rf $PUBLICKEY_PATH" exit
+  kubectl -n kubean-system get configmap kubean-pubkey -ojsonpath='{.data.pk}' | base64 -d > "$PUBLICKEY_PATH"
+fi
+
+if [ ! -s "$PUBLICKEY_PATH" ]; then
+  echo "Cannot get public key, Check PUBLICKEY_PATH env"
+  exit 1
+fi
+
+read -s -r -p "Your password: " password
+echo -en "\nEncrypted password: "
+echo "VAULT;$(echo -n "$password" | openssl pkeyutl -encrypt -pubin -inkey "$PUBLICKEY_PATH" | base64 -w0)"

build/images/spray-job/Dockerfile

@@ -12,3 +12,5 @@ RUN python3 -m pip install toml
 RUN ansible-galaxy collection install sivel.toiletwater
 
 RUN ln -s playbooks/facts.yml facts.yml
+
+RUN apk add --no-cache openssl

charts/kubean/templates/configmap.yaml

@@ -9,3 +9,18 @@ metadata:
 data:
   CLUSTER_OPERATIONS_BACKEND_LIMIT: "{{ .Values.kubeanOperator.operationsBackendLimit }}"
   SPRAY_JOB_IMAGE_REGISTRY: "{{ .Values.sprayJob.image.registry }}"
+  {{- if .Values.kubeanOperator.crypto.skBase64 }}
+  sk: "{{ .Values.kubeanOperator.crypto.skBase64 }}"
+  {{- end }}
+---
+{{- if .Values.kubeanOperator.crypto.pkBase64 -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kubean-pubkey
+  namespace: {{ include "kubean.namespace" . }}
+  labels:
+  {{- include "kubean.labels" . | nindent 4}}
+data:
+  pk: {{ .Values.kubeanOperator.crypto.pkBase64 }}
+{{- end -}}

charts/kubean/values.yaml

@@ -91,6 +91,9 @@ kubeanOperator:
               values:
               - kubean
           topologyKey: kubernetes.io/hostname
+  crypto:
+    skBase64: null
+    pkBase64: null
 
 ## @section kubean admission parameters
 ## @param kubeanAdmission.replicaCount Number of kubean-admission replicas to deploy

cmd/kubean-operator/app/app.go

@@ -18,6 +18,7 @@ import (
 	"github.com/kubean-io/kubean/pkg/controllers/clusterops"
 	"github.com/kubean-io/kubean/pkg/controllers/infomanifest"
 	"github.com/kubean-io/kubean/pkg/controllers/offlineversion"
+	"github.com/kubean-io/kubean/pkg/crypto"
 	"github.com/kubean-io/kubean/pkg/util"
 	"github.com/kubean-io/kubean/pkg/version"
 
@@ -112,6 +113,12 @@ func setupManager(mgr controllerruntime.Manager, opt *Options, stopChan <-chan s
 	if err != nil {
 		return err
 	}
+
+	if crypto.InitConfiguration(ClientSet) != nil {
+		klog.ErrorS(err, "Failed to init crypto configuration")
+		return err
+	}
+
 	clusterClientSet, err := kubeanClusterClientSet.NewForConfig(resetConfig)
 	if err != nil {
 		return err

hack/staticcheck.sh

@@ -8,13 +8,13 @@ set -o nounset
 set -o pipefail
 
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
-GOLANGCI_LINT_PKG="github.com/golangci/golangci-lint/cmd/golangci-lint"
-GOLANGCI_LINT_VER="v1.52.2"
+GOLANGCI_LINT_VER="v2.1.0"
 
 cd "${REPO_ROOT}"
 source "hack/util.sh"
 
-util::install_tools ${GOLANGCI_LINT_PKG} ${GOLANGCI_LINT_VER}
+# binary will be $(go env GOPATH)/bin/golangci-lint
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin $GOLANGCI_LINT_VER
 
 if golangci-lint run --fix --verbose; then
   echo 'Congratulations!  All Go source files have passed staticcheck.'

pkg/controllers/clusterops/controller.go

@@ -13,6 +13,7 @@ import (
 	"time"
 	"unicode"
 
+	"github.com/kubean-io/kubean/pkg/crypto"
 	"github.com/kubean-io/kubean/pkg/util"
 	"github.com/kubean-io/kubean/pkg/util/entrypoint"
 
@@ -382,14 +383,26 @@ func (c *Controller) NewKubesprayJob(clusterOps *clusteroperationv1alpha1.Cluste
 					ServiceAccountName: serviceAccountName,
 					Containers: []corev1.Container{
 						{
-							Name:    SprayJobPodName,
-							Image:   c.ProcessKubeanOperationImage(clusterOps.Spec.Image, c.FetchGlobalManifestImageTag()),
-							Command: []string{"/bin/entrypoint.sh"},
+							Name:            SprayJobPodName,
+							Image:           c.ProcessKubeanOperationImage(clusterOps.Spec.Image, c.FetchGlobalManifestImageTag()),
+							ImagePullPolicy: corev1.PullIfNotPresent,
+							Command:         []string{"/bin/entrypoint.sh"},
 							Env: []corev1.EnvVar{
 								{
 									Name:  "CLUSTER_NAME",
 									Value: clusterOps.Spec.Cluster,
 								},
+								{
+									Name: "VAULT_PRIVATE_KEY",
+									ValueFrom: &corev1.EnvVarSource{
+										ConfigMapKeyRef: &corev1.ConfigMapKeySelector{
+											LocalObjectReference: corev1.LocalObjectReference{
+												Name: constants.KubeanConfigMapName,
+											},
+											Key: crypto.PrivateKey,
+										},
+									},
+								},
 							},
 							VolumeMounts: []corev1.VolumeMount{
 								{
@@ -480,27 +493,6 @@ func (c *Controller) NewKubesprayJob(clusterOps *clusteroperationv1alpha1.Cluste
 				},
 			})
 	}
-	if vaultRef := c.getVaultSecret(clusterOps); vaultRef != nil {
-		if len(job.Spec.Template.Spec.Containers) > 0 && job.Spec.Template.Spec.Containers[0].Name == SprayJobPodName {
-			job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts,
-				corev1.VolumeMount{
-					Name:      "vault-password",
-					MountPath: "/auth/vault-password",
-					SubPath:   "vault-password",
-					ReadOnly:  true,
-				})
-		}
-		job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes,
-			corev1.Volume{
-				Name: "vault-password",
-				VolumeSource: corev1.VolumeSource{
-					Secret: &corev1.SecretVolumeSource{
-						SecretName:  vaultRef.Name,
-						DefaultMode: &PrivatekeyMode,
-					},
-				},
-			})
-	}
 	if clusterOps.Spec.ActiveDeadlineSeconds != nil && *clusterOps.Spec.ActiveDeadlineSeconds > 0 {
 		job.Spec.ActiveDeadlineSeconds = clusterOps.Spec.ActiveDeadlineSeconds
 	}
@@ -638,7 +630,8 @@ func (c *Controller) CreateEntryPointShellConfigMap(clusterOps *clusteroperation
 	if !clusterOps.Spec.EntrypointSHRef.IsEmpty() {
 		return false, nil
 	}
-	entryPointData := entrypoint.NewEntryPoint(c.getVaultSecret(clusterOps) != nil)
+
+	entryPointData := entrypoint.NewEntryPoint()
 	isPrivateKey := !clusterOps.Spec.SSHAuthRef.IsEmpty()
 	builtinActionSource := clusteroperationv1alpha1.BuiltinActionSource
 	for _, action := range clusterOps.Spec.PreHook {
@@ -798,9 +791,8 @@ func (c *Controller) CopyConfigMap(clusterOps *clusteroperationv1alpha1.ClusterO
 			APIVersion: "v1",
 		},
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        newName,
-			Namespace:   namespace,
-			Annotations: oldConfigMap.Annotations,
+			Name:      newName,
+			Namespace: namespace,
 		},
 		Data: oldConfigMap.Data,
 	}
@@ -1010,25 +1002,3 @@ func (c *Controller) CheckClusterDataRef(cluster *clusterv1alpha1.Cluster, clust
 	}
 	return nil
 }
-
-func (c *Controller) getVaultSecret(clusterOps *clusteroperationv1alpha1.ClusterOperation) *apis.SecretRef {
-	if clusterOps.Spec.HostsConfRef.IsEmpty() {
-		return nil
-	}
-	hostsConf, err := c.ClientSet.CoreV1().ConfigMaps(clusterOps.Spec.HostsConfRef.NameSpace).Get(context.Background(), clusterOps.Spec.HostsConfRef.Name, metav1.GetOptions{})
-	if err != nil {
-		return nil
-	}
-	vaultRef, ok := hostsConf.Annotations[constants.AnnotationHostsConfVaultPasswordRef]
-	if !ok || vaultRef == "" {
-		return nil
-	}
-	if !c.CheckSecretExist(util.GetCurrentNSOrDefault(), vaultRef) {
-		klog.Warningf("vault password ref %s not found in namespace %s", vaultRef, util.GetCurrentNSOrDefault())
-		return nil
-	}
-	return &apis.SecretRef{
-		NameSpace: util.GetCurrentNSOrDefault(),
-		Name:      vaultRef,
-	}
-}

pkg/controllers/clusterops/controller_test.go

@@ -2923,102 +2923,3 @@ func (MockManager) GetLogger() logr.Logger { return logr.Logger{} }
 func (MockManager) GetControllerOptions() v1alpha1.ControllerConfigurationSpec {
 	return v1alpha1.ControllerConfigurationSpec{}
 }
-
-func TestGetVaultSecret(t *testing.T) {
-	controller := Controller{
-		Client:    newFakeClient(),
-		ClientSet: clientsetfake.NewSimpleClientset(),
-	}
-	clusterOps := &clusteroperationv1alpha1.ClusterOperation{}
-
-	tests := []struct {
-		name  string
-		setup func()
-		want  *apis.SecretRef
-	}{
-		{
-			name: "No HostsConfRef",
-			setup: func() {
-				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{}
-			},
-			want: nil,
-		},
-		{
-			name: "ConfigMap Not Found",
-			setup: func() {
-				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "nonexistent"}
-			},
-			want: nil,
-		},
-		{
-			name: "No Vault Annotation",
-			setup: func() {
-				cm := &corev1.ConfigMap{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: "default",
-						Name:      "hostsconf",
-					},
-				}
-				controller.ClientSet.CoreV1().ConfigMaps("default").Create(context.Background(), cm, metav1.CreateOptions{})
-				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "hostsconf"}
-			},
-			want: nil,
-		},
-		{
-			name: "Vault Secret Not Found",
-			setup: func() {
-				cm := &corev1.ConfigMap{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace:   "default",
-						Name:        "hostsconf",
-						Annotations: map[string]string{constants.AnnotationHostsConfVaultPasswordRef: "vault-secret"},
-					},
-				}
-				controller.ClientSet.CoreV1().ConfigMaps("default").Create(context.Background(), cm, metav1.CreateOptions{})
-				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "hostsconf"}
-			},
-			want: nil,
-		},
-		{
-			name: "Successful Retrieval",
-			setup: func() {
-				cm := &corev1.ConfigMap{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace:   "default",
-						Name:        "hostsconf1",
-						Annotations: map[string]string{constants.AnnotationHostsConfVaultPasswordRef: "vault-secret"},
-					},
-					Data: map[string]string{
-						"vault-password": "vault-password",
-					},
-				}
-				_, err := controller.ClientSet.CoreV1().ConfigMaps("default").Create(context.Background(), cm, metav1.CreateOptions{})
-				if err != nil {
-					t.Fatalf("failed to create config map: %v", err)
-				}
-				secret := &corev1.Secret{
-					ObjectMeta: metav1.ObjectMeta{
-						Namespace: "default",
-						Name:      "vault-secret",
-					},
-				}
-				_, err = controller.ClientSet.CoreV1().Secrets("default").Create(context.Background(), secret, metav1.CreateOptions{})
-				if err != nil {
-					t.Fatalf("failed to create secret: %v", err)
-				}
-				clusterOps.Spec.HostsConfRef = &apis.ConfigMapRef{NameSpace: "default", Name: "hostsconf1"}
-			},
-			want: &apis.SecretRef{NameSpace: "default", Name: "vault-secret"},
-		},
-	}
-
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			test.setup()
-			got := controller.getVaultSecret(clusterOps)
-			if !reflect.DeepEqual(got, test.want) {
-				t.Fatalf("got %v, want %v", got, test.want)
-			}
-		})
-	}
-}

pkg/crypto/crypto.go

@@ -0,0 +1,56 @@
+package crypto
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	klog "k8s.io/klog/v2"
+
+	"github.com/kubean-io/kubean-api/constants"
+	"github.com/kubean-io/kubean/pkg/util"
+)
+
+const (
+	PrivateKey = "sk"
+	PublicKey  = "pk"
+)
+
+func InitConfiguration(clientset kubernetes.Interface) error {
+	kubeanConfig, err := clientset.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Get(context.Background(), constants.KubeanConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		return err
+	}
+	if privateKey, ok := kubeanConfig.Data[PrivateKey]; ok && privateKey != "" {
+		return nil
+	}
+
+	sk, pk, err := util.GenerateRSAKeyPairB64()
+	if err != nil {
+		return err
+	}
+
+	klog.Infof("inject %s into %s", PrivateKey, constants.KubeanConfigMapName)
+	kubeanConfig.Data[PrivateKey] = sk
+	_, err = clientset.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Update(context.Background(), kubeanConfig, metav1.UpdateOptions{})
+	if err != nil {
+		return err
+	}
+
+	kubeanPubkey := &corev1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      constants.KubeanPubKeyConfigMapName,
+			Namespace: util.GetCurrentNSOrDefault(),
+		},
+		Data: map[string]string{
+			PublicKey: pk,
+		},
+	}
+	klog.Infof("create %s", kubeanPubkey.Name)
+	_, err = clientset.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Create(context.Background(), kubeanPubkey, metav1.CreateOptions{})
+	if err != nil {
+		return err
+	}
+	return nil
+}