Merge pull request #1348 from KubeKyrie/fix-offline-sprayJob-registry

add SprayJobImageRegistry to kubean-config and update manifest kubesprayImage

Author: KubeKyrie

api/cluster/cluster.go

@@ -0,0 +1,25 @@
+package cluster
+
+import (
+	"github.com/kubean-io/kubean-api/constants"
+	"k8s.io/klog/v2"
+	"strconv"
+)
+
+type ConfigProperty struct {
+	ClusterOperationsBackEndLimit string `json:"CLUSTER_OPERATIONS_BACKEND_LIMIT"`
+	SprayJobImageRegistry         string `json:"SPRAY_JOB_IMAGE_REGISTRY"`
+}
+
+func (config *ConfigProperty) GetClusterOperationsBackEndLimit() int {
+	value, _ := strconv.Atoi(config.ClusterOperationsBackEndLimit)
+	if value <= 0 {
+		klog.Warningf("GetClusterOperationsBackEndLimit and use default value %d", constants.DefaultClusterOperationsBackEndLimit)
+		return constants.DefaultClusterOperationsBackEndLimit
+	}
+	if value >= constants.MaxClusterOperationsBackEndLimit {
+		klog.Warningf("GetClusterOperationsBackEndLimit and use max value %d", constants.MaxClusterOperationsBackEndLimit)
+		return constants.MaxClusterOperationsBackEndLimit
+	}
+	return value
+}

api/constants/constants.go

@@ -1,17 +1,22 @@
 package constants
 
-const InfoManifestGlobal = "manifest-global"
+const (
+	InfoManifestGlobal = "manifest-global"
 
-const KubeanClusterLabelKey = "clusterName"
+	KubeanClusterLabelKey = "clusterName"
 
-const Hosts_yml = "hosts.yml"
+	Hosts_yml = "hosts.yml"
 
-const Group_vars_yml = "group_vars.yml"
+	Group_vars_yml = "group_vars.yml"
 
-const SSH_privatekey = "ssh-privatekey"
+	SSH_privatekey = "ssh-privatekey"
 
-const KubeanClusterHasCompleted = "hasCompleted"
+	KubeanClusterHasCompleted = "hasCompleted"
 
-const KeySprayRelease = "kubean.io/sprayRelease"
+	KeySprayRelease = "kubean.io/sprayRelease"
+	KeySprayCommit  = "kubean.io/sprayCommit"
 
-const KeySprayCommit = "kubean.io/sprayCommit"
+	KubeanConfigMapName                  = "kubean-config"
+	DefaultClusterOperationsBackEndLimit = 30
+	MaxClusterOperationsBackEndLimit     = 200
+)

charts/kubean/templates/configmap.yaml

@@ -8,3 +8,4 @@ metadata:
   {{- include "kubean.labels" . | nindent 4}}
 data:
   CLUSTER_OPERATIONS_BACKEND_LIMIT: "{{ .Values.kubeanOperator.operationsBackendLimit }}"
+  SPRAY_JOB_IMAGE_REGISTRY: "{{ .Values.sprayJob.image.registry }}"

pkg/controllers/cluster/controller.go

@@ -5,7 +5,6 @@ package cluster
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"sort"
 	"strconv"
@@ -27,11 +26,9 @@ import (
 )
 
 const (
-	RequeueAfter                         = time.Second * 15
-	KubeanConfigMapName                  = "kubean-config"
-	DefaultClusterOperationsBackEndLimit = 30
-	MaxClusterOperationsBackEndLimit     = 200
-	EliminateScoreAnno                   = "kubean.io/eliminate-score"
+	RequeueAfter        = time.Second * 15
+	KubeanConfigMapName = "kubean-config"
+	EliminateScoreAnno  = "kubean.io/eliminate-score"
 )
 
 type Controller struct {
@@ -149,40 +146,6 @@ func (c *Controller) CleanExcessClusterOps(cluster *clusterv1alpha1.Cluster, Ops
 	return true, nil
 }
 
-type ConfigProperty struct {
-	ClusterOperationsBackEndLimit string `json:"CLUSTER_OPERATIONS_BACKEND_LIMIT"`
-}
-
-func (config *ConfigProperty) GetClusterOperationsBackEndLimit() int {
-	value, _ := strconv.Atoi(config.ClusterOperationsBackEndLimit)
-	if value <= 0 {
-		klog.Warningf("GetClusterOperationsBackEndLimit and use default value %d", DefaultClusterOperationsBackEndLimit)
-		return DefaultClusterOperationsBackEndLimit
-	}
-	if value >= MaxClusterOperationsBackEndLimit {
-		klog.Warningf("GetClusterOperationsBackEndLimit and use max value %d", MaxClusterOperationsBackEndLimit)
-		return MaxClusterOperationsBackEndLimit
-	}
-	return value
-}
-
-func (c *Controller) FetchKubeanConfigProperty() *ConfigProperty {
-	configData, err := c.ClientSet.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Get(context.Background(), KubeanConfigMapName, metav1.GetOptions{})
-	if err != nil {
-		return &ConfigProperty{}
-	}
-	jsonData, err := json.Marshal(configData.Data)
-	if err != nil {
-		return &ConfigProperty{}
-	}
-	result := &ConfigProperty{}
-	err = json.Unmarshal(jsonData, result)
-	if err != nil {
-		return &ConfigProperty{}
-	}
-	return result
-}
-
 func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Request) (controllerruntime.Result, error) {
 	cluster := &clusterv1alpha1.Cluster{}
 	if err := c.Client.Get(ctx, req.NamespacedName, cluster); err != nil {
@@ -192,7 +155,7 @@ func (c *Controller) Reconcile(ctx context.Context, req controllerruntime.Reques
 		klog.ErrorS(err, "failed to get cluster", "cluster", req.String())
 		return controllerruntime.Result{RequeueAfter: RequeueAfter}, nil
 	}
-	OpsBackupNum := c.FetchKubeanConfigProperty().GetClusterOperationsBackEndLimit()
+	OpsBackupNum := util.FetchKubeanConfigProperty(c.ClientSet).GetClusterOperationsBackEndLimit()
 	needRequeue, err := c.CleanExcessClusterOps(cluster, OpsBackupNum)
 	if err != nil {
 		klog.ErrorS(err, "failed to clean excess cluster ops", "cluster", cluster.Name)

pkg/controllers/cluster/controller_test.go

@@ -7,16 +7,10 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"os"
 	"reflect"
 	"testing"
 	"time"
 
-	localartifactsetv1alpha1 "github.com/kubean-io/kubean-api/apis/localartifactset/v1alpha1"
-	manifestv1alpha1 "github.com/kubean-io/kubean-api/apis/manifest/v1alpha1"
-	"github.com/kubean-io/kubean/pkg/util"
-
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
@@ -33,6 +27,8 @@ import (
 	"github.com/kubean-io/kubean-api/apis"
 	clusterv1alpha1 "github.com/kubean-io/kubean-api/apis/cluster/v1alpha1"
 	clusteroperationv1alpha1 "github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1"
+	localartifactsetv1alpha1 "github.com/kubean-io/kubean-api/apis/localartifactset/v1alpha1"
+	manifestv1alpha1 "github.com/kubean-io/kubean-api/apis/manifest/v1alpha1"
 	"github.com/kubean-io/kubean-api/constants"
 	clusterv1alpha1fake "github.com/kubean-io/kubean-api/generated/cluster/clientset/versioned/fake"
 	clusteroperationv1alpha1fake "github.com/kubean-io/kubean-api/generated/clusteroperation/clientset/versioned/fake"
@@ -177,66 +173,6 @@ func TestCompareClusterConditions(t *testing.T) {
 	}
 }
 
-func TestFetchKubeanConfigProperty(t *testing.T) {
-	genController := func() *Controller {
-		return &Controller{
-			Client:              newFakeClient(),
-			ClientSet:           clientsetfake.NewSimpleClientset(),
-			KubeanClusterSet:    clusterv1alpha1fake.NewSimpleClientset(),
-			KubeanClusterOpsSet: clusteroperationv1alpha1fake.NewSimpleClientset(),
-		}
-	}
-	tests := []struct {
-		name       string
-		controller *Controller
-		args       func(controller *Controller)
-		want       int
-	}{
-		{
-			name:       "no config and default value",
-			controller: genController(),
-			args: func(controller *Controller) {
-				// do nothing
-			},
-			want: 30,
-		},
-		{
-			name:       "the config has been set to 25",
-			controller: genController(),
-			args: func(controller *Controller) {
-				os.Setenv("POD_NAMESPACE", "")
-				configMap := &corev1.ConfigMap{}
-				configMap.Name = KubeanConfigMapName
-				configMap.Namespace = util.GetCurrentNSOrDefault()
-				configMap.Data = map[string]string{"CLUSTER_OPERATIONS_BACKEND_LIMIT": "25"}
-				controller.ClientSet.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Create(context.Background(), configMap, metav1.CreateOptions{})
-			},
-			want: 25,
-		},
-		{
-			name:       "the config has been set to 10000",
-			controller: genController(),
-			args: func(controller *Controller) {
-				os.Setenv("POD_NAMESPACE", "")
-				configMap := &corev1.ConfigMap{}
-				configMap.Name = KubeanConfigMapName
-				configMap.Namespace = util.GetCurrentNSOrDefault()
-				configMap.Data = map[string]string{"CLUSTER_OPERATIONS_BACKEND_LIMIT": "10000"}
-				controller.ClientSet.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Create(context.Background(), configMap, metav1.CreateOptions{})
-			},
-			want: 200,
-		},
-	}
-	for _, test := range tests {
-		t.Run(test.name, func(t *testing.T) {
-			test.args(test.controller)
-			if test.controller.FetchKubeanConfigProperty().GetClusterOperationsBackEndLimit() != test.want {
-				t.Fatal()
-			}
-		})
-	}
-}
-
 func TestSortClusterOperationsByCreation(t *testing.T) {
 	controller := &Controller{
 		Client:              newFakeClient(),

pkg/controllers/infomanifest/controller.go

@@ -190,11 +190,8 @@ func (c *Controller) UpdateLocalService(manifests []manifestv1alpha1.Manifest) b
 
 // UpdateLocalAvailableImage update image infos into status.
 func (c *Controller) UpdateLocalAvailableImage(manifests []manifestv1alpha1.Manifest) {
+	imageRepo := util.FetchKubeanConfigProperty(c.ClientSet).SprayJobImageRegistry
 	for _, manifest := range manifests {
-		imageRepo := "ghcr.m.daocloud.io"
-		if len(manifest.Spec.LocalService.GetGHCRImageRepo()) != 0 {
-			imageRepo = manifest.Spec.LocalService.GetGHCRImageRepo() // ghcr.io
-		}
 		var newImageName string
 		sprayRelease := manifest.ObjectMeta.Annotations[constants.KeySprayRelease]
 		sprayCommit := manifest.ObjectMeta.Annotations[constants.KeySprayCommit]

pkg/controllers/infomanifest/controller_test.go

@@ -7,6 +7,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"reflect"
 	"testing"
 	"time"
@@ -37,6 +38,7 @@ import (
 	"github.com/kubean-io/kubean-api/constants"
 	localartifactsetv1alpha1fake "github.com/kubean-io/kubean-api/generated/localartifactset/clientset/versioned/fake"
 	manifestv1alpha1fake "github.com/kubean-io/kubean-api/generated/manifest/clientset/versioned/fake"
+	"github.com/kubean-io/kubean/pkg/util"
 )
 
 func newFakeClient() client.Client {
@@ -252,33 +254,27 @@ func Test_UpdateLocalAvailableImage(t *testing.T) {
 		want string
 	}{
 		{
-			name: "update local kubespray image with ghcr.io",
+			name: "update local kubespray image with ghcr.m.daocloud.io",
 			arg: func() {
 				manifest.Spec = manifestv1alpha1.Spec{
 					KubeanVersion: "123",
 				}
 				manifest, _ := controller.InfoManifestClientSet.KubeanV1alpha1().Manifests().Update(context.Background(), manifest, metav1.UpdateOptions{})
-				controller.UpdateLocalAvailableImage([]manifestv1alpha1.Manifest{*manifest})
-			},
-			want: "ghcr.m.daocloud.io/kubean-io/spray-job:123",
-		},
-		{
-			name: "update local kubespray image with local registry",
-			arg: func() {
-				manifest.Spec = manifestv1alpha1.Spec{
-					LocalService: manifestv1alpha1.LocalService{ImageRepo: map[manifestv1alpha1.ImageRepoType]string{
-						"dockerImageRepo": "abc.io",
-						"githubImageRepo": "ghcr.io",
-					}},
-					KubeanVersion: "123456",
-				}
-				manifest, err := controller.InfoManifestClientSet.KubeanV1alpha1().Manifests().Update(context.Background(), manifest, metav1.UpdateOptions{})
-				if err != nil {
-					t.Error(err)
+				os.Setenv("POD_NAMESPACE", "")
+				configMap := &corev1.ConfigMap{
+					ObjectMeta: metav1.ObjectMeta{
+						Namespace: util.GetCurrentNSOrDefault(),
+						Name:      "kubean-config",
+					},
+					Data: map[string]string{
+						"CLUSTER_OPERATIONS_BACKEND_LIMIT": "10000",
+						"SPRAY_JOB_IMAGE_REGISTRY":         "ghcr.m.daocloud.io",
+					},
 				}
+				controller.ClientSet.CoreV1().ConfigMaps(util.GetCurrentNSOrDefault()).Create(context.Background(), configMap, metav1.CreateOptions{})
 				controller.UpdateLocalAvailableImage([]manifestv1alpha1.Manifest{*manifest})
 			},
-			want: "ghcr.io/kubean-io/spray-job:123456",
+			want: "ghcr.m.daocloud.io/kubean-io/spray-job:123",
 		},
 	}
 

pkg/util/cluster.go

@@ -5,22 +5,24 @@ package util
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"strings"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
 
 	"github.com/kubean-io/kubean-api/apis"
 	clusterv1alpha1 "github.com/kubean-io/kubean-api/apis/cluster/v1alpha1"
 	clusteroperationv1alpha1 "github.com/kubean-io/kubean-api/apis/clusteroperation/v1alpha1"
 	localartifactsetv1alpha1 "github.com/kubean-io/kubean-api/apis/localartifactset/v1alpha1"
 	manifestv1alpha1 "github.com/kubean-io/kubean-api/apis/manifest/v1alpha1"
-
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/kubernetes/scheme"
+	"github.com/kubean-io/kubean-api/cluster"
+	"github.com/kubean-io/kubean-api/constants"
 )
 
 // aggregatedScheme aggregates Kubernetes and extended schemes.
@@ -119,3 +121,20 @@ func GetCurrentRunningPodName() string {
 	}
 	return "default"
 }
+
+func FetchKubeanConfigProperty(client kubernetes.Interface) *cluster.ConfigProperty {
+	configData, err := client.CoreV1().ConfigMaps(GetCurrentNSOrDefault()).Get(context.Background(), constants.KubeanConfigMapName, metav1.GetOptions{})
+	if err != nil {
+		return &cluster.ConfigProperty{}
+	}
+	jsonData, err := json.Marshal(configData.Data)
+	if err != nil {
+		return &cluster.ConfigProperty{}
+	}
+	result := &cluster.ConfigProperty{}
+	err = json.Unmarshal(jsonData, result)
+	if err != nil {
+		return &cluster.ConfigProperty{}
+	}
+	return result
+}