adjust the Priority of General and Accurate Estimator of Karmada Scheduler

Signed-off-by: chaosi-zju <[email protected]>

Author: chaosi-zju

pkg/descheduler/core/helper.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"math"
+	"sort"
 	"time"
 
 	"github.com/kr/pretty"
@@ -62,24 +63,39 @@ func NewSchedulingResultHelper(binding *workv1alpha2.ResourceBinding) *Schedulin
 func (h *SchedulingResultHelper) FillUnschedulableReplicas(unschedulableThreshold time.Duration) {
 	reference := &h.Spec.Resource
 	undesiredClusters, undesiredClusterNames := h.GetUndesiredClusters()
+
 	// Set the boundary.
 	for i := range undesiredClusters {
 		undesiredClusters[i].Unschedulable = math.MaxInt32
 	}
-	// Get the minimum value of MaxAvailableReplicas in terms of all estimators.
+
+	// get estimators and sort it by priority.
 	estimators := estimatorclient.GetUnschedulableReplicaEstimators()
+	sort.Slice(estimators, func(i, j int) bool {
+		return estimators[i].GetPriority() > estimators[j].GetPriority()
+	})
+
+	priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32)
 	ctx := context.WithValue(context.TODO(), util.ContextKeyObject,
 		fmt.Sprintf("kind=%s, name=%s/%s", reference.Kind, reference.Namespace, reference.Name))
+
 	for _, estimator := range estimators {
+		// if higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator.
+		if estimator.GetPriority() < priorityOfAvailableEstimator && clustersFullyEstimated(undesiredClusters) {
+			break
+		}
 		res, err := estimator.GetUnschedulableReplicas(ctx, undesiredClusterNames, reference, unschedulableThreshold)
 		if err != nil {
 			klog.Errorf("Max cluster unschedulable replicas error: %v", err)
 			continue
 		}
+		priorityOfAvailableEstimator = estimator.GetPriority()
 		for i := range res {
+			// if an estimator can only give a part of result of member clusters, same priority estimator will be called.
 			if res[i].Replicas == estimatorclient.UnauthenticReplica {
 				continue
 			}
+			// if two estimators are of the same priority, call both and choose the minimum value of each estimated result.
 			if undesiredClusters[i].ClusterName == res[i].Name && undesiredClusters[i].Unschedulable > res[i].Replicas {
 				undesiredClusters[i].Unschedulable = res[i].Replicas
 			}
@@ -95,6 +111,16 @@ func (h *SchedulingResultHelper) FillUnschedulableReplicas(unschedulableThreshol
 	klog.V(4).Infof("Target undesired cluster of unschedulable replica result: %s", pretty.Sprint(undesiredClusters))
 }
 
+// clustersFullyEstimated whether estimators has gaven a full result of member clusters
+func clustersFullyEstimated(undesiredClusters []*TargetClusterWrapper) bool {
+	for i := range undesiredClusters {
+		if undesiredClusters[i].Unschedulable == math.MaxInt32 {
+			return false
+		}
+	}
+	return true
+}
+
 // GetUndesiredClusters returns the cluster which of ready replicas are not reach the ready ones.
 func (h *SchedulingResultHelper) GetUndesiredClusters() ([]*TargetClusterWrapper, []string) {
 	var clusters []*TargetClusterWrapper

pkg/descheduler/descheduler_test.go

@@ -487,6 +487,7 @@ func TestDescheduler_worker(t *testing.T) {
 			}
 			schedulerEstimator := estimatorclient.NewSchedulerEstimator(desched.schedulerEstimatorCache, 5*time.Second)
 			estimatorclient.RegisterSchedulerEstimator(schedulerEstimator)
+			defer estimatorclient.UnRegisterSchedulerEstimator(schedulerEstimator)
 
 			for _, c := range tt.args.unschedulable {
 				cluster := c

pkg/estimator/client/accurate.go

@@ -32,8 +32,24 @@ import (
 
 // RegisterSchedulerEstimator will register a SchedulerEstimator.
 func RegisterSchedulerEstimator(se *SchedulerEstimator) {
-	replicaEstimators["scheduler-estimator"] = se
-	unschedulableReplicaEstimators["scheduler-estimator"] = se
+	replicaEstimators = append(replicaEstimators, se)
+	unschedulableReplicaEstimators = append(unschedulableReplicaEstimators, se)
+}
+
+// UnRegisterSchedulerEstimator will unregister a SchedulerEstimator.
+func UnRegisterSchedulerEstimator(se *SchedulerEstimator) {
+	for i := 0; i < len(replicaEstimators); i++ {
+		if replicaEstimators[i] == se {
+			replicaEstimators = append(replicaEstimators[:i], replicaEstimators[i+1:]...)
+			i--
+		}
+	}
+	for i := 0; i < len(unschedulableReplicaEstimators); i++ {
+		if unschedulableReplicaEstimators[i] == se {
+			unschedulableReplicaEstimators = append(unschedulableReplicaEstimators[:i], unschedulableReplicaEstimators[i+1:]...)
+			i--
+		}
+	}
 }
 
 type getClusterReplicasFunc func(ctx context.Context, cluster string) (int32, error)
@@ -67,6 +83,10 @@ func (se *SchedulerEstimator) MaxAvailableReplicas(
 	})
 }
 
+func (se *SchedulerEstimator) GetPriority() EstimatorPriority {
+	return Accurate
+}
+
 // GetUnschedulableReplicas gets the unschedulable replicas which belong to a specified workload by calling karmada-scheduler-estimator.
 func (se *SchedulerEstimator) GetUnschedulableReplicas(
 	parentCtx context.Context,

pkg/estimator/client/general.go

@@ -32,7 +32,7 @@ import (
 
 // GeneralEstimator is the default replica estimator.
 func init() {
-	replicaEstimators["general-estimator"] = NewGeneralEstimator()
+	replicaEstimators = append(replicaEstimators, NewGeneralEstimator())
 }
 
 // GeneralEstimator is a normal estimator in terms of cluster ResourceSummary.
@@ -53,6 +53,10 @@ func (ge *GeneralEstimator) MaxAvailableReplicas(_ context.Context, clusters []*
 	return availableTargetClusters, nil
 }
 
+func (ge *GeneralEstimator) GetPriority() EstimatorPriority {
+	return General
+}
+
 func (ge *GeneralEstimator) maxAvailableReplicas(cluster *clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) int32 {
 	resourceSummary := cluster.Status.ResourceSummary
 	if resourceSummary == nil {

pkg/estimator/client/interface.go

@@ -30,26 +30,38 @@ import (
 const UnauthenticReplica = -1
 
 var (
-	replicaEstimators              = map[string]ReplicaEstimator{}
-	unschedulableReplicaEstimators = map[string]UnschedulableReplicaEstimator{}
+	replicaEstimators              []ReplicaEstimator
+	unschedulableReplicaEstimators []UnschedulableReplicaEstimator
 )
 
 // ReplicaEstimator is an estimator which estimates the maximum replicas that can be applied to the target cluster.
 type ReplicaEstimator interface {
 	MaxAvailableReplicas(ctx context.Context, clusters []*clusterv1alpha1.Cluster, replicaRequirements *workv1alpha2.ReplicaRequirements) ([]workv1alpha2.TargetCluster, error)
+	GetPriority() EstimatorPriority
 }
 
 // UnschedulableReplicaEstimator is an estimator which estimates the unschedulable replicas which belong to a specified workload.
 type UnschedulableReplicaEstimator interface {
 	GetUnschedulableReplicas(ctx context.Context, clusters []string, reference *workv1alpha2.ObjectReference, unschedulableThreshold time.Duration) ([]workv1alpha2.TargetCluster, error)
+	GetPriority() EstimatorPriority
 }
 
 // GetReplicaEstimators returns all replica estimators.
-func GetReplicaEstimators() map[string]ReplicaEstimator {
+func GetReplicaEstimators() []ReplicaEstimator {
 	return replicaEstimators
 }
 
 // GetUnschedulableReplicaEstimators returns all unschedulable replica estimators.
-func GetUnschedulableReplicaEstimators() map[string]UnschedulableReplicaEstimator {
+func GetUnschedulableReplicaEstimators() []UnschedulableReplicaEstimator {
 	return unschedulableReplicaEstimators
 }
+
+// EstimatorPriority
+// 1. If two estimators are of the same priority, call both and choose the minimum value of each estimated result.
+// 2. If higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator.
+type EstimatorPriority int32
+
+const (
+	General  EstimatorPriority = 10
+	Accurate EstimatorPriority = 20
+)

pkg/scheduler/core/util.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"math"
+	"sort"
 
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
@@ -68,20 +69,33 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha
 		return availableTargetClusters
 	}
 
-	// Get the minimum value of MaxAvailableReplicas in terms of all estimators.
+	// get estimators and sort it by priority.
 	estimators := estimatorclient.GetReplicaEstimators()
+	sort.Slice(estimators, func(i, j int) bool {
+		return estimators[i].GetPriority() > estimators[j].GetPriority()
+	})
+
+	priorityOfAvailableEstimator := estimatorclient.EstimatorPriority(math.MinInt32)
 	ctx := context.WithValue(context.TODO(), util.ContextKeyObject,
 		fmt.Sprintf("kind=%s, name=%s/%s", spec.Resource.Kind, spec.Resource.Namespace, spec.Resource.Name))
+
 	for _, estimator := range estimators {
+		// if higher-priority estimators have formed a full result of member clusters, no longer to call lower-priority estimator.
+		if estimator.GetPriority() < priorityOfAvailableEstimator && ClustersFullyEstimated(availableTargetClusters) {
+			break
+		}
 		res, err := estimator.MaxAvailableReplicas(ctx, clusters, spec.ReplicaRequirements)
 		if err != nil {
 			klog.Errorf("Max cluster available replicas error: %v", err)
 			continue
 		}
+		priorityOfAvailableEstimator = estimator.GetPriority()
 		for i := range res {
+			// if an estimator can only give a part of result of member clusters, same priority estimator will be called.
 			if res[i].Replicas == estimatorclient.UnauthenticReplica {
 				continue
 			}
+			// if two estimators are of the same priority, call both and choose the minimum value of each estimated result.
 			if availableTargetClusters[i].Name == res[i].Name && availableTargetClusters[i].Replicas > res[i].Replicas {
 				availableTargetClusters[i].Replicas = res[i].Replicas
 			}
@@ -100,6 +114,16 @@ func calAvailableReplicas(clusters []*clusterv1alpha1.Cluster, spec *workv1alpha
 	return availableTargetClusters
 }
 
+// ClustersFullyEstimated whether estimators has gaven a full result of member clusters
+func ClustersFullyEstimated(availableTargetClusters []workv1alpha2.TargetCluster) bool {
+	for i := range availableTargetClusters {
+		if availableTargetClusters[i].Replicas == math.MaxInt32 {
+			return false
+		}
+	}
+	return true
+}
+
 // attachZeroReplicasCluster  attach cluster in clusters into targetCluster
 // The purpose is to avoid workload not appeared in rb's spec.clusters field
 func attachZeroReplicasCluster(clusters []*clusterv1alpha1.Cluster, targetClusters []workv1alpha2.TargetCluster) []workv1alpha2.TargetCluster {