Add min-runtime configuration to queues (#155)

* add min-runtime configuration to queues

* update CHANGELOG

Author: ArmedGuy

CHANGELOG.md

@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ### Added
 - Added support for [k8s pod scheduling gates](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-scheduling-readiness/)
 - nodeSelector, affinity and tolerations configurable with global value definitions
+- Added `PreemptMinRuntime` and `ReclaimMinRuntime` properties to queue CRD
 - Scheduler now adds a "LastStartTimestamp" to podgroup on allocation
 
 ### Changed

deployments/kai-scheduler/crds/scheduling.run.ai_queues.yaml

@@ -58,11 +58,17 @@ spec:
                 type: string
               parentQueue:
                 type: string
+              preemptMinRuntime:
+                description: Minimum runtime of a job in queue before it can be preempted.
+                type: string
               priority:
                 description: |-
                   Priority of the queue. Over-quota resources will be divided first among queues with higher priority. Queues with
                   higher priority will be considerd first for allocation, and last for reclaim. When not set, default is 100.
                 type: integer
+              reclaimMinRuntime:
+                description: Minimum runtime of a job in queue before it can be reclaimed.
+                type: string
               resources:
                 properties:
                   cpu:

pkg/apis/scheduling/v2/queue_types.go

@@ -20,6 +20,14 @@ type QueueSpec struct {
 	// higher priority will be considerd first for allocation, and last for reclaim. When not set, default is 100.
 	// +optional
 	Priority *int `json:"priority,omitempty"`
+
+	// Minimum runtime of a job in queue before it can be preempted.
+	// +optional
+	PreemptMinRuntime *metav1.Duration `json:"preemptMinRuntime,omitempty"`
+
+	// Minimum runtime of a job in queue before it can be reclaimed.
+	// +optional
+	ReclaimMinRuntime *metav1.Duration `json:"reclaimMinRuntime,omitempty"`
 }
 
 // QueueStatus defines the observed state of Queue

pkg/apis/scheduling/v2/zz_generated.deepcopy.go

@@ -21,6 +21,8 @@ type QueueInfo struct {
 	Resources         QueueQuota
 	Priority          int
 	CreationTimestamp metav1.Time
+	PreemptMinRuntime *metav1.Duration
+	ReclaimMinRuntime *metav1.Duration
 }
 
 func NewQueueInfo(queue *enginev2.Queue) *QueueInfo {
@@ -42,6 +44,8 @@ func NewQueueInfo(queue *enginev2.Queue) *QueueInfo {
 		Resources:         getQueueQuota(*queue),
 		Priority:          priority,
 		CreationTimestamp: queue.CreationTimestamp,
+		PreemptMinRuntime: queue.Spec.PreemptMinRuntime,
+		ReclaimMinRuntime: queue.Spec.ReclaimMinRuntime,
 	}
 }
 

pkg/scheduler/api/queue_info/queue_info.go

@@ -5,6 +5,7 @@ package queue_info
 
 import (
 	"testing"
+	"time"
 
 	"gotest.tools/assert"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -89,6 +90,32 @@ func TestNewQueueInfo(t *testing.T) {
 				CreationTimestamp: metav1.Time{},
 			},
 		},
+		{
+			name: "queue with min-runtime",
+			queue: &enginev2.Queue{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "queue",
+				},
+				Spec: enginev2.QueueSpec{
+					DisplayName:       "",
+					ParentQueue:       "",
+					Resources:         nil,
+					PreemptMinRuntime: &metav1.Duration{Duration: 10 * time.Minute},
+					ReclaimMinRuntime: &metav1.Duration{Duration: 10 * time.Minute},
+				},
+			},
+			expected: QueueInfo{
+				UID:               "queue",
+				Name:              "queue",
+				ParentQueue:       "",
+				ChildQueues:       []common_info.QueueID{},
+				Resources:         QueueQuota{},
+				Priority:          100,
+				CreationTimestamp: metav1.Time{},
+				PreemptMinRuntime: &metav1.Duration{Duration: 10 * time.Minute},
+				ReclaimMinRuntime: &metav1.Duration{Duration: 10 * time.Minute},
+			},
+		},
 		{
 			name: "queue with parent",
 			queue: &enginev2.Queue{