Skip to content

Commit 7c43481

Browse files
davidLifdavidLif
authored
Add E2E k8s native integrations (#25)
* Add E2E k8s native integrations
1 parent 675fd90 commit 7c43481

File tree

7 files changed

+587
-4
lines changed

7 files changed

+587
-4
lines changed

‎test/e2e/modules/resources/rd/batch_job.go‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@ import (
88
"context"
99
"fmt"
1010

11-
"github.com/onsi/gomega"
11+
. "github.com/onsi/gomega"
1212
batchv1 "k8s.io/api/batch/v1"
1313
v1 "k8s.io/api/core/v1"
1414
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1515
"k8s.io/client-go/kubernetes"
1616
"knative.dev/pkg/ptr"
17+
"sigs.k8s.io/controller-runtime/pkg/client"
1718
runtimeClient "sigs.k8s.io/controller-runtime/pkg/client"
1819

1920
v2 "github.com/NVIDIA/KAI-scheduler/pkg/apis/scheduling/v2"
@@ -58,7 +59,7 @@ func GetJobPods(ctx context.Context, client *kubernetes.Clientset, job *batchv1.
5859
pods, err := client.CoreV1().Pods(job.Namespace).List(ctx, metav1.ListOptions{
5960
LabelSelector: fmt.Sprintf("%s=%s", BatchJobAppLabel, job.Labels[BatchJobAppLabel]),
6061
})
61-
gomega.Expect(err).To(gomega.Succeed())
62+
Expect(err).To(Succeed())
6263
return pods.Items
6364
}
6465

@@ -68,10 +69,10 @@ func DeleteJob(ctx context.Context, client *kubernetes.Clientset, job *batchv1.J
6869
PropagationPolicy: &propagationPolicy,
6970
GracePeriodSeconds: ptr.Int64(0),
7071
})
71-
gomega.Expect(err).To(gomega.Succeed())
72+
Expect(err).To(Succeed())
7273
}
7374

74-
func DeleteAllJobsInNamespace(ctx context.Context, client runtimeClient.Client, namespace string) error {
75+
func DeleteAllJobsInNamespace(ctx context.Context, client client.Client, namespace string) error {
7576
err := client.DeleteAllOf(
7677
ctx, &batchv1.Job{},
7778
runtimeClient.InNamespace(namespace),

‎test/e2e/modules/resources/rd/cronjob.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
Copyright 2025 NVIDIA CORPORATION
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package rd
6+
7+
import (
8+
batchv1 "k8s.io/api/batch/v1"
9+
v1 "k8s.io/api/core/v1"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"k8s.io/utils/pointer"
12+
13+
"github.com/NVIDIA/KAI-scheduler/pkg/common/constants"
14+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/constant"
15+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/utils"
16+
)
17+
18+
const CronJobAppLabel = "cron-job-app-name"
19+
20+
func CreateCronJobObject(namespace, queueName string) *batchv1.CronJob {
21+
matchLabelValue := utils.GenerateRandomK8sName(10)
22+
23+
return &batchv1.CronJob{
24+
TypeMeta: metav1.TypeMeta{
25+
APIVersion: "batch/v1",
26+
Kind: "CronJob",
27+
},
28+
ObjectMeta: metav1.ObjectMeta{
29+
Name: utils.GenerateRandomK8sName(10),
30+
Namespace: namespace,
31+
Labels: map[string]string{
32+
constants.AppLabelName: "engine-e2e",
33+
CronJobAppLabel: matchLabelValue,
34+
},
35+
},
36+
Spec: batchv1.CronJobSpec{
37+
Schedule: "* * * * *",
38+
JobTemplate: batchv1.JobTemplateSpec{
39+
Spec: batchv1.JobSpec{
40+
Template: v1.PodTemplateSpec{
41+
ObjectMeta: metav1.ObjectMeta{
42+
Labels: map[string]string{
43+
constants.AppLabelName: "engine-e2e",
44+
CronJobAppLabel: matchLabelValue,
45+
"runai/queue": queueName,
46+
},
47+
},
48+
Spec: v1.PodSpec{
49+
RestartPolicy: v1.RestartPolicyNever,
50+
SchedulerName: constant.RunaiSchedulerName,
51+
TerminationGracePeriodSeconds: pointer.Int64(0),
52+
Containers: []v1.Container{
53+
{
54+
Image: "gcr.io/run-ai-lab/ubuntu",
55+
Name: "ubuntu-container",
56+
Args: []string{
57+
"sleep",
58+
"infinity",
59+
},
60+
SecurityContext: DefaultSecurityContext(),
61+
},
62+
},
63+
Tolerations: []v1.Toleration{
64+
{
65+
Key: "nvidia.com/gpu",
66+
Operator: v1.TolerationOpExists,
67+
Effect: v1.TaintEffectNoSchedule,
68+
},
69+
},
70+
},
71+
},
72+
},
73+
},
74+
},
75+
}
76+
}

‎test/e2e/modules/resources/rd/deployment.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
Copyright 2025 NVIDIA CORPORATION
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package rd
6+
7+
import (
8+
appsv1 "k8s.io/api/apps/v1"
9+
v1 "k8s.io/api/core/v1"
10+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"k8s.io/utils/ptr"
12+
13+
"github.com/NVIDIA/KAI-scheduler/pkg/common/constants"
14+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/constant"
15+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/utils"
16+
)
17+
18+
const DeploymentAppLabel = "deployment-app-name"
19+
20+
func CreateDeploymentObject(namespace, queueName string) *appsv1.Deployment {
21+
matchLabelValue := utils.GenerateRandomK8sName(10)
22+
23+
return &appsv1.Deployment{
24+
ObjectMeta: metav1.ObjectMeta{
25+
Name: utils.GenerateRandomK8sName(10),
26+
Namespace: namespace,
27+
Labels: map[string]string{
28+
constants.AppLabelName: "engine-e2e",
29+
DeploymentAppLabel: matchLabelValue,
30+
},
31+
},
32+
Spec: appsv1.DeploymentSpec{
33+
Replicas: ptr.To(int32(1)),
34+
Selector: &metav1.LabelSelector{
35+
MatchLabels: map[string]string{
36+
constants.AppLabelName: "engine-e2e",
37+
DeploymentAppLabel: matchLabelValue,
38+
},
39+
},
40+
Template: v1.PodTemplateSpec{
41+
ObjectMeta: metav1.ObjectMeta{
42+
Labels: map[string]string{
43+
constants.AppLabelName: "engine-e2e",
44+
DeploymentAppLabel: matchLabelValue,
45+
"runai/queue": queueName,
46+
},
47+
},
48+
Spec: v1.PodSpec{
49+
Containers: []v1.Container{
50+
{
51+
Image: "gcr.io/run-ai-lab/ubuntu",
52+
Name: "ubuntu-container",
53+
Args: []string{
54+
"sleep",
55+
"infinity",
56+
},
57+
SecurityContext: DefaultSecurityContext(),
58+
},
59+
},
60+
TerminationGracePeriodSeconds: ptr.To(int64(0)),
61+
SchedulerName: constant.RunaiSchedulerName,
62+
Tolerations: []v1.Toleration{
63+
{
64+
Key: "nvidia.com/gpu",
65+
Operator: v1.TolerationOpExists,
66+
Effect: v1.TaintEffectNoSchedule,
67+
},
68+
},
69+
},
70+
},
71+
},
72+
}
73+
}

‎test/e2e/modules/resources/rd/replicaset.go‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
Copyright 2025 NVIDIA CORPORATION
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package rd
6+
7+
import (
8+
v1 "k8s.io/api/apps/v1"
9+
v12 "k8s.io/api/core/v1"
10+
v13 "k8s.io/apimachinery/pkg/apis/meta/v1"
11+
"k8s.io/utils/pointer"
12+
13+
"github.com/NVIDIA/KAI-scheduler/pkg/common/constants"
14+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/constant"
15+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/utils"
16+
)
17+
18+
const ReplicaSetAppLabel = "replicaset-app-name"
19+
20+
func CreateReplicasetObject(namespace, queueName string) *v1.ReplicaSet {
21+
matchLabelValue := utils.GenerateRandomK8sName(10)
22+
23+
return &v1.ReplicaSet{
24+
ObjectMeta: v13.ObjectMeta{
25+
Name: utils.GenerateRandomK8sName(10),
26+
Namespace: namespace,
27+
Labels: map[string]string{
28+
constants.AppLabelName: "engine-e2e",
29+
ReplicaSetAppLabel: matchLabelValue,
30+
},
31+
},
32+
Spec: v1.ReplicaSetSpec{
33+
Replicas: pointer.Int32(1),
34+
Selector: &v13.LabelSelector{
35+
MatchLabels: map[string]string{
36+
constants.AppLabelName: "engine-e2e",
37+
ReplicaSetAppLabel: matchLabelValue,
38+
},
39+
},
40+
Template: v12.PodTemplateSpec{
41+
ObjectMeta: v13.ObjectMeta{
42+
Labels: map[string]string{
43+
constants.AppLabelName: "engine-e2e",
44+
ReplicaSetAppLabel: matchLabelValue,
45+
"runai/queue": queueName,
46+
},
47+
},
48+
Spec: v12.PodSpec{
49+
Containers: []v12.Container{
50+
{
51+
Image: "gcr.io/run-ai-lab/ubuntu",
52+
Name: "ubuntu-container",
53+
Args: []string{
54+
"sleep",
55+
"infinity",
56+
},
57+
SecurityContext: DefaultSecurityContext(),
58+
},
59+
},
60+
TerminationGracePeriodSeconds: pointer.Int64(0),
61+
SchedulerName: constant.RunaiSchedulerName,
62+
Tolerations: []v12.Toleration{
63+
{
64+
Key: "nvidia.com/gpu",
65+
Operator: v12.TolerationOpExists,
66+
Effect: v12.TaintEffectNoSchedule,
67+
},
68+
},
69+
},
70+
},
71+
},
72+
}
73+
}

‎test/e2e/modules/resources/rd/statefulset.go‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/*
2+
Copyright 2025 NVIDIA CORPORATION
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package rd
7+
8+
import (
9+
v1 "k8s.io/api/apps/v1"
10+
v12 "k8s.io/api/core/v1"
11+
v13 "k8s.io/apimachinery/pkg/apis/meta/v1"
12+
"k8s.io/utils/pointer"
13+
14+
"github.com/NVIDIA/KAI-scheduler/pkg/common/constants"
15+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/constant"
16+
17+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/utils"
18+
)
19+
20+
const StatefulSetAppLabel = "stateful-set-app-name"
21+
22+
func CreateStatefulSetObject(namespace, queueName string) *v1.StatefulSet {
23+
matchLabelValue := utils.GenerateRandomK8sName(10)
24+
25+
return &v1.StatefulSet{
26+
ObjectMeta: v13.ObjectMeta{
27+
Name: utils.GenerateRandomK8sName(10),
28+
Namespace: namespace,
29+
Labels: map[string]string{
30+
constants.AppLabelName: "engine-e2e",
31+
StatefulSetAppLabel: matchLabelValue,
32+
},
33+
},
34+
Spec: v1.StatefulSetSpec{
35+
Replicas: pointer.Int32(1),
36+
Selector: &v13.LabelSelector{
37+
MatchLabels: map[string]string{
38+
constants.AppLabelName: "engine-e2e",
39+
StatefulSetAppLabel: matchLabelValue,
40+
},
41+
},
42+
Template: v12.PodTemplateSpec{
43+
ObjectMeta: v13.ObjectMeta{
44+
Labels: map[string]string{
45+
constants.AppLabelName: "engine-e2e",
46+
StatefulSetAppLabel: matchLabelValue,
47+
"runai/queue": queueName,
48+
},
49+
},
50+
Spec: v12.PodSpec{
51+
Containers: []v12.Container{
52+
{
53+
Image: "gcr.io/run-ai-lab/ubuntu",
54+
Name: "ubuntu-container",
55+
Args: []string{
56+
"sleep",
57+
"infinity",
58+
},
59+
SecurityContext: DefaultSecurityContext(),
60+
},
61+
},
62+
TerminationGracePeriodSeconds: pointer.Int64(0),
63+
SchedulerName: constant.RunaiSchedulerName,
64+
Tolerations: []v12.Toleration{
65+
{
66+
Key: "nvidia.com/gpu",
67+
Operator: v12.TolerationOpExists,
68+
Effect: v12.TaintEffectNoSchedule,
69+
},
70+
},
71+
},
72+
},
73+
},
74+
}
75+
}

‎test/e2e/suites/integrations/k8s_native/k8s_native_suite_test.go‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/*
2+
Copyright 2025 NVIDIA CORPORATION
3+
SPDX-License-Identifier: Apache-2.0
4+
*/
5+
package k8s_native
6+
7+
import (
8+
"testing"
9+
10+
"github.com/NVIDIA/KAI-scheduler/test/e2e/modules/utils"
11+
12+
. "github.com/onsi/ginkgo/v2"
13+
. "github.com/onsi/gomega"
14+
)
15+
16+
func TestK8sNativeIntegrations(t *testing.T) {
17+
utils.SetLogger()
18+
RegisterFailHandler(Fail)
19+
RunSpecs(t, "K8S Native Integrations Suite")
20+
}

0 commit comments

Comments
 (0)