Skip to content

Commit 18c449b

Browse files
t0mmylamt0mmylam
authored
feat: add e2e tests for deployment policy (#112)
* feat: add e2e tests for deployment policy * fix shell issue * update wording * fix webhook timing * replace scripts with Chainsaw assertions in deployment policy tests * consolidate test jobs in ci * add ordered assertions for batch states to tests
1 parent 9c4d961 commit 18c449b

37 files changed

+1589
-82
lines changed

‎.github/workflows/operator-ci.yaml‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,14 +47,22 @@ env:
4747
PLATFORMS: linux/amd64,linux/arm64
4848

4949
jobs:
50-
# Test operator across supported Kubernetes versions
50+
# Test operator across supported Kubernetes versions and test suites
5151
tests:
5252
runs-on: ubuntu-latest
5353
strategy:
5454
matrix:
55-
# Test on all supported K8s versions (matches docs/kubernetes-support.md)
55+
# Standard E2E tests on all supported K8s versions
5656
k8s-version: ["1.31.12", "1.32.8", "1.33.4", "1.34.0"]
57+
test-suite: ["e2e"]
58+
include:
59+
# Deployment policy tests on 15-node cluster (K8s 1.34 only)
60+
- k8s-version: "1.34.0"
61+
test-suite: deployment-policy
62+
kind-config: k8s-tests/chainsaw/deployment-policy/kind-config.yaml
63+
make-target: deployment-policy-tests
5764
fail-fast: false # Continue testing other versions if one fails
65+
name: ${{ matrix.test-suite }}-tests (k8s-${{ matrix.k8s-version }})
5866
steps:
5967
- uses: actions/checkout@v4
6068
with:
@@ -71,13 +79,13 @@ jobs:
7179
registry: ${{ env.REGISTRY }}
7280
username: ${{ github.actor }}
7381
password: ${{ secrets.GITHUB_TOKEN }}
74-
- name: Kubernetes KinD Cluster v${{ matrix.k8s-version }}
82+
- name: Create Kubernetes KinD Cluster v${{ matrix.k8s-version }}
7583
id: kind
7684
uses: helm/kind-action@v1
7785
with:
7886
version: v0.30.0
7987
node_image: kindest/node:v${{ matrix.k8s-version }}
80-
config: operator/config/local-dev/kind-config.yaml
88+
config: ${{ matrix.kind-config || 'operator/config/local-dev/kind-config.yaml' }}
8189
cluster_name: kind
8290
# Cache build tools and dependencies for faster builds
8391
- name: Restore cached Binaries
@@ -103,17 +111,21 @@ jobs:
103111
path: |
104112
${{ github.workspace }}/operator/bin
105113
~/.cache/go-build
106-
# Run full test suite including e2e tests
107-
- name: end-to-end-tests
114+
# Run test suite
115+
- name: Run ${{ matrix.test-suite }} tests
108116
run: |
109117
cd operator
110-
make setup-kind-cluster
111-
make test
118+
if [ "${{ matrix.test-suite }}" = "e2e" ]; then
119+
make setup-kind-cluster
120+
make test
121+
else
122+
make ${{ matrix.make-target }}
123+
fi
112124
113125
# Build multi-platform container image and push to registry
114126
build-and-push-operator:
115127
runs-on: ubuntu-latest
116-
needs: [tests] # Don't run the build and push if the k8s tests fail
128+
needs: [tests] # Don't run the build and push if tests fail
117129
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
118130
permissions:
119131
contents: read

‎k8s-tests/chainsaw/deployment-policy/kind-config.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
# Shared Kind cluster configuration for all deployment policy tests
21+
# 15 worker nodes + 1 control-plane
22+
# - Multi-compartment test uses all 15 nodes
23+
# - Linear strategy test uses first 8 nodes
24+
# - Overlapping selectors test uses first 6 nodes
25+
26+
kind: Cluster
27+
apiVersion: kind.x-k8s.io/v1alpha4
28+
nodes:
29+
- role: control-plane
30+
- role: worker
31+
- role: worker
32+
- role: worker
33+
- role: worker
34+
- role: worker
35+
- role: worker
36+
- role: worker
37+
- role: worker
38+
- role: worker
39+
- role: worker
40+
- role: worker
41+
- role: worker
42+
- role: worker
43+
- role: worker
44+
- role: worker

‎k8s-tests/chainsaw/deployment-policy/label-nodes.sh‎

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
19+
set -e
20+
21+
# Usage: label-nodes.sh <operation> <node_range> <label1=value1> [label2=value2] ...
22+
# Examples:
23+
# label-nodes.sh add 0-4 priority=critical skyhook.nvidia.com/test-node=skyhooke2e
24+
# label-nodes.sh remove 0-14 priority env region
25+
# label-nodes.sh clean-all skyhook.nvidia.com/test-node
26+
27+
OPERATION=$1
28+
shift
29+
30+
if [ "$OPERATION" = "clean-all" ]; then
31+
LABEL_PREFIX=$1
32+
echo "Cleaning all labels matching: $LABEL_PREFIX"
33+
kubectl label nodes --all "${LABEL_PREFIX}-" --overwrite 2>/dev/null || true
34+
echo "✓ Cleanup complete"
35+
exit 0
36+
fi
37+
38+
NODE_RANGE=$1
39+
shift
40+
41+
# Get all worker nodes (excluding control-plane)
42+
WORKERS=($(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | grep -v control-plane | sort))
43+
44+
# Parse node range
45+
if [[ $NODE_RANGE == *-* ]]; then
46+
START=$(echo $NODE_RANGE | cut -d'-' -f1)
47+
END=$(echo $NODE_RANGE | cut -d'-' -f2)
48+
else
49+
START=$NODE_RANGE
50+
END=$NODE_RANGE
51+
fi
52+
53+
# Validate we have enough nodes
54+
if [ ${#WORKERS[@]} -lt $((END + 1)) ]; then
55+
echo "ERROR: Need at least $((END + 1)) worker nodes for this operation"
56+
echo "Found: ${#WORKERS[@]} workers"
57+
exit 1
58+
fi
59+
60+
case "$OPERATION" in
61+
add)
62+
LABELS="$@"
63+
echo "Adding labels to nodes [$START-$END]: $LABELS"
64+
for i in $(seq $START $END); do
65+
if [ -n "${WORKERS[$i]}" ]; then
66+
kubectl label node ${WORKERS[$i]} $LABELS --overwrite
67+
fi
68+
done
69+
;;
70+
remove)
71+
LABELS_TO_REMOVE=""
72+
for label in "$@"; do
73+
LABELS_TO_REMOVE="$LABELS_TO_REMOVE ${label}-"
74+
done
75+
echo "Removing labels from nodes [$START-$END]: $@"
76+
for i in $(seq $START $END); do
77+
if [ -n "${WORKERS[$i]}" ]; then
78+
kubectl label node ${WORKERS[$i]} $LABELS_TO_REMOVE --overwrite 2>/dev/null || true
79+
fi
80+
done
81+
;;
82+
*)
83+
echo "ERROR: Unknown operation: $OPERATION"
84+
echo "Usage: $0 {add|remove|clean-all} ..."
85+
exit 1
86+
;;
87+
esac
88+
89+
echo "✓ Operation complete"
90+

‎k8s-tests/chainsaw/deployment-policy/legacy-compatibility/assert-default-compartment.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
apiVersion: skyhook.nvidia.com/v1alpha1
21+
kind: Skyhook
22+
metadata:
23+
name: legacy-interruption-budget-test
24+
status:
25+
compartmentStatuses:
26+
__default__:
27+
matched: 6
28+
ceiling: 3

‎k8s-tests/chainsaw/deployment-policy/legacy-compatibility/chainsaw-test.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
21+
apiVersion: chainsaw.kyverno.io/v1alpha1
22+
kind: Test
23+
metadata:
24+
name: legacy-interruption-budget-compatibility
25+
spec:
26+
description: |
27+
Tests backwards compatibility with legacy InterruptionBudget.
28+
- Creates Skyhook with interruptionBudget (count: 3) instead of deploymentPolicy
29+
- Verifies that a synthetic __default__ compartment is created
30+
- Verifies that the budget ceiling is respected (max 3 nodes in progress)
31+
- Ensures existing customers' configurations continue to work
32+
timeouts:
33+
exec: 180s
34+
assert: 120s
35+
steps:
36+
- name: setup-nodes
37+
try:
38+
- script:
39+
content: |
40+
chmod +x ../label-nodes.sh
41+
# Clean up any existing labels from previous tests
42+
../label-nodes.sh clean-all skyhook.nvidia.com/test-node
43+
# Label first 6 worker nodes
44+
../label-nodes.sh add 0-5 skyhook.nvidia.com/test-node=skyhooke2e
45+
echo "✓ Node labeling complete"
46+
kubectl get nodes -L skyhook.nvidia.com/test-node --sort-by=.metadata.name | head -8
47+
48+
- name: apply-skyhook
49+
try:
50+
- apply:
51+
file: skyhook.yaml
52+
- sleep:
53+
duration: 10s
54+
55+
- name: verify-default-compartment
56+
try:
57+
- assert:
58+
file: assert-default-compartment.yaml
59+
60+
- name: verify-metrics
61+
try:
62+
- script:
63+
content: |
64+
echo "=== Verifying legacy compatibility metrics ==="
65+
66+
# Verify metrics for synthetic __default__ compartment
67+
../../metrics_test.py skyhook_rollout_matched_nodes 6 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
68+
../../metrics_test.py skyhook_rollout_ceiling 3 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
69+
70+
# Verify that the legacy policy name is "legacy" and not a real policy
71+
echo "✓ Legacy compatibility metrics verified!"
72+
echo "✓ Metrics use policy_name=legacy for backwards compatibility"
73+
74+
- name: cleanup
75+
try:
76+
- script:
77+
content: |
78+
# Clean up node annotations
79+
../../skyhook/rest_test.sh legacy-interruption-budget-test
80+
# Clean up labels
81+
../label-nodes.sh clean-all skyhook.nvidia.com/test-node

‎k8s-tests/chainsaw/deployment-policy/legacy-compatibility/skyhook.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
apiVersion: skyhook.nvidia.com/v1alpha1
21+
kind: Skyhook
22+
metadata:
23+
name: legacy-interruption-budget-test
24+
spec:
25+
priority: 100
26+
# Use legacy InterruptionBudget instead of DeploymentPolicy
27+
interruptionBudget:
28+
count: 3 # Max 3 nodes at once
29+
nodeSelectors:
30+
matchLabels:
31+
skyhook.nvidia.com/test-node: skyhooke2e
32+
packages:
33+
test-pkg:
34+
version: "6.2.0"
35+
image: "ghcr.io/nvidia/skyhook/agentless"
36+

‎k8s-tests/chainsaw/deployment-policy/linear-strategy/assert-batch-1.yaml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
# Assert: Batch 1 - At least 1 node completed (initial batch)
21+
apiVersion: skyhook.nvidia.com/v1alpha1
22+
kind: Skyhook
23+
metadata:
24+
name: linear-strategy-test
25+
status:
26+
compartmentStatuses:
27+
production:
28+
(completed >= `1`): true

0 commit comments

Comments
 (0)