Skip to content

Commit aedd4f8

Browse files
t0mmylamt0mmylam
committed
add e2e tests for deployment policy
1 parent cfb05df commit aedd4f8

File tree

23 files changed

+1743
-1
lines changed

23 files changed

+1743
-1
lines changed

‎.github/workflows/operator-ci.yaml‎

Lines changed: 58 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,67 @@ jobs:
110110
make setup-kind-cluster
111111
make test
112112
113+
# Test deployment policies on 15-node cluster
114+
deployment-policy-tests:
115+
runs-on: ubuntu-latest
116+
steps:
117+
- uses: actions/checkout@v4
118+
with:
119+
fetch-tags: true
120+
fetch-depth: 0
121+
- name: Setup Go ${{ env.GO_VERSION }}
122+
uses: actions/setup-go@v5
123+
with:
124+
go-version: ${{ env.GO_VERSION }}
125+
cache-dependency-path: operator/go.sum
126+
- name: Log in to the Container registry
127+
uses: docker/login-action@v3
128+
with:
129+
registry: ${{ env.REGISTRY }}
130+
username: ${{ github.actor }}
131+
password: ${{ secrets.GITHUB_TOKEN }}
132+
- name: Create 15-node Kind Cluster
133+
id: kind
134+
uses: helm/kind-action@v1
135+
with:
136+
version: v0.30.0
137+
node_image: kindest/node:v1.33.4
138+
config: k8s-tests/chainsaw/deployment-policy/kind-config.yaml
139+
cluster_name: skyhook-dp-test
140+
# Cache build tools and dependencies for faster builds
141+
- name: Restore cached Binaries
142+
id: cached-binaries
143+
uses: actions/cache/restore@v4
144+
with:
145+
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
146+
restore-keys: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-
147+
path: |
148+
${{ github.workspace }}/operator/bin
149+
~/.cache/go-build
150+
- name: Install dependencies
151+
if: steps.cached-binaries.outputs.cache-hit != 'true'
152+
run: |
153+
cd operator
154+
make install-deps
155+
- name: Save cached Binaries
156+
id: save-cached-binaries
157+
if: steps.cached-binaries.outputs.cache-hit != 'true'
158+
uses: actions/cache/save@v4
159+
with:
160+
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
161+
path: |
162+
${{ github.workspace }}/operator/bin
163+
~/.cache/go-build
164+
# Run deployment policy E2E tests
165+
- name: deployment-policy-e2e-tests
166+
run: |
167+
cd operator
168+
make deployment-policy-tests
169+
113170
# Build multi-platform container image and push to registry
114171
build-and-push-operator:
115172
runs-on: ubuntu-latest
116-
needs: [tests] # Don't run the build and push if the k8s tests fail
173+
needs: [tests, deployment-policy-tests] # Don't run the build and push if tests fail
117174
# Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job.
118175
permissions:
119176
contents: read

‎k8s-tests/chainsaw/deployment-policy/kind-config.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
# Shared Kind cluster configuration for all deployment policy tests
21+
# 15 worker nodes + 1 control-plane
22+
# - Multi-compartment test uses all 15 nodes
23+
# - Linear strategy test uses first 8 nodes
24+
# - Overlapping selectors test uses first 6 nodes
25+
26+
kind: Cluster
27+
apiVersion: kind.x-k8s.io/v1alpha4
28+
nodes:
29+
- role: control-plane
30+
- role: worker
31+
- role: worker
32+
- role: worker
33+
- role: worker
34+
- role: worker
35+
- role: worker
36+
- role: worker
37+
- role: worker
38+
- role: worker
39+
- role: worker
40+
- role: worker
41+
- role: worker
42+
- role: worker
43+
- role: worker
44+
- role: worker

‎k8s-tests/chainsaw/deployment-policy/legacy-compatibility/chainsaw-test.yaml‎

Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
21+
apiVersion: chainsaw.kyverno.io/v1alpha1
22+
kind: Test
23+
metadata:
24+
name: legacy-interruption-budget-compatibility
25+
spec:
26+
description: |
27+
Tests backwards compatibility with legacy InterruptionBudget.
28+
- Creates Skyhook with interruptionBudget (count: 3) instead of deploymentPolicy
29+
- Verifies that a synthetic __default__ compartment is created
30+
- Verifies that the budget ceiling is respected (max 3 nodes in progress)
31+
- Ensures existing customers' configurations continue to work
32+
timeouts:
33+
exec: 180s
34+
assert: 120s
35+
steps:
36+
- name: setup-nodes
37+
try:
38+
- script:
39+
content: |
40+
chmod +x setup-nodes.sh cleanup-nodes.sh
41+
./setup-nodes.sh
42+
43+
- name: apply-skyhook
44+
try:
45+
- apply:
46+
file: skyhook.yaml
47+
- script:
48+
content: |
49+
echo "Waiting for Skyhook to initialize..."
50+
sleep 10
51+
52+
- name: verify-default-compartment-created
53+
try:
54+
- script:
55+
content: |
56+
echo "=== Verifying synthetic __default__ compartment ==="
57+
58+
COMPARTMENTS=$(kubectl get skyhook legacy-interruption-budget-test -o jsonpath='{.status.compartmentStatuses}' | jq -r 'keys[]')
59+
echo "Compartments found: $COMPARTMENTS"
60+
61+
DEFAULT=$(echo "$COMPARTMENTS" | grep -c "__default__" || echo "0")
62+
63+
if [ "$DEFAULT" != "1" ]; then
64+
echo "ERROR: Expected __default__ compartment to be created"
65+
echo "Found compartments: $COMPARTMENTS"
66+
kubectl get skyhook legacy-interruption-budget-test -o yaml
67+
exit 1
68+
fi
69+
70+
echo "✓ Synthetic __default__ compartment created"
71+
72+
- name: verify-compartment-uses-fixed-strategy
73+
try:
74+
- script:
75+
content: |
76+
echo "=== Verifying FixedStrategy is used ==="
77+
78+
# Get the compartment details from status
79+
MATCHED=$(kubectl get skyhook legacy-interruption-budget-test \
80+
-o jsonpath='{.status.compartmentStatuses.__default__.matched}')
81+
82+
echo "Nodes matched: $MATCHED (expected: 6)"
83+
84+
if [ "$MATCHED" != "6" ]; then
85+
echo "ERROR: Expected 6 nodes in default compartment, got $MATCHED"
86+
kubectl get skyhook legacy-interruption-budget-test -o yaml
87+
exit 1
88+
fi
89+
90+
echo "✓ All nodes assigned to __default__ compartment"
91+
92+
- name: verify-budget-ceiling-configured
93+
try:
94+
- script:
95+
content: |
96+
echo "=== Verifying budget ceiling is configured ==="
97+
98+
CEILING=$(kubectl get skyhook legacy-interruption-budget-test \
99+
-o jsonpath='{.status.compartmentStatuses.__default__.ceiling}')
100+
101+
echo "Configured ceiling: $CEILING (expected: 3)"
102+
103+
if [ "$CEILING" != "3" ]; then
104+
echo "ERROR: Expected ceiling of 3 (from interruptionBudget count), got $CEILING"
105+
kubectl get skyhook legacy-interruption-budget-test -o yaml
106+
exit 1
107+
fi
108+
109+
echo "✓ Budget ceiling configured correctly from InterruptionBudget"
110+
111+
- name: verify-final-state
112+
try:
113+
- script:
114+
content: |
115+
echo "=== Verifying legacy compatibility ==="
116+
117+
CEILING=$(kubectl get skyhook legacy-interruption-budget-test -o jsonpath='{.status.compartmentStatuses.__default__.ceiling}')
118+
MATCHED=$(kubectl get skyhook legacy-interruption-budget-test -o jsonpath='{.status.compartmentStatuses.__default__.matched}')
119+
120+
echo "Final verification:"
121+
echo " - Synthetic __default__ compartment: ✓"
122+
echo " - Budget ceiling from interruptionBudget: $CEILING"
123+
echo " - Matched nodes: $MATCHED"
124+
125+
if [ "$CEILING" != "3" ] || [ "$MATCHED" != "6" ]; then
126+
echo "ERROR: Legacy compatibility verification failed"
127+
exit 1
128+
fi
129+
130+
echo ""
131+
echo "✓✓✓ Legacy InterruptionBudget compatibility verified! ✓✓✓"
132+
echo "Existing customers can continue using interruptionBudget safely."
133+
- script:
134+
content: |
135+
echo "=== Verifying legacy compatibility metrics ==="
136+
137+
# Verify metrics for synthetic __default__ compartment
138+
../metrics_test.py skyhook_rollout_matched_nodes 6 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
139+
../metrics_test.py skyhook_rollout_ceiling 3 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
140+
141+
# Verify that the legacy policy name is "legacy" and not a real policy
142+
echo "✓ Legacy compatibility metrics verified!"
143+
echo "✓ Metrics use policy_name=legacy for backwards compatibility"
144+
145+
- name: cleanup
146+
try:
147+
- script:
148+
content: |
149+
./cleanup-nodes.sh || true

‎k8s-tests/chainsaw/deployment-policy/legacy-compatibility/cleanup-nodes.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
19+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
20+
# SPDX-License-Identifier: Apache-2.0
21+
22+
set -e
23+
24+
echo "Cleaning up node labels..."
25+
26+
# Remove test labels from all nodes
27+
kubectl label nodes --all skyhook.nvidia.com/test-node- --overwrite || true
28+
29+
echo "✓ Cleanup complete!"

‎k8s-tests/chainsaw/deployment-policy/legacy-compatibility/setup-nodes.sh‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4+
# SPDX-License-Identifier: Apache-2.0
5+
#
6+
#
7+
# Licensed under the Apache License, Version 2.0 (the "License");
8+
# you may not use this file except in compliance with the License.
9+
# You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing, software
14+
# distributed under the License is distributed on an "AS IS" BASIS,
15+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
# See the License for the specific language governing permissions and
17+
# limitations under the License.
18+
19+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
20+
# SPDX-License-Identifier: Apache-2.0
21+
22+
set -e
23+
24+
echo "Cleaning up any existing labels from previous tests..."
25+
kubectl label nodes --all skyhook.nvidia.com/test-node- --overwrite 2>/dev/null || true
26+
27+
echo "Labeling nodes for legacy compatibility test..."
28+
29+
# Get all worker nodes (excluding control-plane) and label the first 6
30+
WORKERS=($(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | grep -v control-plane | sort))
31+
32+
# Label first 6 worker nodes for the test
33+
echo "Labeling test nodes (first 6 workers)..."
34+
for i in {0..5}; do
35+
if [ -n "${WORKERS[$i]}" ]; then
36+
kubectl label node/${WORKERS[$i]} skyhook.nvidia.com/test-node=skyhooke2e --overwrite
37+
fi
38+
done
39+
40+
echo ""
41+
echo "✓ Node labeling complete!"
42+
echo ""
43+
kubectl get nodes -L skyhook.nvidia.com/test-node --sort-by=.metadata.name | head -8

‎k8s-tests/chainsaw/deployment-policy/legacy-compatibility/skyhook.yaml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
#
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
18+
# SPDX-License-Identifier: Apache-2.0
19+
20+
apiVersion: skyhook.nvidia.com/v1alpha1
21+
kind: Skyhook
22+
metadata:
23+
name: legacy-interruption-budget-test
24+
spec:
25+
priority: 100
26+
# Use legacy InterruptionBudget instead of DeploymentPolicy
27+
interruptionBudget:
28+
count: 3 # Max 3 nodes at once
29+
nodeSelectors:
30+
matchLabels:
31+
skyhook.nvidia.com/test-node: skyhooke2e
32+
packages:
33+
test-pkg:
34+
version: "6.2.0"
35+
image: "ghcr.io/nvidia/skyhook/agentless"
36+

0 commit comments

Comments
 (0)