add ordered assertions for batch states to tests

Author: t0mmylam

k8s-tests/chainsaw/deployment-policy/legacy-compatibility/chainsaw-test.yaml

@@ -64,8 +64,8 @@ spec:
           echo "=== Verifying legacy compatibility metrics ==="
           
           # Verify metrics for synthetic __default__ compartment
-          ../metrics_test.py skyhook_rollout_matched_nodes 6 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
-          ../metrics_test.py skyhook_rollout_ceiling 3 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
+          ../../metrics_test.py skyhook_rollout_matched_nodes 6 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
+          ../../metrics_test.py skyhook_rollout_ceiling 3 -t skyhook_name=legacy-interruption-budget-test -t policy_name=legacy -t compartment_name=__default__ -t strategy=fixed
           
           # Verify that the legacy policy name is "legacy" and not a real policy
           echo "✓ Legacy compatibility metrics verified!"
@@ -75,4 +75,7 @@ spec:
     try:
     - script:
         content: |
+          # Clean up node annotations
+          ../../skyhook/rest_test.sh legacy-interruption-budget-test
+          # Clean up labels
           ../label-nodes.sh clean-all skyhook.nvidia.com/test-node

k8s-tests/chainsaw/deployment-policy/linear-strategy/assert-batch-1.yaml

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Assert: Batch 1 - At least 1 node completed (initial batch)
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: linear-strategy-test
+status:
+  compartmentStatuses:
+    production:
+      (completed >= `1`): true

k8s-tests/chainsaw/deployment-policy/linear-strategy/assert-batch-2.yaml

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Assert: Batch 2 - At least 2 nodes completed (1 + delta 1)
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: linear-strategy-test
+status:
+  compartmentStatuses:
+    production:
+      (completed >= `2`): true
+      batchState:
+        (currentBatch >= `2`): true

k8s-tests/chainsaw/deployment-policy/linear-strategy/assert-batch-3.yaml

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Assert: Batch 3 - At least 3 nodes completed (1 + 1 + delta 1)
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: linear-strategy-test
+status:
+  compartmentStatuses:
+    production:
+      (completed >= `3`): true
+      batchState:
+        (currentBatch >= `3`): true

k8s-tests/chainsaw/deployment-policy/linear-strategy/assert-batch-4.yaml

@@ -0,0 +1,30 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Assert: Batch 4 - At least 4 nodes completed (ceiling reached)
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: linear-strategy-test
+status:
+  compartmentStatuses:
+    production:
+      (completed >= `4`): true
+      batchState:
+        (currentBatch >= `4`): true

k8s-tests/chainsaw/deployment-policy/linear-strategy/chainsaw-test.yaml

@@ -59,12 +59,23 @@ spec:
           echo "=== Verifying initial metrics ==="
           
           # Verify matched nodes and ceiling for production compartment
-          ../metrics_test.py skyhook_rollout_matched_nodes 8 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
+          ../../metrics_test.py skyhook_rollout_matched_nodes 8 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
           # Budget is count=4
-          ../metrics_test.py skyhook_rollout_ceiling 4 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
+          ../../metrics_test.py skyhook_rollout_ceiling 4 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
           
           echo "✓ Initial metrics verified"
   
+  - name: track-batch-progression
+    try:
+    - assert:
+        file: assert-batch-1.yaml
+    - assert:
+        file: assert-batch-2.yaml
+    - assert:
+        file: assert-batch-3.yaml
+    - assert:
+        file: assert-batch-4.yaml
+  
   - name: wait-for-completion
     try:
     - assert:
@@ -78,23 +89,26 @@ spec:
           echo "=== Verifying final metrics ==="
           
           # All nodes should be completed
-          ../metrics_test.py skyhook_rollout_completed 8 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
-          ../metrics_test.py skyhook_rollout_in_progress 0 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
-          ../metrics_test.py skyhook_rollout_progress_percent 100 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
+          ../../metrics_test.py skyhook_rollout_completed 8 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
+          ../../metrics_test.py skyhook_rollout_in_progress 0 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
+          ../../metrics_test.py skyhook_rollout_progress_percent 100 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
           
           # Verify no consecutive failures
-          ../metrics_test.py skyhook_rollout_consecutive_failures 0 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
+          ../../metrics_test.py skyhook_rollout_consecutive_failures 0 -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
           
           # Verify current_batch metric exists and is greater than 0
           CURRENT_BATCH=$(kubectl get skyhook linear-strategy-test -o jsonpath='{.status.compartmentStatuses.production.batchState.currentBatch}')
-          ../metrics_test.py skyhook_rollout_current_batch "$CURRENT_BATCH" -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
+          ../../metrics_test.py skyhook_rollout_current_batch "$CURRENT_BATCH" -t skyhook_name=linear-strategy-test -t policy_name=linear-ramp-policy -t compartment_name=production -t strategy=linear
           
           echo "✓ All final metrics verified!"
   
   - name: cleanup
     try:
     - script:
         content: |
+          # Clean up node annotations
+          ../../skyhook/rest_test.sh linear-strategy-test
+          # Clean up labels
           ../label-nodes.sh clean-all tier
           ../label-nodes.sh clean-all skyhook.nvidia.com/test-node
     - delete:

k8s-tests/chainsaw/deployment-policy/multi-compartment/assert-batch-1.yaml

@@ -0,0 +1,32 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Assert: Batch 1 - Initial batch started for all compartments
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: multi-compartment-test
+status:
+  compartmentStatuses:
+    critical:
+      (completed >= `1`): true
+    standard:
+      (completed >= `1`): true
+    test:
+      (completed >= `2`): true  # initialBatch: 2

k8s-tests/chainsaw/deployment-policy/multi-compartment/assert-batch-2.yaml

@@ -0,0 +1,36 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Assert: Batch 2 - Exponential growth (1 -> 2, 1 -> 2, 2 -> 4)
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: multi-compartment-test
+status:
+  compartmentStatuses:
+    critical:
+      (completed >= `2`): true  # 1 * 2 = 2, but ceiling is 2
+      batchState:
+        (currentBatch >= `2`): true
+    standard:
+      (completed >= `1`): true  # 1 * 2 = 2, but ceiling is 1
+    test:
+      (completed >= `4`): true  # 2 * 2 = 4, ceiling is 2
+      batchState:
+        (currentBatch >= `2`): true