Add Salt 3008 test coverage, driven off generate.py's version list

.github/workflows/ci.yml

@@ -1,7 +1,11 @@
 name: CI
 on:
-  - push
-  - pull_request
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    # Weekly full test run (all versions), regardless of changed files
+    - cron: "0 0 * * 0"
 
 concurrency:
   # If changes are pushed to a PR, stop all running workflows before starting new ones
@@ -56,7 +60,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: collect-changed-files
 
-    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || needs.collect-changed-files.outputs.run-tests == 'true'
     steps:
       - uses: actions/checkout@v6
 
@@ -93,7 +97,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: collect-changed-files
     container: koalaman/shellcheck-alpine:latest
-    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || needs.collect-changed-files.outputs.run-tests == 'true'
     steps:
       - uses: actions/checkout@v6
       - name: ShellCheck
@@ -117,7 +121,7 @@ jobs:
 
   windows-2022:
     name: Windows 2022
-    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || needs.collect-changed-files.outputs.run-tests == 'true'
     uses: ./.github/workflows/test-windows.yml
     needs:
       - lint
@@ -128,13 +132,13 @@ jobs:
       container-slug: windows-2022
       timeout: 20
       runs-on: windows-2022
-      instances: '["3006", "3006-15", "3007", "3007-7"]'
+      instances: '["3006", "3006-15", "3007", "3007-7", "3008", "3008-1", "upgrade-3007", "upgrade-3008"]'
 
 
 
   rockylinux-9:
     name: Rocky Linux 9
-    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || needs.collect-changed-files.outputs.run-tests == 'true'
     uses: ./.github/workflows/test-linux.yml
     needs:
       - lint
@@ -144,7 +148,7 @@ jobs:
       display-name: Rocky Linux 9
       container-slug: systemd-rockylinux-9
       timeout: 20
-      instances: '["3006", "3006-15", "3007", "3007-7"]'
+      instances: '["3006", "3006-15", "3007", "3007-7", "3008", "3008-1", "upgrade-3007", "upgrade-3008"]'
 
 
   set-pipeline-exit-status:

.github/workflows/templates/ci.yml

@@ -1,7 +1,11 @@
 name: CI
 on:
-  - push
-  - pull_request
+  push:
+  pull_request:
+  workflow_dispatch:
+  schedule:
+    # Weekly full test run (all versions), regardless of changed files
+    - cron: "0 0 * * 0"
 
 concurrency:
   # If changes are pushed to a PR, stop all running workflows before starting new ones
@@ -56,7 +60,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: collect-changed-files
 
-    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || needs.collect-changed-files.outputs.run-tests == 'true'
     steps:
       - uses: actions/checkout@v6
 
@@ -93,7 +97,7 @@ jobs:
     runs-on: ubuntu-latest
     needs: collect-changed-files
     container: koalaman/shellcheck-alpine:latest
-    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'
+    if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || needs.collect-changed-files.outputs.run-tests == 'true'
     steps:
       - uses: actions/checkout@v6
       - name: ShellCheck

.github/workflows/templates/generate.py

@@ -3,6 +3,7 @@
 import json
 import os
 import pathlib
+import sys
 
 os.chdir(os.path.abspath(os.path.dirname(__file__)))
 
@@ -21,16 +22,68 @@
     "3006-15",
     "3007",
     "3007-7",
+    "3008",
+    "3008-1",
 ]
 
 VERSION_DISPLAY_NAMES = {
     "3006": "v3006",
     "3006-15": "v3006.15",
     "3007": "v3007",
     "3007-7": "v3007.7",
+    "3008": "v3008",
+    "3008-1": "v3008.1",
 }
 
 
+def get_version_pairs():
+    # Derive (major, exact) pairs from SALT_VERSIONS' dash convention, e.g.
+    # "3006-15" -> ("3006", "3006.15"). This is the single source of truth
+    # test suites read to know which exact version to test per major.
+    pairs = []
+    for entry in SALT_VERSIONS:
+        if "-" in entry:
+            major, minor = entry.split("-", 1)
+            pairs.append((major, f"{major}.{minor}"))
+    return pairs
+
+
+def get_major_order():
+    # Ordered list of distinct majors, in the order first seen in
+    # SALT_VERSIONS.
+    seen = []
+    for entry in SALT_VERSIONS:
+        major = entry.split("-", 1)[0]
+        if major not in seen:
+            seen.append(major)
+    return seen
+
+
+def get_upgrade_steps():
+    # One (from_exact, to_major, to_exact) tuple per adjacent major pair,
+    # e.g. ("3006.15", "3007", "3007.7"). Used to derive one CI job per
+    # upgrade step, and for test suites to look up their assigned step.
+    exact_by_major = dict(get_version_pairs())
+    majors = get_major_order()
+    steps = []
+    for prev_major, next_major in zip(majors, majors[1:]):
+        if prev_major in exact_by_major and next_major in exact_by_major:
+            steps.append(
+                (exact_by_major[prev_major], next_major, exact_by_major[next_major])
+            )
+    return steps
+
+
+def print_version_pairs():
+    for major, exact in get_version_pairs():
+        print(f"{major} {exact}")
+
+
+def print_upgrade_steps():
+    for from_exact, to_major, to_exact in get_upgrade_steps():
+        print(f"{from_exact} {to_major} {to_exact}")
+
+
 # TODO: Revert the commit relating to this section, once the Git-based builds
 #       have been fixed for the distros listed below
 #
@@ -61,6 +114,21 @@
 TIMEOUT_OVERRIDES = {}
 VERSION_ONLY_OVERRIDES = []
 
+# Test jobs run on every push, on manual (workflow_dispatch) and scheduled
+# (weekly cron, see templates/ci.yml) runs, and on PRs where the
+# collect-changed-files job found relevant files changed.
+RUN_TESTS_IF = (
+    "\n    if: github.event_name == 'push' || "
+    "github.event_name == 'workflow_dispatch' || "
+    "github.event_name == 'schedule' || "
+    "needs.collect-changed-files.outputs.run-tests == 'true'"
+)
+RUN_ALWAYS_IF = (
+    "\n    if: github.event_name == 'push' || "
+    "github.event_name == 'workflow_dispatch' || "
+    "github.event_name == 'schedule'"
+)
+
 TEMPLATE = """
   {distro}:
     name: {display_name}{ifcheck}
@@ -85,7 +153,7 @@ def generate_test_jobs():
     for distro in WINDOWS:
         test_jobs += "\n"
         runs_on = f"\n      runs-on: {distro}"
-        ifcheck = "\n    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'"
+        ifcheck = RUN_TESTS_IF
         uses = "./.github/workflows/test-windows.yml"
         instances = []
         timeout_minutes = (
@@ -96,6 +164,8 @@ def generate_test_jobs():
 
         for salt_version in SALT_VERSIONS:
            instances.append(salt_version)
+        for _, to_major, _ in get_upgrade_steps():
+           instances.append(f"upgrade-{to_major}")
 
         if instances:
             needs.append(distro)
@@ -114,7 +184,7 @@ def generate_test_jobs():
     for distro in LINUX_DISTROS:
         test_jobs += "\n"
         runs_on = ""
-        ifcheck = "\n    if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'"
+        ifcheck = RUN_TESTS_IF
         uses = "./.github/workflows/test-linux.yml"
         instances = []
         timeout_minutes = (
@@ -123,10 +193,12 @@ def generate_test_jobs():
             else TIMEOUT_DEFAULT
         )
         if distro in VERSION_ONLY_OVERRIDES:
-            ifcheck = "\n    if: github.event_name == 'push'"
+            ifcheck = RUN_ALWAYS_IF
 
         for salt_version in SALT_VERSIONS:
            instances.append(salt_version)
+        for _, to_major, _ in get_upgrade_steps():
+           instances.append(f"upgrade-{to_major}")
 
         if instances:
             needs.append(distro)
@@ -152,4 +224,9 @@ def generate_test_jobs():
 
 
 if __name__ == "__main__":
-    generate_test_jobs()
+    if "--print-versions" in sys.argv:
+        print_version_pairs()
+    elif "--print-upgrade-steps" in sys.argv:
+        print_upgrade_steps()
+    else:
+        generate_test_jobs()

.github/workflows/test-linux.yml

@@ -47,6 +47,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: VMTools Salt
+        if: ${{ !startsWith(matrix.instance, 'upgrade-') }}
         run: |
           # sed 1st - becomes space, 2nd - becomes dot
           bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./')
@@ -55,6 +56,8 @@ jobs:
           bash -x ./linux/svtminion.sh "$bt_arg1" "$bt_arg2"
 
       - name: Test VMTools
+        env:
+          SALT_TEST_VERSION: ${{ matrix.instance }}
         run: |
           bash -x ./tests/linux/test-linux.sh
 

.github/workflows/test-windows.yml

@@ -47,6 +47,8 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Test SVT Minion Script
+        env:
+          SALT_TEST_VERSION: ${{ matrix.instance }}
         run: |
           # Make sure we can run the script
           Write-Host "Run Script (no parameters)"

tests/linux/test-linux.sh

@@ -181,19 +181,77 @@ else
 fi
 ./svtminion.sh --remove || { _retn=$?; echo "test failed, did not uninstall the salt-minion, returned '${_retn}'"; exit 1; }
 
-./svtminion.sh --source ${oldpwd}/tests/testarea --install master=192.168.0.5 --loglevel debug --minionversion 3007
-./svtminion.sh --status --loglevel debug || { _retn=$?; if [[ ${_retn} -eq 100 ]]; then echo "test correct"; else echo "test failed, salt-minion should be installed, returned '${_retn}'"; exit 1; fi; }
-sleep 1
-cat /etc/salt/minion
-cat /etc/salt/minion | grep 'master:\ 192.168.0.5' 1>/dev/null
-## wait for RC with 3008
-## ./svtminion.sh --remove || { _retn=$?; echo "test failed, did not uninstall the salt-minion, returned '${_retn}'"; }
-## ./svtminion.sh --install master=192.168.0.5 --loglevel debug --source https://packages.broadcom.com/artifactory/saltproject-generic/onedir
-## ./svtminion.sh --status --loglevel debug || { _retn=$?; if [[ ${_retn} -eq 100 ]]; then echo "test correct"; else echo "test failed, salt-minion should be installed, returned '${_retn}'"; exit 1; fi; }
-## sleep 1
-cat /etc/salt/minion
-cat /etc/salt/minion | grep 'master:\ 192.168.0.5' 1>/dev/null
-./svtminion.sh --remove || { _retn=$?; echo "test failed, did not uninstall the salt-minion, returned '${_retn}'"; }
+## Major-version, exact-version, and upgrade-chain coverage, data-driven off
+## generate.py's SALT_VERSIONS (the single source of truth for which Salt
+## versions this repo tests). Sourced from the real network default so this
+## also exercises real packages.broadcom.com resolution, not local testarea
+## fixtures. Gated by $SALT_TEST_VERSION so each CI matrix job exercises
+## only its assigned entry; unset (local ad-hoc run) exercises all of them.
+_generate_py="${oldpwd}/.github/workflows/templates/generate.py"
+
+_run_major_check() {
+    local _major="$1"
+    ./svtminion.sh --install master=192.168.0.5 --loglevel debug --minionversion "${_major}"
+    ./svtminion.sh --status --loglevel debug || { _retn=$?; if [[ ${_retn} -eq 100 ]]; then echo "test correct"; else echo "test failed, salt-minion should be installed (major ${_major}), returned '${_retn}'"; exit 1; fi; }
+    cat /etc/salt/minion
+    cat /etc/salt/minion | grep 'master:\ 192.168.0.5' 1>/dev/null
+    ./svtminion.sh --remove || { _retn=$?; echo "test failed, did not uninstall the salt-minion, returned '${_retn}'"; exit 1; }
+}
+
+_run_exact_check() {
+    local _exact="$1"
+    ./svtminion.sh --install master=192.168.0.5 --loglevel debug --minionversion "${_exact}"
+    ./svtminion.sh --status --loglevel debug || { _retn=$?; if [[ ${_retn} -eq 100 ]]; then echo "test correct"; else echo "test failed, salt-minion should be installed (exact ${_exact}), returned '${_retn}'"; exit 1; fi; }
+    _ver_out=$(/usr/bin/salt-call --local test.version --out=txt 2>/dev/null || true)
+    if echo "${_ver_out}" | grep -q "${_exact}"; then echo "test correct"; else echo "test failed: expected ${_exact} in test.version, got '${_ver_out}'"; exit 1; fi
+    ./svtminion.sh --remove || { _retn=$?; echo "test failed, did not uninstall the salt-minion, returned '${_retn}'"; exit 1; }
+}
+
+_run_upgrade_check() {
+    local _from="$1" _to="$2"
+    # Thin check before the upgrade - just confirm the starting version.
+    ./svtminion.sh --install master=192.168.0.5 id="tup" --loglevel debug --minionversion "${_from}"
+    if [[ "$(/usr/bin/salt-call --local test.version --out=pprint | awk '{print $2}' | cut -d "'" -f 2)" != "${_from}" ]]; then echo "test failed, wrong starting version for upgrade ${_from} -> ${_to}"; exit 1; fi
+
+    ./svtminion.sh --upgrade --install --loglevel debug --minionversion "${_to}"
+
+    # Only check things that relate to the upgrade itself - binaries
+    # present and ping are already covered by the fresh-install tests and
+    # don't exercise anything upgrade-specific.
+    ./svtminion.sh --status --loglevel debug || { _retn=$?; if [[ ${_retn} -eq 100 ]]; then echo "test correct"; else echo "test failed, salt-minion should be installed after upgrade ${_from} -> ${_to}, returned '${_retn}'"; exit 1; fi; }
+    systemctl is-active salt-minion || { echo "test failed, salt-minion service not active after upgrade ${_from} -> ${_to}"; exit 1; }
+    # An upgrade preserves the existing config - the guest vars passed to
+    # the upgrade call above are expected to be ignored.
+    cat /etc/salt/minion | grep 'master:\ 192.168.0.5' 1>/dev/null || { echo "test failed, master not preserved after upgrade ${_from} -> ${_to}"; exit 1; }
+    cat /etc/salt/minion | grep 'id:\ tup' 1>/dev/null || { echo "test failed, id not preserved after upgrade ${_from} -> ${_to}"; exit 1; }
+    if [[ "$(/usr/bin/salt-call --local test.version --out=pprint | awk '{print $2}' | cut -d "'" -f 2)" != "${_to}" ]]; then echo "test failed, wrong version after upgrade ${_from} -> ${_to}"; exit 1; fi
+
+    ./svtminion.sh --remove || { _retn=$?; echo "test failed, did not uninstall the salt-minion, returned '${_retn}'"; exit 1; }
+}
+
+if [[ "${SALT_TEST_VERSION:-}" =~ ^upgrade-([0-9]+)$ ]]; then
+    _to_major="${BASH_REMATCH[1]}"
+    while read -r _from_exact _to_major_row _to_exact; do
+        if [[ "${_to_major_row}" == "${_to_major}" ]]; then
+            _run_upgrade_check "${_from_exact}" "${_to_exact}"
+        fi
+    done < <(python3 "${_generate_py}" --print-upgrade-steps)
+elif [[ -n "${SALT_TEST_VERSION:-}" ]]; then
+    if [[ "${SALT_TEST_VERSION}" =~ ^([0-9]+)-([0-9]+)$ ]]; then
+        _run_exact_check "${BASH_REMATCH[1]}.${BASH_REMATCH[2]}"
+    elif [[ "${SALT_TEST_VERSION}" =~ ^[0-9]+$ ]]; then
+        _run_major_check "${SALT_TEST_VERSION}"
+    fi
+else
+    while read -r _major _exact; do
+        _run_major_check "${_major}"
+        _run_exact_check "${_exact}"
+    done < <(python3 "${_generate_py}" --print-versions)
+    while read -r _from_exact _to_major _to_exact; do
+        _run_upgrade_check "${_from_exact}" "${_to_exact}"
+    done < <(python3 "${_generate_py}" --print-upgrade-steps)
+fi
+
 # test stop and start
 ./svtminion.sh --install master=192.168.0.5 --loglevel debug --source https://packages.broadcom.com/artifactory/saltproject-generic/onedir
 ./svtminion.sh --status --loglevel debug || { _retn=$?; if [[ ${_retn} -eq 100 ]]; then echo "test correct"; else echo "test failed, salt-minion should be installed, returned '${_retn}'"; exit 1; fi; }
@@ -218,23 +276,10 @@ cat /etc/salt/minion
 cat /etc/salt/minion | grep 'master:\ 192.168.0.7' 1>/dev/null
 ps -ef | grep salt
 systemctl is-active salt-minion
-# test 3006-3007 and upgrade
+## The 3006->3007->3008 upgrade chain is now covered above by the
+## data-driven _run_upgrade_check loop.
 ./svtminion.sh --remove || { _retn=$?; echo "test failed, did not uninstall the salt-minion, returned '${_retn}'"; }
 sleep 1
-./svtminion.sh --source ${oldpwd}/tests/testarea --install master=192.168.0.5 id="tup" --loglevel debug --minionversion 3006
-cat /etc/salt/minion
-cat /etc/salt/minion | grep 'master:\ 192.168.0.5' 1>/dev/null
-cat /etc/salt/minion | grep 'id:\ tup' 1>/dev/null
-ps -ef | grep salt
-systemctl is-active salt-minion
-if [[ $(/usr/bin/salt-call --local test.version --out=pprint | awk '{print $2}' | cut -d "'" -f 2 | awk -F "." '{print $1}') -eq 3006 ]]; then echo "test correct"; else echo "test failed, wrong major version for salt-minion"; exit 1; fi
-./svtminion.sh --source ${oldpwd}/tests/testarea --upgrade --install --loglevel debug --minionversion 3007
-cat /etc/salt/minion
-cat /etc/salt/minion | grep 'master:\ 192.168.0.5' 1>/dev/null
-cat /etc/salt/minion | grep 'id:\ tup' 1>/dev/null
-ps -ef | grep salt
-systemctl is-active salt-minion
-if [[ $(/usr/bin/salt-call --local test.version --out=pprint | awk '{print $2}' | cut -d "'" -f 2 | awk -F "." '{print $1}') -eq 3007 ]]; then echo "test correct"; else echo "test failed, wrong major version for salt-minion"; exit 1; fi
 ./svtminion.sh --source ${oldpwd}/tests/testarea --install master=192.168.0.5 --loglevel debug
 cat /etc/salt/minion
 cat /etc/salt/minion | grep 'master:\ 192.168.0.5' 1>/dev/null