NMS-19648: CircleCI: Store failed result of flaky tests, improve a test case

.circleci/epoch

@@ -1,2 +1,2 @@
 # increment this to force a full build
-26
+27

.circleci/main/commands/executions/run-build.yml

@@ -47,7 +47,7 @@ commands:
                 ;;
             esac
             echo "export OPENNMS_VERSION=\"$OPENNMS_VERSION\"" >> $BASH_ENV
-            ./compile.pl -DskipTests=true -Dbuild.skip.tarball=false \
+            ./compile.pl -DskipTests=true -DskipITs=true -Dbuild.skip.tarball=false \
               -Daether.connector.resumeDownloads=false \
               -Daether.connector.basic.threads=1 \
               -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn \

.circleci/main/commands/executions/run-integration-tests.yml

@@ -57,12 +57,31 @@ commands:
 
             (date '+%T' || :)                       > ~/test-results/system-logs/docker_stats 2>&1
             (docker stats --all --no-stream || :) >> ~/test-results/system-logs/docker_stats 2>&1
-            (docker ps ---all || :)               >> ~/test-results/system-logs/docker_stats 2>&1
+            (docker ps --all || :)               >> ~/test-results/system-logs/docker_stats 2>&1
             for CONTAINER in `docker ps --all --quiet`; do
               ((docker logs --timestamps "$CONTAINER" 2>&1 | tail -n 20 ) || :) > ~/test-results/system-logs/"docker-${CONTAINER}.log" 2>&1
             done
+      - run:
+          name: Collect flaky test evidence
+          when: always
+          command: |
+            mkdir -p ~/test-results/flaky-evidence
+            if [ -d /tmp/flaky-evidence ] && [ "$(find /tmp/flaky-evidence -name '*.xml' | wc -l)" -gt 0 ]; then
+              (
+                cd /tmp/flaky-evidence &&
+                find . -name '*.xml' -exec cp --parents {} ~/test-results/flaky-evidence/ \;
+              )
+              echo "#### Flaky tests detected (passed only after retry):"
+              find /tmp/flaky-evidence -name '*.xml' | sed 's|.*/TEST-||;s|\.xml||' | sort -u
+            else
+              echo "No flaky tests detected."
+            fi
       - store_test_results:
           path: ~/test-results
+      - store_artifacts:
+          when: always
+          path: ~/test-results/flaky-evidence
+          destination: flaky-evidence
       - run:
           name: Copy artifacts on failure
           when: on_fail
@@ -72,5 +91,6 @@ commands:
             cp -R ~/build-results ~/failure-artifacts/ || true
             cp -R ~/generated-tests ~/failure-artifacts/ || true
       - store_artifacts:
+          when: on_fail
           path: ~/failure-artifacts
           destination: failure-artifacts

.circleci/main/commands/executions/run-smoke-tests.yml

@@ -8,6 +8,10 @@ commands:
       rerun-failtest-count:
         default: 0
         type: integer
+      allow-failures:
+        description: "Exit 0 regardless of test outcome (use for quarantined suites)"
+        default: false
+        type: boolean
     steps:
       - run:
           name: Enable swap
@@ -20,13 +24,17 @@ commands:
             cat /proc/sys/vm/swappiness
       - load-oci:
           match: amd64
-      - restore-maven-cache
       - run:
           name: Smoke Tests
-          no_output_timeout: 30m
+          no_output_timeout: 60m
           command: |
             export CCI_RERUN_FAILTEST=<< parameters.rerun-failtest-count >>
+            <<# parameters.allow-failures >>
+            .circleci/scripts/smoke.sh << parameters.suite >> || true
+            <</ parameters.allow-failures >>
+            <<^ parameters.allow-failures >>
             .circleci/scripts/smoke.sh << parameters.suite >>
+            <</ parameters.allow-failures >>
       - run:
           name: Gather system logs
           when: always
@@ -38,7 +46,7 @@ commands:
 
             (date '+%T' || :)                      > ~/test-results/system-logs/docker_stats 2>&1
             (docker stats --all --no-stream || :) >> ~/test-results/system-logs/docker_stats 2>&1
-            (docker ps ---all || :)               >> ~/test-results/system-logs/docker_stats 2>&1
+            (docker ps --all || :)               >> ~/test-results/system-logs/docker_stats 2>&1
             for CONTAINER in `docker ps --all --quiet`; do
               ((docker logs --timestamps "$CONTAINER" 2>&1 | tail -n 20 ) || :) > ~/test-results/system-logs/"docker-${CONTAINER}.log" 2>&1
             done
@@ -55,8 +63,29 @@ commands:
             cp -R ~/project/smoke-test/target/*.{flv,mp4} ~/test-artifacts/recordings || true
             cp -R ~/project/smoke-test/target/screenshots ~/test-artifacts/ || true
             cp -R ~/project/smoke-test/target/logs ~/test-artifacts/ || true
+      - run:
+          name: Collect flaky test evidence
+          when: always
+          command: |
+            mkdir -p ~/test-results/flaky-evidence
+            if [ -d /tmp/flaky-evidence ] && [ "$(find /tmp/flaky-evidence -name '*.xml' | wc -l)" -gt 0 ]; then
+              (
+                cd /tmp/flaky-evidence || exit 1
+                find . -name '*.xml' -type f | while read -r xml; do
+                  xml="${xml#./}"
+                  mkdir -p ~/test-results/flaky-evidence/"$(dirname "$xml")"
+                  cp "$xml" ~/test-results/flaky-evidence/"$xml"
+                done
+              )
+              echo "#### Flaky tests detected (passed only after retry):"
+              find /tmp/flaky-evidence -name '*.xml' | sed 's|.*/TEST-||;s|\.xml||' | sort -u
+            fi
       - store_test_results:
           path: ~/test-results
+      - store_artifacts:
+          when: always
+          path: ~/test-results/flaky-evidence
+          destination: flaky-evidence
       - run:
           name: Copy artifacts on failure
           when: on_fail
@@ -65,5 +94,6 @@ commands:
             cp -R ~/test-results ~/failure-artifacts/ || true
             cp -R ~/test-artifacts ~/failure-artifacts/ || true
       - store_artifacts:
+          when: on_fail
           path: ~/failure-artifacts
           destination: failure-artifacts

.circleci/main/commands/generic/generic.yml

@@ -24,10 +24,12 @@ commands:
               do_sudo chmod a+x /usr/local/bin/download-artifacts.pl
             fi
 
-            do_sudo apt-get -y --allow-releaseinfo-change update && \
-              do_sudo apt-get -y -q --no-install-recommends install \
-                libdatetime-format-iso8601-perl \
-                libjson-pp-perl \
-                libwww-perl \
-                liblwp-protocol-https-perl \
-                liblwp-useragent-determined-perl
+            if ! dpkg -s libdatetime-format-iso8601-perl libjson-pp-perl libwww-perl liblwp-protocol-https-perl liblwp-useragent-determined-perl >/dev/null 2>&1; then
+              do_sudo apt-get -y --allow-releaseinfo-change update && \
+                do_sudo apt-get -y -q --no-install-recommends install \
+                  libdatetime-format-iso8601-perl \
+                  libjson-pp-perl \
+                  libwww-perl \
+                  liblwp-protocol-https-perl \
+                  liblwp-useragent-determined-perl
+            fi

.circleci/main/commands/oci/trivy-analyze.yml

@@ -1,7 +1,6 @@
 commands:
   trivy-analyze:
     steps:
-      - download-download-artifacts
       - cached-checkout
       - attach_workspace:
           at: ~/project

.circleci/main/jobs/tarball-assembly-only.yml

@@ -4,7 +4,7 @@ jobs:
     resource_class: large
     parameters:
       number-vcpu:
-        default: 6
+        default: 4
         type: integer
       vaadin-javamaxmem:
         default: 1g
@@ -18,8 +18,7 @@ jobs:
           name: Assemble tarballs and related artifacts
           command: |
             ulimit -n 65536 || :
-            export MAVEN_OPTS="-Xmx8g -XX:ReservedCodeCacheSize=1g -XX:+TieredCompilation"
-            export MAVEN_ARGS="install"
+            export MAVEN_OPTS="-Xmx6g -XX:ReservedCodeCacheSize=1g -XX:+TieredCompilation"
             export MAVEN_ARGS="install"
             # release branches should enable extra "production" stuff like license indexing
             case "${CIRCLE_BRANCH}" in

.circleci/main/jobs/tests/smoke/smoke-test-flaky.yml

@@ -0,0 +1,13 @@
+jobs:
+  smoke-test-flaky:
+    executor: smoke-test-executor
+    parallelism: 2
+    resource_class: medium+
+    steps:
+      - cached-checkout
+      - attach_workspace:
+          at: ~/
+      - run-smoke-tests:
+          suite: flaky
+          rerun-failtest-count: 2
+          allow-failures: true

.circleci/main/jobs/tests/smoke/smoke-test-minion.yml

@@ -1,7 +1,7 @@
  jobs:
   smoke-test-minion:
     executor: smoke-test-executor
-    parallelism: 4
+    parallelism: 10
     resource_class: medium+
     steps:
       - cached-checkout

.circleci/main/jobs/tests/smoke/smoke-test-sentinel.yml

@@ -1,7 +1,7 @@
  jobs:
   smoke-test-sentinel:
     executor: smoke-test-executor
-    parallelism: 4
+    parallelism: 8
     resource_class: medium+
     steps:
       - cached-checkout

.circleci/main/jobs/tests/smoke/smoke-test.index

@@ -2,3 +2,4 @@
   #jobs:tests/smoke/smoke-test-minion#
   #jobs:tests/smoke/smoke-test-sentinel#
   #jobs:tests/smoke/smoke-test-minimal#
+  #jobs:tests/smoke/smoke-test-flaky#

.circleci/main/workflows/workflows_v2.json

@@ -253,6 +253,17 @@
                 "sentinel-image-single-arch-linux-amd64"
             ]
         },
+        "smoke-test-flaky": {
+            "context": [
+                "CircleCI",
+                "docker-publish-account"
+            ],
+            "requires": [
+                "horizon-image-single-arch-linux-amd64",
+                "minion-image-single-arch-linux-amd64",
+                "sentinel-image-single-arch-linux-amd64"
+            ]
+        },
         "tarball-assembly-only": {
             "context": [
                 "CircleCI"

.circleci/scripts/find-tests/find-tests.py

@@ -1,4 +1,4 @@
-#/usr/bin/env python3
+#!/usr/bin/env python3
 import argparse
 import os
 import re
@@ -55,7 +55,6 @@ def generate_test_lists(maven_project_root, changes_only=True, unit_test_output=
         for file_changed in other_files_changed:
             print(file_changed)
 
-        print(list(test_files_changed))
         modules_with_test_changes = project.get_modules_related_to(test_files_changed)
         print("Modules with test changes:")
         for test_module_with_changes in modules_with_test_changes:

.circleci/scripts/find-tests/git.py

@@ -14,6 +14,8 @@ def get_parent_branch(repo_path):
             match = re.match(r'parent_branch: (.*)$', line, re.M | re.I)
             if match:
                 parent_branch = match.group(1)
+    if parent_branch is None:
+        raise ValueError("Could not find 'parent_branch:' entry in .nightly file")
     return parent_branch.strip()
 
 

.circleci/scripts/itest.sh

@@ -4,6 +4,8 @@ set -e
 set -o pipefail
 
 FIND_TESTS_DIR="target/find-tests"
+NODE_INDEX="${CIRCLE_NODE_INDEX:-0}"
+NODE_TOTAL="${CIRCLE_NODE_TOTAL:-1}"
 
 # attempt to work around repository flakiness
 retry()
@@ -49,9 +51,15 @@ echo "#### Determining tests to run"
 perl -pi -e "s,/home/circleci,${HOME},g" target/structure-graph.json
 find_tests
 if [ ! -s /tmp/this_node_projects ]; then
-  echo "No tests to run."
+  echo "#### Node $NODE_INDEX/$NODE_TOTAL: No tests assigned, skipping"
   exit 0
 fi
+PROJECT_COUNT=$(wc -l < /tmp/this_node_projects)
+TEST_COUNT=$(wc -l < /tmp/this_node_tests 2>/dev/null || echo 0)
+IT_COUNT=$(wc -l < /tmp/this_node_it_tests 2>/dev/null || echo 0)
+echo "#### Node $NODE_INDEX/$NODE_TOTAL: $PROJECT_COUNT projects | $TEST_COUNT unit tests | $IT_COUNT integration tests"
+
+echo "#### Node $NODE_INDEX/$NODE_TOTAL: Setting up local dependencies"
 
 echo "#### Set loopback to 127.0.0.1"
 sudo sed -i 's/127.0.1.1/127.0.0.1/g' /etc/hosts
@@ -65,39 +73,39 @@ echo "#### Setting up Postgres"
 echo "#### Installing other dependencies"
 # limit the sources we need to update
 sudo rm -f /etc/apt/sources.list.d/*
- 
-# kill other apt commands first to avoid problems locking /var/lib/apt/lists/lock - see https://discuss.circleci.com/t/could-not-get-lock-var-lib-apt-lists-lock/28337/6
+
+# kill other apt commands first to avoid problems locking /var/lib/apt/lists/lock
 sudo killall -9 apt || true && \
-            retry sudo apt update && \
-            retry sudo env DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install \
-                ca-certificates \
-                tzdata \
-                software-properties-common \
-                debconf-utils
- 
+      retry sudo apt update && \
+      retry sudo env DEBIAN_FRONTEND=noninteractive apt -y --no-install-recommends install \
+        ca-certificates \
+        tzdata \
+        software-properties-common \
+        debconf-utils
+
 # install some keys
 curl -sSf https://cloud.r-project.org/bin/linux/ubuntu/marutter_pubkey.asc | sudo tee -a /etc/apt/trusted.gpg.d/cran_ubuntu_key.asc
 curl -sSf https://debian.opennms.org/OPENNMS-GPG-KEY | sudo tee -a /etc/apt/trusted.gpg.d/opennms_key.asc
- 
+
 # limit more sources and add mirrors
 echo "deb mirror://mirrors.ubuntu.com/mirrors.txt $(lsb_release -cs) main restricted universe multiverse
 deb http://archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted" | sudo tee -a /etc/apt/sources.list
 sudo add-apt-repository -y 'deb http://debian.opennms.org stable main'
- 
+
 # add the R repository
 sudo add-apt-repository -y "deb https://cloud.r-project.org/bin/linux/ubuntu $(lsb_release -cs)-cran40/"
 
 retry sudo apt update && \
-            RRDTOOL_VERSION=$(apt-cache show rrdtool | grep Version: | grep -v opennms | awk '{ print $2 }') && \
-            echo '* libraries/restart-without-asking boolean true' | sudo debconf-set-selections && \
-            retry sudo env DEBIAN_FRONTEND=noninteractive apt -f --no-install-recommends install \
-                openjdk-17-jdk-headless \
-                r-base \
-                "rrdtool=$RRDTOOL_VERSION" \
-                jrrd2 \
-                jicmp \
-                jicmp6 \
-            || exit 1
+      RRDTOOL_VERSION=$(apt-cache show rrdtool | grep Version: | grep -v opennms | awk '{ print $2 }') && \
+      echo '* libraries/restart-without-asking boolean true' | sudo debconf-set-selections && \
+      retry sudo env DEBIAN_FRONTEND=noninteractive apt -f --no-install-recommends install \
+        openjdk-17-jdk-headless \
+        r-base \
+        "rrdtool=$RRDTOOL_VERSION" \
+        jrrd2 \
+        jicmp \
+        jicmp6 \
+      || exit 1
 
 export JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64
 export MAVEN_OPTS="$MAVEN_OPTS -Xmx4g -XX:ReservedCodeCacheSize=1g"
@@ -132,9 +140,7 @@ else
   fi
 fi
 
-MAVEN_COMMANDS=("install")
-
-echo "#### Building Assembly Dependencies"
+echo "#### Node $NODE_INDEX/$NODE_TOTAL: Building Assembly Dependencies"
 ./compile.pl "${MAVEN_ARGS[@]}" \
            -P'!checkstyle' \
            -P'!production' \
@@ -164,7 +170,7 @@ ionice nice ./compile.pl "${MAVEN_ARGS[@]}" \
            -DskipITs=false \
            --batch-mode \
            "${CCI_FAILURE_OPTION:--fail-fast}" \
-           -Dorg.opennms.core.test-api.dbCreateThreads=1 \
+           -Dorg.opennms.core.test-api.dbCreateThreads=8 \
            -Dorg.opennms.core.test-api.snmp.useMockSnmpStrategy=false \
            -Dtest="$(< /tmp/this_node_tests paste -s -d, -)" \
            -Dit.test="$(< /tmp/this_node_it_tests paste -s -d, -)" \
@@ -196,8 +202,14 @@ while [ "$TEST_EXIT" -ne 0 ] && [ "$RETRIES_LEFT" -gt 0 ]; do
     echo "#### Failed tests: $FAILED_TESTS"
     RETRIED_TESTS="$FAILED_TESTS"
 
-    # Clean failed test XML reports so fresh results are written
+    # Preserve failing XMLs as flaky evidence before overwriting with retry results
+    FLAKY_EVIDENCE_DIR="/tmp/flaky-evidence/attempt-${ATTEMPT}"
+    mkdir -p "${FLAKY_EVIDENCE_DIR}"
     set +e +o pipefail
+    find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \
+      -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \
+      | xargs -I{} cp {} "${FLAKY_EVIDENCE_DIR}/"
+    # Now delete originals so fresh results are written by the retry
     find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \
       -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \
       | xargs rm -f
@@ -234,7 +246,7 @@ while [ "$TEST_EXIT" -ne 0 ] && [ "$RETRIES_LEFT" -gt 0 ]; do
                -DskipITs=false \
                --batch-mode \
                --fail-at-end \
-               -Dorg.opennms.core.test-api.dbCreateThreads=1 \
+               -Dorg.opennms.core.test-api.dbCreateThreads=8 \
                -Dorg.opennms.core.test-api.snmp.useMockSnmpStrategy=false \
                --projects "$(< /tmp/this_node_projects paste -s -d, -)" \
                install