NMS-19137: Rerun tests and move Integration tests to large

.circleci/main/commands/executions/run-smoke-tests.yml

@@ -5,6 +5,9 @@ commands:
       suite:
         default: core
         type: string
+      rerun-failtest-count:
+        default: 0
+        type: integer
     steps:
       - run:
           name: Enable swap
@@ -22,6 +25,7 @@ commands:
           name: Smoke Tests
           no_output_timeout: 30m
           command: |
+            export CCI_RERUN_FAILTEST=<< parameters.rerun-failtest-count >>
             .circleci/scripts/smoke.sh << parameters.suite >>
       - run:
           name: Gather system logs

.circleci/main/jobs/tests/integration/integration-test.yml

@@ -2,7 +2,7 @@ jobs:
   integration-test:
     executor: integration-test-executor
     parallelism: 8
-    resource_class: xlarge
+    resource_class: large
     steps:
       - cached-checkout
       - attach_workspace:

.circleci/main/jobs/tests/smoke/smoke-test-core.yml

@@ -9,3 +9,4 @@
           at: ~/
       - run-smoke-tests:
           suite: core
+          rerun-failtest-count: 1

.circleci/main/jobs/tests/smoke/smoke-test-minion.yml

@@ -9,3 +9,4 @@
           at: ~/
       - run-smoke-tests:
           suite: minion
+          rerun-failtest-count: 1

.circleci/main/jobs/tests/smoke/smoke-test-sentinel.yml

@@ -9,3 +9,4 @@
           at: ~/
       - run-smoke-tests:
           suite: sentinel
+          rerun-failtest-count: 1

.circleci/scripts/itest.sh

@@ -124,8 +124,13 @@ if [ ! -s /tmp/this_node_it_tests ]; then
   MAVEN_ARGS+=("-DskipFailsafe=true")
 fi
 
-if [ "${CCI_FAILURE_OPTION:--fail-fast}" = "--fail-fast" ]; then
-  MAVEN_ARGS+=("-Dfailsafe.skipAfterFailureCount=1")
+# When retries are enabled, use --fail-at-end so all tests run before retrying
+if [ "${CCI_RERUN_FAILTEST:-0}" -gt 0 ]; then
+  CCI_FAILURE_OPTION="--fail-at-end"
+else
+  if [ "${CCI_FAILURE_OPTION:--fail-fast}" = "--fail-fast" ]; then
+    MAVEN_ARGS+=("-Dfailsafe.skipAfterFailureCount=1")
+  fi
 fi
 
 MAVEN_COMMANDS=("install")
@@ -146,6 +151,7 @@ echo "#### Building Assembly Dependencies"
            install
 
 echo "#### Executing tests"
+set +e
 ionice nice ./compile.pl "${MAVEN_ARGS[@]}" \
            -P'!checkstyle' \
            -P'!production' \
@@ -165,3 +171,92 @@ ionice nice ./compile.pl "${MAVEN_ARGS[@]}" \
            -Dit.test="$(< /tmp/this_node_it_tests paste -s -d, -)" \
            --projects "$(< /tmp/this_node_projects paste -s -d, -)" \
            install
+TEST_EXIT=$?
+set -e
+
+# Retry failed tests if configured
+RETRIES_LEFT="${CCI_RERUN_FAILTEST:-0}"
+MAX_RETRIES="$RETRIES_LEFT"
+RETRIED_TESTS=""
+while [ "$TEST_EXIT" -ne 0 ] && [ "$RETRIES_LEFT" -gt 0 ]; do
+    ATTEMPT=$((MAX_RETRIES - RETRIES_LEFT + 1))
+    echo "#### Finding failed tests for re-run (attempt $ATTEMPT of $MAX_RETRIES)"
+
+    set +e +o pipefail
+    FAILED_TESTS=$(find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \
+      -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \
+      | sed 's|.*/TEST-||;s|\.xml||' \
+      | sort -u)
+    set -e -o pipefail
+
+    if [ -z "$FAILED_TESTS" ]; then
+        echo "#### No failed tests found in reports, skipping retry"
+        break
+    fi
+
+    echo "#### Failed tests: $FAILED_TESTS"
+    RETRIED_TESTS="$FAILED_TESTS"
+
+    # Clean failed test XML reports so fresh results are written
+    set +e +o pipefail
+    find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \
+      -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \
+      | xargs rm -f
+    set -e -o pipefail
+
+    # Split into ITs vs unit tests
+    FAILED_ITS=$(echo "$FAILED_TESTS" | grep -E 'IT$' | paste -s -d, - || true)
+    FAILED_UNITS=$(echo "$FAILED_TESTS" | grep -vE 'IT$' | paste -s -d, - || true)
+
+    RERUN_ARGS=("${MAVEN_ARGS[@]}")
+    if [ -n "$FAILED_UNITS" ]; then
+      RERUN_ARGS+=("-Dtest=$FAILED_UNITS")
+    else
+      RERUN_ARGS+=("-DskipSurefire=true")
+    fi
+    if [ -n "$FAILED_ITS" ]; then
+      RERUN_ARGS+=("-Dit.test=$FAILED_ITS")
+    else
+      RERUN_ARGS+=("-DskipFailsafe=true")
+    fi
+
+    echo "#### Re-running failed tests (unit: ${FAILED_UNITS:-none}, IT: ${FAILED_ITS:-none})"
+    set +e
+    ionice nice ./compile.pl "${RERUN_ARGS[@]}" \
+               -P'!checkstyle' \
+               -P'!production' \
+               -Pbuild-bamboo \
+               -Pcoverage \
+               -Dbuild.skip.tarball=true \
+               -DfailIfNoTests=false \
+               -Dsurefire.failIfNoSpecifiedTests=false \
+               -Dfailsafe.failIfNoSpecifiedTests=false \
+               -DrunPingTests=false \
+               -DskipITs=false \
+               --batch-mode \
+               --fail-at-end \
+               -Dorg.opennms.core.test-api.dbCreateThreads=1 \
+               -Dorg.opennms.core.test-api.snmp.useMockSnmpStrategy=false \
+               --projects "$(< /tmp/this_node_projects paste -s -d, -)" \
+               install
+    TEST_EXIT=$?
+    set -e
+
+    RETRIES_LEFT=$((RETRIES_LEFT - 1))
+done
+
+# Print retry summary
+if [ -n "$RETRIED_TESTS" ]; then
+    echo ""
+    echo "========================================"
+    echo "#### Retry Summary"
+    echo "#### Retried tests: $(echo "$RETRIED_TESTS" | paste -s -d, -)"
+    if [ "$TEST_EXIT" -eq 0 ]; then
+        echo "#### Result: PASSED on retry (attempt $ATTEMPT)"
+    else
+        echo "#### Result: FAILED after $MAX_RETRIES retry attempt(s)"
+    fi
+    echo "========================================"
+fi
+
+exit $TEST_EXIT

.circleci/scripts/smoke.sh

@@ -1,4 +1,6 @@
-#!/bin/sh -e
+#!/bin/bash
+set -e
+set -o pipefail
 
 SUITE="$1"; shift
 if [ -z "$SUITE" ]; then
@@ -67,17 +69,97 @@ fi
 
 sudo apt update && sudo apt -y install openjdk-11-jdk-headless
 
+# When retries are enabled, use --fail-at-end so all tests run before retrying
+FAILURE_MODE="--fail-fast"
+SKIP_AFTER_FAILURE=""
+if [ "${CCI_RERUN_FAILTEST:-0}" -gt 0 ]; then
+  FAILURE_MODE="--fail-at-end"
+else
+  SKIP_AFTER_FAILURE="-Dfailsafe.skipAfterFailureCount=1"
+fi
+
 # When we are ready to collect coverge on smoke tests, add "-Pcoverage" below
+set +e
 ionice nice ../compile.pl \
   -DskipTests=false \
   -DskipITs=false \
   -DfailIfNoTests=false \
   -Dtest.fork.count=1 \
   -Dit.test="$IT_TESTS" \
-  --fail-fast \
+  $FAILURE_MODE \
   --batch-mode \
-  -Dfailsafe.skipAfterFailureCount=1 \
+  $SKIP_AFTER_FAILURE \
   -N \
   '-P!smoke.all' \
   "-Psmoke.$SUITE" \
   install
+TEST_EXIT=$?
+set -e
+
+# Retry failed tests if configured
+RETRIES_LEFT="${CCI_RERUN_FAILTEST:-0}"
+MAX_RETRIES="$RETRIES_LEFT"
+RETRIED_TESTS=""
+while [ "$TEST_EXIT" -ne 0 ] && [ "$RETRIES_LEFT" -gt 0 ]; do
+    ATTEMPT=$((MAX_RETRIES - RETRIES_LEFT + 1))
+    echo "#### Finding failed smoke tests for re-run (attempt $ATTEMPT of $MAX_RETRIES)"
+
+    set +e +o pipefail
+    FAILED_TESTS=$(find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \
+      -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \
+      | sed 's|.*/TEST-||;s|\.xml||' \
+      | sort -u)
+    set -e -o pipefail
+
+    if [ -z "$FAILED_TESTS" ]; then
+        echo "#### No failed tests found in reports, skipping retry"
+        break
+    fi
+
+    echo "#### Failed tests: $FAILED_TESTS"
+    RETRIED_TESTS="$FAILED_TESTS"
+
+    # Clean failed test XML reports so fresh results are written
+    set +e +o pipefail
+    find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \
+      -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \
+      | xargs rm -f
+    set -e -o pipefail
+
+    FAILED_ITS=$(echo "$FAILED_TESTS" | paste -s -d, -)
+
+    echo "#### Re-running failed smoke tests: $FAILED_ITS"
+    set +e
+    ionice nice ../compile.pl \
+      -DskipTests=false \
+      -DskipITs=false \
+      -DfailIfNoTests=false \
+      -Dtest.fork.count=1 \
+      -Dit.test="$FAILED_ITS" \
+      --fail-at-end \
+      --batch-mode \
+      -N \
+      '-P!smoke.all' \
+      "-Psmoke.$SUITE" \
+      install
+    TEST_EXIT=$?
+    set -e
+
+    RETRIES_LEFT=$((RETRIES_LEFT - 1))
+done
+
+# Print retry summary
+if [ -n "$RETRIED_TESTS" ]; then
+    echo ""
+    echo "========================================"
+    echo "#### Retry Summary"
+    echo "#### Retried tests: $(echo "$RETRIED_TESTS" | paste -s -d, -)"
+    if [ "$TEST_EXIT" -eq 0 ]; then
+        echo "#### Result: PASSED on retry (attempt $ATTEMPT)"
+    else
+        echo "#### Result: FAILED after $MAX_RETRIES retry attempt(s)"
+    fi
+    echo "========================================"
+fi
+
+exit $TEST_EXIT

features/enlinkd/tests/src/test/resources/log4j2-test.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+  <Appenders>
+    <Console name="Console" target="SYSTEM_OUT">
+      <PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n"/>
+    </Console>
+  </Appenders>
+  <Loggers>
+    <Logger name="org.opennms.netmgt.snmp" level="WARN" />
+    <Logger name="org.opennms.netmgt.snmp.snmp4j" level="WARN" />
+    <Root level="INFO">
+      <AppenderRef ref="Console"/>
+    </Root>
+  </Loggers>
+</Configuration>