diff --git a/.circleci/main/commands/executions/run-smoke-tests.yml b/.circleci/main/commands/executions/run-smoke-tests.yml index d7e041e490cf..7bb1405c6280 100644 --- a/.circleci/main/commands/executions/run-smoke-tests.yml +++ b/.circleci/main/commands/executions/run-smoke-tests.yml @@ -5,6 +5,9 @@ commands: suite: default: core type: string + rerun-failtest-count: + default: 0 + type: integer steps: - run: name: Enable swap @@ -22,6 +25,7 @@ commands: name: Smoke Tests no_output_timeout: 30m command: | + export CCI_RERUN_FAILTEST=<< parameters.rerun-failtest-count >> .circleci/scripts/smoke.sh << parameters.suite >> - run: name: Gather system logs diff --git a/.circleci/main/jobs/tests/integration/integration-test.yml b/.circleci/main/jobs/tests/integration/integration-test.yml index 593629d877b8..1abd15bb1088 100644 --- a/.circleci/main/jobs/tests/integration/integration-test.yml +++ b/.circleci/main/jobs/tests/integration/integration-test.yml @@ -2,7 +2,7 @@ jobs: integration-test: executor: integration-test-executor parallelism: 8 - resource_class: xlarge + resource_class: large steps: - cached-checkout - attach_workspace: diff --git a/.circleci/main/jobs/tests/smoke/smoke-test-core.yml b/.circleci/main/jobs/tests/smoke/smoke-test-core.yml index f10cc0692b32..d4b44f48f6e8 100644 --- a/.circleci/main/jobs/tests/smoke/smoke-test-core.yml +++ b/.circleci/main/jobs/tests/smoke/smoke-test-core.yml @@ -9,3 +9,4 @@ at: ~/ - run-smoke-tests: suite: core + rerun-failtest-count: 1 diff --git a/.circleci/main/jobs/tests/smoke/smoke-test-minion.yml b/.circleci/main/jobs/tests/smoke/smoke-test-minion.yml index 2f36cceae9e5..58ec45e5ba8b 100644 --- a/.circleci/main/jobs/tests/smoke/smoke-test-minion.yml +++ b/.circleci/main/jobs/tests/smoke/smoke-test-minion.yml @@ -9,3 +9,4 @@ at: ~/ - run-smoke-tests: suite: minion + rerun-failtest-count: 1 diff --git a/.circleci/main/jobs/tests/smoke/smoke-test-sentinel.yml b/.circleci/main/jobs/tests/smoke/smoke-test-sentinel.yml index a7ca7a2ccecf..e6dac7f38097 100644 --- a/.circleci/main/jobs/tests/smoke/smoke-test-sentinel.yml +++ b/.circleci/main/jobs/tests/smoke/smoke-test-sentinel.yml @@ -9,3 +9,4 @@ at: ~/ - run-smoke-tests: suite: sentinel + rerun-failtest-count: 1 diff --git a/.circleci/scripts/itest.sh b/.circleci/scripts/itest.sh index 16127c1696ba..341db1718c59 100755 --- a/.circleci/scripts/itest.sh +++ b/.circleci/scripts/itest.sh @@ -124,8 +124,13 @@ if [ ! -s /tmp/this_node_it_tests ]; then MAVEN_ARGS+=("-DskipFailsafe=true") fi -if [ "${CCI_FAILURE_OPTION:--fail-fast}" = "--fail-fast" ]; then - MAVEN_ARGS+=("-Dfailsafe.skipAfterFailureCount=1") +# When retries are enabled, use --fail-at-end so all tests run before retrying +if [ "${CCI_RERUN_FAILTEST:-0}" -gt 0 ]; then + CCI_FAILURE_OPTION="--fail-at-end" +else + if [ "${CCI_FAILURE_OPTION:--fail-fast}" = "--fail-fast" ]; then + MAVEN_ARGS+=("-Dfailsafe.skipAfterFailureCount=1") + fi fi MAVEN_COMMANDS=("install") @@ -146,6 +151,7 @@ echo "#### Building Assembly Dependencies" install echo "#### Executing tests" +set +e ionice nice ./compile.pl "${MAVEN_ARGS[@]}" \ -P'!checkstyle' \ -P'!production' \ @@ -165,3 +171,92 @@ ionice nice ./compile.pl "${MAVEN_ARGS[@]}" \ -Dit.test="$(< /tmp/this_node_it_tests paste -s -d, -)" \ --projects "$(< /tmp/this_node_projects paste -s -d, -)" \ install +TEST_EXIT=$? +set -e + +# Retry failed tests if configured +RETRIES_LEFT="${CCI_RERUN_FAILTEST:-0}" +MAX_RETRIES="$RETRIES_LEFT" +RETRIED_TESTS="" +while [ "$TEST_EXIT" -ne 0 ] && [ "$RETRIES_LEFT" -gt 0 ]; do + ATTEMPT=$((MAX_RETRIES - RETRIES_LEFT + 1)) + echo "#### Finding failed tests for re-run (attempt $ATTEMPT of $MAX_RETRIES)" + + set +e +o pipefail + FAILED_TESTS=$(find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \ + -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \ + | sed 's|.*/TEST-||;s|\.xml||' \ + | sort -u) + set -e -o pipefail + + if [ -z "$FAILED_TESTS" ]; then + echo "#### No failed tests found in reports, skipping retry" + break + fi + + echo "#### Failed tests: $FAILED_TESTS" + RETRIED_TESTS="$FAILED_TESTS" + + # Clean failed test XML reports so fresh results are written + set +e +o pipefail + find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \ + -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \ + | xargs rm -f + set -e -o pipefail + + # Split into ITs vs unit tests + FAILED_ITS=$(echo "$FAILED_TESTS" | grep -E 'IT$' | paste -s -d, - || true) + FAILED_UNITS=$(echo "$FAILED_TESTS" | grep -vE 'IT$' | paste -s -d, - || true) + + RERUN_ARGS=("${MAVEN_ARGS[@]}") + if [ -n "$FAILED_UNITS" ]; then + RERUN_ARGS+=("-Dtest=$FAILED_UNITS") + else + RERUN_ARGS+=("-DskipSurefire=true") + fi + if [ -n "$FAILED_ITS" ]; then + RERUN_ARGS+=("-Dit.test=$FAILED_ITS") + else + RERUN_ARGS+=("-DskipFailsafe=true") + fi + + echo "#### Re-running failed tests (unit: ${FAILED_UNITS:-none}, IT: ${FAILED_ITS:-none})" + set +e + ionice nice ./compile.pl "${RERUN_ARGS[@]}" \ + -P'!checkstyle' \ + -P'!production' \ + -Pbuild-bamboo \ + -Pcoverage \ + -Dbuild.skip.tarball=true \ + -DfailIfNoTests=false \ + -Dsurefire.failIfNoSpecifiedTests=false \ + -Dfailsafe.failIfNoSpecifiedTests=false \ + -DrunPingTests=false \ + -DskipITs=false \ + --batch-mode \ + --fail-at-end \ + -Dorg.opennms.core.test-api.dbCreateThreads=1 \ + -Dorg.opennms.core.test-api.snmp.useMockSnmpStrategy=false \ + --projects "$(< /tmp/this_node_projects paste -s -d, -)" \ + install + TEST_EXIT=$? + set -e + + RETRIES_LEFT=$((RETRIES_LEFT - 1)) +done + +# Print retry summary +if [ -n "$RETRIED_TESTS" ]; then + echo "" + echo "========================================" + echo "#### Retry Summary" + echo "#### Retried tests: $(echo "$RETRIED_TESTS" | paste -s -d, -)" + if [ "$TEST_EXIT" -eq 0 ]; then + echo "#### Result: PASSED on retry (attempt $ATTEMPT)" + else + echo "#### Result: FAILED after $MAX_RETRIES retry attempt(s)" + fi + echo "========================================" +fi + +exit $TEST_EXIT diff --git a/.circleci/scripts/smoke.sh b/.circleci/scripts/smoke.sh index 6b5f5140dff6..055a932d6fb4 100755 --- a/.circleci/scripts/smoke.sh +++ b/.circleci/scripts/smoke.sh @@ -1,4 +1,6 @@ -#!/bin/sh -e +#!/bin/bash +set -e +set -o pipefail SUITE="$1"; shift if [ -z "$SUITE" ]; then @@ -67,17 +69,97 @@ fi sudo apt update && sudo apt -y install openjdk-11-jdk-headless +# When retries are enabled, use --fail-at-end so all tests run before retrying +FAILURE_MODE="--fail-fast" +SKIP_AFTER_FAILURE="" +if [ "${CCI_RERUN_FAILTEST:-0}" -gt 0 ]; then + FAILURE_MODE="--fail-at-end" +else + SKIP_AFTER_FAILURE="-Dfailsafe.skipAfterFailureCount=1" +fi + # When we are ready to collect coverge on smoke tests, add "-Pcoverage" below +set +e ionice nice ../compile.pl \ -DskipTests=false \ -DskipITs=false \ -DfailIfNoTests=false \ -Dtest.fork.count=1 \ -Dit.test="$IT_TESTS" \ - --fail-fast \ + $FAILURE_MODE \ --batch-mode \ - -Dfailsafe.skipAfterFailureCount=1 \ + $SKIP_AFTER_FAILURE \ -N \ '-P!smoke.all' \ "-Psmoke.$SUITE" \ install +TEST_EXIT=$? +set -e + +# Retry failed tests if configured +RETRIES_LEFT="${CCI_RERUN_FAILTEST:-0}" +MAX_RETRIES="$RETRIES_LEFT" +RETRIED_TESTS="" +while [ "$TEST_EXIT" -ne 0 ] && [ "$RETRIES_LEFT" -gt 0 ]; do + ATTEMPT=$((MAX_RETRIES - RETRIES_LEFT + 1)) + echo "#### Finding failed smoke tests for re-run (attempt $ATTEMPT of $MAX_RETRIES)" + + set +e +o pipefail + FAILED_TESTS=$(find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \ + -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \ + | sed 's|.*/TEST-||;s|\.xml||' \ + | sort -u) + set -e -o pipefail + + if [ -z "$FAILED_TESTS" ]; then + echo "#### No failed tests found in reports, skipping retry" + break + fi + + echo "#### Failed tests: $FAILED_TESTS" + RETRIED_TESTS="$FAILED_TESTS" + + # Clean failed test XML reports so fresh results are written + set +e +o pipefail + find . \( -path "*/failsafe-reports/TEST-*.xml" -o -path "*/surefire-reports/TEST-*.xml" \) \ + -exec grep -l -E 'failures="[1-9]|errors="[1-9]' {} + 2>/dev/null \ + | xargs rm -f + set -e -o pipefail + + FAILED_ITS=$(echo "$FAILED_TESTS" | paste -s -d, -) + + echo "#### Re-running failed smoke tests: $FAILED_ITS" + set +e + ionice nice ../compile.pl \ + -DskipTests=false \ + -DskipITs=false \ + -DfailIfNoTests=false \ + -Dtest.fork.count=1 \ + -Dit.test="$FAILED_ITS" \ + --fail-at-end \ + --batch-mode \ + -N \ + '-P!smoke.all' \ + "-Psmoke.$SUITE" \ + install + TEST_EXIT=$? + set -e + + RETRIES_LEFT=$((RETRIES_LEFT - 1)) +done + +# Print retry summary +if [ -n "$RETRIED_TESTS" ]; then + echo "" + echo "========================================" + echo "#### Retry Summary" + echo "#### Retried tests: $(echo "$RETRIED_TESTS" | paste -s -d, -)" + if [ "$TEST_EXIT" -eq 0 ]; then + echo "#### Result: PASSED on retry (attempt $ATTEMPT)" + else + echo "#### Result: FAILED after $MAX_RETRIES retry attempt(s)" + fi + echo "========================================" +fi + +exit $TEST_EXIT diff --git a/features/enlinkd/tests/src/test/resources/log4j2-test.xml b/features/enlinkd/tests/src/test/resources/log4j2-test.xml new file mode 100644 index 000000000000..f0026a47a8a4 --- /dev/null +++ b/features/enlinkd/tests/src/test/resources/log4j2-test.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + +