diff --git a/docs/bundle_script_overview.md b/docs/bundle_script_overview.md new file mode 100644 index 00000000..aa805482 --- /dev/null +++ b/docs/bundle_script_overview.md @@ -0,0 +1,253 @@ +# Scripts to help with the running of integtests + +The following scripts are intended to help developers find and run _integtests_ (regression and integration tests) in all repositories: + +* `dunedaq_integtest_bundle.sh` - runs sets ("bundles") of integtests. This script supports the running of integtests from repositories that have been cloned into a local software area as well as repositories that are being used from a base release on CVMFS. +* `list_available_integtests.sh` +* `list_repos_with_integtests.sh` + +Users may choose to primarily use the `dunedaq_integtest_bundle.sh` script, but the two "`list`" scripts may be occasionally useful, so they are also described here. + +All of these scripts support the "`--help`" command-line option to provide information on how they should be run. Here is what is currently shown for each of the three scripts: + +### dunedaq_integtest_bundle.sh --help + +``` +Usage: +dunedaq_integtest_bundle.sh [option(s)] + +Options: + -h, --help : prints out usage information + -r + - this can be the name of a single repo + - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' + - it can have the special value of "all" - integtests in all repos will be run + - it can have the special value of "local" - integtests in locally-cloned repos will be run + -k, --include + -x, --exclude + -n + -N + --stop-on-failure : causes the script to stop when one of the integtests reports a failure + --concise-output : suppresses run control and DAQApp messages in order to focus on test results + --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' + --list-only : list the tests that match the requested patterns without running them +``` + +### list_available_integtests.sh --help + +``` +Usage: list_available_integtests.sh [optional list of repo names|local|all] + e.g. list_available_integtests.sh daqsystemtest + If no repo name is specified, integtests for all repos are listed. + If a special repo name of "local" is specified, only integtests for repos + in the local software area are listed. + If a special repo name of "all" is specified, integtests for all repos are listed. +``` + +### list_repos_with_integtests.sh --help + +``` +Usage: list_repos_with_integtests.sh [optional "local" keyword] + Lists the software repositories that have integration tests (integtests) in them. + Searches the base releases, local install dir, and local sourcecode dir, + unless "local" is passed as an argument. In that case, only the local + install and sourcecode directories are searched. +``` + +Here are examples of using each of the scripts: + +### dunedaq_integtest_bundle.sh + +``` +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh --list-only + +Integtests from the _daqsystemtest_ repo will be run... + +The following tests will be run: + daqsystemtest/3ru_1df_multirun_test.py + daqsystemtest/3ru_3df_multirun_test.py + daqsystemtest/example_system_test.py + daqsystemtest/fake_data_producer_test.py + daqsystemtest/long_window_readout_test.py + daqsystemtest/minimal_system_quick_test.py + daqsystemtest/readout_type_scan_test.py + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpreplay_test.py + daqsystemtest/tpstream_writing_test.py + daqsystemtest/trigger_bitwords_test.py + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r listrev --list-only + +The following tests will be run: + listrev/listrev_test.py + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r local --list-only + +Building the list of _local_ integtests... + +The following tests will be run: + daqsystemtest/3ru_1df_multirun_test.py + daqsystemtest/3ru_3df_multirun_test.py + daqsystemtest/example_system_test.py + daqsystemtest/fake_data_producer_test.py + daqsystemtest/long_window_readout_test.py + daqsystemtest/minimal_system_quick_test.py + daqsystemtest/readout_type_scan_test.py + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpreplay_test.py + daqsystemtest/tpstream_writing_test.py + daqsystemtest/trigger_bitwords_test.py + dfmodules/disabled_output_test.py + dfmodules/hdf5_compression_test.py + dfmodules/insufficient_disk_space_test.py + dfmodules/large_trigger_record_test.py + dfmodules/max_file_size_test.py + dfmodules/multiple_data_writers_test.py + dfmodules/offline_prod_run_test.py + dfmodules/trmonrequestor_test.py + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r "asiolibs|crtmodules" --list-only + +The following tests will be run: + asiolibs/socket_reader_test.py + crtmodules/crt_reader_test.py + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r local -k tp --list-only + +Building the list of _local_ integtests... + +The following tests will be run: + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpreplay_test.py + daqsystemtest/tpstream_writing_test.py + dfmodules/disabled_output_test.py + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r local -k tp -x "disabled|tpreplay" --list-only + +Building the list of _local_ integtests... + +The following tests will be run: + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpstream_writing_test.py +``` + +### list_available_integtests.sh + +``` +(dbt) [biery@daq]$ list_available_integtests.sh + +Looking for integtests in _all_ repos... + +asiolibs/socket_reader_test.py +crtmodules/crt_reader_test.py +daqsystemtest/3ru_1df_multirun_test.py +daqsystemtest/3ru_3df_multirun_test.py +daqsystemtest/example_system_test.py +daqsystemtest/fake_data_producer_test.py +daqsystemtest/long_window_readout_test.py +daqsystemtest/minimal_system_quick_test.py +daqsystemtest/readout_type_scan_test.py +daqsystemtest/small_footprint_quick_test.py +daqsystemtest/tpg_state_collection_test.py +daqsystemtest/tpreplay_test.py +daqsystemtest/tpstream_writing_test.py +daqsystemtest/trigger_bitwords_test.py +dfmodules/disabled_output_test.py +dfmodules/hdf5_compression_test.py +dfmodules/insufficient_disk_space_test.py +dfmodules/large_trigger_record_test.py +dfmodules/max_file_size_test.py +dfmodules/multiple_data_writers_test.py +dfmodules/offline_prod_run_test.py +dfmodules/trmonrequestor_test.py +hsilibs/iceberg_real_hsi_test.py +listrev/listrev_test.py +snbmodules/simple_transform_test.py +snbmodules/snb_1node_1app_rclone_http_system_quick_test.py +snbmodules/snb_1node_1app_torrent_system_quick_test.py +snbmodules/snb_1node_multiclientapps_rclone_http_system_quick_test.py +snbmodules/snb_minimal_system_test.py +trigger/change_rate_test.py +trigger/tc_time_outside_window_test.py +trigger/td_leakage_between_runs_test.py + +(dbt) [biery@daq]$ list_available_integtests.sh local + +Looking for integtests in _local_ repos... + +daqsystemtest/3ru_1df_multirun_test.py +daqsystemtest/3ru_3df_multirun_test.py +daqsystemtest/example_system_test.py +daqsystemtest/fake_data_producer_test.py +daqsystemtest/long_window_readout_test.py +daqsystemtest/minimal_system_quick_test.py +daqsystemtest/readout_type_scan_test.py +daqsystemtest/small_footprint_quick_test.py +daqsystemtest/tpg_state_collection_test.py +daqsystemtest/tpreplay_test.py +daqsystemtest/tpstream_writing_test.py +daqsystemtest/trigger_bitwords_test.py +dfmodules/disabled_output_test.py +dfmodules/hdf5_compression_test.py +dfmodules/insufficient_disk_space_test.py +dfmodules/large_trigger_record_test.py +dfmodules/max_file_size_test.py +dfmodules/multiple_data_writers_test.py +dfmodules/offline_prod_run_test.py +dfmodules/trmonrequestor_test.py + +(dbt) [biery@daq]$ list_available_integtests.sh asiolibs crtmodules + +Looking for integtests in the _asiolibs crtmodules_ repo(s)... + +asiolibs/socket_reader_test.py +crtmodules/crt_reader_test.py + +(dbt) [biery@daq]$ list_available_integtests.sh asdf jklp + +Looking for integtests in the _asdf jklp_ repo(s)... + +-> "asdf" does not appear to be a valid repository name. +-> "jklp" does not appear to be a valid repository name. +``` + +### list_repos_with_integtests.sh + +``` +(dbt) [biery@daq]$ list_repos_with_integtests.sh + +Looking for _all_ repositories with integtests in them... + +asiolibs +crtmodules +daqsystemtest +dfmodules +hsilibs +listrev +snbmodules +trigger + +(dbt) [biery@daq]$ list_repos_with_integtests.sh local + +Looking for _local_ repositories with integtests in them... + +daqsystemtest +dfmodules + +(dbt) [biery@daq]$ list_repos_with_integtests.sh asdf + +Looking for _all_ repositories with integtests in them... + +asiolibs +crtmodules +daqsystemtest +dfmodules +hsilibs +listrev +snbmodules +trigger +``` diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh new file mode 100755 index 00000000..6ec85113 --- /dev/null +++ b/scripts/dunedaq_integtest_bundle.sh @@ -0,0 +1,358 @@ +#!/bin/bash +# 10-Oct-2023, KAB + +integtest_list=() + +usage() { + declare -r script_name=$(basename "$0") + echo """ +Usage: +"${script_name}" [option(s)] + +Options: + -h, --help : prints out usage information + -r + - this can be the name of a single repo + - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' + - it can have the special value of \"all\" - integtests in all repos will be run + - it can have the special value of \"local\" - integtests in locally-cloned repos will be run + -k, --include + -x, --exclude + -n + -N + --stop-on-failure : causes the script to stop when one of the integtests reports a failure + --concise-output : suppresses run control and DAQApp messages in order to focus on test results + --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' + --list-only : list the tests that match the requested patterns without running them +""" +} + +# 29-Dec-2025, KAB: Determine if a non-standard pytest tmpdir has been specified +# in the linux shell environment in which this script is being run. We need to know +# this value in order to direct functionality in this script to the right place. +# A user-specified command-line value for the tmpdir over-rides the value determined here. +tmpdir_root=`dst_get_pytest_tmpdir` + +# Removes the ANSI characters associated with formatting, including color coding and font styling +CaptureOutputNoANSI() { + tee -a >(sed -u 's/\x1b\[[0-9;]*m//g' >> "$1") +} +# Captures the output to the specified file, without changing the output +CaptureOutput() { + tee -a $1 +} + +GETOPT_TEMP=`getopt -o hr:k:x:n:N: --long help,stop-on-failure,concise-output,include:,exclude:,tmpdir:,list-only -- "$@"` +if [ $? -ne 0 ]; then + exit 1 +fi +eval set -- "$GETOPT_TEMP" + +let individual_test_requested_iterations=1 +let full_set_requested_interations=1 +let stop_on_failure=0 +requested_test_names= +excluded_test_names= +only_list_tests="" +PYTEST_COMMAND="pytest -s --tb=short" # our core pytest command, with DAQ printout included and short pytest traceback + +while true; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -r) + if [[ "$2" == "all" ]]; then + echo "" + echo "Building the list of _all_ integtests..." + integtest_list=(`list_available_integtests.sh 2>/dev/null`) + if [[ ${#integtest_list[@]} -eq 0 ]]; then + echo "" + echo "*** No integtests were found!" + echo "" + exit 3 + fi + elif [[ "$2" == "local" ]]; then + echo "" + echo "Building the list of _local_ integtests..." + integtest_list=(`list_available_integtests.sh local 2>/dev/null`) + if [[ ${#integtest_list[@]} -eq 0 ]]; then + echo "" + echo "*** No integtests were found in local repositories!" + echo "" + exit 3 + fi + else + repo_list_string=`echo $2 | sed 's/|/ /g'` + integtest_list=(`list_available_integtests.sh ${repo_list_string} 2>/dev/null`) + if [[ ${#integtest_list[@]} -eq 0 ]]; then + echo "" + echo "*** No integtests were found in the \"${repo_list_string}\" repo(s)." + echo "" + exit 3 + fi + fi + shift 2 + ;; + -k|--include) + requested_test_names=$2 + shift 2 + ;; + -x|--exclude) + excluded_test_names=$2 + shift 2 + ;; + -n) + let individual_test_requested_iterations=$2 + shift 2 + ;; + -N) + let full_set_requested_interations=$2 + shift 2 + ;; + --stop-on-failure) + let stop_on_failure=1 + PYTEST_COMMAND="${PYTEST_COMMAND} -x" # add the -x option to our pytest command to have it exit on first error + shift + ;; + --concise-output) + PYTEST_COMMAND="`echo ${PYTEST_COMMAND} | sed 's/ -s//'`" # remove the -s option to turn off messages from DAQ processes + shift + ;; + --tmpdir) + tmpdir_root=$2 + export PYTEST_DEBUG_TEMPROOT=${tmpdir_root} + shift 2 + ;; + --list-only) + only_list_tests="yes" + shift + ;; + --) + shift + break + ;; + esac +done + +# run the integtests from the daqsystemtest repo if no repo was specified +if [[ "${integtest_list}" == "" ]]; then + integtest_list=(`list_available_integtests.sh daqsystemtest 2>/dev/null`) + echo "" + echo "Integtests from the _daqsystemtest_ repo will be run..." +fi + +# check if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "*********************************************************************" + echo "*** DANGER, DANGER, 'numad' appears to be running on this computer!" + echo "*** 'ps' output: ${numad_grep_output}" + echo "*** now if you want to abort this testing." + echo "*********************************************************************" + sleep 3 +fi + +# other setup +INITIAL_TIMESTAMP=`date '+%Y%m%d%H%M%S'` +# 30-Dec-2025, KAB: check that the specified tmpdir exists and is writeable +if [[ ! -d ${tmpdir_root} ]]; then + echo "" + echo "*** ERROR: directory \"${tmpdir_root}\" does not exist." + echo "" + exit 1 +fi +if [[ ! -w ${tmpdir_root} ]]; then + echo "" + echo "*** ERROR: directory \"${tmpdir_root}\" is not writeable in the current environment." + echo "" + exit 1 +fi +pytest_user_dir=${tmpdir_root}/pytest-of-${USER} +mkdir -p ${pytest_user_dir} +ITGRUNNER_LOG_FILE="${pytest_user_dir}/dunedaq_integtest_bundle_${INITIAL_TIMESTAMP}.log" +CURRENT_PID=$$ + +if [[ "$only_list_tests" != "" ]]; then + echo "" + echo "The following tests will be run:" +fi +let number_of_individual_tests=0 +let test_index=0 +for FULL_TEST_NAME in "${integtest_list[@]}"; do + test_name=`basename ${FULL_TEST_NAME}` + requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` + excluded_test=`echo ${test_name} | egrep -i ${excluded_test_names:-nullnullnull}` + if [[ "${requested_test}" != "" ]] && [[ "${excluded_test}" == "" ]]; then + let number_of_individual_tests=${number_of_individual_tests}+1 + if [[ "$only_list_tests" != "" ]]; then + echo " ${FULL_TEST_NAME}" + fi + fi + let test_index=${test_index}+1 +done +let total_number_of_tests=${number_of_individual_tests}*${individual_test_requested_iterations}*${full_set_requested_interations} +if [[ "$only_list_tests" != "" ]]; then + exit 0 +fi + +# run the tests +let overall_test_index=0 # this is only used for user feedback +let full_set_loop_count=0 +while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do + let test_index=0 + for FULL_TEST_NAME in "${integtest_list[@]}"; do + test_repo=`dirname ${FULL_TEST_NAME}` + test_name=`basename ${FULL_TEST_NAME}` + CURRENT_TIMESTAMP=`date '+%Y%m%d%H%M%S'` + # 15-Dec-2025, KAB: added the export of the following enviromental variable. This is used + # by the integrationtest infrastructure to put a bread-crumb file in the directory where + # the test results are located. That file, in turn, allows this script to find the directory + # for the current test, and make a copy of it if the test fails. + export DUNEDAQ_INTEGTEST_BUNDLE_INFO="${INITIAL_TIMESTAMP};${CURRENT_PID};${CURRENT_TIMESTAMP}" + requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` + excluded_test=`echo ${test_name} | egrep -i ${excluded_test_names:-nullnullnull}` + if [[ "${requested_test}" != "" ]] && [[ "${excluded_test}" == "" ]]; then + let individual_loop_count=0 + while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do + let overall_test_index=${overall_test_index}+1 + echo "" + echo -e "\U0001F535 \033[0;34mStarting test ${overall_test_index} of ${total_number_of_tests}...\033[0m \U0001F535" | CaptureOutput ${ITGRUNNER_LOG_FILE} + + echo -e "\u2B95 \033[0;1mRunning ${FULL_TEST_NAME}\033[0m \u2B05" | CaptureOutput ${ITGRUNNER_LOG_FILE} + if [[ -e "./${test_name}" ]]; then + ${PYTEST_COMMAND} ./${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + elif [[ -e "${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name}" ]]; then + if [[ -w "${DBT_AREA_ROOT}" ]]; then + ${PYTEST_COMMAND} ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + else + ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + fi + else + share_envvar_name="${test_repo^^}_SHARE" # double caret converts env var to uppercase + ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${!share_envvar_name}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + fi + let pytest_return_code=${PIPESTATUS[0]} + + let individual_loop_count=${individual_loop_count}+1 + + # check if the test failed + if [[ ${pytest_return_code} -ne 0 ]]; then + # 15-Dec-2025, KAB: if the test failed for a reason other than it + # couldn't be found, make a copy of the pytest directory. This allows + # testers to take a look at the results within a reasonable time frame. + # (If we can't find the "jq" JSON utility, we simply note that fact + # and continue.) + # This code makes use of a bread-crumb file that is created by the + # integrationtest infrastructure. + if [[ ${pytest_return_code} -ne 4 ]]; then + if [[ "`which jq 2>/dev/null`" != "" ]]; then + current_pytest_rundir="" + mapfile -t bundle_info_files < <(find "${pytest_user_dir}" -type f -name "bundle_script_info.json" -printf '%T@ %p\n' | grep -v 'failed-' | sort -nr | awk '{print $2}') + for info_file in "${bundle_info_files[@]}"; do + script_start_time=`jq -r .bundle_script_start_time ${info_file}` + script_pid=`jq -r .bundle_script_process_id ${info_file}` + individual_test_start_time=`jq -r .individual_test_start_time ${info_file}` + if [[ ${script_start_time} -eq ${INITIAL_TIMESTAMP} ]] && \ + [[ ${script_pid} -eq ${CURRENT_PID} ]] && \ + [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then + current_pytest_rundir=$info_file + break + fi + done + + was_successfully_copied="" + if [[ "${current_pytest_rundir}" != "" ]]; then + pytest_tmpdir=`echo ${current_pytest_rundir} | xargs -r dirname | xargs -r dirname` + if [[ "${pytest_tmpdir}" != "" ]]; then + pytest_rootdir=`echo ${pytest_tmpdir} | xargs -r dirname` + pytest_basedir=`echo ${pytest_tmpdir} | xargs -r basename` + if [[ "${pytest_rootdir}" != "" ]] && [[ "${pytest_basedir}" != "" ]]; then + new_dir="${pytest_rootdir}/failed-${pytest_basedir}" + echo "" + echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535" + echo -e "\U1F535 Please note that copied directories from failed tests typically get cleaned up after 26 hours, \U1F535" + echo -e "\U1F535 or when 10 newer failures happen, whichever comes first. \U1F535" + cp -pR "${pytest_tmpdir}" "${new_dir}" + if [[ $? == 0 ]]; then + was_successfully_copied="yes" + # 18-Dec-2025, KAB: added the removal of the "current" symbolic links + # from inside the copied directory (since they get broken in the copying) + rm -f "${new_dir}/configcurrent" + rm -f "${new_dir}/runcurrent" + fi + fi + fi + fi + if [[ "${was_successfully_copied}" == "" ]]; then + echo "" + echo -e "\U1f7e1 WARNING: Unable to copy the pytest directory for this failed test (${current_pytest_rundir}). \U1f7e1" + fi + else + echo "" + echo -e "\U1f7e1 WARNING: Unable to find the 'jq' utility which is needed to help identify which pytest directory to copy for this failed test. \U1f7e1" + fi + + # remove stale and surplus directories from failed tests + test_dirs_to_remove=() + mapfile -t all_failed_test_dirs < <(find ${pytest_user_dir} -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk '{print $2}' | grep 'failed-') + surplus_dirs=("${all_failed_test_dirs[@]:10}") + for test_dir in "${surplus_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + stale_failed_test_dirs=(`find ${pytest_user_dir} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`) + for test_dir in "${stale_failed_test_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then + echo "" + echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" + for test_dir in "${test_dirs_to_remove[@]}"; do + if [[ -e "${test_dir}" ]]; then + rm -rf "${test_dir}" + fi + done + fi + fi + + # exit out of this script if the user has requested that we stop on a failure + if [[ ${stop_on_failure} -gt 0 ]]; then + break 3 + fi + fi + done + fi + let test_index=${test_index}+1 + done + + let full_set_loop_count=${full_set_loop_count}+1 +done + +# print out summary information +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "+++++++++++++++++++++++++++++++++++++++++++++++++" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "++++++++++++++++++++ SUMMARY ++++++++++++++++++++" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "+++++++++++++++++++++++++++++++++++++++++++++++++" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +date | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "Log file is: ${ITGRUNNER_LOG_FILE}" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +summary_string="`egrep $'=====|\u2B95' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running'`" +colorized_summary_string="`echo \"${summary_string}\" | sed 's/passed/passed \\\\U2705/' | sed 's/failed/failed \\\\U274c/' | sed 's/no tests ran/no tests ran \\\\U274c/' | sed 's/skipped/skipped \\\\U1f7e1/'`" +echo -e "${colorized_summary_string}" | CaptureOutput ${ITGRUNNER_LOG_FILE} + +# check again if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** WARNING: 'numad' appears to be running on this computer!" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** 'ps' output: ${numad_grep_output}" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** This daemon can adversely affect the running of these tests, especially ones" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** that are resource intensive in the Readout Apps. This is because numad moves" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** processes (threads?) to different cores/numa nodes periodically, and that" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** context switch can disrupt the stable running of the DAQ processes." | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | CaptureOutput ${ITGRUNNER_LOG_FILE} +fi diff --git a/scripts/list_available_integtests.sh b/scripts/list_available_integtests.sh new file mode 100755 index 00000000..9e9dc26c --- /dev/null +++ b/scripts/list_available_integtests.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# 19-Dec-2025, KAB + +if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then + echo + echo "Usage: `basename $0` [optional list of repo names|local|all]" + echo " e.g. `basename $0` daqsystemtest" + echo " If no repo name is specified, integtests for all repos are listed." + echo " If a special repo name of \"local\" is specified, only integtests for repos" + echo " in the local software area are listed." + echo " If a special repo name of \"all\" is specified, integtests for all repos are listed." + echo + exit +fi + +echo "" >&2 +repo_list=() +if [[ $# -ge 1 ]]; then + if [[ "$1" == "local" ]]; then + echo "Looking for integtests in _local_ repos..." >&2 + echo "" >&2 + repo_list=(`list_repos_with_integtests.sh local 2>/dev/null`) + elif [[ "$1" == "all" ]]; then + echo "Looking for integtests in _all_ repos..." >&2 + echo "" >&2 + repo_list=(`list_repos_with_integtests.sh 2>/dev/null`) + else + echo "Looking for integtests in the _$@_ repo(s)..." >&2 + echo "" >&2 + repo_list=("$@") + fi +else + echo "Looking for integtests in _all_ repos..." >&2 + echo "" >&2 + repo_list=(`list_repos_with_integtests.sh 2>/dev/null`) +fi + +for repo_name in "${repo_list[@]}"; do + share_envvar_name="${repo_name^^}_SHARE" # double caret converts env var to uppercase + # ${!var} returns what var points to + if [[ -e "${!share_envvar_name}/integtest" ]] || [[ -e "${DBT_AREA_ROOT}/sourcecode/${repo_name}/integtest" ]]; then + integtest_list=(`ls -1 ${!share_envvar_name}/integtest/*_test.py ${DBT_AREA_ROOT}/sourcecode/${repo_name}/integtest/*_test.py 2>/dev/null | xargs -r -n 1 basename | sort -u`) + if [[ ${#integtest_list[@]} -gt 0 ]]; then + for test_name in "${integtest_list[@]}"; do + echo "${repo_name}/${test_name}" + done + else + echo "-> No integtests were found for repository \"${repo_name}\"." >&2 + fi + else + if [[ -e "${DBT_AREA_ROOT}/sourcecode/${repo_name}" ]]; then + echo "-> No integtest directory was found in ${DBT_AREA_ROOT}/sourcecode/${repo_name}." >&2 + fi + if [[ "${!share_envvar_name}" == "" ]]; then + echo "-> \"${repo_name}\" does not appear to be a valid repository name." >&2 + else + echo "-> No integtest directory was found in ${share_envvar_name} (${!share_envvar_name})." >&2 + fi + fi +done diff --git a/scripts/list_repos_with_integtests.sh b/scripts/list_repos_with_integtests.sh new file mode 100755 index 00000000..2e973e96 --- /dev/null +++ b/scripts/list_repos_with_integtests.sh @@ -0,0 +1,49 @@ +#!/bin/bash +# 19-Dec-2025, KAB + +if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then + echo + echo "Usage: `basename $0` [optional \"local\" keyword]" + echo " Lists the software repositories that have integration tests (integtests) in them." + echo " Searches the base releases, local install dir, and local sourcecode dir," + echo " unless \"local\" is passed as an argument. In that case, only the local" + echo " install and sourcecode directories are searched." + echo + exit +fi + +# initialization +all_repo_paths=() + +# skip repos in the base release, if the user has specified "local" +echo "" >&2 +if [[ $# -eq 0 ]] || [[ "$1" != "local" ]]; then + echo "Looking for _all_ repositories with integtests in them..." >&2 + echo "" >&2 + + # determine the base release directory + release_dir=`dbt-info release | grep 'Release dir:' | cut -d' ' -f3` + base_release_name=`dbt-info release | grep 'Base release name:' | cut -d' ' -f4` + base_release_dir="${release_dir}/../${base_release_name}" + + # look up the paths of all of the repositories in the core and detector-specific categories + det_rel_repo_paths=(`ls -1d ${release_dir}/spack-installation/opt/spack/*linux*/gcc-*/*/*/integtest/*_test.py`) + base_rel_repo_paths=(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*linux*/gcc-*/*/*/integtest/*_test.py`) + all_repo_paths=("${det_rel_repo_paths[@]}" "${base_rel_repo_paths[@]}") +else + echo "Looking for _local_ repositories with integtests in them..." >&2 + echo "" >&2 +fi + +# add in the paths of the repositories in the local install and sourcecode dirs +if [[ "$DBT_AREA_ROOT" != "" ]]; then + install_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/install/*/share/integtest/*_test.py`) + sourcecode_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/sourcecode/*/integtest/*_test.py`) + all_repo_paths=("${all_repo_paths[@]}" "${install_dir_repo_paths[@]}" "${sourcecode_dir_repo_paths[@]}") +fi + +repos_with_integtests=(`echo "${all_repo_paths[@]}" | sed 's,/share,,g' | xargs -r -n 1 dirname | xargs -r -n 1 dirname | xargs -r -n 1 basename | cut -d'-' -f1 | sort -u`) + +for repo in "${repos_with_integtests[@]}"; do + echo "$repo" +done