From 617d2ef5ee6958c4cf4fc9eff0ce3ca69d27b071 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Sat, 20 Dec 2025 18:21:13 -0600 Subject: [PATCH 01/15] Initial utility scripts for a global bundle script. --- scripts/list_all_repos_with_integtests.sh | 16 ++++++++++++ scripts/list_available_integtests.sh | 32 +++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100755 scripts/list_all_repos_with_integtests.sh create mode 100755 scripts/list_available_integtests.sh diff --git a/scripts/list_all_repos_with_integtests.sh b/scripts/list_all_repos_with_integtests.sh new file mode 100755 index 0000000..e21b13e --- /dev/null +++ b/scripts/list_all_repos_with_integtests.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# 19-Dec-2025, KAB + +release_dir=`dbt-info release | grep 'Release dir:' | cut -d' ' -f3` +base_release_name=`dbt-info release | grep 'Base release name:' | cut -d' ' -f4` +base_release_dir="${release_dir}/../${base_release_name}" + +groupA="(`ls -1d ${release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest`)" +groupB="(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest`)" +groupC=("${groupA}"+" "+"${groupB}") + +the_list=("`echo ${groupC} | xargs -r -n 1 dirname | xargs -r -n 1 dirname | xargs -r -n 1 basename | cut -d'-' -f1 | sort`") + +for pkg in ${the_list[@]}; do + echo $pkg +done diff --git a/scripts/list_available_integtests.sh b/scripts/list_available_integtests.sh new file mode 100755 index 0000000..61af74b --- /dev/null +++ b/scripts/list_available_integtests.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# 19-Dec-2025, KAB + +if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then + echo + echo "Usage: $0 [optional_repo_name]" + echo " e.g. $0 daqsystemtest" + echo " If no repo name is specified, integtests for all repos are listed." + echo + exit +fi + +repo_list=() +if [ $# -ge 1 ]; then + repo_list=("$1") +else + echo + echo "Finding the repositories that have integtests..." + repo_list=("`list_all_repos_with_integtests.sh`") +fi + +for repo in ${repo_list[@]}; do + echo + echo "*** ${repo} ***" + echo + share_envvar_name="${repo^^}_SHARE" + if [[ -e ${!share_envvar_name}/integtest ]]; then + ls -1 ${!share_envvar_name}/integtest/*_test.py | xargs -n 1 basename + else + echo "-> No integtest directory was found in \$${share_envvar_name}." + fi +done From 3d2930d62cfe287ed80271dd90e9c1ffc4215168 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 7 Jan 2026 12:44:37 -0600 Subject: [PATCH 02/15] Made some changes to the preliminary scripts for finding integtests. --- scripts/list_all_repos_with_integtests.sh | 30 +++++++++++++++---- scripts/list_available_integtests.sh | 35 ++++++++++++++--------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/scripts/list_all_repos_with_integtests.sh b/scripts/list_all_repos_with_integtests.sh index e21b13e..a35965f 100755 --- a/scripts/list_all_repos_with_integtests.sh +++ b/scripts/list_all_repos_with_integtests.sh @@ -1,16 +1,34 @@ #!/bin/bash # 19-Dec-2025, KAB +if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then + echo + echo "Usage: `basename $0`" + echo " Lists the software repositories that have integration tests (integtests) in them." + echo " Searches the base releases, local install dir, and local sourcecode dir." + echo + exit +fi + +# determine the base release directory release_dir=`dbt-info release | grep 'Release dir:' | cut -d' ' -f3` base_release_name=`dbt-info release | grep 'Base release name:' | cut -d' ' -f4` base_release_dir="${release_dir}/../${base_release_name}" -groupA="(`ls -1d ${release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest`)" -groupB="(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest`)" -groupC=("${groupA}"+" "+"${groupB}") +# look up the paths of all of the repositories in the core and detector-specific categories +det_rel_repo_paths=(`ls -1d ${release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) +base_rel_repo_paths=(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) +all_repo_paths=("${det_rel_repo_paths[@]}" "${base_rel_repo_paths[@]}") + +# add in the paths of the repositories in the local install and sourcecode dirs +if [[ "$DBT_AREA_ROOT" != "" ]]; then + install_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/install/*/share/integtest/*_test.py`) + sourcecode_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/sourcecode/*/integtest/*_test.py`) + all_repo_paths=("${all_repo_paths[@]}" "${install_dir_repo_paths[@]}" "${sourcecode_dir_repo_paths[@]}") +fi -the_list=("`echo ${groupC} | xargs -r -n 1 dirname | xargs -r -n 1 dirname | xargs -r -n 1 basename | cut -d'-' -f1 | sort`") +repos_with_integtests=(`echo "${all_repo_paths[@]}" | sed 's,/share,,g' | xargs -r -n 1 dirname | xargs -r -n 1 dirname | xargs -r -n 1 basename | cut -d'-' -f1 | sort -u`) -for pkg in ${the_list[@]}; do - echo $pkg +for repo in "${repos_with_integtests[@]}"; do + echo "$repo" done diff --git a/scripts/list_available_integtests.sh b/scripts/list_available_integtests.sh index 61af74b..b0296ad 100755 --- a/scripts/list_available_integtests.sh +++ b/scripts/list_available_integtests.sh @@ -3,8 +3,8 @@ if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then echo - echo "Usage: $0 [optional_repo_name]" - echo " e.g. $0 daqsystemtest" + echo "Usage: `basename $0` [optional_repo_name]" + echo " e.g. `basename $0` daqsystemtest" echo " If no repo name is specified, integtests for all repos are listed." echo exit @@ -12,21 +12,28 @@ fi repo_list=() if [ $# -ge 1 ]; then - repo_list=("$1") + repo_list=("$@") else - echo - echo "Finding the repositories that have integtests..." - repo_list=("`list_all_repos_with_integtests.sh`") + repo_list=(`list_all_repos_with_integtests.sh`) fi -for repo in ${repo_list[@]}; do - echo - echo "*** ${repo} ***" - echo - share_envvar_name="${repo^^}_SHARE" - if [[ -e ${!share_envvar_name}/integtest ]]; then - ls -1 ${!share_envvar_name}/integtest/*_test.py | xargs -n 1 basename +for repo_name in "${repo_list[@]}"; do + share_envvar_name="${repo_name^^}_SHARE" # double caret converts env var to uppercase + # ${!var} returns what var points to + if [[ -e "${!share_envvar_name}/integtest" ]] || [[ -e "${DBT_AREA_ROOT}/sourcecode/${repo_name}/integtest" ]]; then + integtest_list=(`ls -1 ${!share_envvar_name}/integtest/*_test.py ${DBT_AREA_ROOT}/sourcecode/${repo_name}/integtest/*_test.py 2>/dev/null | xargs -r -n 1 basename | sort -u`) + echo "KAB ${#integtest_list[@]}" + if [[ ${#integtest_list[@]} -gt 0 ]]; then + for test_name in "${integtest_list[@]}"; do + echo "${repo_name}/${test_name}" + done + else + echo "-> No integtests were found for repository \"${repo_name}\"." + fi else - echo "-> No integtest directory was found in \$${share_envvar_name}." + if [[ -e "${DBT_AREA_ROOT}/sourcecode/${repo_name}" ]]; then + echo "-> No integtest directory was found in ${DBT_AREA_ROOT}/sourcecode/${repo_name}." + fi + echo "-> No integtest directory was found in ${share_envvar_name} (${!share_envvar_name})." fi done From 3b8757c32515abef9e9f4cfbb79040f15c238c77 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 21 Jan 2026 19:14:58 -0600 Subject: [PATCH 03/15] next round of changes for a global bundle script --- scripts/dunedaq_integtest_bundle.sh | 329 ++++++++++++++++++++++++++ scripts/list_available_integtests.sh | 25 +- scripts/list_repos_with_integtests.sh | 43 ++++ 3 files changed, 389 insertions(+), 8 deletions(-) create mode 100755 scripts/dunedaq_integtest_bundle.sh create mode 100755 scripts/list_repos_with_integtests.sh diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh new file mode 100755 index 0000000..b2c5426 --- /dev/null +++ b/scripts/dunedaq_integtest_bundle.sh @@ -0,0 +1,329 @@ +#!/bin/bash +# 10-Oct-2023, KAB + +integtest_list=() + +usage() { + declare -r script_name=$(basename "$0") + echo """ +Usage: +"${script_name}" [option(s)] + +Options: + -h, --help : prints out usage information + -r + - this can be the name of a single repo + - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' + - it can have the special value of \"all\" - integtests in all repos will be run + - it can have the special value of \"local\" - integtests in locally-cloned repos will be run + -k + -n + -N + --stop-on-failure : causes the script to stop when one of the integtests reports a failure + --concise-output : suppresses run control and DAQApp messages in order to focus on test results + --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' +""" +} + +# 29-Dec-2025, KAB: Determine if a non-standard pytest tmpdir has been specified +# in the linux shell environment in which this script is being run. We need to know +# this value in order to direct functionality in this script to the right place. +# A user-specified command-line value for the tmpdir over-rides the value determined here. +tmpdir_root=`dst_get_pytest_tmpdir` + +# Removes the ANSI characters associated with formatting, including color coding and font styling +CaptureOutputNoANSI() { + tee -a >(sed -u 's/\x1b\[[0-9;]*m//g' >> "$1") +} +# Captures the output to the specified file, without changing the output +CaptureOutput() { + tee -a $1 +} + +GETOPT_TEMP=`getopt -o hr:k:n:N: --long help,stop-on-failure,concise-output,tmpdir: -- "$@"` +if [ $? -ne 0 ]; then + exit 1 +fi +eval set -- "$GETOPT_TEMP" + +let individual_test_requested_iterations=1 +let full_set_requested_interations=1 +let stop_on_failure=0 +requested_test_names= +PYTEST_COMMAND="pytest -s --tb=short" # our core pytest command, with DAQ printout included and short pytest traceback + +while true; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + -r) + if [[ "$2" == "all" ]]; then + echo "" + echo "Building the list of all integtests..." + integtest_list=(`list_available_integtests.sh 2>/dev/null`) + if [[ ${#integtest_list[@]} -eq 0 ]]; then + echo "" + echo "*** No integtests were found!" + echo "" + exit 3 + fi + elif [[ "$2" == "local" ]]; then + echo "" + echo "Building the list of local integtests..." + integtest_list=(`list_available_integtests.sh local 2>/dev/null`) + if [[ ${#integtest_list[@]} -eq 0 ]]; then + echo "" + echo "*** No integtests were found in local repositories!" + echo "" + exit 3 + fi + else + repo_list_string=`echo $2 | sed 's/|/ /g'` + integtest_list=(`list_available_integtests.sh ${repo_list_string} 2>/dev/null`) + if [[ ${#integtest_list[@]} -eq 0 ]]; then + echo "" + echo "*** No integtests were found in the \"${repo_list_string}\" repo(s)." + echo "" + exit 3 + fi + fi + shift 2 + ;; + -k) + requested_test_names=$2 + shift 2 + ;; + -n) + let individual_test_requested_iterations=$2 + shift 2 + ;; + -N) + let full_set_requested_interations=$2 + shift 2 + ;; + --stop-on-failure) + let stop_on_failure=1 + PYTEST_COMMAND="${PYTEST_COMMAND} -x" # add the -x option to our pytest command to have it exit on first error + shift + ;; + --concise-output) + PYTEST_COMMAND="`echo ${PYTEST_COMMAND} | sed 's/ -s//'`" # remove the -s option to turn off messages from DAQ processes + shift + ;; + --tmpdir) + tmpdir_root=$2 + export PYTEST_DEBUG_TEMPROOT=${tmpdir_root} + shift 2 + ;; + --) + shift + break + ;; + esac +done + +# run the integtests from the daqsystemtest repo if no repo was specified +if [[ "${integtest_list}" == "" ]]; then + integtest_list=(`list_available_integtests.sh daqsystemtest 2>/dev/null`) +fi + +# check if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "*********************************************************************" + echo "*** DANGER, DANGER, 'numad' appears to be running on this computer!" + echo "*** 'ps' output: ${numad_grep_output}" + echo "*** now if you want to abort this testing." + echo "*********************************************************************" + sleep 3 +fi + +# other setup +INITIAL_TIMESTAMP=`date '+%Y%m%d%H%M%S'` +# 30-Dec-2025, KAB: check that the specified tmpdir exists and is writeable +if [[ ! -d ${tmpdir_root} ]]; then + echo "" + echo "*** ERROR: directory \"${tmpdir_root}\" does not exist." + echo "" + exit 1 +fi +if [[ ! -w ${tmpdir_root} ]]; then + echo "" + echo "*** ERROR: directory \"${tmpdir_root}\" is not writeable in the current environment." + echo "" + exit 1 +fi +pytest_user_dir=${tmpdir_root}/pytest-of-${USER} +mkdir -p ${pytest_user_dir} +ITGRUNNER_LOG_FILE="${pytest_user_dir}/dunedaq_integtest_bundle_${INITIAL_TIMESTAMP}.log" +CURRENT_PID=$$ + +let number_of_individual_tests=0 +let test_index=0 +for FULL_TEST_NAME in "${integtest_list[@]}"; do + test_name=`basename ${FULL_TEST_NAME}` + requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` + if [[ "${requested_test}" != "" ]]; then + let number_of_individual_tests=${number_of_individual_tests}+1 + fi + let test_index=${test_index}+1 +done +let total_number_of_tests=${number_of_individual_tests}*${individual_test_requested_iterations}*${full_set_requested_interations} + +# run the tests +let overall_test_index=0 # this is only used for user feedback +let full_set_loop_count=0 +while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do + let test_index=0 + for FULL_TEST_NAME in "${integtest_list[@]}"; do + test_repo=`dirname ${FULL_TEST_NAME}` + test_name=`basename ${FULL_TEST_NAME}` + CURRENT_TIMESTAMP=`date '+%Y%m%d%H%M%S'` + # 15-Dec-2025, KAB: added the export of the following enviromental variable. This is used + # by the integrationtest infrastructure to put a bread-crumb file in the directory where + # the test results are located. That file, in turn, allows this script to find the directory + # for the current test, and make a copy of it if the test fails. + export DUNEDAQ_INTEGTEST_BUNDLE_INFO="${INITIAL_TIMESTAMP};${CURRENT_PID};${CURRENT_TIMESTAMP}" + requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` + if [[ "${requested_test}" != "" ]]; then + let individual_loop_count=0 + while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do + let overall_test_index=${overall_test_index}+1 + echo "" + echo -e "\U0001F535 \033[0;34mStarting test ${overall_test_index} of ${total_number_of_tests}...\033[0m \U0001F535" | CaptureOutput ${ITGRUNNER_LOG_FILE} + + echo -e "\u2B95 \033[0;1mRunning ${FULL_TEST_NAME}\033[0m \u2B05" | CaptureOutput ${ITGRUNNER_LOG_FILE} + if [[ -e "./${test_name}" ]]; then + ${PYTEST_COMMAND} ./${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + elif [[ -e "${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name}" ]]; then + if [[ -w "${DBT_AREA_ROOT}" ]]; then + ${PYTEST_COMMAND} ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + else + ${PYTEST_COMMAND} -p no:cacheprovider ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + fi + else + share_envvar_name="${test_repo^^}_SHARE" # double caret converts env var to uppercase + ${PYTEST_COMMAND} -p no:cacheprovider ${!share_envvar_name}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + fi + let pytest_return_code=${PIPESTATUS[0]} + + let individual_loop_count=${individual_loop_count}+1 + + # check if the test failed + if [[ ${pytest_return_code} -ne 0 ]]; then + # 15-Dec-2025, KAB: if the test failed for a reason other than it + # couldn't be found, make a copy of the pytest directory. This allows + # testers to take a look at the results within a reasonable time frame. + # (If we can't find the "jq" JSON utility, we simply note that fact + # and continue.) + # This code makes use of a bread-crumb file that is created by the + # integrationtest infrastructure. + if [[ ${pytest_return_code} -ne 4 ]]; then + if [[ "`which jq 2>/dev/null`" != "" ]]; then + current_pytest_rundir="" + mapfile -t bundle_info_files < <(find "${pytest_user_dir}" -type f -name "bundle_script_info.json" -printf '%T@ %p\n' | grep -v 'failed-' | sort -nr | awk '{print $2}') + for info_file in "${bundle_info_files[@]}"; do + script_start_time=`jq -r .bundle_script_start_time ${info_file}` + script_pid=`jq -r .bundle_script_process_id ${info_file}` + individual_test_start_time=`jq -r .individual_test_start_time ${info_file}` + if [[ ${script_start_time} -eq ${INITIAL_TIMESTAMP} ]] && \ + [[ ${script_pid} -eq ${CURRENT_PID} ]] && \ + [[ ${individual_test_start_time} -eq ${CURRENT_TIMESTAMP} ]]; then + current_pytest_rundir=$info_file + break + fi + done + + was_successfully_copied="" + if [[ "${current_pytest_rundir}" != "" ]]; then + pytest_tmpdir=`echo ${current_pytest_rundir} | xargs -r dirname | xargs -r dirname` + if [[ "${pytest_tmpdir}" != "" ]]; then + pytest_rootdir=`echo ${pytest_tmpdir} | xargs -r dirname` + pytest_basedir=`echo ${pytest_tmpdir} | xargs -r basename` + if [[ "${pytest_rootdir}" != "" ]] && [[ "${pytest_basedir}" != "" ]]; then + new_dir="${pytest_rootdir}/failed-${pytest_basedir}" + echo "" + echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535" + cp -pR "${pytest_tmpdir}" "${new_dir}" + if [[ $? == 0 ]]; then + was_successfully_copied="yes" + # 18-Dec-2025, KAB: added the removal of the "current" symbolic links + # from inside the copied directory (since they get broken in the copying) + rm -f "${new_dir}/configcurrent" + rm -f "${new_dir}/runcurrent" + fi + fi + fi + fi + if [[ "${was_successfully_copied}" == "" ]]; then + echo "" + echo -e "\U1f7e1 WARNING: Unable to copy the pytest directory for this failed test (${current_pytest_rundir}). \U1f7e1" + fi + else + echo "" + echo -e "\U1f7e1 WARNING: Unable to find the 'jq' utility which is needed to help identify which pytest directory to copy for this failed test. \U1f7e1" + fi + + # remove stale and surplus directories from failed tests + test_dirs_to_remove=() + mapfile -t all_failed_test_dirs < <(find ${pytest_user_dir} -maxdepth 1 -type d -printf '%T@ %p\n' | sort -nr | awk '{print $2}' | grep 'failed-') + surplus_dirs=("${all_failed_test_dirs[@]:10}") + for test_dir in "${surplus_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + stale_failed_test_dirs=(`find ${pytest_user_dir} -maxdepth 1 -type d -name 'failed-*' -cmin +1560 -print`) + for test_dir in "${stale_failed_test_dirs[@]}"; do + test_dirs_to_remove+=(${test_dir}) + done + if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then + echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" + for test_dir in "${test_dirs_to_remove[@]}"; do + if [[ -e "${test_dir}" ]]; then + rm -rf "${test_dir}" + fi + done + fi + fi + + # exit out of this script if the user has requested that we stop on a failure + if [[ ${stop_on_failure} -gt 0 ]]; then + break 3 + fi + fi + done + fi + let test_index=${test_index}+1 + done + + let full_set_loop_count=${full_set_loop_count}+1 +done + +# print out summary information +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "+++++++++++++++++++++++++++++++++++++++++++++++++" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "++++++++++++++++++++ SUMMARY ++++++++++++++++++++" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "+++++++++++++++++++++++++++++++++++++++++++++++++" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +date | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "Log file is: ${ITGRUNNER_LOG_FILE}" | CaptureOutput ${ITGRUNNER_LOG_FILE} +echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} +summary_string="`egrep $'=====|\u2B95' ${ITGRUNNER_LOG_FILE} | egrep ' in |Running'`" +colorized_summary_string="`echo \"${summary_string}\" | sed 's/passed/passed \\\\U2705/' | sed 's/failed/failed \\\\U274c/' | sed 's/no tests ran/no tests ran \\\\U274c/' | sed 's/skipped/skipped \\\\U1f7e1/'`" +echo -e "${colorized_summary_string}" | CaptureOutput ${ITGRUNNER_LOG_FILE} + +# check again if the numad daemon is running +numad_grep_output=`ps -ef | grep numad | grep -v grep` +if [[ "${numad_grep_output}" != "" ]]; then + echo "" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** WARNING: 'numad' appears to be running on this computer!" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** 'ps' output: ${numad_grep_output}" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** This daemon can adversely affect the running of these tests, especially ones" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** that are resource intensive in the Readout Apps. This is because numad moves" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** processes (threads?) to different cores/numa nodes periodically, and that" | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "*** context switch can disrupt the stable running of the DAQ processes." | CaptureOutput ${ITGRUNNER_LOG_FILE} + echo "********************************************************************************" | CaptureOutput ${ITGRUNNER_LOG_FILE} +fi diff --git a/scripts/list_available_integtests.sh b/scripts/list_available_integtests.sh index b0296ad..e665b8a 100755 --- a/scripts/list_available_integtests.sh +++ b/scripts/list_available_integtests.sh @@ -3,18 +3,24 @@ if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then echo - echo "Usage: `basename $0` [optional_repo_name]" + echo "Usage: `basename $0` [optional list of repo names]" echo " e.g. `basename $0` daqsystemtest" echo " If no repo name is specified, integtests for all repos are listed." + echo " If a special repo name of \"local\" is specified, only integtests for repos" + echo " in the local software area are listed." echo exit fi repo_list=() -if [ $# -ge 1 ]; then - repo_list=("$@") +if [[ $# -ge 1 ]]; then + if [[ "$1" == "local" ]]; then + repo_list=(`list_repos_with_integtests.sh local`) + else + repo_list=("$@") + fi else - repo_list=(`list_all_repos_with_integtests.sh`) + repo_list=(`list_repos_with_integtests.sh`) fi for repo_name in "${repo_list[@]}"; do @@ -22,18 +28,21 @@ for repo_name in "${repo_list[@]}"; do # ${!var} returns what var points to if [[ -e "${!share_envvar_name}/integtest" ]] || [[ -e "${DBT_AREA_ROOT}/sourcecode/${repo_name}/integtest" ]]; then integtest_list=(`ls -1 ${!share_envvar_name}/integtest/*_test.py ${DBT_AREA_ROOT}/sourcecode/${repo_name}/integtest/*_test.py 2>/dev/null | xargs -r -n 1 basename | sort -u`) - echo "KAB ${#integtest_list[@]}" if [[ ${#integtest_list[@]} -gt 0 ]]; then for test_name in "${integtest_list[@]}"; do echo "${repo_name}/${test_name}" done else - echo "-> No integtests were found for repository \"${repo_name}\"." + echo "-> No integtests were found for repository \"${repo_name}\"." >&2 fi else if [[ -e "${DBT_AREA_ROOT}/sourcecode/${repo_name}" ]]; then - echo "-> No integtest directory was found in ${DBT_AREA_ROOT}/sourcecode/${repo_name}." + echo "-> No integtest directory was found in ${DBT_AREA_ROOT}/sourcecode/${repo_name}." >&2 + fi + if [[ "${!share_envvar_name}" == "" ]]; then + echo "-> \"${repo_name}\" does not appear to be a valid repository name." >&2 + else + echo "-> No integtest directory was found in ${share_envvar_name} (${!share_envvar_name})." >&2 fi - echo "-> No integtest directory was found in ${share_envvar_name} (${!share_envvar_name})." fi done diff --git a/scripts/list_repos_with_integtests.sh b/scripts/list_repos_with_integtests.sh new file mode 100755 index 0000000..877a453 --- /dev/null +++ b/scripts/list_repos_with_integtests.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# 19-Dec-2025, KAB + +if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then + echo + echo "Usage: `basename $0` [optional \"local\" keyword]" + echo " Lists the software repositories that have integration tests (integtests) in them." + echo " Searches the base releases, local install dir, and local sourcecode dir," + echo " unless \"local\" is passed as an argument. In that case, only the local" + echo " install and sourcecode directories are searched." + echo + exit +fi + +# initialization +all_repo_paths=() + +# skip repos in the base release, if the user has specified "local" +if [[ $# -eq 0 ]] || [[ "$1" != "local" ]]; then + + # determine the base release directory + release_dir=`dbt-info release | grep 'Release dir:' | cut -d' ' -f3` + base_release_name=`dbt-info release | grep 'Base release name:' | cut -d' ' -f4` + base_release_dir="${release_dir}/../${base_release_name}" + + # look up the paths of all of the repositories in the core and detector-specific categories + det_rel_repo_paths=(`ls -1d ${release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) + base_rel_repo_paths=(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) + all_repo_paths=("${det_rel_repo_paths[@]}" "${base_rel_repo_paths[@]}") +fi + +# add in the paths of the repositories in the local install and sourcecode dirs +if [[ "$DBT_AREA_ROOT" != "" ]]; then + install_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/install/*/share/integtest/*_test.py`) + sourcecode_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/sourcecode/*/integtest/*_test.py`) + all_repo_paths=("${all_repo_paths[@]}" "${install_dir_repo_paths[@]}" "${sourcecode_dir_repo_paths[@]}") +fi + +repos_with_integtests=(`echo "${all_repo_paths[@]}" | sed 's,/share,,g' | xargs -r -n 1 dirname | xargs -r -n 1 dirname | xargs -r -n 1 basename | cut -d'-' -f1 | sort -u`) + +for repo in "${repos_with_integtests[@]}"; do + echo "$repo" +done From f8f4db27c88ce15d07cf55b36aaedfa05317d8c4 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 21 Jan 2026 19:15:18 -0600 Subject: [PATCH 04/15] next round of changes for a global bundle script --- scripts/list_all_repos_with_integtests.sh | 34 ----------------------- 1 file changed, 34 deletions(-) delete mode 100755 scripts/list_all_repos_with_integtests.sh diff --git a/scripts/list_all_repos_with_integtests.sh b/scripts/list_all_repos_with_integtests.sh deleted file mode 100755 index a35965f..0000000 --- a/scripts/list_all_repos_with_integtests.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -# 19-Dec-2025, KAB - -if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then - echo - echo "Usage: `basename $0`" - echo " Lists the software repositories that have integration tests (integtests) in them." - echo " Searches the base releases, local install dir, and local sourcecode dir." - echo - exit -fi - -# determine the base release directory -release_dir=`dbt-info release | grep 'Release dir:' | cut -d' ' -f3` -base_release_name=`dbt-info release | grep 'Base release name:' | cut -d' ' -f4` -base_release_dir="${release_dir}/../${base_release_name}" - -# look up the paths of all of the repositories in the core and detector-specific categories -det_rel_repo_paths=(`ls -1d ${release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) -base_rel_repo_paths=(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) -all_repo_paths=("${det_rel_repo_paths[@]}" "${base_rel_repo_paths[@]}") - -# add in the paths of the repositories in the local install and sourcecode dirs -if [[ "$DBT_AREA_ROOT" != "" ]]; then - install_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/install/*/share/integtest/*_test.py`) - sourcecode_dir_repo_paths=(`ls -1 ${DBT_AREA_ROOT}/sourcecode/*/integtest/*_test.py`) - all_repo_paths=("${all_repo_paths[@]}" "${install_dir_repo_paths[@]}" "${sourcecode_dir_repo_paths[@]}") -fi - -repos_with_integtests=(`echo "${all_repo_paths[@]}" | sed 's,/share,,g' | xargs -r -n 1 dirname | xargs -r -n 1 dirname | xargs -r -n 1 basename | cut -d'-' -f1 | sort -u`) - -for repo in "${repos_with_integtests[@]}"; do - echo "$repo" -done From cae83c8c8502a1597ae103cff744b496d4d18cf4 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 27 Jan 2026 13:12:05 -0600 Subject: [PATCH 05/15] Added --exclude option to dunedaq_integtest_bundle.sh and hopefully prevented the printout of the pytest duration summary when running integtests from the base release --- scripts/dunedaq_integtest_bundle.sh | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh index b2c5426..33068b1 100755 --- a/scripts/dunedaq_integtest_bundle.sh +++ b/scripts/dunedaq_integtest_bundle.sh @@ -16,7 +16,8 @@ Options: - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' - it can have the special value of \"all\" - integtests in all repos will be run - it can have the special value of \"local\" - integtests in locally-cloned repos will be run - -k + -k, --include + -x, --exclude -n -N --stop-on-failure : causes the script to stop when one of the integtests reports a failure @@ -40,7 +41,7 @@ CaptureOutput() { tee -a $1 } -GETOPT_TEMP=`getopt -o hr:k:n:N: --long help,stop-on-failure,concise-output,tmpdir: -- "$@"` +GETOPT_TEMP=`getopt -o hr:k:x:n:N: --long help,stop-on-failure,concise-output,include,exclude,tmpdir: -- "$@"` if [ $? -ne 0 ]; then exit 1 fi @@ -50,6 +51,7 @@ let individual_test_requested_iterations=1 let full_set_requested_interations=1 let stop_on_failure=0 requested_test_names= +excluded_test_names= PYTEST_COMMAND="pytest -s --tb=short" # our core pytest command, with DAQ printout included and short pytest traceback while true; do @@ -91,10 +93,14 @@ while true; do fi shift 2 ;; - -k) + -k|--include) requested_test_names=$2 shift 2 ;; + -x|--exclude) + excluded_test_names=$2 + shift 2 + ;; -n) let individual_test_requested_iterations=$2 shift 2 @@ -165,7 +171,8 @@ let test_index=0 for FULL_TEST_NAME in "${integtest_list[@]}"; do test_name=`basename ${FULL_TEST_NAME}` requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` - if [[ "${requested_test}" != "" ]]; then + excluded_test=`echo ${test_name} | egrep -i ${excluded_test_names:-nullnullnull}` + if [[ "${requested_test}" != "" ]] && [[ "${excluded_test}" == "" ]]; then let number_of_individual_tests=${number_of_individual_tests}+1 fi let test_index=${test_index}+1 @@ -187,7 +194,8 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do # for the current test, and make a copy of it if the test fails. export DUNEDAQ_INTEGTEST_BUNDLE_INFO="${INITIAL_TIMESTAMP};${CURRENT_PID};${CURRENT_TIMESTAMP}" requested_test=`echo ${test_name} | egrep -i ${requested_test_names:-${test_name}}` - if [[ "${requested_test}" != "" ]]; then + excluded_test=`echo ${test_name} | egrep -i ${excluded_test_names:-nullnullnull}` + if [[ "${requested_test}" != "" ]] && [[ "${excluded_test}" == "" ]]; then let individual_loop_count=0 while [[ ${individual_loop_count} -lt ${individual_test_requested_iterations} ]]; do let overall_test_index=${overall_test_index}+1 @@ -201,11 +209,11 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do if [[ -w "${DBT_AREA_ROOT}" ]]; then ${PYTEST_COMMAND} ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} else - ${PYTEST_COMMAND} -p no:cacheprovider ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${DBT_AREA_ROOT}/sourcecode/${test_repo}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} fi else share_envvar_name="${test_repo^^}_SHARE" # double caret converts env var to uppercase - ${PYTEST_COMMAND} -p no:cacheprovider ${!share_envvar_name}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} + ${PYTEST_COMMAND} -p no:cacheprovider --no-summary ${!share_envvar_name}/integtest/${test_name} | CaptureOutputNoANSI ${ITGRUNNER_LOG_FILE} fi let pytest_return_code=${PIPESTATUS[0]} From 42e9a917f94005e0b65c1076e6b133e5c4bdf048 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 28 Jan 2026 08:04:26 -0600 Subject: [PATCH 06/15] Added a list-only option to dunedaq_integtest_bundle.sh; fixed bugs in a couple of long options; added some echo statements to give feedback to the user on what is happening. --- scripts/dunedaq_integtest_bundle.sh | 24 +++++++++++++++++++++--- scripts/list_available_integtests.sh | 18 +++++++++++++++--- scripts/list_repos_with_integtests.sh | 6 ++++++ 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh index 33068b1..cb47cd2 100755 --- a/scripts/dunedaq_integtest_bundle.sh +++ b/scripts/dunedaq_integtest_bundle.sh @@ -23,6 +23,7 @@ Options: --stop-on-failure : causes the script to stop when one of the integtests reports a failure --concise-output : suppresses run control and DAQApp messages in order to focus on test results --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' + --list-only : list the tests that match the requested patterns without running them """ } @@ -41,7 +42,7 @@ CaptureOutput() { tee -a $1 } -GETOPT_TEMP=`getopt -o hr:k:x:n:N: --long help,stop-on-failure,concise-output,include,exclude,tmpdir: -- "$@"` +GETOPT_TEMP=`getopt -o hr:k:x:n:N: --long help,stop-on-failure,concise-output,include:,exclude:,tmpdir:,list-only -- "$@"` if [ $? -ne 0 ]; then exit 1 fi @@ -52,6 +53,7 @@ let full_set_requested_interations=1 let stop_on_failure=0 requested_test_names= excluded_test_names= +only_list_tests="" PYTEST_COMMAND="pytest -s --tb=short" # our core pytest command, with DAQ printout included and short pytest traceback while true; do @@ -63,7 +65,7 @@ while true; do -r) if [[ "$2" == "all" ]]; then echo "" - echo "Building the list of all integtests..." + echo "Building the list of _all_ integtests..." integtest_list=(`list_available_integtests.sh 2>/dev/null`) if [[ ${#integtest_list[@]} -eq 0 ]]; then echo "" @@ -73,7 +75,7 @@ while true; do fi elif [[ "$2" == "local" ]]; then echo "" - echo "Building the list of local integtests..." + echo "Building the list of _local_ integtests..." integtest_list=(`list_available_integtests.sh local 2>/dev/null`) if [[ ${#integtest_list[@]} -eq 0 ]]; then echo "" @@ -123,6 +125,10 @@ while true; do export PYTEST_DEBUG_TEMPROOT=${tmpdir_root} shift 2 ;; + --list-only) + only_list_tests="yes" + shift + ;; --) shift break @@ -133,6 +139,8 @@ done # run the integtests from the daqsystemtest repo if no repo was specified if [[ "${integtest_list}" == "" ]]; then integtest_list=(`list_available_integtests.sh daqsystemtest 2>/dev/null`) + echo "" + echo "Integtests from the _daqsystemtest_ repo will be run..." fi # check if the numad daemon is running @@ -166,6 +174,10 @@ mkdir -p ${pytest_user_dir} ITGRUNNER_LOG_FILE="${pytest_user_dir}/dunedaq_integtest_bundle_${INITIAL_TIMESTAMP}.log" CURRENT_PID=$$ +if [[ "$only_list_tests" != "" ]]; then + echo "" + echo "The following tests will be run:" +fi let number_of_individual_tests=0 let test_index=0 for FULL_TEST_NAME in "${integtest_list[@]}"; do @@ -174,10 +186,16 @@ for FULL_TEST_NAME in "${integtest_list[@]}"; do excluded_test=`echo ${test_name} | egrep -i ${excluded_test_names:-nullnullnull}` if [[ "${requested_test}" != "" ]] && [[ "${excluded_test}" == "" ]]; then let number_of_individual_tests=${number_of_individual_tests}+1 + if [[ "$only_list_tests" != "" ]]; then + echo " ${FULL_TEST_NAME}" + fi fi let test_index=${test_index}+1 done let total_number_of_tests=${number_of_individual_tests}*${individual_test_requested_iterations}*${full_set_requested_interations} +if [[ "$only_list_tests" != "" ]]; then + exit 0 +fi # run the tests let overall_test_index=0 # this is only used for user feedback diff --git a/scripts/list_available_integtests.sh b/scripts/list_available_integtests.sh index e665b8a..9e9dc26 100755 --- a/scripts/list_available_integtests.sh +++ b/scripts/list_available_integtests.sh @@ -3,24 +3,36 @@ if [[ "$1" == "--help" ]] || [[ "$1" == "-h" ]] || [[ "$1" == "-?" ]]; then echo - echo "Usage: `basename $0` [optional list of repo names]" + echo "Usage: `basename $0` [optional list of repo names|local|all]" echo " e.g. `basename $0` daqsystemtest" echo " If no repo name is specified, integtests for all repos are listed." echo " If a special repo name of \"local\" is specified, only integtests for repos" echo " in the local software area are listed." + echo " If a special repo name of \"all\" is specified, integtests for all repos are listed." echo exit fi +echo "" >&2 repo_list=() if [[ $# -ge 1 ]]; then if [[ "$1" == "local" ]]; then - repo_list=(`list_repos_with_integtests.sh local`) + echo "Looking for integtests in _local_ repos..." >&2 + echo "" >&2 + repo_list=(`list_repos_with_integtests.sh local 2>/dev/null`) + elif [[ "$1" == "all" ]]; then + echo "Looking for integtests in _all_ repos..." >&2 + echo "" >&2 + repo_list=(`list_repos_with_integtests.sh 2>/dev/null`) else + echo "Looking for integtests in the _$@_ repo(s)..." >&2 + echo "" >&2 repo_list=("$@") fi else - repo_list=(`list_repos_with_integtests.sh`) + echo "Looking for integtests in _all_ repos..." >&2 + echo "" >&2 + repo_list=(`list_repos_with_integtests.sh 2>/dev/null`) fi for repo_name in "${repo_list[@]}"; do diff --git a/scripts/list_repos_with_integtests.sh b/scripts/list_repos_with_integtests.sh index 877a453..61659bf 100755 --- a/scripts/list_repos_with_integtests.sh +++ b/scripts/list_repos_with_integtests.sh @@ -16,7 +16,10 @@ fi all_repo_paths=() # skip repos in the base release, if the user has specified "local" +echo "" >&2 if [[ $# -eq 0 ]] || [[ "$1" != "local" ]]; then + echo "Looking for _all_ repositories with integtests in them..." >&2 + echo "" >&2 # determine the base release directory release_dir=`dbt-info release | grep 'Release dir:' | cut -d' ' -f3` @@ -27,6 +30,9 @@ if [[ $# -eq 0 ]] || [[ "$1" != "local" ]]; then det_rel_repo_paths=(`ls -1d ${release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) base_rel_repo_paths=(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) all_repo_paths=("${det_rel_repo_paths[@]}" "${base_rel_repo_paths[@]}") +else + echo "Looking for _local_ repositories with integtests in them..." >&2 + echo "" >&2 fi # add in the paths of the repositories in the local install and sourcecode dirs From a4db2a20c5062d150364d86785781c5744a924ce Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 28 Jan 2026 08:15:00 -0600 Subject: [PATCH 07/15] Adding a bundle_script_description.md file --- docs/bundle_script_description.md | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 docs/bundle_script_description.md diff --git a/docs/bundle_script_description.md b/docs/bundle_script_description.md new file mode 100644 index 0000000..2dc7336 --- /dev/null +++ b/docs/bundle_script_description.md @@ -0,0 +1,10 @@ +# daqsystemtest integtest scripts + +The following scripts are intended to help developers find and run integtests in all repositories: + +* list_repos_with_integtests.sh +* list_available_integtests.sh +* dunedaq_integtest_bundle.sh + +All of these scripts support the "--help" command-line option to provide information on how they should be run. Here is what is currently shown for each of the three scripts: + From b13bb3646c44f92bfb9fc096b1777a8a6c887185 Mon Sep 17 00:00:00 2001 From: bieryAtFnal <36311946+bieryAtFnal@users.noreply.github.com> Date: Wed, 28 Jan 2026 08:25:00 -0600 Subject: [PATCH 08/15] Enhance documentation for integtest scripts Expanded the documentation for integtest scripts, including detailed usage information for each script's help command. --- docs/bundle_script_description.md | 55 +++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/docs/bundle_script_description.md b/docs/bundle_script_description.md index 2dc7336..155d52e 100644 --- a/docs/bundle_script_description.md +++ b/docs/bundle_script_description.md @@ -1,10 +1,53 @@ -# daqsystemtest integtest scripts +# Scripts to help with the running of integtests -The following scripts are intended to help developers find and run integtests in all repositories: +The following scripts are intended to help developers find and run _integtests_ (regression and integration tests) in all repositories: -* list_repos_with_integtests.sh -* list_available_integtests.sh -* dunedaq_integtest_bundle.sh +* `list_repos_with_integtests.sh` +* `list_available_integtests.sh` +* `dunedaq_integtest_bundle.sh` -All of these scripts support the "--help" command-line option to provide information on how they should be run. Here is what is currently shown for each of the three scripts: +All of these scripts support the "`--help`" command-line option to provide information on how they should be run. Here is what is currently shown for each of the three scripts: +### list_repos_with_integtests.sh --help + +``` +Usage: list_repos_with_integtests.sh [optional "local" keyword] + Lists the software repositories that have integration tests (integtests) in them. + Searches the base releases, local install dir, and local sourcecode dir, + unless "local" is passed as an argument. In that case, only the local + install and sourcecode directories are searched. +``` + +### list_available_integtests.sh --help + +``` +Usage: list_available_integtests.sh [optional list of repo names|local|all] + e.g. list_available_integtests.sh daqsystemtest + If no repo name is specified, integtests for all repos are listed. + If a special repo name of "local" is specified, only integtests for repos + in the local software area are listed. + If a special repo name of "all" is specified, integtests for all repos are listed. +``` + +### dunedaq_integtest_bundle.sh --help + +``` +Usage: +dunedaq_integtest_bundle.sh [option(s)] + +Options: + -h, --help : prints out usage information + -r + - this can be the name of a single repo + - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' + - it can have the special value of "all" - integtests in all repos will be run + - it can have the special value of "local" - integtests in locally-cloned repos will be run + -k, --include + -x, --exclude + -n + -N + --stop-on-failure : causes the script to stop when one of the integtests reports a failure + --concise-output : suppresses run control and DAQApp messages in order to focus on test results + --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' + --list-only : list the tests that match the requested patterns without running them +``` From 4df6b2c244f9318e5df80dc62cee941a359265f3 Mon Sep 17 00:00:00 2001 From: bieryAtFnal <36311946+bieryAtFnal@users.noreply.github.com> Date: Wed, 28 Jan 2026 08:40:00 -0600 Subject: [PATCH 09/15] Enhance bundle script documentation with usage examples Expanded the documentation for integtest scripts to include detailed usage instructions and examples for each script. --- docs/bundle_script_description.md | 198 +++++++++++++++++++++++++++--- 1 file changed, 179 insertions(+), 19 deletions(-) diff --git a/docs/bundle_script_description.md b/docs/bundle_script_description.md index 155d52e..c5ee4ed 100644 --- a/docs/bundle_script_description.md +++ b/docs/bundle_script_description.md @@ -2,12 +2,38 @@ The following scripts are intended to help developers find and run _integtests_ (regression and integration tests) in all repositories: +* `dunedaq_integtest_bundle.sh` - runs sets ("bundles") of integtests. This script supports the running of integtests from repositories that have been cloned into a local software area as well as repositories that are being used from a base release on CVMFS. * `list_repos_with_integtests.sh` * `list_available_integtests.sh` -* `dunedaq_integtest_bundle.sh` + +Users may choose to primarily use the `dunedaq_integtest_bundle.sh` script, but the two "`list`" scripts may be occasionally useful, so they are also described here. All of these scripts support the "`--help`" command-line option to provide information on how they should be run. Here is what is currently shown for each of the three scripts: +### dunedaq_integtest_bundle.sh --help + +``` +Usage: +dunedaq_integtest_bundle.sh [option(s)] + +Options: + -h, --help : prints out usage information + -r + - this can be the name of a single repo + - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' + - it can have the special value of "all" - integtests in all repos will be run + - it can have the special value of "local" - integtests in locally-cloned repos will be run + -k, --include + -x, --exclude + -n + -N + --stop-on-failure : causes the script to stop when one of the integtests reports a failure + --concise-output : suppresses run control and DAQApp messages in order to focus on test results + --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' + --list-only : list the tests that match the requested patterns without running them +``` + +Here are examples of run ### list_repos_with_integtests.sh --help ``` @@ -29,25 +55,159 @@ Usage: list_available_integtests.sh [optional list of repo names|local|all] If a special repo name of "all" is specified, integtests for all repos are listed. ``` -### dunedaq_integtest_bundle.sh --help +Here are examples of using each of the scripts: + +### dunedaq_integtest_bundle.sh ``` -Usage: -dunedaq_integtest_bundle.sh [option(s)] +(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh --list-only -Options: - -h, --help : prints out usage information - -r - - this can be the name of a single repo - - it can be a pipe-delimited string with a list of repos, e.g. 'dfmodules|trigger' - - it can have the special value of "all" - integtests in all repos will be run - - it can have the special value of "local" - integtests in locally-cloned repos will be run - -k, --include - -x, --exclude - -n - -N - --stop-on-failure : causes the script to stop when one of the integtests reports a failure - --concise-output : suppresses run control and DAQApp messages in order to focus on test results - --tmpdir : specifies a root directory to use for test output, e.g. a directory instead of '/tmp' - --list-only : list the tests that match the requested patterns without running them +Integtests from the _daqsystemtest_ repo will be run... + +The following tests will be run: + daqsystemtest/3ru_1df_multirun_test.py + daqsystemtest/3ru_3df_multirun_test.py + daqsystemtest/example_system_test.py + daqsystemtest/fake_data_producer_test.py + daqsystemtest/long_window_readout_test.py + daqsystemtest/minimal_system_quick_test.py + daqsystemtest/readout_type_scan_test.py + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpreplay_test.py + daqsystemtest/tpstream_writing_test.py + daqsystemtest/trigger_bitwords_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ +(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r listrev --list-only + +The following tests will be run: + listrev/listrev_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r local --list-only + +Building the list of _local_ integtests... + +The following tests will be run: + daqsystemtest/3ru_1df_multirun_test.py + daqsystemtest/3ru_3df_multirun_test.py + daqsystemtest/example_system_test.py + daqsystemtest/fake_data_producer_test.py + daqsystemtest/long_window_readout_test.py + daqsystemtest/minimal_system_quick_test.py + daqsystemtest/readout_type_scan_test.py + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpreplay_test.py + daqsystemtest/tpstream_writing_test.py + daqsystemtest/trigger_bitwords_test.py + dfmodules/disabled_output_test.py + dfmodules/hdf5_compression_test.py + dfmodules/insufficient_disk_space_test.py + dfmodules/large_trigger_record_test.py + dfmodules/max_file_size_test.py + dfmodules/multiple_data_writers_test.py + dfmodules/offline_prod_run_test.py + dfmodules/trmonrequestor_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r "asiolibs|crtmodules" --list-only + +The following tests will be run: + asiolibs/socket_reader_test.py + crtmodules/crt_reader_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r local -k tp --list-only + +Building the list of _local_ integtests... + +The following tests will be run: + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpreplay_test.py + daqsystemtest/tpstream_writing_test.py + dfmodules/disabled_output_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r local -k tp -x "disabled|tpreplay" --list-only + +Building the list of _local_ integtests... + +The following tests will be run: + daqsystemtest/small_footprint_quick_test.py + daqsystemtest/tpg_state_collection_test.py + daqsystemtest/tpstream_writing_test.py ``` + +### list_available_integtests.sh + +``` +(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh + +Looking for integtests in _all_ repos... + +asiolibs/socket_reader_test.py +crtmodules/crt_reader_test.py +daqsystemtest/3ru_1df_multirun_test.py +daqsystemtest/3ru_3df_multirun_test.py +daqsystemtest/example_system_test.py +daqsystemtest/fake_data_producer_test.py +daqsystemtest/long_window_readout_test.py +daqsystemtest/minimal_system_quick_test.py +daqsystemtest/readout_type_scan_test.py +daqsystemtest/small_footprint_quick_test.py +daqsystemtest/tpg_state_collection_test.py +daqsystemtest/tpreplay_test.py +daqsystemtest/tpstream_writing_test.py +daqsystemtest/trigger_bitwords_test.py +dfmodules/disabled_output_test.py +dfmodules/hdf5_compression_test.py +dfmodules/insufficient_disk_space_test.py +dfmodules/large_trigger_record_test.py +dfmodules/max_file_size_test.py +dfmodules/multiple_data_writers_test.py +dfmodules/offline_prod_run_test.py +dfmodules/trmonrequestor_test.py +hsilibs/iceberg_real_hsi_test.py +listrev/listrev_test.py +snbmodules/simple_transform_test.py +snbmodules/snb_1node_1app_rclone_http_system_quick_test.py +snbmodules/snb_1node_1app_torrent_system_quick_test.py +snbmodules/snb_1node_multiclientapps_rclone_http_system_quick_test.py +snbmodules/snb_minimal_system_test.py +trigger/change_rate_test.py +trigger/tc_time_outside_window_test.py +trigger/td_leakage_between_runs_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh local + +Looking for integtests in _local_ repos... + +daqsystemtest/3ru_1df_multirun_test.py +daqsystemtest/3ru_3df_multirun_test.py +daqsystemtest/example_system_test.py +daqsystemtest/fake_data_producer_test.py +daqsystemtest/long_window_readout_test.py +daqsystemtest/minimal_system_quick_test.py +daqsystemtest/readout_type_scan_test.py +daqsystemtest/small_footprint_quick_test.py +daqsystemtest/tpg_state_collection_test.py +daqsystemtest/tpreplay_test.py +daqsystemtest/tpstream_writing_test.py +daqsystemtest/trigger_bitwords_test.py +dfmodules/disabled_output_test.py +dfmodules/hdf5_compression_test.py +dfmodules/insufficient_disk_space_test.py +dfmodules/large_trigger_record_test.py +dfmodules/max_file_size_test.py +dfmodules/multiple_data_writers_test.py +dfmodules/offline_prod_run_test.py +dfmodules/trmonrequestor_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ +(dbt) [biery@daq 28JanFDDevInstrTest]$ +(dbt) [biery@daq 28JanFDDevInstrTest]$ +(dbt) [biery@daq 28JanFDDevInstrTest]$ +(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh asiolibs crtmodules + +Looking for integtests in the _asiolibs crtmodules_ repo(s)... + +asiolibs/socket_reader_test.py +crtmodules/crt_reader_test.py +(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh asdf jklp + +Looking for integtests in the _asdf jklp_ repo(s)... + +-> "asdf" does not appear to be a valid repository name. +-> "jklp" does not appear to be a valid repository name.``` From 51362b2ddb9cd9bb3f0dc2d7d2c57acb1cf75bde Mon Sep 17 00:00:00 2001 From: bieryAtFnal <36311946+bieryAtFnal@users.noreply.github.com> Date: Wed, 28 Jan 2026 08:41:44 -0600 Subject: [PATCH 10/15] Update bundle_script_description.md with script examples Added examples of using the list_repos_with_integtests.sh script to find repositories with integration tests. --- docs/bundle_script_description.md | 38 ++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/bundle_script_description.md b/docs/bundle_script_description.md index c5ee4ed..9cb3b3d 100644 --- a/docs/bundle_script_description.md +++ b/docs/bundle_script_description.md @@ -210,4 +210,40 @@ crtmodules/crt_reader_test.py Looking for integtests in the _asdf jklp_ repo(s)... -> "asdf" does not appear to be a valid repository name. --> "jklp" does not appear to be a valid repository name.``` +-> "jklp" does not appear to be a valid repository name. +``` + +### list_repos_with_integtests.sh + +``` +(dbt) [biery@daq 28JanFDDevInstrTest]$ list_repos_with_integtests.sh + +Looking for _all_ repositories with integtests in them... + +asiolibs +crtmodules +daqsystemtest +dfmodules +hsilibs +listrev +snbmodules +trigger +(dbt) [biery@daq 28JanFDDevInstrTest]$ list_repos_with_integtests.sh local + +Looking for _local_ repositories with integtests in them... + +daqsystemtest +dfmodules +(dbt) [biery@daq 28JanFDDevInstrTest]$ list_repos_with_integtests.sh asdf + +Looking for _all_ repositories with integtests in them... + +asiolibs +crtmodules +daqsystemtest +dfmodules +hsilibs +listrev +snbmodules +trigger +``` From 22d62d64cda3391d72c03ba4d5672ebb81eb77f1 Mon Sep 17 00:00:00 2001 From: bieryAtFnal <36311946+bieryAtFnal@users.noreply.github.com> Date: Wed, 28 Jan 2026 08:45:22 -0600 Subject: [PATCH 11/15] Update command examples in bundle_script_description --- docs/bundle_script_description.md | 41 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/docs/bundle_script_description.md b/docs/bundle_script_description.md index 9cb3b3d..da6b22c 100644 --- a/docs/bundle_script_description.md +++ b/docs/bundle_script_description.md @@ -60,7 +60,7 @@ Here are examples of using each of the scripts: ### dunedaq_integtest_bundle.sh ``` -(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh --list-only +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh --list-only Integtests from the _daqsystemtest_ repo will be run... @@ -77,12 +77,13 @@ The following tests will be run: daqsystemtest/tpreplay_test.py daqsystemtest/tpstream_writing_test.py daqsystemtest/trigger_bitwords_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ -(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r listrev --list-only + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r listrev --list-only The following tests will be run: listrev/listrev_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r local --list-only + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r local --list-only Building the list of _local_ integtests... @@ -107,12 +108,14 @@ The following tests will be run: dfmodules/multiple_data_writers_test.py dfmodules/offline_prod_run_test.py dfmodules/trmonrequestor_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r "asiolibs|crtmodules" --list-only + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r "asiolibs|crtmodules" --list-only The following tests will be run: asiolibs/socket_reader_test.py crtmodules/crt_reader_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r local -k tp --list-only + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r local -k tp --list-only Building the list of _local_ integtests... @@ -122,7 +125,8 @@ The following tests will be run: daqsystemtest/tpreplay_test.py daqsystemtest/tpstream_writing_test.py dfmodules/disabled_output_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ dunedaq_integtest_bundle.sh -r local -k tp -x "disabled|tpreplay" --list-only + +(dbt) [biery@daq]$ dunedaq_integtest_bundle.sh -r local -k tp -x "disabled|tpreplay" --list-only Building the list of _local_ integtests... @@ -135,7 +139,7 @@ The following tests will be run: ### list_available_integtests.sh ``` -(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh +(dbt) [biery@daq]$ list_available_integtests.sh Looking for integtests in _all_ repos... @@ -171,7 +175,8 @@ snbmodules/snb_minimal_system_test.py trigger/change_rate_test.py trigger/tc_time_outside_window_test.py trigger/td_leakage_between_runs_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh local + +(dbt) [biery@daq]$ list_available_integtests.sh local Looking for integtests in _local_ repos... @@ -195,17 +200,15 @@ dfmodules/max_file_size_test.py dfmodules/multiple_data_writers_test.py dfmodules/offline_prod_run_test.py dfmodules/trmonrequestor_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ -(dbt) [biery@daq 28JanFDDevInstrTest]$ -(dbt) [biery@daq 28JanFDDevInstrTest]$ -(dbt) [biery@daq 28JanFDDevInstrTest]$ -(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh asiolibs crtmodules + +(dbt) [biery@daq]$ list_available_integtests.sh asiolibs crtmodules Looking for integtests in the _asiolibs crtmodules_ repo(s)... asiolibs/socket_reader_test.py crtmodules/crt_reader_test.py -(dbt) [biery@daq 28JanFDDevInstrTest]$ list_available_integtests.sh asdf jklp + +(dbt) [biery@daq]$ list_available_integtests.sh asdf jklp Looking for integtests in the _asdf jklp_ repo(s)... @@ -216,7 +219,7 @@ Looking for integtests in the _asdf jklp_ repo(s)... ### list_repos_with_integtests.sh ``` -(dbt) [biery@daq 28JanFDDevInstrTest]$ list_repos_with_integtests.sh +(dbt) [biery@daq]$ list_repos_with_integtests.sh Looking for _all_ repositories with integtests in them... @@ -228,13 +231,15 @@ hsilibs listrev snbmodules trigger -(dbt) [biery@daq 28JanFDDevInstrTest]$ list_repos_with_integtests.sh local + +(dbt) [biery@daq]$ list_repos_with_integtests.sh local Looking for _local_ repositories with integtests in them... daqsystemtest dfmodules -(dbt) [biery@daq 28JanFDDevInstrTest]$ list_repos_with_integtests.sh asdf + +(dbt) [biery@daq]$ list_repos_with_integtests.sh asdf Looking for _all_ repositories with integtests in them... From 190d8d6dacb97c862e39a06e1aae5c10179520d3 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Wed, 28 Jan 2026 08:48:34 -0600 Subject: [PATCH 12/15] renamed bundle_script_description.md as bundle_script_overview.md --- docs/{bundle_script_description.md => bundle_script_overview.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{bundle_script_description.md => bundle_script_overview.md} (100%) diff --git a/docs/bundle_script_description.md b/docs/bundle_script_overview.md similarity index 100% rename from docs/bundle_script_description.md rename to docs/bundle_script_overview.md From eaeefa255491cffbab3a1e745c2b124e577a069b Mon Sep 17 00:00:00 2001 From: bieryAtFnal <36311946+bieryAtFnal@users.noreply.github.com> Date: Wed, 28 Jan 2026 09:33:03 -0600 Subject: [PATCH 13/15] Re-ordered some of the sections Added usage example for list_repos_with_integtests.sh script and removed redundant section. --- docs/bundle_script_overview.md | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/docs/bundle_script_overview.md b/docs/bundle_script_overview.md index da6b22c..aa80548 100644 --- a/docs/bundle_script_overview.md +++ b/docs/bundle_script_overview.md @@ -3,8 +3,8 @@ The following scripts are intended to help developers find and run _integtests_ (regression and integration tests) in all repositories: * `dunedaq_integtest_bundle.sh` - runs sets ("bundles") of integtests. This script supports the running of integtests from repositories that have been cloned into a local software area as well as repositories that are being used from a base release on CVMFS. -* `list_repos_with_integtests.sh` * `list_available_integtests.sh` +* `list_repos_with_integtests.sh` Users may choose to primarily use the `dunedaq_integtest_bundle.sh` script, but the two "`list`" scripts may be occasionally useful, so they are also described here. @@ -33,17 +33,6 @@ Options: --list-only : list the tests that match the requested patterns without running them ``` -Here are examples of run -### list_repos_with_integtests.sh --help - -``` -Usage: list_repos_with_integtests.sh [optional "local" keyword] - Lists the software repositories that have integration tests (integtests) in them. - Searches the base releases, local install dir, and local sourcecode dir, - unless "local" is passed as an argument. In that case, only the local - install and sourcecode directories are searched. -``` - ### list_available_integtests.sh --help ``` @@ -55,6 +44,16 @@ Usage: list_available_integtests.sh [optional list of repo names|local|all] If a special repo name of "all" is specified, integtests for all repos are listed. ``` +### list_repos_with_integtests.sh --help + +``` +Usage: list_repos_with_integtests.sh [optional "local" keyword] + Lists the software repositories that have integration tests (integtests) in them. + Searches the base releases, local install dir, and local sourcecode dir, + unless "local" is passed as an argument. In that case, only the local + install and sourcecode directories are searched. +``` + Here are examples of using each of the scripts: ### dunedaq_integtest_bundle.sh From 9875e65adefbd312b71fea83d500c097c8a3d21e Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Mon, 2 Feb 2026 15:13:14 -0600 Subject: [PATCH 14/15] Added printout of information to dunedaq_integtest_bundle.sh to tell users when directories from failed tests get cleaned up. --- scripts/dunedaq_integtest_bundle.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/dunedaq_integtest_bundle.sh b/scripts/dunedaq_integtest_bundle.sh index cb47cd2..6ec8511 100755 --- a/scripts/dunedaq_integtest_bundle.sh +++ b/scripts/dunedaq_integtest_bundle.sh @@ -272,6 +272,8 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do new_dir="${pytest_rootdir}/failed-${pytest_basedir}" echo "" echo -e "\U1F535 Copying the files from failed test ${pytest_tmpdir} to ${new_dir}. \U1F535" + echo -e "\U1F535 Please note that copied directories from failed tests typically get cleaned up after 26 hours, \U1F535" + echo -e "\U1F535 or when 10 newer failures happen, whichever comes first. \U1F535" cp -pR "${pytest_tmpdir}" "${new_dir}" if [[ $? == 0 ]]; then was_successfully_copied="yes" @@ -304,6 +306,7 @@ while [[ ${full_set_loop_count} -lt ${full_set_requested_interations} ]]; do test_dirs_to_remove+=(${test_dir}) done if [[ ${#test_dirs_to_remove[@]} -gt 0 ]];then + echo "" echo -e "\U1F535 Removing ${#test_dirs_to_remove[@]} old failed test directory(ies). \U1F535" for test_dir in "${test_dirs_to_remove[@]}"; do if [[ -e "${test_dir}" ]]; then From d8c2d3c133b8f37419aecd5c2893d11d030fe220 Mon Sep 17 00:00:00 2001 From: Kurt Biery Date: Tue, 3 Feb 2026 14:00:19 -0600 Subject: [PATCH 15/15] As suggested by Eric, avoid limiting the lookup of base release packages in list_repos_with_integtests.sh to only almalinux9 ones --- scripts/list_repos_with_integtests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/list_repos_with_integtests.sh b/scripts/list_repos_with_integtests.sh index 61659bf..2e973e9 100755 --- a/scripts/list_repos_with_integtests.sh +++ b/scripts/list_repos_with_integtests.sh @@ -27,8 +27,8 @@ if [[ $# -eq 0 ]] || [[ "$1" != "local" ]]; then base_release_dir="${release_dir}/../${base_release_name}" # look up the paths of all of the repositories in the core and detector-specific categories - det_rel_repo_paths=(`ls -1d ${release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) - base_rel_repo_paths=(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*almalinux9*/gcc-*/*/*/integtest/*_test.py`) + det_rel_repo_paths=(`ls -1d ${release_dir}/spack-installation/opt/spack/*linux*/gcc-*/*/*/integtest/*_test.py`) + base_rel_repo_paths=(`ls -1d ${base_release_dir}/spack-installation/opt/spack/*linux*/gcc-*/*/*/integtest/*_test.py`) all_repo_paths=("${det_rel_repo_paths[@]}" "${base_rel_repo_paths[@]}") else echo "Looking for _local_ repositories with integtests in them..." >&2