|
| 1 | +#!/bin/bash |
| 2 | + |
| 3 | +set -uo pipefail |
| 4 | + |
| 5 | +usage() { |
| 6 | + echo "Run tests in axlearn with specified options." |
| 7 | + echo "" |
| 8 | + echo "Usage: $0 [OPTIONS]" |
| 9 | + echo "" |
| 10 | + echo " OPTIONS DESCRIPTION" |
| 11 | + echo " -d, --directory DIR Directory to run tests in." |
| 12 | + echo " Default: 'axlearn/axlearn/common'." |
| 13 | + echo " -t, --test-files FILES Pattern for test files to run." |
| 14 | + echo " Default: '*_test.py'." |
| 15 | + echo " -o, --output DIRECTORY Output directory for logs and summary." |
| 16 | + echo " Default: 'test_runs/<timestamp>'." |
| 17 | + echo " -h, --help Show this help message and exit." |
| 18 | + exit 1 |
| 19 | +} |
| 20 | + |
| 21 | +# Default values |
| 22 | +DIR='axlearn/axlearn/common' |
| 23 | +TEST_FILES=() |
| 24 | +OUTPUT_DIRECTORY='' |
| 25 | + |
| 26 | +# Parse args manually |
| 27 | +while [[ $# -gt 0 ]]; do |
| 28 | + key="$1" |
| 29 | + case $key in |
| 30 | + -d|--directory) |
| 31 | + if [[ -z "$2" ]]; then |
| 32 | + echo "Error: --directory requires an argument." |
| 33 | + usage |
| 34 | + fi |
| 35 | + DIR="$2" |
| 36 | + shift 2 |
| 37 | + ;; |
| 38 | + -t|--test-files) |
| 39 | + shift |
| 40 | + # Collect all arguments until the next option (starting with '-') |
| 41 | + if [[ $# -eq 0 ]]; then |
| 42 | + echo "Error: --test-files requires at least one file pattern." |
| 43 | + usage |
| 44 | + fi |
| 45 | + echo "Option -t|--test-files with arguments:" |
| 46 | + while [[ $# -gt 0 && ! "$1" =~ ^- ]]; do |
| 47 | + echo " $1" |
| 48 | + TEST_FILES+=("$1") |
| 49 | + shift |
| 50 | + done |
| 51 | + ;; |
| 52 | + -o|--output) |
| 53 | + if [[ -z "$2" ]]; then |
| 54 | + echo "Error: --output requires an argument." |
| 55 | + usage |
| 56 | + fi |
| 57 | + OUTPUT_DIRECTORY="$2" |
| 58 | + shift 2 |
| 59 | + ;; |
| 60 | + -h|--help) |
| 61 | + usage |
| 62 | + ;; |
| 63 | + *) |
| 64 | + echo "Unknown option: $1" |
| 65 | + usage |
| 66 | + ;; |
| 67 | + esac |
| 68 | +done |
| 69 | + |
| 70 | + |
| 71 | +if [ -z "$OUTPUT_DIRECTORY" ]; then |
| 72 | + timestamp=$(date +%Y%m%d_%H%M%S) |
| 73 | + OUTPUT_DIRECTORY="test_runs/${timestamp}" |
| 74 | +fi |
| 75 | +LOG_DIRECTORY="${OUTPUT_DIRECTORY}/logs" |
| 76 | + |
| 77 | +mkdir -p "${LOG_DIRECTORY}" |
| 78 | + |
| 79 | +# Print out config for sanity check |
| 80 | +echo "Configuration:" |
| 81 | +echo " Directory: $DIR" |
| 82 | +if [ "${#TEST_FILES[@]}" -gt 0 ]; then |
| 83 | + echo " Test Files:" |
| 84 | + for f in "${TEST_FILES[@]}"; do |
| 85 | + echo " $f" |
| 86 | + done |
| 87 | +else |
| 88 | + echo " Test Files Pattern: '*_test.py' (default)" |
| 89 | +fi |
| 90 | +echo " Output Directory: $OUTPUT_DIRECTORY" |
| 91 | + |
| 92 | +cd "$DIR" || exit 1 |
| 93 | + |
| 94 | +echo "Running tests..." |
| 95 | + |
| 96 | +pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu |
| 97 | +pip install timm transformers scikit-learn |
| 98 | + |
| 99 | + |
| 100 | +if [ "${#TEST_FILES[@]}" -eq 0 ]; then |
| 101 | + TEST_FILES=("*_test.py") |
| 102 | +fi |
| 103 | + |
| 104 | +expanded_test_files=() |
| 105 | +for pattern in "${TEST_FILES[@]}"; do |
| 106 | + # retrieve all the files |
| 107 | + files=( $pattern ) |
| 108 | + if [ "${#files[@]}" -gt 0 ]; then |
| 109 | + expanded_test_files+=( "${files[@]}" ) |
| 110 | + else |
| 111 | + echo "Warning: No files matched pattern '$pattern'" |
| 112 | + fi |
| 113 | +done |
| 114 | + |
| 115 | +if [ "${#expanded_test_files[@]}" -eq 0 ]; then |
| 116 | + echo "No test files found to run." |
| 117 | + exit 1 |
| 118 | +fi |
| 119 | + |
| 120 | +# in case we have the exclusion list file |
| 121 | +EXCLUDE_LIST_FILE="$DIR/exclusion_list.txt" |
| 122 | +EXCLUDE_PATTERNS=() |
| 123 | + |
| 124 | +if [ -f "$EXCLUDE_LIST_FILE" ]; then |
| 125 | + echo "Reading exclusion list from '$EXCLUDE_LIST_FILE'" |
| 126 | + mapfile -t EXCLUDE_PATTERNS < "$EXCLUDE_LIST_FILE" |
| 127 | +else |
| 128 | + echo "Exclusion list file not found at '$EXCLUDE_LIST_FILE'" |
| 129 | +fi |
| 130 | + |
| 131 | +final_test_files=() |
| 132 | + |
| 133 | +for test_file in "${expanded_test_files[@]}"; do |
| 134 | + exclude=false |
| 135 | + for pattern in "${EXCLUDE_PATTERNS[@]}"; do |
| 136 | + if [[ "$(basename "$test_file")" == "$(basename "$pattern")" ]]; then |
| 137 | + exclude=true |
| 138 | + break |
| 139 | + fi |
| 140 | + done |
| 141 | + if [ "$exclude" = false ]; then |
| 142 | + final_test_files+=("$test_file") |
| 143 | + fi |
| 144 | +done |
| 145 | + |
| 146 | +# Initialize counters for test |
| 147 | +failures=0 |
| 148 | +passed=0 |
| 149 | +SUMMARY_FILE="${OUTPUT_DIRECTORY}/summary.txt" |
| 150 | + |
| 151 | + |
| 152 | +for test_file in "${final_test_files[@]}"; do |
| 153 | + echo "Running: ${test_file}" |
| 154 | + log_file_name=$(echo "${test_file%.py}" | sed 's/\//__/g').log |
| 155 | + log_file="${LOG_DIRECTORY}/${log_file_name}" |
| 156 | + # run the tests and save them as *.log |
| 157 | + pytest "${test_file}" --capture=tee-sys | tee "${log_file}" |
| 158 | + exit_code=${PIPESTATUS[0]} |
| 159 | + echo $exit_code |
| 160 | + # write number of tests passed and failed |
| 161 | + if [ $exit_code -eq 0 ]; then |
| 162 | + echo "${test_file}: PASSED" >> "${SUMMARY_FILE}" |
| 163 | + ((passed++)) |
| 164 | + else |
| 165 | + echo "${test_file}: FAILED (Exit code: $exit_code)" >> "${SUMMARY_FILE}" |
| 166 | + ((failures++)) |
| 167 | + fi |
| 168 | + echo "" |
| 169 | +done |
0 commit comments