fetch.sh

#!/bin/bash
set -eu

WEBKIT_RESULTS_URL="http://build.webkit.org/results"

# The GTK test bot is now "GTK Linux 64-bit Release (Tests)".
# The others were the names of this test bot in the past.
# We fetch all the data from the actual test bot as also the past ones, to have a complete history
declare -A webkitbots_map
webkitbots_map=(
   ["gtk-release"]="GTK-Linux-64-bit-Release-Tests"
   ["gtk-debug"]="GTK-Linux-64-bit-Debug-Tests"
   ["wpe-release"]="WPE-Linux-64-bit-Release-Tests"
   ["wpe-debug"]="WPE-Linux-64-bit-Debug-Tests"
   ["wpe-arm64-release"]="WPE-Linux-ARM64-bit-Release-Tests"
   ["wpe-arm64-252-release"]="WPE-Linux-ARM64-bit-Release-v252-BuildAndTest"
   ["wpe-arm64-254-release"]="WPE-Linux-ARM64-bit-Release-v254-BuildAndTest"
)

# Bots that track a WebKit stable branch store their results one directory level
# deeper than the trunk bots: each "<base>.<seq>@<branch>/" result directory
# contains a single "<version> (<build>)/" subdir with the full_results.json, and
# the revision identifier is "<base>.<seq>@<branch>" (not "<N>@main"). The value
# below is the branch name used to validate the revision (e.g. "webkitglib").
declare -A webkitbots_branch
webkitbots_branch=(
   ["WPE-Linux-ARM64-bit-Release-v252-BuildAndTest"]="webkitglib"
   ["WPE-Linux-ARM64-bit-Release-v254-BuildAndTest"]="webkitglib"
)

webkitbots_values=("${webkitbots_map[@]}")

fatal () {
    echo "Fatal: $@"
    exit 1
}

usage () {
   local exit_code="$1"
   local program_name=$(basename "$0")

   echo "$program_name [BOT] ..."
   echo "Retrieves tests data from a set of bots. In case no bot is set, data is retrieved from all bots."
   echo ""
   echo "ARGUMENTS:"
   echo -e "\tBOT\t${!webkitbots_map[@]}"

   exit $exit_code
}

urlencode () {
    python3 -c "import urllib.parse; print(urllib.parse.quote(\"${@}\"))"
}

urldecode () {
    python3 -c "import urllib.parse; print(urllib.parse.unquote(\"${@}\"))"
}

print_revision_from_json_results () {
    python3 -c "import json, os; json_data=open(\"${1}\").read().split('ADD_RESULTS(')[1].split(');')[0]; print(json.loads(json_data)['revision'])"
}

json_result_test_has_revision_key () {
    python3 -c "import json, os, sys; jd=json.loads(open(\"${1}\").read().split('ADD_RESULTS(')[1].split(');')[0]); exit_code = 0 if 'revision' in jd.keys() else 1; sys.exit(exit_code)"
}

json_result_test_run_was_interrupted () {
    python3 -c "import json, os, sys; jd=json.loads(open(\"${1}\").read().split('ADD_RESULTS(')[1].split(');')[0]); exit_code = 0 if 'interrupted' in jd.keys() and jd['interrupted'] else 1; sys.exit(exit_code)"
}

fetch_bot_results () {
    local webkitbot="$1"

    curl -L -s "${WEBKIT_RESULTS_URL}/${webkitbot}/" | grep -P "href=\"[^\"]+/"| cut -d\" -f2 | grep -v  "\.\./" | sort -u
}

parse_args () {
    local values=()
    for arg in $@; do
        if [[ "$arg" == "-h" || "$arg" == "--help" ]]; then
            usage 0
        elif [[ "$arg" == -* ]]; then
            fatal "Unrecognized option: '$arg'"
        # Is a valid bot identifier?
        elif [[ " ${!webkitbots_map[@]} " =~ " $arg " ]]; then
            values+=( "${webkitbots_map[$arg]}" )
        else
            fatal "Invalid argument: '$arg'"
        fi
    done
    # Override default webkitbots_values.
    if [[ ${#values[@]} -gt 0 ]]; then
        webkitbots_values=("${values[@]}")
    fi
}

parse_args $@

alreadytried=".cache_already_tried"
islegacybot=".is_legacy_bot"
cd jsonresults
for webkitbot in "${webkitbots_values[@]}"; do
    test -d "${webkitbot}" || mkdir "${webkitbot}"
    cd "${webkitbot}"
    if test -f "${islegacybot}"; then
        echo -e "\nFetching results for bot ${webkitbot} skipped because this bot is not longer active and we already have all the results for it fetched."
        echo -e "If this is not the case, then please remove the file $(pwd)/${islegacybot} and run this script again."
        cd ..
        continue
    else
        echo -e "\nFetching results for bot: ${webkitbot}"
    fi
    test -f "${alreadytried}" || touch "${alreadytried}"
    branch_flavor="${webkitbots_branch[$webkitbot]:-}"
    tempjsonresultfile="$(mktemp)"
    webkitbot="$(urlencode "${webkitbot}")"
    fetch_bot_results "$webkitbot" | while read resultsdir; do
        if [[ -n "${branch_flavor}" ]] && ! [[ "${resultsdir}" =~ ^[0-9]+\.[0-9]+(@|%40)${branch_flavor}/?$ ]]; then
            # Result directories of a stable-branch bot that don't carry a
            # "<base>.<seq>@<branch>" identifier are from the builds made before the
            # bot was switched over to the branch (they are plain "<N>@main" runs at
            # the branch point). They live in the trunk revision namespace, so mixing
            # them with the branch sequence would break the revision ordering: ignore.
            continue
        fi
        # These are computed lazily (once, only when we actually need to download)
        # so that we don't pay an extra request per already-fetched directory.
        downloadurl=""
        buildnum_src="${resultsdir}"
        revision_override=""
        tries=1
        while true; do
            if grep -qx "${resultsdir}" "${alreadytried}"; then
                echo -n "."
                break
            fi
            if [[ -z "${downloadurl}" ]]; then
                if [[ -n "${branch_flavor}" ]]; then
                    # Stable-branch bot: descend into the single "<version> (<build>)/"
                    # subdir. The build number lives there, and the revision comes from
                    # the result directory name (the JSON revision carries a trailing
                    # "/<version>" that can't be part of a filename).
                    builddir="$(fetch_bot_results "${webkitbot}/${resultsdir%/}" | head -n1)"
                    downloadurl="${WEBKIT_RESULTS_URL}/${webkitbot}/${resultsdir}${builddir}full_results.json"
                    buildnum_src="${builddir}"
                    revision_override="$(urldecode "${resultsdir%/}")"
                else
                    downloadurl="${WEBKIT_RESULTS_URL}/${webkitbot}/${resultsdir}full_results.json"
                fi
            fi
            if test -f "${tempjsonresultfile}" && grep -qP "^ADD_RESULTS\(.*\);$" "${tempjsonresultfile}" ; then
                # Get the revision number from the json data and move the file to its place.
                # Is important that the file in disk is stored as "r${revision}_b${buildnumber}" because the
                # python wktesthunter code assumes that the revision on the filename is right.
                buildnum="$(echo ${buildnum_src}| awk -F'%28' '{print $2}'|awk -F'%29' '{print $1}')"
                if [[ -n "${revision_override}" ]]; then
                    # Stable-branch bot: the directory name is the authoritative revision.
                    revision="${revision_override}"
                elif json_result_test_has_revision_key "${tempjsonresultfile}"; then
                    revision="$(print_revision_from_json_results ${tempjsonresultfile})"
                else
                    # If the test run deadlocks then it may lack the revision in the json data.
                    # Workaround the problem by guessing the revision number from the URL
                    # But also ensure that the test is marked as interrupted in that case
                    # See example:
                    # Original URL http://build.webkit.org/results/WPE-Linux-64-bit-Debug-Tests/257598%40main%20%284216%29/full_results.json
                    # Saved as: jsonresults/WPE-Linux-64-bit-Debug-Tests/full_results_257598@main_b4216.json
                    # Logs: https://build.webkit.org/#/builders/14/builds/4216
                    if json_result_test_run_was_interrupted "${tempjsonresultfile}"; then
                        revision="$(urldecode $(echo ${resultsdir} | awk -F'%20' '{print $1}'))"
                        echo -e "\nWARNING: Revision ${revision} guessed from the URL because it is not in the json data from ${downloadurl}"
                    else
                        echo -e "\FATAL: ${downloadurl} has no revision data and test run was not interrupted. This is unexpected."
                        exit 1
                    fi
                fi
                # Sanity checks
                echo "${buildnum}" | grep -Pq "^[0-9]+$" || fatal "Buildnum should be numeric and I got buildnum \"${buildnum}\" for ${downloadurl}"
                if [[ -n "${branch_flavor}" ]]; then
                    echo "${revision}" | grep -Pq "^[0-9]+\.[0-9]+@${branch_flavor}$" || fatal "Revision should be in the format: number.number@${branch_flavor} and I got revision \"${revision}\" for ${downloadurl}"
                else
                    echo "${revision}" | grep -Pq "^[0-9]+@main$" || fatal "Revision should be in the format: number@main and I got revision \"${revision}\" for ${downloadurl}"
                fi
                echo -n "${revision}... "
                mv "${tempjsonresultfile}" "full_results_${revision}_b${buildnum}.json"
                # store the resultsdir on the cache to not retry this download
                echo -n ":"
                echo "${resultsdir}" >> "${alreadytried}"
                break
            fi
            if [[ ${tries} -gt 3 ]]; then
                httpcode="$(curl -L -w "%{http_code}" -s "${downloadurl}" -o /dev/null)"
                if [[ "${httpcode}" == "404" ]]; then
                    echo "${resultsdir}" >> "${alreadytried}"
                fi
                echo -e "\nERROR: After ${tries} tries I was unable to fetch resource: ${downloadurl}. HTTP Error code was: ${httpcode}"
                break
            fi
            #echo "Downloading results for revision $revision buildnum $buildnum ..."
            [[ ${tries} -eq 1 ]] && echo
            rm -f "${tempjsonresultfile}"
            curl -L -s "${downloadurl}" -o "${tempjsonresultfile}"
            tries=$(( ${tries} + 1 ))
        done
    done
    cd ..
done