diff options
-rw-r--r-- | jenkins-helpers.sh | 261 | ||||
-rwxr-xr-x | pw-apply.sh | 104 | ||||
-rwxr-xr-x | pw-report.sh | 92 | ||||
-rwxr-xr-x | pw-trigger.sh | 87 | ||||
-rw-r--r-- | round-robin.sh | 297 |
5 files changed, 769 insertions, 72 deletions
diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index ce225e9d..83de76fd 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -34,7 +34,7 @@ assert_with_msg () local failure_message=$1 shift - eval "$@" || (echo "$failure_message" && exit 1) + eval "$@" || (echo "$failure_message" >&2 && exit 1) ) } @@ -518,6 +518,14 @@ clone_or_update_repo_no_checkout () # Delete stale locks -- especially .git/refs/remotes/REMOTE/BRANCH.lock # These occur when builds are aborted during "git remote update" or similar. find "$dir/.git" -name "*.lock" -delete + + # Recover from any previous am/cherry-pick/rebase. + # In pre-commit CI we apply patches with "git am", which can fail + # and leave clone in a bad state. + local i + for i in am cherry-pick rebase; do + git -C "$dir" "$i" --abort &>/dev/null || true + done fi git_set_remote "$dir" "$remote" "$url" "$single_branch" @@ -944,9 +952,18 @@ get_baseline_git () { ( set -euf -o pipefail + + local base_artifacts + + if test_array rr && [[ -v rr[base_artifacts] ]]; then + base_artifacts=${rr[base_artifacts]} + else + base_artifacts="base-artifacts" + fi + assert_with_msg "ERROR: No $1 in baseline git" \ - [ -f "${rr[base_artifacts]}/git/$1" ] - cat "${rr[base_artifacts]}/git/$1" + [ -f "$base_artifacts/git/$1" ] + cat "$base_artifacts/git/$1" ) } @@ -1793,3 +1810,241 @@ describe_sha1 () echo "$component#$(git -C "$component" rev-parse --short)" fi } + +pw_init () +{ + ( + set -euf -o pipefail + local project="$1" + + git -C "$project" config pw.server \ + "https://patchwork.sourceware.org/api/1.2/" + git -C "$project" config pw.project "$project" + + set +x + local token="$2" + if [ "$token" != "" ]; then + git -C "$project" config pw.token "$token" + fi + ) +} + +pw_yaml_get () +{ + ( + set -euf -o pipefail + local project="$1" + local section="$2" + local id="$3" + local match_field="$4" + local match_value="$5" + local get_field="$6" + local match_stop="${7-}" + local match_num="${8-}" + + # Reduce noise in the logs + set +x + + local yaml + yaml=$(mktemp) + # shellcheck disable=SC2064 + trap "rm -f $yaml" EXIT + + local -a cmd + case "$id" in + "--owner"*) + # shellcheck disable=SC2206 + cmd=(list $id) + ;; + *) + cmd=(show "$id") + ;; + esac + + git -C "$project" pw "$section" "${cmd[@]}" -f yaml > "$yaml" + + local len + len=$(shyaml get-length < "$yaml") + while [ "$len" -gt "0" ]; do + len=$(($len - 1)) + if [ "$match_value" != ".*" ]; then + if shyaml get-value "$len.$match_field" < "$yaml" \ + | grep "$match_value" >/dev/null; then + if [ "$match_num" = "" ] || [ "$match_num" = "0" ]; then + shyaml get-value "$len.$get_field" < "$yaml" + return 0 + fi + match_num=$(($match_num - 1)) + fi + else + # Special case for $match_value == ".*". + # Only check that $match_field exist, and don't look at the value. + # This handles empty values without involving grep, which can't + # match EOF generated by empty value. + if shyaml get-value "$len.$match_field" < "$yaml" >/dev/null; then + if [ "$match_num" = "" ] || [ "$match_num" = "0" ]; then + shyaml get-value "$len.$get_field" < "$yaml" + return 0 + fi + match_num=$(($match_num - 1)) + fi + fi + if [ "$match_stop" != "" ] \ + && shyaml get-value "$len.$match_field" < "$yaml" \ + | grep "$match_stop" >/dev/null; then + return 1 + fi + done + + assert_with_msg "Missing $match_field == $match_value" false + ) +} + +pw_series_complete_p () +{ + ( + set -euf -o pipefail + local project="$1" + local series_id="$2" + + local value + value=$(pw_yaml_get "$project" series "$series_id" property Complete value) + if [ "$value" = "True" ]; then + return 0 + fi + return 1 + ) +} + +pw_get_patch_from_bundle_or_series () +{ + ( + set -euf -o pipefail + local project="$1" + local bundle_or_series="$2" + local id="$3" + local num="$4" + + local patch_id + patch_id=$(pw_yaml_get "$project" $bundle_or_series "$id" property ".*" \ + value Patches "$num" | cut -d" " -f 1) + if [ "$patch_id" = "" ]; then + return 1 + fi + assert_with_msg "Invalid patch_id $patch_id" \ + git -C "$project" pw patch show "$patch_id" > /dev/null + echo "$patch_id" + ) +} + +pw_bundle_name () +{ + ( + set -euf -o pipefail + local action="$1" + local ci_bot="$2" + + # Patchworks limits bundle names to 50 symbols, so reduce $ci_bot name + # to 39 bytes: 30 bytes of the original $ci_bot + "-" + 8-byte hash. + if [ ${#ci_bot} -gt "39" ]; then + local hash + hash=$(echo "$ci_bot" | md5sum | cut -b 1-8) + ci_bot="${ci_bot::29}-$hash" + fi + + # Then append 1 byte of "-" + at most 9 bytes of $action. + assert_with_msg "action too long" \ + [ ${#action} -le 9 ] + ci_bot="$ci_bot-$action" + + assert_with_msg "bundle name too long" \ + [ ${#ci_bot} -le 49 ] + + echo "$ci_bot" + ) +} + +pw_bundle_id () +{ + ( + set -euf -o pipefail + local project="$1" + local name="$2" + + pw_yaml_get "$project" bundle "--owner linaro-tcwg-bot" name "$name" id + ) +} + +pw_bundle_has_patch_p () +{ + ( + set -euf -o pipefail + local project="$1" + local bundle_id="$2" + local patch_id="$3" + + if [ "$bundle_id" = "" ]; then + return 1 + fi + + pw_yaml_get "$project" bundle "$bundle_id" value "^$patch_id " value \ + "^[^0-9]" >/dev/null 2>&1 + ) +} + +pw_ci_bot_bundle_check_and_add () +{ + ( + set -euf -o pipefail + local project="$1" + local bundle_type="$2" + local ci_bot="$3" + local patch_id="$4" + local skip_check="$5" + + local bundle_name bundle_id + bundle_name=$(pw_bundle_name "$bundle_type" "$ci_bot") + bundle_id=$(pw_bundle_id "$project" "$bundle_name" || true) + + if [ "$bundle_id" != "" ]; then + if ! $skip_check \ + && pw_bundle_has_patch_p \ + "$project" "$bundle_id" "$patch_id"; then + return 1 + fi + git -C "$project" pw bundle add -f yaml "$bundle_id" "$patch_id" + else + git -C "$project" pw bundle create --public -f yaml \ + "$bundle_name" "$patch_id" + fi + ) +} + +pw_ci_bot_bundle_remove () +{ + ( + set -euf -o pipefail + local project="$1" + local bundle_type="$2" + local ci_bot="$3" + local patch_id="$4" + local allow_delete="${5-true}" + + local bundle_name bundle_id + bundle_name=$(pw_bundle_name "$bundle_type" "$ci_bot") + bundle_id=$(pw_bundle_id "$project" "$bundle_name" || true) + + if ! pw_bundle_has_patch_p "$project" "$bundle_id" "$patch_id"; then + return + fi + + local only_patch + only_patch=$(pw_yaml_get "$project" bundle "$bundle_id" property ".*" \ + property Patches) + if [ "$only_patch" != "Patches" ]; then + git -C "$project" pw bundle remove -f yaml "$bundle_id" "$patch_id" + elif $allow_delete; then + git -C "$project" pw bundle delete -f yaml "$bundle_id" + fi + ) +} diff --git a/pw-apply.sh b/pw-apply.sh new file mode 100755 index 00000000..67cf6d20 --- /dev/null +++ b/pw-apply.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +set -euf -o pipefail +x + +scripts=$(dirname $0) +# shellcheck source=jenkins-helpers.sh +. $scripts/jenkins-helpers.sh + +convert_args_to_variables "$@" + +obligatory_variables ci_bot project pw_url pw_token pw_dir build_url +declare ci_bot project pw_url pw_token pw_dir build_url + +verbose="${verbose-true}" + +if $verbose; then + set -x +fi + +case "$pw_url" in + "pw://series/"*) + series_id=$(echo "$pw_url" | cut -d/ -f 4) + ;; + *) assert_with_msg "pw_url does not match pw://series/*" false ;; +esac + +retrigger=false +case "$pw_url/" in + "pw://series/"*"/retrigger/"*) retrigger=true ;; +esac + +url=$(get_baseline_git "${project}_url") +rev=$(get_baseline_git "${project}_rev") +echo "Fetching baseline $url#$rev" +clone_or_update_repo "$project" "$rev" "$url" > /dev/null + +(set +x; pw_init "$project" "$pw_token") + +if ! pw_series_complete_p "$project" "$series_id"; then + echo "ERROR: Series $series_id is not complete" + exit 2 +fi + +num_patch=0 +while true; do + patch_id=$(pw_get_patch_from_bundle_or_series \ + "$project" series "$series_id" "$num_patch" || true) + if [ "$patch_id" = "" ]; then + break + fi + + pw_ci_bot_bundle_check_and_add \ + "$project" triggered "$ci_bot" "$patch_id" "$retrigger" & + res=0 && wait $! || res=$? + if [ $res = 0 ]; then + break + fi + + num_patch=$(($num_patch + 1)) +done + +if [ "$patch_id" = "" ]; then + echo "ERROR: All patches in series $series_id are already tested" + exit 3 +fi + +# Remove the patch from queue bundle, unless it is the last one. +# We avoid deleting and re-creatthe queued bundles to conserve bundle IDs. +pw_ci_bot_bundle_remove "$project" queued "$ci_bot" "$patch_id" false + +clone_or_update_repo glibc-cicd main \ + https://gitlab.com/djdelorie/glibc-cicd.git > /dev/null +( + set +x; + if [ "$pw_token" != "" ]; then + cat >> glibc-cicd/cicd-config.py <<EOF + +patchwork_token = "$pw_token" +EOF + fi +) +pw_check_cmd=(glibc-cicd/check.py --patch_id "$patch_id" --context "$ci_bot") + +mkdir -p "$pw_dir" +cat > "$pw_dir/$project" <<EOF +project="$project" +patch_id="$patch_id" +ci_bot="$ci_bot" +pw_check_cmd=(${pw_check_cmd[@]}) +build_url="$build_url" +EOF + +$scripts/pw-report.sh --check triggered --result pass --pw_dir "$pw_dir" + +if ! git -C "$project" pw series apply "$series_id"; then + $scripts/pw-report.sh --check apply --result fail --pw_dir "$pw_dir" + echo "ERROR: Series $series_id did not apply" + exit 4 +fi + +git -C "$project" checkout --detach HEAD~$num_patch + +$scripts/pw-report.sh --check apply --result pass --pw_dir "$pw_dir" + diff --git a/pw-report.sh b/pw-report.sh new file mode 100755 index 00000000..f14ebf46 --- /dev/null +++ b/pw-report.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +set -euf -o pipefail +x + +scripts=$(dirname $0) +# shellcheck source=jenkins-helpers.sh +. $scripts/jenkins-helpers.sh + +convert_args_to_variables "$@" + +obligatory_variables check result pw_dir +declare check result pw_dir + +verbose="${verbose-true}" + +if $verbose; then + set -x +fi + +if ! [ -d "$pw_dir" ]; then + exit 0 +fi + +bundle_types=({appl,test}-{fail,pass}) +# ... and "triggered" -- all bundle types are <= 9 characters long. + +while IFS= read -r -d '' pw_file; do + ( + source "$pw_file" + obligatory_variables project patch_id ci_bot pw_check_cmd build_url + declare project patch_id ci_bot pw_check_cmd build_url + + case "$ci_bot" in + tcwg_*_check) pw_check_cmd=(echo DRYRUN: "${pw_check_cmd[@]}") ;; + esac + + case "$check-$result" in + triggered-*) + # Remove $patch_id from *-pass and *-fail bundles. + for i in "${bundle_types[@]}"; do + pw_ci_bot_bundle_remove "$project" "$i" "$ci_bot" \ + "$patch_id" + done + "${pw_check_cmd[@]}" --state pending \ + --description "Test started" \ + --url "${build_url}console" + ;; + apply-fail|apply-pass|test-fail|test-pass) + if [ "$check" = "apply" ]; then + # Make apply-pass and apply-fail fit into 9 characters. + check="appl" + fi + pw_ci_bot_bundle_check_and_add \ + "$project" "$check-$result" "$ci_bot" "$patch_id" true + + case "$check-$result" in + appl-fail) + desc="Patch failed to apply" + url="${build_url}artifact/artifacts/jenkins/pw-apply.log" + state="fail" + ;; + appl-pass) + desc="Patch applied" + url="${build_url}console" + state="pending" + ;; + test-fail) + desc="Testing failed" + url="${build_url}artifact/artifacts/" + state="fail" + ;; + test-pass) + desc="Testing passed" + url="$build_url" + state="success" + ;; + esac + "${pw_check_cmd[@]}" --state "$state" \ + --description "$desc" --url "$url" + ;; + test-ignore) + # Testing failed due to sporadic failure. Allow retrigger testing + # of this patch in the next round. + pw_ci_bot_bundle_remove "$project" "triggered" "$ci_bot" \ + "$patch_id" + desc="Build did not complete; will be retriggered" + "${pw_check_cmd[@]}" --state warning \ + --description "$desc" --url "$build_url" + ;; + esac + ) +done < <(find "$pw_dir" -type f -print0) diff --git a/pw-trigger.sh b/pw-trigger.sh new file mode 100755 index 00000000..1d8c1181 --- /dev/null +++ b/pw-trigger.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +set -euf -o pipefail +x + +scripts=$(dirname $0) +# shellcheck source=jenkins-helpers.sh +. $scripts/jenkins-helpers.sh + +convert_args_to_variables "$@" + +obligatory_variables ci_bot project pw_token out_dir +declare ci_bot project pw_token out_dir + +verbose="${verbose-true}" + +if $verbose; then + set -x +fi + +(set +x; pw_init "$project" "$pw_token") + +# Delete patches from the queue bundle. The fact that we are here means +# that the jenkins queue is empty, and no more patches will be tested. +# This can happen if a new version of series was uploaded and we tested +# a different top-of-series patch. +# We avoid deleting and re-creatthe queued bundles to conserve bundle IDs. +queued_name=$(pw_bundle_name queued "$ci_bot") +queued_id=$(pw_bundle_id "$project" "$queued_name" || true) +while [ "$queued_id" != "" ]; do + patch_id=$(pw_get_patch_from_bundle_or_series \ + "$project" bundle "$queued_id" 0) + git -C "$project" pw bundle remove -f yaml "$queued_id" "$patch_id" & + if ! wait $!; then + # "bundle remove" will refuse to delete the last patch. + break + fi +done + +triggered_name=$(pw_bundle_name triggered "$ci_bot") +triggered_id=$(pw_bundle_id "$project" "$triggered_name" || true) + +yaml=$(mktemp) +trap "rm -f $yaml" EXIT + +git -C "$project" pw series list -f yaml > "$yaml" +mkdir -p "$out_dir" + +len=$(shyaml get-length < "$yaml") +i=$len +j=0 +# Go through series with the oldest first. +while [ "$i" -gt "0" ]; do + i=$(($i - 1 )) + series_id=$(shyaml get-value $i.id < "$yaml") + + if ! pw_series_complete_p "$project" "$series_id"; then + continue + fi + + num_patch=0 + while true; do + patch_id=$(pw_get_patch_from_bundle_or_series \ + "$project" series "$series_id" "$num_patch" || true) + if [ "$patch_id" = "" ]; then + break + fi + + if ! pw_bundle_has_patch_p "$project" "$triggered_id" "$patch_id"; then + break + fi + num_patch=$(($num_patch + 1)) + done + + if [ "$patch_id" = "" ]; then + continue + fi + + j=$(($j + 1)) + cat > "$out_dir/trigger-precommit-$project-$j-$series_id" <<EOF +${project}_git=pw://series/$series_id +EOF + + pw_ci_bot_bundle_check_and_add \ + "$project" queued "$ci_bot" "$patch_id" true +done + +echo "Processed $len series and created $j pre-commit triggers" diff --git a/round-robin.sh b/round-robin.sh index ed5e1a78..b8832d7e 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -58,7 +58,7 @@ rr[breakup_changed_components]=breakup_changed_components # Abe's repository and branch to use for the build. rr[abe_repo]="https://git-us.linaro.org/toolchain/abe.git" -rr[abe_branch]="master" +rr[abe_branch]="new-flaky" # Host compiler defaults to /usr/bin/gcc and g++ rr[host_cc]="/usr/bin/gcc" @@ -160,14 +160,24 @@ clone_repo () local url branch - if [ x"${rr[${project}_git]}" != x"baseline" ]; then - # Fetch and checkout from the specified repo. - url="${rr[${project}_git]%#*}" - branch="${rr[${project}_git]#*#}" - else - url=$(get_baseline_git ${project}_url) - branch=$(get_baseline_git ${project}_rev) - fi + case "${rr[${project}_git]}" in + *"#"*) + # Fetch from specified remote repo. + url="${rr[${project}_git]%#*}" + branch="${rr[${project}_git]#*#}" + ;; + "baseline") + # Fetch from remote repo specified in the baseline. + url=$(get_baseline_git ${project}_url) + branch=$(get_baseline_git ${project}_rev) + ;; + *) + # Use revision in the existing local repo. + # Most likely it is "HEAD" in precommit testing. + url="" + branch="${rr[${project}_git]}" + ;; + esac # Allow manifest override for $url url="${rr[${project}_url]-$url}" @@ -175,27 +185,36 @@ clone_repo () branch="${rr[${project}_rev]-$branch}" if [ x"${rr[mode]}" = x"bisect" ]; then - # Cleanup current checkout in bisect mode. + # FIXME: We should be able to merge this into below "local repo" clause. + # In bisect builds we should specify HEAD for the bisected project, + # and it should be OK to fetch "baseline" for all other components. + git_clean "$project" + elif [ "$url" = "" ]; then + # In local mode -- clean the project directory. git_clean "$project" + # Don't use git_checkout(), which prefers remote resolution of refs. + git -C "$project" checkout --detach "$branch" else - clone_or_update_repo_no_checkout "$project" "$url" auto "" origin \ - > /dev/null - git_checkout "$project" "$branch" origin + clone_or_update_repo "$project" "$branch" "$url" > /dev/null fi local cur_rev debug_project_date cur_rev=$(git -C $project rev-parse HEAD) debug_project_date=$(git -C $project show --no-patch --pretty="%ct # %cr" HEAD) - # Store git info info in the manifest. - cat <<EOF | manifest_out + # Store git info in the manifest and git data into artifacts. + # Git data in artifacts is then used by subsequent builds to fetch + # baseline commits. + if [ "$url" != "" ]; then + cat <<EOF | manifest_out rr[${project}_url]=$url +EOF + echo "$url" | set_current_git ${project}_url + fi + cat <<EOF | manifest_out rr[${project}_rev]=$cur_rev rr[debug_${project}_date]=$debug_project_date EOF - # Store baseline git data. This is then used by - # subsequent builds to fetch baseline commits. - echo "$url" | set_current_git ${project}_url echo "$cur_rev" | set_current_git ${project}_rev ) } @@ -206,7 +225,7 @@ prepare_abe () ( set -euf -o pipefail - clone_or_update_repo abe ${rr[abe_branch]} ${rr[abe_repo]} > /dev/null + clone_or_update_repo abe ${rr[abe_branch]} ${rr[abe_repo]} > /dev/null # We use our modified version of GCC's comparison script clone_or_update_repo gcc-compare-results master \ @@ -379,43 +398,41 @@ build_abe () mkdir "$run_step_top_artifacts/sumfiles" \ "$run_step_top_artifacts/00-sumfiles" - # Construct $xfails from - # - failed tests from base-artifacts/sumfiles/ and - # - flaky tests seen in history of base-artifacts/sumfiles/flaky.xfail. - # These are passed to ABE to speed-up test convergence - # and to .sum comparison in tcwg_gnu-build.sh:no_regression_p(). - local xfails="$run_step_top_artifacts/sumfiles/xfails.xfail" - + # Construct $baseline_fails from base-artifacts/sumfiles/. + # These and $flaky_tests are passed to ABE to speed-up test convergence + # and then to .sum comparison in tcwg_gnu-build.sh:no_regression_p(). + local baseline_fails="$run_step_artifacts/baseline.xfail" if [ -d base-artifacts/sumfiles ]; then gcc-compare-results/contrib/testsuite-management/validate_failures.py \ --build_dir=base-artifacts/sumfiles --produce_manifest \ - --manifest "$xfails" + --manifest "$baseline_fails" else - touch "$xfails" + touch "$baseline_fails" fi # Fetch flaky tests from base-artifacts history. - local history_flaky history_root="" + local history_flaky history_root="" flaky_tests + flaky_tests=$(mktemp) while read history_flaky; do if [ "$history_root" = "" ]; then history_root="$history_flaky" continue fi - (echo; cat "$history_flaky") >> "$xfails" + (echo; cat "$history_flaky") >> "$flaky_tests" done < <(get_git_history 0 base-artifacts sumfiles/flaky.xfail) rm -rf "$history_root" - # Record flaky tests in current run in $flaky_tests; these will then be - # included in subsequent runs as $xfails. + # Record flaky tests in current run in $new_flaky. local new_flaky="$run_step_artifacts/flaky.xfail" - cat > "$new_flaky" <<EOF -# From ${BUILD_URL-$(pwd)}: -EOF + + # ABE will re-write the $new_flaky file to contain entries only + # for the new flaky tests detected in this run. + cp "$flaky_tests" "$new_flaky" rerun_failed_tests=("--rerun-failed-tests" "--gcc-compare-results" "$PWD/gcc-compare-results" - "--expected-failures" "$xfails" + "--expected-failures" "$baseline_fails" "--flaky-failures" "$new_flaky") fi @@ -647,23 +664,50 @@ EOF sumfiles="$run_step_top_artifacts/00-sumfiles" fi - # Post-process flaky tests: add "flaky" attibute, and set the entry - # to expire in 1 month. The upside of expiration is that we will - # have an up-to-date list of flaky tests, which we can address. - # The downside is that we will spend a bit more CPU cycles - # to re-detect flaky tests. - local expire - expire=$(date -d "now+1 month" +%Y%m%d) - expire="expire=$expire" - - cp "$new_flaky" "$new_flaky.bak" - sed -i -e "s/^\([A-Z]\+: \)/flaky,$expire \| \1/" "$new_flaky" - if ! diff -q "$new_flaky" "$new_flaky.bak" > /dev/null; then + local xfails="$sumfiles/xfails.xfail" + + if [ -s "$new_flaky" ]; then + # Mark new flaky entries to expire in 1 month. + # The upside of expiration is that we will + # have an up-to-date list of flaky tests, which we can address. + # The downside is that we will spend a bit more CPU cycles + # to re-detect flaky tests. + local expire + expire=$(date -d "now+1 month" +%Y%m%d) + expire="expire=$expire" + sed -i -e "s#^flaky | #flaky,$expire | #" "$new_flaky" + # Move flaky fails to the sumfiles so that they will be # fetched by next run's get_git_history(). - mv "$new_flaky" "$sumfiles/flaky.xfail" + echo "# From ${BUILD_URL-$(pwd)}:" > "$sumfiles/flaky.xfail" + cat "$new_flaky" >> "$sumfiles/flaky.xfail" + + # Add newly-detected flaky tests to the xfails to be used in + # tcwg_gnu-build.sh:no_regression_p(). + # Strictly speaking, this is not necessary, since all tests + # detected as flaky in the current run will show up as PASSed + # test in the final merged .sum files. + cat "$sumfiles/flaky.xfail" >> "$xfails" fi - rm "$new_flaky.bak" + + # Add known flaky tests and baseline_fails to the xfails to be used in + # tcwg_gnu-build.sh:no_regression_p(). + # + # Note #1: We generate $baseline_fails without regard for flaky + # tests. Therefore, validate_failures in no_regression_p() will + # see same tests with and without flaky attributes. + # Validate_failure uses python sets to store results, so the first + # entry wins. Therefore, we need to put lists of flaky tests before + # lists of expected fails -- $baseline_fails. + # + # Note #2: Order of expired and non-expired flaky tests is less of + # an issue because expired entries are discarded by validate_failures + # before adding them to the ResultSet. Still, for uniformity, we + # put fresh flaky entries before older ones. + cat "$flaky_tests" >> "$xfails" + cat "$baseline_fails" >> "$xfails" + + rm "$new_flaky" "$baseline_fails" fi # Wait for logs to compress. @@ -843,14 +887,16 @@ create_trigger_files () if [ ${#changed_components[@]} -gt 1 ] \ || { [ x"${rr[mode]}" = x"bisect" ] \ - && ! [ $score -lt 0 ] 2>/dev/null; }; then + && ! [ "$score" -lt 0 ] 2>/dev/null; }; then # If we have several changed components, then trigger individual builds # for these components to narrow down the problem to a single component. # Also generate trigger-build-* files as a favor to # round-robin-bisect.sh script to distinguish "bad" from "skip" builds # and to provide templates for triggering follow-up builds. - # Note that the negative "! [ $score -lt 0 ]" condition is to handle - # non-numeric scores like "all" and "boot". + # Note that we can't use "[ "$score" -ge 0 ]" condition above and below + # because it will give wrong answer for non-numeric scores like "all" + # and "boot". "-lt" comparison, however, will produce correct answer, + # albeit, with an occasional harmless error message. local -a update_components while read -a update_components; do @@ -860,6 +906,8 @@ create_trigger_files () for c in ${rr[components]}; do if echo "${update_components[@]}" | tr ' ' '\n' \ | grep "^$c\$" >/dev/null; then + assert_with_msg "Should never happen for precommit builds" \ + [ "${rr[${c}_git]}" != "HEAD" ] echo "${c}_git=${rr[${c}_git]}" fi done > $trigger_dest/trigger-build-$update_components2 @@ -875,14 +923,9 @@ create_trigger_files () fi done < <(${rr[breakup_changed_components]}) elif [ ${#changed_components[@]} = 1 ] \ - && ! [ $score -lt 0 ] 2>/dev/null; then + && ! [ "$score" -lt 0 ] 2>/dev/null; then local single_component="${changed_components[0]}" - if [ x"${rr[update_baseline]}" = x"ignore" ]; then - # Don't trigger bisect for "ignore" builds. - return - fi - # Trigger bisect failures in a single changed component in all steps # with a positive result. If $score is less-than 0, then # the regression is not very interesting, so don't bisect. @@ -902,7 +945,107 @@ EOF } -# Check if current build regressed compared to the baseline +# Check if current build regressed compared to the baseline. +# +# As inputs we have: +# $score -- last line of artifacts/results +# $res -- exit code of no_regression_p() +# +# Additionally, ${rr[update_baseline]} and ${rr[mode]} affect how the rest +# of the build will proceed -- via create_trigger_files(). +# +# Decision matrix: +# +# 1. score >= 0 && res == 0: return 0 +# OK; new results good for baseline; +# update baseline if != ignore (e.g., not bisect test or precommit) +# - round-robin-notify.sh compares results in artifacts/ vs +# base-artifacts#HEAD^ +# don't update baseline if == ignore (e.g., bisect test or precommit) +# - round-robin-notify.sh compares results in artifacts/ vs +# base-artifacts#HEAD +# send precommit feedback if precommit testing is active; +# (see [R] below) send regression report if notify==onregression +# push baseline if != ignore (e.g., not precommit build); +# trigger pre-commit testing if SCM build; +# +# 2. score >= 0 && res == E: return E +# sporadic fail; wait for next build; don't trigger anything; +# don't update baseline even with force or init; +# Can happen in, e.g., tcwg_native_build if git server fails while +# cloning repo for one of the components; +# +# 3. score >= 0 && res != {0,E}: return I (or 0 if update_baseline==force) +# regression (or expected regression if update_baseline==force); +# if update_baseline == onsuccess -- trigger reduce or bisect; +# [R] if update_baseline == force -- succeed, return 0, treat as (1). +# if update_baseline == ignore -- nothing +# - if mode==bisect, then create trigger-* files as favor for +# round-robin-bisect.sh script. +# - if pre-commit testing is active, then send notifications +# similar to "update_baseline==force". See [P] below. +# case +# +# 4. score < 0 && res == 0: return E +# should not happen; baseline is bad and current build is no better; +# don't update the baseline or reduce or bisect; send error-mail +# +# 5. score < 0 && res == E: return E +# sporadic fail; wait for next build; don't trigger anything; +# don't update baseline even with force or init. +# +# 6. score < 0 && res != {0,E}: return I +# uninteresting fail; trigger reduce, but no bisect +# don't update baseline even with force or init. +# +# 7. score == -E: return E +# special case indicating a sporadic failure; same as (5) but without +# calling no_regression_p() to get "res". +# +# "E" -- EXTERNAL_FAILURE; "I" -- INTERNAL_FAILURE +# +# Notes: +# - If score < 0 then we never update baseline -- even with force or init. +# This allows us to always have a reasonable "interesting" baseline, +# which is necessary for pre-commit testing. E.g., we don't want to +# pre-commit test upstream patches against a baseline that doesn't build. +# +# - If we return E -- exit, do nothing, wait for the next timed trigger. +# Hopefully by that time infra fixes itself. +# In the .yaml files we have a choice on how to implement this: +# a. Mark the build as FAILURE, which will prevent any subsequent steps +# from running. Pros: simple, doesn't require conditional-steps; +# cons: the build is shown as "red" failure. +# b. Mark the build as UNSTABLE, and then condition all subsequent steps +# to run only for current-status==SUCCESS, which is what we are doing +# now. Pros: the build is shown as "yellow"; cons: all subsequent +# steps need to be wrapped in conditional-step. +# +# - [P] It's not clear how to send notifications for failed pre-commit +# builds. At the moment we run trigger-followup-builds for failed +# builds and exit, thus avoiding notify-and-push step for failed builds. +# It seems we need to +# a. Split notify-and-push into "notify" and "push" +# b. Move "notify" before trigger-followup-builds and leave "push" +# after trigger-followup-builds. +# c. We need to make sure "notify" does not send anything when it is +# running after a failed build. +# d. Trigger of pre-commit testing needs to happen only after "push", +# so we know that we have a good recent baseline. +# +# - If we return E during precommit testing -- we should, ideally, +# remove the patch from "triggered" bundle, so that we will process +# it again later. +# +# - If we return N != {0, I, E}, then the exit was due to script error, +# so send error-mail and follow case of returning "E". +# +# - If we returned "0", then build should be marked as SUCCESS. +# +# - If we returned "I", then build should be marked FAILURE +# +# - If we returned "E" or anything else, then build should be marked +# as UNSTABLE. check_regression () { ( @@ -911,6 +1054,8 @@ check_regression () local score score=$(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1) + # FIXME: notify-round-robin.sh should get information on status of + # the build via exit code -- 0, I, E -- not via check-regression-status.txt. mkdir -p ${rr[top_artifacts]}/notify if [ x"$score" = x"-$EXTERNAL_FAIL" ]; then echo "ERROR: We have encountered some infrastructure problem (e.g.," @@ -920,7 +1065,7 @@ check_regression () # Exit now and don't update baseline artifacts. # By not creating trigger-build-* files, we signal # round-robin-bisect.sh to skip this build/revision. - exit $EXTERNAL_FAIL + return $EXTERNAL_FAIL fi local res @@ -945,11 +1090,22 @@ check_regression () echo "$res" > ${rr[top_artifacts]}/notify/check-regression-status.txt if [ $res = 0 ]; then - # All good, no regression - return + if [ "$score" -lt 0 ] 2>/dev/null; then + # Case (4) in the comment above. + if [ -d ${rr[top_artifacts]}/jenkins ]; then + echo "maxim.kuvyrkov@linaro.org, laurent.alfonsi@linaro.org" \ + > artifacts/jenkins/error-mail-recipients.txt + echo -e "${BUILD_URL-}\nERROR: case (4) in check_regression" \ + >> artifacts/jenkins/error-mail-body.txt + fi + return $EXTERNAL_FAIL + else + # All good, no regression + return 0 + fi elif [ $res = $EXTERNAL_FAIL ]; then # Comparison failed to produce a meaningful result - exit $EXTERNAL_FAIL + return $EXTERNAL_FAIL fi assert_with_msg "no_regression_p should succeed in init baseline mode" \ @@ -957,7 +1113,8 @@ check_regression () # We've got a regression. Generate trigger-* files. local trigger_dest - if [ x"${rr[update_baseline]}" != x"force" ]; then + if [ "${rr[update_baseline]}" = "onsuccess" ] \ + || [ "${rr[mode]}" = "bisect" ]; then # We are seeing a failure, so instead of updating baseline start # reducing/bisecting the failure. Create trigger-* files at top level # where jenkins expects them -- and trigger the followup builds. @@ -971,10 +1128,12 @@ check_regression () create_trigger_files "$trigger_dest" "$score" - if [ x"${rr[update_baseline]}" != x"force" ]; then - echo "Detected a regression!" - false + if [ "${rr[update_baseline]}" = "force" ]; then + return 0 fi + + echo "Detected a regression!" + return $INTERNAL_FAIL ) } @@ -1108,7 +1267,7 @@ update_baseline () set -euf -o pipefail if [ x"${rr[update_baseline]}" = x"ignore" ]; then - return + return 0 fi trim_base_artifacts |