#!/bin/bash # shellcheck source=jenkins-helpers.sh . "$(dirname $0)"/jenkins-helpers.sh # Round-Robin associative array. declare -gA rr # PROJECT's git url#branch or url#SHA1 revision parsable by git rev-parse. # A special value "baseline" means that PROJECT is not being updated # in this build, and its baseline branch should be used. # In a successful build "update_baseline" step will update baseline # branches of all PROJECTs to the current values, thus setting # a baseline for the next build. #rr[PROJECT_git] # PROJECT's git SHA1 revision. These are mostly used in manifests. #rr[PROJECT_rev] # Baseline branch name for current configuration. These branches should # be present in all above git repos (if ${rr[init_configuration]} is false). #rr[baseline_branch]="${rr[ci_project]}/${rr[ci_config]}" # Run mode: bisect or non-bisect. In bisect mode we do a couple things # slightly differently (e.g., don't touch repo in clone_repo() ). #rr[mode]="$mode" # How to handle baseline: # - init: use "empty" results for base-artifacts, which will make current # build successful. Push our artifacts as the one and only entry. # - update: update baseline branches of base-artifacts and components' repos # on success (generate trigger files for bisection on failure). # - reset: ignore failures in check_regression(), which will make current # build successful. Push our artifacts to the top of base-artifacts/. # - push: push results from current build (whatever they are, regressions are # ignored) as new commit to base-artifacts, and update baseline # branches. This is useful for user projects, e.g., to generate # historical results for several toolchain versions. # - rebase: treat results of this build as historically eldest, and # rebase base-artifacts commits on top of this build's artifacts. #rr[update_baseline]=update/reset/init/rebase # Target architecture to build for: arm or aarch64. #rr[target]="$target" # Top-level artifacts directory. #rr[top_artifacts]="$top_artifacts" # Predicate function to determine whether there is regression between # 2 sets of results. # shellcheck disable=SC2154 rr[no_regression_p]=no_regression_p # Hook to break up updated component (see print_updated_components) into # smaller sets: print one set per line. By default, breakup into singletons. # shellcheck disable=SC2154 rr[breakup_updated_components]=breakup_updated_components # Print round-robin components that are being updated in this build # (the ones using non-baseline branches). print_updated_components () { ( set -euf -o pipefail local delim="" local c for c in ${rr[components]}; do if [ x"${rr[${c}_git]}" != x"baseline" ]; then echo -ne "$delim$c" delim=${1- } fi done echo ) } # By default, print each component on its own line. breakup_updated_components () { print_updated_components "\n" } # Print the single round-robin component being updated in this build. # Print nothing if multiple components are being updated. print_single_updated_component () { ( set -euf -o pipefail local update_components IFS=" " read -r -a updated_components <<< "$(print_updated_components)" if [ ${#updated_components[@]} -eq 1 ]; then echo "${updated_components[0]}" fi ) } # Reset artifacts to an empty state. ${rr[top_artifacts]}/results is the most # important artifact, since it records the metric of how successful the build # is. reset_artifacts () { ( set -euf -o pipefail # Clean ${rr[top_artifacts]} but preserve # - ${rr[top_artifacts]}/console.log and $run_step_artifacts/console.log, which # are being written to by run_step(). # - ${rr[top_artifacts]}/jenkins/*, which is cleaned by tcwg_kernel.yaml. # shellcheck disable=SC2154 fresh_dir $run_step_top_artifacts \ $run_step_top_artifacts/console.log \ $run_step_artifacts/console.log \ "$run_step_top_artifacts/jenkins/*" # Clone base-artifacts here so that bisect runs (which skip this step) # don't overwrite it. # base-artifacts repo is big and changes all the time, so we # fetch only the $baseline_branch, instead of all branches. local single_branch if [ x"${rr[update_baseline]}" = x"init" ]; then single_branch=empty else single_branch=${rr[baseline_branch]} fi rr[base-artifacts_rev]="${rr[base-artifacts_rev]-$single_branch}" clone_or_update_repo base-artifacts ${rr[base-artifacts_rev]} https://git-us.linaro.org/toolchain/ci/base-artifacts.git auto $single_branch cat < /dev/null # Allow manifest override branch="${rr[${project}_rev]-$branch}" git_checkout "$project" "$branch" origin local cur_rev cur_rev=$(git -C $project rev-parse HEAD) cat < /dev/null cd abe # Add ccache wrappers. # shellcheck disable=SC2115 rm -rf "$(pwd)/bin" mkdir "$(pwd)/bin" cat > "$(pwd)/bin/gcc" < "$(pwd)/bin/g++" < "$(pwd)/bin/cc" < "$(pwd)/bin/c++" < "$(pwd)/bin/ninja" < $run_step_artifacts/results.regressions <> ${rr[top_artifacts]}/results fi # We've got a regression. Generate trigger-* files. local single_component single_component=$(print_single_updated_component) local trigger_dest if [ x"${rr[update_baseline]}" = x"update" ]; then # When "updating" baseline place trigger-* files at top level # where jenkins expects them -- and trigger the followup builds. trigger_dest="${rr[top_artifacts]}" else # We don't want to trigger follow up builds when "pushing" # baseline. So, for the record, place trigger-* files in # the step's artifacts directory. trigger_dest="$run_step_artifacts" fi # 1. If $score is less-than 0, then the regression is not very # interesting, so reduce down to component, but don't bisect. This # allows non-broken components to advance their revisions. # 2. Otherwise, trigger builds with narrowed-down list of updated # components -- by generating trigger-build-* files. Also generate # these files as a favor to round-robin-bisect.sh script. # 3. Otherwise, we have narrowed-down regression to a single component, # so trigger bisect build for this component -- by generating # trigger-bisect file. Note that the negative "! [ $score -lt 0 ]" # condition is to handle non-numeric scores like "all" and "boot". if [ $score -lt 0 ] 2>/dev/null && [ x"${rr[mode]}" = x"bisect" ]; then # Don't bisect uninteresting regressions. : elif [ x"$single_component" = x"" ] || [ x"${rr[mode]}" = x"bisect" ]; then local -a update_components while read -a update_components; do local c update_components2 update_components2=$(echo "${update_components[@]}" | tr ' ' '-') for c in ${rr[components]}; do if echo "${update_components[@]}" | tr ' ' '\n' | grep -q "^$c\$"; then echo "${c}_git=${rr[${c}_git]}" else echo "${c}_git=baseline" fi done > $trigger_dest/trigger-build-$update_components2 done < <(${rr[breakup_updated_components]}) elif ! [ $score -lt 0 ] 2>/dev/null; then # Bisect failures in all steps after "-1" step. local cur_rev cur_rev=$(git -C $single_component rev-parse HEAD) cat > $trigger_dest/trigger-bisect <> $trigger_dest/trigger-bisect fi fi if [ x"${rr[update_baseline]}" = x"update" ]; then echo "Detected a regression in \"update\" mode!" false else # Compare results to generate logs and other artifacts, # but we don't really care about regressions. : fi fi ) } # Commit current result and artifacts to the baseline repository update_baseline () { ( set -euf -o pipefail local amend="" local rebase_head rebase_tail if [ x"${rr[update_baseline]}" = x"init" ]; then amend="--amend" elif [ x"${rr[update_baseline]}" = x"push" ]; then : elif [ x"${rr[update_baseline]}" = x"rebase" ]; then rebase_head=$(git -C base-artifacts rev-parse HEAD) rebase_tail=$(git -C base-artifacts rev-list --max-parents=0 HEAD) git -C base-artifacts reset --hard $rebase_tail amend="--amend" else # ${rr[update_baseline]} == update, reset local prev_head="" # We discard baseline entries for results worse or same than # the current one, but keep entries for results that are better # (so that we have a record of pending regressions). while true; do ${rr[no_regression_p]} base-artifacts ${rr[top_artifacts]} & if ! wait $!; then break fi prev_head="" if git -C base-artifacts rev-parse HEAD^ >/dev/null 2>&1; then # For every regression we want to keep artifacts for the first-bad # build, so reset to the most relevant regression (marked by reset-baseline). if [ -f base-artifacts/reset-baseline ] \ && [ x"${rr[update_baseline]}" = x"update" ]; then prev_head=$(git -C base-artifacts rev-parse HEAD) fi git -C base-artifacts reset --hard HEAD^ else # We got to the beginning of git history, so amend the current # commit. The initial state of baseline is "empty" branch, # which we treat as worst possible in ${rr[no_regression_p]}(). amend="--amend" break fi done if [ x"$prev_head" != x"" ]; then git -C base-artifacts reset --hard $prev_head fi fi # Rsync current artifacts. Make sure to use -I rsync option since # quite often size and timestamp on artifacts/results will be the same # as on base-artifacts/results due to "git reset --hard HEAD^" below. # This caused rsync's "quick check" heuristic to skip "results" file. # !!! From this point on, logs and other artifacts won't be included # in base-artifacts.git repo (though they will be uploaded to jenkins). rsync -aI --del --exclude /.git ${rr[top_artifacts]}/ base-artifacts/ local rev_count if [ x"$amend" = x"" ]; then rev_count=$(git -C base-artifacts rev-list --count HEAD) else rev_count="0" fi local msg_title="$rev_count: ${rr[update_baseline]}" if [ x"${rr[update_baseline]}" = x"reset" ]; then # Create a marker for builds that reset baselines (these are builds # for bisected regressions). touch base-artifacts/reset-baseline fi local single_component single_component=$(print_single_updated_component) if [ x"$single_component" != x"" ]; then local single_rev single_rev=$(git -C $single_component rev-parse HEAD) msg_title="$msg_title: $single_component-$single_rev" else msg_title="$msg_title: $(print_updated_components "-")" fi msg_title="$msg_title: $(grep -v "^#" ${rr[top_artifacts]}/results | tail -n1)" git -C base-artifacts add . git -C base-artifacts commit $amend -m "$msg_title BUILD_URL: ${BUILD_URL-$(pwd)} results: $(cat ${rr[top_artifacts]}/results)" if [ x"${rr[update_baseline]}" = x"rebase" ]; then for rebase_tail in $(git -C base-artifacts rev-list --reverse $rebase_head); do git -C base-artifacts checkout $rebase_tail -- . git -C base-artifacts commit -C $rebase_tail done fi ) } # Push to baseline branches and to base-artifacts repo. push_baseline () { ( set -euf -o pipefail git_init_linaro_local_remote base-artifacts baseline false git_push base-artifacts baseline ${rr[baseline_branch]} if [ x"${rr[update_baseline]}" = x"rebase" ]; then return fi local url local c for c in $(print_updated_components); do # Clone (but don't checkout) always-present "empty" branch of # the baseline repo. This initializes read/write "baseline" remote. url=$(print_baseline_repo "$c" false) git_set_remote "$c" baseline "$url" git_push $c baseline ${rr[baseline_branch]} done ) }