diff options
author | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2022-08-08 13:53:04 +0000 |
---|---|---|
committer | Maxim Kuvyrkov <maxim.kuvyrkov@linaro.org> | 2022-08-08 15:36:16 +0000 |
commit | 939a28516aea819976982d935bb6401bca14f94e (patch) | |
tree | 2c8a4ea298129ac1b38c8ebd89a8f43269b72f46 | |
parent | 2460b3594356409f0eab5cdbea28e4c6700abb71 (diff) |
round-robin.sh: Rework handling of manifests
Change meaning of "%% manifest" option to define artifacts
directory, which has manifest.sh at the top level:
"%%artifacts_var artifacts_dir" will
- define $artifacts_var and set it to "artifacts_dir", and
- start manifest.sh file at artifacts_dir/manifest.sh .
This is all in preparation for switching to using manifests for
baselines.
Change-Id: I1ea3597f4934363a062c0d040bf44e1d5f8351e0
-rw-r--r-- | jenkins-helpers.sh | 29 | ||||
-rwxr-xr-x | round-robin-bisect.sh | 45 | ||||
-rw-r--r-- | round-robin.sh | 1 |
3 files changed, 42 insertions, 33 deletions
diff --git a/jenkins-helpers.sh b/jenkins-helpers.sh index 89ce07e5..db9169d0 100644 --- a/jenkins-helpers.sh +++ b/jenkins-helpers.sh @@ -968,10 +968,11 @@ convert_arg_source () # "==arr[key] value" defines shell associative array "$arr[@]" and sets # "${arr[key]}" to "value". # "@@ file" sources file. -# "%% file" starts manifest in file. Also see "^^ true". -# "^^ true/false %% manifest" whether to reproduce the build using manifest. -# If "true" -- source manifest instead of generating it, then discard -# all following options at to separator "--". +# "%%artifacts_var dir" defines artifacts directory and starts manifest in +# dir/manifest.sh. Also see "^^ true". +# "^^ true/false %%artifacts_var dir" whether to reproduce the build using manifest. +# If "true" -- source dir/manifest.sh instead of generating it, then discard +# all following options up to separator "--". # If "false" -- do nothing and proceed as usual. # # Shell array $CONVERTED_ARGS is set to the arguments processed. @@ -1021,16 +1022,22 @@ convert_args_to_variables () convert_arg_source "$2" num=2 ;; - "%%") + "%%"*) assert_with_msg "ERROR: Parameter value not provided for $1." \ [ $# -ge 2 ] - manifest_push "$2" + manifest_push "$2/manifest.sh" cat <<EOF | manifest_out declare -g "jenkins_scripts_rev=$(git -C "$(dirname "$0")" rev-parse HEAD)" +# Artifacts directory EOF + name="${1#%%}" + case "$name" in + *"["*"]") convert_arg_assarr "$name" "$2" ;; + *) convert_arg_var "$name" "$2" ;; + esac cat <<EOF | manifest_out -# Recording parameters to manifest: $2 +# Recording parameters to manifest: $2/manifest.sh EOF num=2 ;; @@ -1040,8 +1047,12 @@ EOF if [ x"$2" = x"true" ]; then assert_with_msg "ERROR: manifest does not exist: $4/manifest.sh" \ [ -f $4/manifest.sh ] - - convert_arg_source "$4" + name="${3#%%}" + case "$name" in + *"["*"]") convert_arg_assarr "$name" "$4" ;; + *) convert_arg_var "$name" "$4" ;; + esac + convert_arg_source "$4/manifest.sh" # Skip processing all following arguments. num=0 diff --git a/round-robin-bisect.sh b/round-robin-bisect.sh index 6ba5af74..7855baed 100755 --- a/round-robin-bisect.sh +++ b/round-robin-bisect.sh @@ -6,13 +6,6 @@ scripts=$(dirname $0) # shellcheck source=jenkins-helpers.sh . $scripts/jenkins-helpers.sh -# Relative artifacts are used for generation of manifests and reproduction -# instructions. -rel_artifacts=artifacts -artifacts=$(pwd)/$rel_artifacts - -fresh_dir $artifacts "$artifacts/manifests/*" "$artifacts/jenkins/*" - declare -A rr # Process bisect-only args @@ -22,15 +15,26 @@ shift "$SHIFT_CONVERTED_ARGS" obligatory_variables bad_git build_script current_project declare bad_git build_script current_project +# Relative artifacts are used for generation of manifests and reproduction +# instructions. +rel_artifacts="${rel_artifacts-artifacts}" +artifacts=$(pwd)/$rel_artifacts + +fresh_dir $artifacts \ + "$artifacts/manifest.sh" \ + "$artifacts/build-parameters/manifest.sh" \ + "$artifacts/jenkins/*" + BUILD_URL="${BUILD_URL:-$(pwd)}" replay_log="${replay_log-}" reproduce_bisect="${reproduce_bisect:-false}" # Process build args and record them in build-parameters.sh -convert_args_to_variables ^^ $reproduce_bisect %% $artifacts/manifests/build-parameters.sh "$@" +convert_args_to_variables ^^ $reproduce_bisect %%build_parameters $artifacts/build-parameters "$@" $reproduce_bisect || manifest_pop -obligatory_variables rr[ci_project] rr[ci_config] +obligatory_variables build_parameters rr[ci_project] rr[ci_config] +declare build_parameters verbose="${verbose-true}" @@ -73,11 +77,10 @@ esac echo "Testing baseline (should be success)" $build_script \ ^^ $reproduce_bisect \ - %% $rel_artifacts/manifests/build-baseline.sh \ - @@ $rel_artifacts/manifests/build-parameters.sh \ + %%rr[top_artifacts] "$rel_artifacts/build-baseline" \ + @@ $build_parameters/manifest.sh \ ==rr[mode] build \ ==rr[update_baseline] force \ - ==rr[top_artifacts] "$rel_artifacts/build-baseline" \ --verbose "$verbose" \ "${rebase_workaround_opts[@]}" @@ -87,7 +90,6 @@ declare -g baseline_rev=$baseline_rev EOF ln -f -s "build-baseline" "$artifacts/build-$baseline_rev" -ln -f -s "build-baseline.sh" "$artifacts/manifests/build-$baseline_rev.sh" case "${rr[ci_project]}" in tcwg_gcc_check*|tcwg_gnu_*_check_*) @@ -197,12 +199,11 @@ rsync -a --del ${baseline_exclude[@]} ./bisect/baseline/ ./ $build_script \ ^^ $reproduce_bisect \ - %% $rel_artifacts/manifests/build-\$rev.sh \ - @@ $rel_artifacts/manifests/build-parameters.sh \ + %%rr[top_artifacts] $rel_artifacts/build-\$rev \ + @@ $rel_artifacts/build-parameters/manifest.sh \ ==rr[mode] bisect \ ==rr[update_baseline] ignore \ ==rr[${current_project}_git] "$bad_url#\$rev" \ - ==rr[top_artifacts] $rel_artifacts/build-\$rev \ --verbose "$verbose" & res=0 && wait \$! || res=\$? @@ -309,7 +310,6 @@ if ! git -C $current_project bisect log \ | grep -q "^git bisect .* $bad_rev\$"; then git -C $current_project bisect bad $bad_rev ln -f -s "build-$bad_rev" "$artifacts/build-bad" - ln -f -s "build-$bad_rev.sh" "$artifacts/manifests/build-bad.sh" fi # Clone interesting-commits.git repo, which contains a list of SHA1s @@ -649,10 +649,7 @@ if [ x"$first_bad" != x"" ]; then sed -i -e "s/\$/-$first_bad/" $artifacts/jenkins/build-name ln -f -s "build-$first_bad" "$artifacts/build-first_bad" - ln -f -s "build-$first_bad.sh" "$artifacts/manifests/build-first_bad.sh" - ln -f -s "build-$last_good" "$artifacts/build-last_good" - ln -f -s "build-$last_good.sh" "$artifacts/manifests/build-last_good.sh" else notify_devs=false fi @@ -756,14 +753,14 @@ cd investigate-$current_project-$first_bad git clone https://git.linaro.org/toolchain/jenkins-scripts # Fetch manifests and test.sh script -mkdir -p $rel_artifacts/manifests -curl -o $rel_artifacts/manifests/build-baseline.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-baseline.sh --fail -curl -o $rel_artifacts/manifests/build-parameters.sh ${BUILD_URL}artifact/$rel_artifacts/manifests/build-parameters.sh --fail +mkdir -p $rel_artifacts/build-baseline $rel_artifacts/build-parameters +curl -o $rel_artifacts/build-baseline/manifest.sh ${BUILD_URL}artifact/$rel_artifacts/build-baseline/manifest.sh --fail +curl -o $rel_artifacts/build-parameters/manifest.sh ${BUILD_URL}artifact/$rel_artifacts/build-parameters/manifest.sh --fail curl -o $rel_artifacts/test.sh ${BUILD_URL}artifact/$rel_artifacts/test.sh --fail chmod +x $rel_artifacts/test.sh # Reproduce the baseline build (build all pre-requisites) -$build_script @@ $rel_artifacts/manifests/build-baseline.sh +$build_script ^^ true %%rr[top_artifacts] $rel_artifacts/build-baseline # Save baseline build state (which is then restored in $rel_artifacts/test.sh) mkdir -p ./bisect diff --git a/round-robin.sh b/round-robin.sh index 86161d8f..04683e59 100644 --- a/round-robin.sh +++ b/round-robin.sh @@ -113,6 +113,7 @@ reset_artifacts () fresh_dir $run_step_top_artifacts \ $run_step_top_artifacts/console.log \ $run_step_artifacts/console.log \ + $run_step_top_artifacts/manifest.sh \ "$run_step_top_artifacts/jenkins/*" if [ ${rr[update_baseline]} = "onsuccess" ] && |