diff options
Diffstat (limited to 'pw-helpers.sh')
| -rwxr-xr-x | pw-helpers.sh | 375 |
1 files changed, 375 insertions, 0 deletions
diff --git a/pw-helpers.sh b/pw-helpers.sh new file mode 100755 index 00000000..5f081c09 --- /dev/null +++ b/pw-helpers.sh @@ -0,0 +1,375 @@ +#!/bin/bash + +# Helper functions for accessing patchwork (pw) API. Almost all access +# is implemented via git-pw's commands and its YAML output. + +# Scripts that use below functions: +# - pw-trigger.sh -- looks for new patch series to test in patchwork +# and creates trigger-* files for jenkins. +# - pw-apply.sh -- fetches and applies a series to a local git clone. +# - pw-report.sh -- sends "check" feedback back to patchwork. +# +# The general workflow is: +# 1. At the end of successful post-commit testing, after baseline was +# updated, pw-trigger.sh generates trigger-* files for jenkins. This +# populates jenkins queue with jobs for testing patches posted since +# the last successful post-commit build. Pw-trigger.sh looks at the state +# of the latest "check", and triggers builds for all "pending" patches. +# +# 2. Jenkins starts a build for a given patch series, and applies to +# the local git checkout -- this is done with pw-apply.sh. As soon +# as pw-apply.sh has a patch ID, it calls pw-report.sh to add a "pending" +# check to patchwork indicating that testing has started. +# +# 3. If patch applied successfully, pw-apply.sh generates a state file +# (artifacts/jenkins/<project>), which has information necessary for +# pw-report.sh to send subsequent "check" feedback to patchwork. +# +# 4. The build proceeds as a normal post-commit build. +# +# 5. Once the build finishes pw-report.sh is called to send the final +# "check" -- whether patch passed or failed testing. Only the final +# pw-report.sh sets check state to something other than "pending". +# Therefore, if for whatever reason the final pw-report.sh does not run, +# patch testing will be retriggered on the next round. + +# Initialize git-pw in $project. +# $1 -- existing git clone of $project +pw_init () +{ + ( + set -euf -o pipefail + local project="$1" + + git -C "$project" config pw.server \ + "https://patchwork.sourceware.org/api/1.2/" + git -C "$project" config pw.project "$project" + + pw_clear_cache + ) +} + +# De-initialize git-pw in $project. +# $1 -- existing git clone of $project +pw_deinit () +{ + ( + set -euf -o pipefail + local project="$1" + + rm -rf "/tmp/pw-yaml-cache-$$" + ) +} + +# Clear pw_yaml cache. +pw_clear_cache () +{ + ( + set -euf -o pipefail + + rm -rf "/tmp/pw-yaml-cache-$$" + mkdir "/tmp/pw-yaml-cache-$$" + ) +} + +# Get specified piece of data from git-pw yaml output. +# This is reasonably unstable and relies heavily on git-pw's yaml format and +# field names not changing. +# +# $1 -- $project git directory +# $2 -- git-pw section: series, patch, etc. +# $3 -- identifier of object in the section; usually series or patch ID. +# $4, $5, $6 -- find object with field name $4 has value $5, and return value +# field $6 from this entry: if (data.$5 == data.$5) return data.$6; +# A special match_value ".*" selects the first entry that has match_field +# regardless of match_value. +# $7 -- optional value that stops search for object with match_field==match_value. +# This is necessary to avoid going "outside" of our data of interest and +# matching a random object that happens to have similarly named fields. +# $8 -- optional index of the entry to match; first N-1 matching entries +# will be skipped. +# +# Note: we match entries starting from the tail, since that is where +# the interesting stuff is most of the time. +pw_yaml_get () +{ + ( + set -euf -o pipefail + local project="$1" + local section="$2" + local id="$3" + local match_field="$4" + local match_value="$5" + local get_field="$6" + local match_stop="${7-}" + local match_num="${8-0}" + + # Reduce noise in the logs + set +x + + local -a cmd + case "$id" in + "--owner"*) + # shellcheck disable=SC2206 + cmd=(list $id) + ;; + *) + cmd=(show "$id") + ;; + esac + + local -a git_cmd + git_cmd=(git -C "$project" pw "$section" "${cmd[@]}" -f yaml) + + local yaml + yaml=$(IFS="_"; echo "${git_cmd[*]}" | tr "/" "_") + yaml="/tmp/pw-yaml-cache-$$/$yaml" + if [ -f "$yaml" ]; then + touch "$yaml" + else + # Timeout if PW throttles connection. Otherwise, this would hang + # indefinitely. + timeout 1m "${git_cmd[@]}" > "$yaml" & + local res + res=0 && wait $! || res=$? + if [ $res != 0 ]; then + rm -f "$yaml" + return $res + fi + fi + + local len + len=$(shyaml get-length < "$yaml") + while [ "$len" -gt "0" ]; do + len=$(($len - 1)) + if [ "$match_value" != ".*" ]; then + if shyaml get-value "$len.$match_field" < "$yaml" \ + | grep "$match_value" >/dev/null; then + if [ "$match_num" = "0" ]; then + shyaml get-value "$len.$get_field" < "$yaml" + return 0 + fi + match_num=$(($match_num - 1)) + fi + else + # Special case for $match_value == ".*". + # Only check that $match_field exist, and don't look at the value. + # This handles empty values without involving grep, which can't + # match EOF generated by empty value. + if shyaml get-value "$len.$match_field" < "$yaml" >/dev/null; then + if [ "$match_num" = "0" ]; then + shyaml get-value "$len.$get_field" < "$yaml" + return 0 + fi + match_num=$(($match_num - 1)) + fi + fi + if [ "$match_stop" != "" ] \ + && shyaml get-value "$len.$match_field" < "$yaml" \ + | grep "$match_stop" >/dev/null; then + return 1 + fi + done + + assert_with_msg "Missing $match_field == $match_value" false + ) +} + +# Return true if patch series is complete. +pw_series_complete_p () +{ + ( + set -euf -o pipefail + local project="$1" + local series_id="$2" + + local value + value=$(pw_yaml_get "$project" series "$series_id" property Complete value) + if [ "$value" = "True" ]; then + return 0 + fi + return 1 + ) +} + +# Return patch ID at specified index from series. +# $2 -- series ID +# $3 -- index of the patch *from the end* of series; "0" should always +# work. +pw_get_patch_from_series () +{ + ( + set -euf -o pipefail + local project="$1" + local id="$2" + local num="$3" + + local patch_id + patch_id=$(pw_yaml_get "$project" series "$id" property ".*" \ + value Patches "$num" | cut -d" " -f 1) + if [ "$patch_id" = "" ]; then + return 1 + fi + echo "$patch_id" + ) +} + +# Fetch patch entry data +# $2 -- patch ID +# $3 -- data field +pw_get_patch_data () +{ + ( + set -euf -o pipefail + local project="$1" + local patch_id="$2" + local field="$3" + + pw_yaml_get "$project" patch "$patch_id" property "$field" value + ) +} + +# Fetch current state of $patch_id's check for $ci_bot configuration. +# Prints out "pending/warning/fail/success"; with no matching checks +# prints out "pending". +pw_patch_check_state () +{ + ( + set -euf -o pipefail + local patch_id="$1" + local ci_owner_bot="$2" + + # Split $ci_owner_bot into [optional] $ci_owner and $ci_bot. + local ci_owner ci_bot + ci_owner="$(echo "$ci_owner_bot" | cut -s -d/ -f1)" + ci_bot="$(echo "$ci_owner_bot" | cut -s -d/ -f2)" + if [ "$ci_bot" = "" ]; then + ci_owner="linaro-tcwg-bot" + ci_bot="$ci_owner_bot" + fi + + local json1 json2 + json1=$(mktemp) + json2=$(mktemp) + # shellcheck disable=SC2064 + trap "rm $json1 $json2" EXIT + + curl -s \ + "https://patchwork.sourceware.org/api/1.2/patches/$patch_id/checks/" \ + > "$json1" + + local i="-1" cur_date="0" cur_state="pending" + local username context date + while true; do + i=$(($i + 1)) + + jq -r ".[$i]" < "$json1" > "$json2" + if [ "$(cat "$json2")" = "null" ]; then + break + fi + + username=$(jq -r ".user.username" < "$json2") + if [ "$username" != "$ci_owner" ]; then + continue + fi + + context=$(jq -r ".context" < "$json2") + if [ "$context" != "$ci_bot" ]; then + continue + fi + + date=$(jq -r ".date" < "$json2") + date=$(date -d "$date" +%s) + if [ "$cur_date" -le "$date" ]; then + cur_date="$date" + cur_state=$(jq -r ".state" < "$json2") + fi + done + + echo "$cur_state" + ) +} + +# Apply a patch series +# $1: project +# $2: method (either 'am' for plain git, or 'pw' for patchwork interface) +# $3: series dir (for 'am') or ID (for 'pw') +# $4: series name +# $5+: Optional "git am" options, e.g., "-p0" or "-p1". +apply_series() +{ + ( + set -euf -o pipefail + local project="$1" + local method="$2" + local series_id="$3" + local series_name="$4" + + shift 4 + + local res=0 + local subcommand="" + + case "$method" in + am) subcommand="am" ;; + pw) subcommand="pw series apply" ;; + *) + echo "ERROR: method $method not supported by apply_series()" + return 4 + ;; + esac + + # Apply the whole series and then roll-back to the desired patch. + if ! git -C "$project" $subcommand "$series_id" "$@"; then + echo "WARNING: Series $series_name did not apply cleanly" + # "git am" sometimes detects email text as a patch, and complains that it + # has no actual code changes. Workaround this by skipping empty patches. + res=4 + patch_file="$project/.git/rebase-apply/patch" + while [ "$res" = "4" ] \ + && [ -f "$patch_file" ] && ! [ -s "$patch_file" ]; do + # The patch is empty, so skip it. + res=0 + if ! git -C "$project" am --skip; then + res=4 + fi + done + fi + return $res + ) +} + +# Apply a patch series first with -p1, retry with -p0 if needed +# $1: project +# $2: prev_head +# $3: method (either 'am' for plain git, or 'pw' for patchwork interface) +# $4: series dir (for 'am') or ID (for 'pw') +# $5: optional series_url when $2='am' +apply_series_with_retry() +{ + ( + set -euf -o pipefail + local project="$1" + local prev_head="$2" + local method="$3" + local series_id="$4" + local series_name="$5" + local res=0 + + local try + for try in "" -p1 -p0; do + apply_series "$project" "$method" "$series_id" "$series_name" $try & + res=0 && wait $! || res=$? + + if [ $res = 0 ]; then + break + fi + + # Restore a clean state + git -C "$project" am --abort || true + git -C "$project" checkout --detach $prev_head + done + + return $res + ) +} |