path: root/pw-helpers.sh

#!/bin/bash

# Helper functions for accessing patchwork (pw) API.  Almost all access
# is implemented via git-pw's commands and its YAML output.

# Scripts that use below functions:
# - pw-trigger.sh -- looks for new patch series to test in patchwork
#   and creates trigger-* files for jenkins.
# - pw-apply.sh -- fetches and applies a series to a local git clone.
# - pw-report.sh -- sends "check" feedback back to patchwork.
#
# The general workflow is:
# 1. At the end of successful post-commit testing, after baseline was
#    updated, pw-trigger.sh generates trigger-* files for jenkins.  This
#    populates jenkins queue with jobs for testing patches posted since
#    the last successful post-commit build.  Pw-trigger.sh looks at the state
#    of the latest "check", and triggers builds for all "pending" patches.
#
# 2. Jenkins starts a build for a given patch series, and applies to
#    the local git checkout -- this is done with pw-apply.sh.  As soon
#    as pw-apply.sh has a patch ID, it calls pw-report.sh to add a "pending"
#    check to patchwork indicating that testing has started.
#
# 3. If patch applied successfully, pw-apply.sh generates a state file
#    (artifacts/jenkins/<project>), which has information necessary for
#    pw-report.sh to send subsequent "check" feedback to patchwork.
#
# 4. The build proceeds as a normal post-commit build.
#
# 5. Once the build finishes pw-report.sh is called to send the final
#    "check" -- whether patch passed or failed testing.  Only the final
#    pw-report.sh sets check state to something other than "pending".
#    Therefore, if for whatever reason the final pw-report.sh does not run,
#    patch testing will be retriggered on the next round.

# Initialize git-pw in $project.
# $1 -- existing git clone of $project
# $2 -- PW API token; without it modifications fail
pw_init ()
{
    (
    set -euf -o pipefail
    local project="$1"

    git -C "$project" config pw.server \
	"https://patchwork.sourceware.org/api/1.2/"
    git -C "$project" config pw.project "$project"

    pw_clear_cache

    # BE CAREFUL WITH $token
    set +x
    local token="$2"
    if [ "$token" != "" ]; then
	git -C "$project" config pw.token "$token"
    fi
    )
}

# De-initialize git-pw in $project.
# $1 -- existing git clone of $project
pw_deinit ()
{
    (
    set -euf -o pipefail
    local project="$1"

    rm -rf "/tmp/pw-yaml-cache-$$"

    # ignore error if pw.token is not set.
    git -C "$project" config --unset pw.token || true
    )
}

# Clear pw_yaml cache.
pw_clear_cache ()
{
    (
    set -euf -o pipefail

    rm -rf "/tmp/pw-yaml-cache-$$"
    mkdir "/tmp/pw-yaml-cache-$$"
    )
}

# Get specified piece of data from git-pw yaml output.
# This is reasonably unstable and relies heavily on git-pw's yaml format and
# field names not changing.
#
# $1 -- $project git directory
# $2 -- git-pw section: series, patch, etc.
# $3 -- identifier of object in the section; usually series or patch ID.
# $4, $5, $6 -- find object with field name $4 has value $5, and return value
#       field $6 from this entry: if (data.$5 == data.$5) return data.$6;
#       A special match_value ".*" selects the first entry that has match_field
#       regardless of match_value.
# $7 -- optional value that stops search for object with match_field==match_value.
#       This is necessary to avoid going "outside" of our data of interest and
#       matching a random object that happens to have similarly named fields.
# $8 -- optional index of the entry to match; first N-1 matching entries
#       will be skipped.
#
# Note: we match entries starting from the tail, since that is where
# the interesting stuff is most of the time.
pw_yaml_get ()
{
    (
    set -euf -o pipefail
    local project="$1"
    local section="$2"
    local id="$3"
    local match_field="$4"
    local match_value="$5"
    local get_field="$6"
    local match_stop="${7-}"
    local match_num="${8-0}"

    # Reduce noise in the logs
    set +x

    local -a cmd
    case "$id" in
	"--owner"*)
	    # shellcheck disable=SC2206
	    cmd=(list $id)
	    ;;
	*)
	    cmd=(show "$id")
	    ;;
    esac

    local -a git_cmd
    git_cmd=(git -C "$project" pw "$section" "${cmd[@]}" -f yaml)

    local yaml
    yaml=$(IFS="_"; echo "${git_cmd[*]}" | tr "/" "_")
    yaml="/tmp/pw-yaml-cache-$$/$yaml"
    if [ -f "$yaml" ]; then
	touch "$yaml"
    else
	# Timeout if PW throttles connection.  Otherwise, this would hang
	# indefinitely.
	timeout 1m "${git_cmd[@]}" > "$yaml" &
	local res
	res=0 && wait $! || res=$?
	if [ $res != 0 ]; then
	    rm -f "$yaml"
	    return $res
	fi
    fi

    local len
    len=$(shyaml get-length < "$yaml")
    while [ "$len" -gt "0" ]; do
	len=$(($len - 1))
	if [ "$match_value" != ".*" ]; then
	    if shyaml get-value "$len.$match_field" < "$yaml" \
		    | grep "$match_value" >/dev/null; then
		if [ "$match_num" = "0" ]; then
		    shyaml get-value "$len.$get_field" < "$yaml"
		    return 0
		fi
		match_num=$(($match_num - 1))
	    fi
	else
	    # Special case for $match_value == ".*".
	    # Only check that $match_field exist, and don't look at the value.
	    # This handles empty values without involving grep, which can't
	    # match EOF generated by empty value.
	    if shyaml get-value "$len.$match_field" < "$yaml" >/dev/null; then
		if [ "$match_num" = "0" ]; then
		    shyaml get-value "$len.$get_field" < "$yaml"
		    return 0
		fi
		match_num=$(($match_num - 1))
	    fi
	fi
	if [ "$match_stop" != "" ] \
	       && shyaml get-value "$len.$match_field" < "$yaml" \
		   | grep "$match_stop" >/dev/null; then
	    return 1
	fi
    done

    assert_with_msg "Missing $match_field == $match_value" false
    )
}

# Return true if patch series is complete.
pw_series_complete_p ()
{
    (
    set -euf -o pipefail
    local project="$1"
    local series_id="$2"

    local value
    value=$(pw_yaml_get "$project" series "$series_id" property Complete value)
    if [ "$value" = "True" ]; then
	return 0
    fi
    return 1
    )
}

# Return patch ID at specified index from series.
# $2 -- series ID
# $3 -- index of the patch *from the end* of series; "0" should always
#       work.
pw_get_patch_from_series ()
{
    (
    set -euf -o pipefail
    local project="$1"
    local id="$2"
    local num="$3"

    local patch_id
    patch_id=$(pw_yaml_get "$project" series "$id" property ".*" \
			   value Patches "$num" | cut -d" " -f 1)
    if [ "$patch_id" = "" ]; then
	return 1
    fi
    echo "$patch_id"
    )
}

# Fetch patch entry data
# $2 -- patch ID
# $3 -- data field
pw_get_patch_data ()
{
    (
    set -euf -o pipefail
    local project="$1"
    local patch_id="$2"
    local field="$3"

    pw_yaml_get "$project" patch "$patch_id" property "$field" value
    )
}

# Fetch current state of $patch_id's check for $ci_bot configuration.
# Prints out "pending/warning/fail/success"; with no matching checks
# prints out "pending".
pw_patch_check_state ()
{
    (
    set -euf -o pipefail
    local patch_id="$1"
    local ci_owner_bot="$2"

    # Split $ci_owner_bot into [optional] $ci_owner and $ci_bot.
    local ci_owner ci_bot
    ci_owner="$(echo "$ci_owner_bot" | cut -s -d/ -f1)"
    ci_bot="$(echo "$ci_owner_bot" | cut -s -d/ -f2)"
    if [ "$ci_bot" = "" ]; then
	ci_owner="linaro-tcwg-bot"
	ci_bot="$ci_owner_bot"
    fi

    local json1 json2
    json1=$(mktemp)
    json2=$(mktemp)
    # shellcheck disable=SC2064
    trap "rm $json1 $json2" EXIT

    curl -s \
	 "https://patchwork.sourceware.org/api/1.2/patches/$patch_id/checks/" \
	 > "$json1"

    local i="-1" cur_date="0" cur_state="pending"
    local username context date
    while true; do
	i=$(($i + 1))

	jq -r ".[$i]" < "$json1" > "$json2"
	if [ "$(cat "$json2")" = "null" ]; then
	    break
	fi

	username=$(jq -r ".user.username" < "$json2")
	if [ "$username" != "$ci_owner" ]; then
	    continue
	fi

	context=$(jq -r ".context" < "$json2")
	if [ "$context" != "$ci_bot" ]; then
	    continue
	fi

	date=$(jq -r ".date" < "$json2")
	date=$(date -d "$date" +%s)
	if [ "$cur_date" -le "$date" ]; then
	    cur_date="$date"
	    cur_state=$(jq -r ".state" < "$json2")
	fi
    done

    echo "$cur_state"
    )
}

# Apply a patch series
# $1: project
# $2: method (either 'am' for plain git, or 'pw' for patchwork interface)
# $3: series dir (for 'am') or ID (for 'pw')
# $4: "patch paths" option (eg -p0 or -p1, passed to 'git am')
# $5: optional series_url when $2='am'
apply_series()
{
    (
    set -euf -o pipefail
    local project="$1"
    local method="$2"
    local series_id="$3"
    local patch_paths="$4"
    local series_url=""
    local res=0
    local subcommand=""
    local series_name=""

    case "$method" in
	am)
	    subcommand="am"
	    series_url="$5"
	    series_name="$series_url"
	    ;;
	pw)
	    subcommand="pw series apply"
	    series_name="$series_id"
	    ;;
	*)
	    echo "ERROR: method $method not supported by apply_series()"
	    return 4
	    ;;
    esac

    # Apply the whole series and then roll-back to the desired patch.
    if ! git -C "$project" $subcommand "$series_id" "$patch_paths"; then
	echo "WARNING: Series $series_name did not apply cleanly"
	# "git am" sometimes detects email text as a patch, and complains that it
	# has no actual code changes.  Workaround this by skipping empty patches.
	res=4
	patch_file="$project/.git/rebase-apply/patch"
	while [ "$res" = "4" ] \
		  && [ -f "$patch_file" ] && ! [ -s "$patch_file" ]; do
	    # The patch is empty, so skip it.
	    res=0
	    if ! git -C "$project" am --skip; then
		res=4
	    fi
	done
    fi
    return $res
    )
}

# Apply a patch series first with -p1, retry with -p0 if needed
# $1: project
# $2: prev_head
# $3: method (either 'am' for plain git, or 'pw' for patchwork interface)
# $4: series dir (for 'am') or ID (for 'pw')
# $5: optional series_url when $2='am'
apply_series_with_retry()
{
    (
    set -euf -o pipefail
    local project="$1"
    local prev_head="$2"
    local method="$3"
    local series_id="$4"
    local series_url=""
    local res=0

    if [ "$method" = "am" ]; then
	series_url="$5"
    fi

    # Try to apply patches with -p1 first
    apply_series "$project" $method "$series_id" "-p1" "$series_url" &
    res=0 && wait $! || res=$?

    # It can happen that the patch series was merged between the trigger and
    # this build.  Make sure that we have applied enough patches to test
    # something interesting.
    patches_applied=$(git -C "$project" rev-list --count HEAD "^$prev_head")

    # If we couldn't apply any patch, retry with -p0
    if [ "$res" != "0" ] && [ "$patches_applied" = "0" ]; then
	# Restore a clean state
	git -C "$project" am --abort || true
	git -C "$project" checkout --detach $prev_head
	apply_series "$project" $method "$series_id" "-p0" "$series_url" &
	res=0 && wait $! || res=$?
    fi

    return $res
    )
}