path: root/lnt-utils.sh

#!/usr/bin/env bash

# ==============================================================================
# is_first_lnt_entry : is used in both report_header() and report_test_entry()
declare is_first_lnt_entry

report_header()
{
    local machine_name="$1"
    local date_time="$2"
    shift 2
    local run_infos=("$@")
    declare -g is_first_lnt_entry

    cat <<-EOF
	{
	    "Machine": {
	        "Info": {},
	        "Name": "$machine_name"
	    },
	    "Run": {
	        "Info": {
EOF

    for i in "${run_infos[@]}"; do
        tag=$(echo $i | cut -d: -f1)
        val=$(echo $i | cut -d: -f2-)
        echo "            \"$tag\": \"$val\","
    done

    cat <<-EOF
	            "__report_version__": "1"
	        },
	        "Start Time": "$date_time"
	    },
	    "Tests": [
	EOF
    is_first_lnt_entry=true
}

report_footer()
{
    cat <<-EOF
	    ]
	}
	EOF
}

report_test_entry()
{
    local name="$1"
    local value="$2"
    declare -g is_first_lnt_entry

    $is_first_lnt_entry || echo "	   ,"

    cat <<-EOF
	        {
	            "Data": [
	                $value
	            ],
	            "Info": {},
	            "Name": "$name"
	        }
	EOF
    is_first_lnt_entry=false
}

get_current_component_url()
{
    c=$1

    local url
    url=$(get_current_git ${c}_url)
    rev=$(get_current_git ${c}_rev)

    if [[ "$url" =~ git://sourceware.org/git/ ]]; then
	    url="${url#git://sourceware.org/git/}"
	    url="https://sourceware.org/git/?p=$url"
	    echo "$url;a=commit;h=$rev"
    elif [[ "$url" =~ https://github.com/ ]] \
	     || [[ "$url" =~ https://gitlab.com/ ]]; then
	    echo "${url%.git}/commit/$rev"
    elif [[ "$url" =~ https://git.linaro.org/ ]]; then
	    echo "${url}/commit/?id=$rev"
    else
	    echo "$url ($rev)"
    fi
}

get_component_changes()
{
    local base_rev cur_rev
    # In "init" mode, no component_changes annotation
    if [ x"$(get_current_manifest "{rr[update_baseline]}")" != x"init" ]; then
        base_rev=$(get_baseline_git ${c}_rev)
        cur_rev=$(get_current_git ${c}_rev)
        if [ "$base_rev" = "$cur_rev" ]; then
            echo "(unchanged)"
        else
            echo "(+ $(git -C $c rev-list --count $base_rev..$cur_rev || echo "??") commits)"
        fi
    fi
}

get_describe_pad()
{
    local nb_changed_components=0 describe_pad=""

    for c in $(get_current_manifest "{rr[components]}"); do
        local base_rev="" cur_rev
        # in "init" mode, the base_rev will be empty, and considered as
        # different than cur_rev. the results of the function will be as
        # if every component changed.
        [ "$(get_current_manifest "{rr[update_baseline]}")" = "init" ] || \
           base_rev=$(get_baseline_git ${c}_rev || true)
        cur_rev=$(get_current_git ${c}_rev)
        if [ "$base_rev" != "$cur_rev" ]; then
            nb_changed_components=$((nb_changed_components+1))
            if [ $nb_changed_components = 1 ]; then
                describe=$(describe_sha1 "$c" "$cur_rev" false)
                describe_pad=$(
                    echo "$describe" \
                     | sed 's/\(.*\)-\(.*\)-\(.*\)$/\1 \2 \3/' \
                     | awk '{ $2 = sprintf("%05d", $2); print $1"-"$2"-"$3}'
                    )
            else
                describe_pad="$nb_changed_components-changed-components"
            fi
        fi
    done

    if [ $nb_changed_components = 0 ]; then
       echo "no-change"
    else
       echo "$describe_pad"
    fi
}

# ==============================================================================

# create a lnt json report for the given sumfiles
generate_lnt_gnu_check_report()
{
    local build_url=${1:?}
    local ci_project=${2:?} # tcwg_gnu_native_check_gcc
    local ci_config=${3:?} # master-aarch64
    local results_date=${4:?}
    local jira_key=${5:?}
    local sumfiles_dir=${6:?}
    local output_file=${7:?}

    local -a sumfiles
    readarray -t -d '' sumfiles < <(find "$sumfiles_dir" -name '*.sum' -print0)
    # do not generate any lnt report if there is no sumfile
    [ ${#sumfiles[@]} = 0 ] && return

    local lnt_testsuite=${ci_project}
    local machine_name=${ci_config}

    local build_number
    build_number=$(basename "$build_url")

    (
        # Generate a header with useful infos for the report
        additional_run_infos=("tag:$lnt_testsuite" "test_url:$build_url")
        additional_run_infos+=("run_order:$(printf "%04d" "$build_number") ($(get_describe_pad))")
        [ "$jira_key" != "-" ] && additional_run_infos+=("regression:https://linaro.atlassian.net/browse/$jira_key")

        # disable traces. This function is too verbose
        set +x

        for c in $(get_current_manifest "{rr[components]}"); do
            additional_run_infos+=("git_${c}:$(get_current_component_url $c) $(get_component_changes)")
        done

        report_header "$machine_name" "$results_date" "${additional_run_infos[@]}"

        # an entry and a total value must be generated for each of these fields
        # (even if the field is missing in the sum file)
        local all_fields=(
            FAIL UNRESOLVED UNTESTED ERROR XPASS OTHER
            KFAIL XFAIL PASS UNSUPPORTED
            good bad
        )

        for field in "${all_fields[@]}"; do
            eval "total_$field=0"
        done

        for sumfile in "${sumfiles[@]}"; do
            suite_name=$(basename "$sumfile" .sum) # gcc, g++, libatomic, ...
            fields=("${all_fields[@]}")

            for field in "${fields[@]}"; do
                eval "$field=0"
            done

            while read -r kind; do
                [[ ! ${fields[*]} =~ $kind ]] && kind="OTHER"
                eval "(( $kind+=1 ))"
                eval "(( total_$kind+=1 ))"
                case "$kind" in
                    KFAIL|XFAIL|PASS|UNSUPPORTED)
                        eval "(( good+=1 ))"
                        eval "(( total_good+=1 ))"
                    ;;
                    FAIL|UNRESOLVED|UNTESTED|ERROR|XPASS|OTHER)
                        eval "(( bad+=1 ))"
                        eval "(( total_bad+=1 ))"
                    ;;
                    *)
                    ;;
                esac
            done < <(grep -E '^[A-Z]+:' "$sumfile" | sed 's/:.*//')

            for field in "${fields[@]}"; do
                # tcwg_test_gcc_check.gcc.FAIL.nb_FAIL
                field_name="$suite_name.$field.nb_$field"
                report_test_entry "$lnt_testsuite.$field_name" "$(eval "echo \$$field")"
            done
        done

        for field in "${all_fields[@]}"; do
            # tcwg_test_gcc_check.TOTAL.FAIL.nb_FAIL
            field_name="TOTAL.$field.nb_$field"
            report_test_entry "$lnt_testsuite.$field_name" "$(eval "echo \$total_$field")"
        done

        report_footer
    ) > "$output_file"
}


# ==============================================================================

# create a lnt json report for the given benchmark results
generate_lnt_bmk_report()
{
    local build_url=${1:?}
    local ci_project=${2:?} # tcwg_bmk-code_speed-cpu2017rate
    local ci_config=${3:?} # llvm-aarch64-master-O3
    local results_date=${4:?}
    local jira_key=${5:?}
    local size_csv=${6:?}
    local perf_csv=${7:?}
    local status_csv=${8:?}
    local variability_csv=${9:?}
    local output_file=${10:?}

    local lnt_testsuite=${ci_project}
    local machine_name=${ci_config}

    local build_number
    build_number=$(basename "$build_url")

    [ -f "$size_csv" ] || return
    [ -f "$perf_csv" ] || return

    (
    # Generate a header with useful infos for the report
    additional_run_infos=("tag:$lnt_testsuite" "test_url:$build_url")
    additional_run_infos+=("run_order:$(printf "%04d" "$build_number") ($(get_describe_pad))")
    [ "$jira_key" != "-" ] && additional_run_infos+=("regression:$jira_key")

    for c in $(get_current_manifest "{rr[components]}"); do
        additional_run_infos+=("git_${c}:$(get_current_component_url $c) $(get_component_changes)")
    done

    report_header "$machine_name" "$results_date" "${additional_run_infos[@]}"

    # disable traces locally because too verbose
    set +x

	## Inspect size.csv
	# 531.deepsjeng_r,deepsjeng_r_base.default,90926
	while IFS="," read -r benchmark symbol size; do
	    [[ "$symbol" == *base.default ]] || continue

	    report_test_entry "$lnt_testsuite.$benchmark.code_size" "$size"

	done < <(tail -n +2 "$size_csv" | tr -d '\r')

	## Inspect perf.csv
	# 531.deepsjeng_r,deepsjeng_r_base.default,8692,na
	local -A execution_time
	while IFS="," read -r benchmark symbol sample _dso; do
	    [[ "$symbol" == *base.default ]] || continue

	    execution_time[$symbol]="$sample"
	    report_test_entry "$lnt_testsuite.$benchmark.execution" "$sample"

	done < <(tail -n +2 "$perf_csv" | tr -d '\r')

	## Inspect bmk-specific-variability.csv
	# 531.deepsjeng_r,deepsjeng_r_base.default,0.96,0.0,,
	if [ -f "$variability_csv" ]; then
	    # shellcheck disable=SC2034
	    while IFS="," read -r benchmark symbol sample_var sve_var vect_var; do
	        [[ "$symbol" == *base.default ]] || continue

	        if [ "${execution_time[$symbol]+abc}" ]; then
	            local var

                var=$(echo "$sample_var * ${execution_time[$symbol]} / 100" | bc)
	            report_test_entry "$lnt_testsuite.$benchmark.execution_variation" "$var"
	        fi
	    done < <(tail -n +2 "$variability_csv" | tr -d '\r')
	fi

	score_success_compile=0
	score_success_execution=0
	score_total_test=0

	## Inspect status.csv
	# 531.deepsjeng_r,deepsjeng_r_base.default,failed-to-build
	while IFS="," read -r benchmark symbol status; do
	    [[ "$symbol" == *base.default ]] || continue

	    local compile_success execution_success
        case "$status" in
	       success)
	            compile_success=1
	            execution_success=1
	            ;;
	        failed-to-run)
	            compile_success=1
	            execution_success=0
	            ;;
	        failed-to-build)
	            compile_success=0
	            execution_success=0
	              ;;
	        *)
	            assert_with_msg "unknown status in $status_csv"
	            ;;
	    esac

        report_test_entry "$lnt_testsuite.$benchmark.compile_status" "$((compile_success==0))"
	    report_test_entry "$lnt_testsuite.$benchmark.execution_status" "$((execution_success==0))"

        score_success_compile=$((score_success_compile+compile_success))
        score_success_execution=$((score_success_execution+execution_success))

	    score_total_test=$((score_total_test+1))

	done < <(tail -n +2 "$status_csv" | tr -d '\r')

    report_test_entry "$lnt_testsuite.nb_compile_successful.score" "$score_success_compile"
    report_test_entry "$lnt_testsuite.nb_execution_successful.score" "$score_success_execution"

    report_test_entry "$lnt_testsuite.nb_total_tests.score" "$score_total_test"

	report_footer
    ) > "$output_file"
}


# ==============================================================================