android: add tests of stringbench and libc-bench

and update the tests of bionic-benchmarks

Change-Id: I548b1620487802654c7f32dedd5a81221f71ad0f
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

7 files changed, 462 insertions, 42 deletions

diff --git a/android/bionic-benchmark.yaml b/android/bionic-benchmark.yaml
index ac8b135..e6f522f 100644
--- a/android/bionic-benchmark.yaml
+++ b/android/bionic-benchmark.yaml
@@ -12,8 +12,9 @@ metadata:
         - juno
 
 params:
-    LOOP_TIMES: 1
+    RECORD_RESULT_LOCAL: "FALSE"
+    LOOP_COUNT: "1"
 
 run:
     steps:
-        - ./android/scripts/bionic-benchmarks.sh $LOOP_TIMES
+        - ./android/scripts/bionic-benchmarks.sh --record-csv ${RECORD_RESULT_LOCAL} --loop-count ${LOOP_COUNT}
diff --git a/android/libc-bench.yaml b/android/libc-bench.yaml
new file mode 100644
index 0000000..fd64f49
--- /dev/null
+++ b/android/libc-bench.yaml
@@ -0,0 +1,23 @@
+metadata:
+    name: libc-bench
+    format: "Lava-Test-Shell Test Definition 1.0"
+    description: "Run libc-bench command to benchmark the
+                  performance of bionic.
+                  The source is here now:
+                  git://android.git.linaro.org/platform/external/libc-bench"
+    maintainer:
+        - yongqin.liu@linaro.org
+    os:
+        - android
+    scope:
+        - performance
+    devices:
+        - juno
+
+params:
+    RECORD_RESULT_LOCAL: "FALSE"
+    LOOP_COUNT: "1"
+
+run:
+    steps:
+        - ./android/scripts/libc-bench.sh --record-csv ${RECORD_RESULT_LOCAL} --loop-count ${LOOP_COUNT}
diff --git a/android/scripts/bionic-benchmarks.sh b/android/scripts/bionic-benchmarks.sh
index 9839179..b3d27fb 100755
--- a/android/scripts/bionic-benchmarks.sh
+++ b/android/scripts/bionic-benchmarks.sh
@@ -1,75 +1,54 @@
 #!/system/bin/sh
 
-loop_times=$1
-if [ -z "${loop_times}" ]; then
-    loop_times=12
-fi
-raw_data="/data/local/tmp/bionic-benchmarks-raw.csv"
-rm -fr ${raw_data}
-
-add_lava_result_entry(){
-    local cmd="lava-test-case"
-    if [ -n "$(which $cmd)" ];then
-        $cmd "$@"
-    else
-        echo "$cmd $@"
-    fi
-}
+local_file_path="$0"
+local_file_parent=$(cd $(dirname ${local_file_path}); pwd)
+. ${local_file_parent}/common.sh
 
 test_bionic_benchmark(){
-    local arch=$1 && shift
+    local arch=$1
     if [ "X$arch" != "X32" ] && [ "X$arch" != "X64" ]; then
         echo "The specified $arch is not specified!"
         return
     fi
-    local excluded_test=$1 && shift
+    local excluded_test=$2
     local cmd="bionic-benchmarks${arch}"
     if [ -n "$(which ${cmd})" ]; then
         for line in $(${cmd} --help 2>&1|grep BM_); do
             if [ -n "$excluded_test" ]; then
                 if echo "${excluded_test}"|grep -q ${line}; then
-                    echo "${arch}_${line} skip"
-                    add_lava_result_entry "${arch}_${line}" "--result" "skip"
+                    output_test_result "${arch}_${line}" "skip"
                     continue
                 fi
             fi
-            echo "--------------start $line--------"
             local hasResult=false
-            for  res_line in $(${cmd} ${line}|grep "BM_"|tr -s ' '|tr ' ' ','); do
-                echo "${arch}_$line pass"
-                add_lava_result_entry "${arch}_${line}" "--result" "pass"
+            for  res_line in $(${cmd} "^${line}$"|grep "BM_"|tr -s ' '|tr ' ' ','); do
+                output_test_result "${arch}_${line}" "pass"
                 local key=$(echo $res_line|cut -d, -f1|tr '/' '_')
                 local iterations=$(echo $res_line|cut -d, -f2)
                 local ns_time=$(echo $res_line|cut -d, -f3)
                 local throughput=$(echo $res_line|cut -d, -f4)
                 local throughput_units=$(echo $res_line|cut -d, -f5)
-                echo "${arch}_${key}_time pass $ns_time ns/op"
-                add_lava_result_entry "${arch}_${key}_time" "--result" "pass" '--measurement' "${ns_time}"  '--units' "ns/op"
-                echo "${arch}_${key}_time,${ns_time}">>${raw_data}
+                output_test_result "${arch}_${key}_time" "pass" "${ns_time}"  "ns/op"
                 if [ -n "${throughput_units}" ];then
-                    echo "${arch}_${key}_throught pass ${throughput} ${throughput_units}"
-                    add_lava_result_entry "${arch}_${key}_throught" "--result" "pass" '--measurement' "${throughput}"  '--units' "${throughput_units}"
-                    echo "${arch}_${key}_throught,${throughput}">>${raw_data}
+                    output_test_result "${arch}_${key}_throught" "pass" "${throughput}" "${throughput_units}"
                 fi
                 hasResult=true
             done
             if ! $hasResult; then
-                echo "${arch}_$line fail"
-                add_lava_result_entry "${arch}_${line}" "--result" "fail"
+                output_test_result "${arch}_${line}" "fail"
             fi
-            echo "--------------finished $line--------"
         done
     fi
 }
 
-loop_index=0
-while [ ${loop_index} -lt ${loop_times} ]; do
+test_func(){
     test_bionic_benchmark "64" "BM_property_serial"
     test_bionic_benchmark "32" "BM_property_serial BM_property_read"
-    loop_index=$((loop_index + 1))
-done
+}
+
+main(){
+    var_test_func="test_func"
+    run_test "$@"
+}
 
-attach_cmd="lava-test-run-attach"
-if [ -n "$(which ${attach_cmd})" ] && [ -f "${raw_data}" ]; then
-    ${attach_cmd} ${raw_data} text/plain
-fi
+main "$@"
diff --git a/android/scripts/common.sh b/android/scripts/common.sh
new file mode 100644
index 0000000..bc4a526
--- /dev/null
+++ b/android/scripts/common.sh
@@ -0,0 +1,292 @@
+#!/system/bin/sh
+
+G_LOOP_COUNT=12
+G_RECORD_LOCAL_CSV=TRUE
+G_VERBOSE_OUTPUT=FALSE
+F_RAW_DATA_CSV="/data/local/tmp/lava_test_shell_raw_data.csv"
+F_STATISTIC_DATA_CSV="/data/local/tmp/lava_test_shell_statistic_data.csv"
+
+## Description:
+##    output the max value of the passed 2 parameters
+## Usage:
+##    f_max "${val1}" "${val2}"
+## Example:
+##    max=$(f_max "1.5" "2.0")
+f_max(){
+    local val1=$1
+    local val2=$2
+    [ -z "$val1" ] && echo $val2
+    [ -z "$val2" ] && echo $val1
+
+    echo "$val1,$val2"|awk -F, '{if($1<$2) print $2; else print $1}'
+}
+
+## Description:
+##    output the min value of the passed 2 parameters
+## Usage:
+##    f_min "${val1}" "${val2}"
+## Example:
+##    min=$(f_min "1.5" "2.0")
+f_min(){
+    local val1=$1
+    local val2=$2
+    [ -z "$val1" ] && echo $val1
+    [ -z "$val2" ] && echo $val2
+
+    echo "$val1,$val2"|awk -F, '{if($1>$2) print $2; else print $1}'
+}
+
+## Description:
+##   calculate the average value for specified csv file.
+##   The first field of that csv file should be the key/name of that line,
+##   Lines have the same key should be together.
+## Usage:
+##    statistic "${csv_file_path}" "${file_number}"
+## Example:
+##    statistic "$f_res_starttime" 2
+## Note:
+##    if less than 4 samples for that key/item there, average will be calculated as total/count
+##    if 4 or more samples for that key/item there, average will be calculated with max and min excluded
+statistic(){
+    local f_data=$1
+    if ! [ -f "$f_data" ]; then
+        return
+    fi
+    local field_no=$2
+    if [ -z "$field_no" ]; then
+        field_no=2
+    fi
+    local total=0
+    local max=0
+    local min=0
+    local old_key=""
+    local new_key=""
+    local count=0
+    local units=""
+    sort ${f_data} >${f_data}.sort
+    for line in $(cat "${f_data}.sort" |tr ' ' '~'); do
+        if ! echo "$line"|grep -q ,; then
+            continue
+        fi
+        new_key=$(echo $line|cut -d, -f1)
+        local measurement_units=$(echo $line|cut -d, -f${field_no})
+        if echo ${measurement_units}|grep -q '~'; then
+            value=$(echo ${measurement_units}|cut -d~ -f1)
+        else
+            value=${measurement_units}
+        fi
+
+        if [ "X${new_key}" = "X${old_key}" ]; then
+            total=$(echo "${total},${value}"|awk -F, '{printf "%.2f",$1+$2;}')
+            count=$(echo "${count},1"|awk -F, '{printf $1+$2;}')
+            max=$(f_max "$max" "$value")
+            min=$(f_min "$min" "$value")
+        else
+            if [ "X${old_key}" != "X" ]; then
+                if [ $count -ge 4 ]; then
+                    average=$(echo "${total},${max},${min},$count"|awk -F, '{printf "%.2f",($1-$2-$3)/($4-2);}')
+                else
+                    average=$(echo "${total},$count"|awk -F, '{printf "%.2f",$1/$2;}')
+                fi
+                if [ -z "${units}" ]; then
+                    echo "${old_key}=${average}"
+                else
+                    echo "${old_key}=${average},${units}"
+                fi
+            fi
+            total="${value}"
+            max="${value}"
+            min="${value}"
+            old_key="${new_key}"
+            count=1
+            if echo ${measurement_units}|grep -q '~'; then
+                units=$(echo ${measurement_units}|cut -d~ -f2)
+            else
+                units=""
+            fi
+        fi
+    done
+    if [ "X${new_key}" != "X" ]; then
+        if [ $count -ge 4 ]; then
+            average=$(echo "${total},${max},${min},$count"|awk -F, '{printf "%.2f",($1-$2-$3)/($4-2);}')
+        else
+            average=$(echo "${total},$count"|awk -F, '{printf "%.2f",$1/$2;}')
+        fi
+        if [ -z "${units}" ]; then
+            echo "${new_key}=${average}"
+        else
+            echo "${new_key}=${average},${units}"
+        fi
+    fi
+    rm "${f_data}.sort"
+}
+
+## Description:
+##   output the test result to console and add for lava-test-shell,
+##   also write into one csv file for comparing manually
+## Usage:
+##    output_test_result $test_name $result [ $measurement [ $units ] ]
+## Note:
+##    G_RECORD_LOCAL_CSV: when this environment variant is set to "TRUE",
+##         the result will be recorded in a csv file in the following path:
+##              rawdata/final_result.csv
+##    G_VERBOSE_OUTPUT: when this environment variant is set to "TRUE", and only it is TRUE,
+##         the verbose informain about the result will be outputed
+output_test_result(){
+    local test_name=$1
+    local result=$2
+    local measurement=$3
+    local units=$4
+
+    if [ -z "${test_name}" ] || [ -z "$result" ]; then
+        return
+    fi
+    local output=""
+    local lava_paras=""
+    local output_csv=""
+    if [ -z "$units" ]; then
+        units="points"
+    fi
+    if [ -z "${measurement}" ]; then
+        output="${test_name}=${result}"
+        lava_paras="${test_name} --result ${result}"
+    else
+        output="${test_name}=${measurement} ${units}"
+        lava_paras="${test_name} --result ${result} --measurement ${measurement} --units ${units}"
+        output_csv="${test_name},${measurement} ${units}"
+    fi
+
+    if [ "X${G_VERBOSE_OUTPUT}" = "XTRUE" ];then
+        echo "${output}"
+    fi
+
+    local cmd="lava-test-case"
+    if [ -n "$(which $cmd)" ];then
+        $cmd ${lava_paras}
+    elif [ "X${G_VERBOSE_OUTPUT}" = "XTRUE" ];then
+        echo "$cmd ${lava_paras}"
+    fi
+    if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then
+        if [ -n "${output_csv}" ]; then
+            echo "${output_csv}">>${F_RAW_DATA_CSV}
+        fi
+    fi
+}
+
+func_print_usage_common(){
+    echo "$(basename $0) [--record-csv TRUE|others] [--loop-count LOOP_COUNT]"
+    echo "     --record-csv: specify if record the result in csv format in file ${F_RAW_DATA_CSV}"
+    echo "                   Only record the file when TRUE is specified."
+    echo "     --loop-count: specify the number that how many times should be run for each application to get the average result, default is 12"
+    echo "     --verbose-output: output the result and lava-test-case command for each test case each time it is run"
+    echo "$(basename $0) [--help|-h]"
+    echo "     print out this usage."
+}
+
+func_parse_parameters_common(){
+    local para_loop_count=""
+    while [ -n "$1" ]; do
+        case "X$1" in
+            X--record-csv)
+                para_record_csv=$2
+                if [ -z "${para_record_csv}" ]; then
+                    echo "Please specify the value for --record-csv option"
+                    exit 1
+                fi
+                shift 2
+                ;;
+            X--verbose-output)
+                para_verbose_output=$2
+                if [ -z "${para_verbose_output}" ]; then
+                    echo "Please specify the value for --verbose-output option"
+                    exit 1
+                fi
+                shift 2
+                ;;
+            X--loop-count)
+                para_loop_count=$2
+                if [ -z "${para_loop_count}" ]; then
+                    echo "Please specify the value for --loop-count option"
+                    exit 1
+                fi
+                shift 2
+                ;;
+            X-h|X--help)
+                func_print_usage_common
+                exit 1
+                ;;
+            X-*)
+                echo "Unknown option: $1"
+                func_print_usage_common
+                exit 1
+                ;;
+            X*)
+                func_print_usage_common
+                exit 1
+                ;;
+        esac
+    done
+
+    if [ -n "${para_loop_count}" ]; then
+        local tmp_str=$(echo ${para_loop_count}|tr -d '[:digit:]')
+        if [ -z "${tmp_str}" ]; then
+            G_LOOP_COUNT=${para_loop_count}
+        else
+            echo "The specified LOOP_COUNT(${para_loop_count}) is not valid"
+            exit 1
+        fi
+    fi
+
+    if [ -n "${para_record_csv}" ] && [ "X${para_record_csv}" = "XTRUE" ];then
+        G_RECORD_LOCAL_CSV=TRUE
+    elif [ -n "${para_record_csv}" ];then
+        G_RECORD_LOCAL_CSV=FALSE
+    fi
+
+    if [ -n "${para_verbose_output}" ] && [ "X${para_verbose_output}" = "XTRUE" ];then
+        G_VERBOSE_OUTPUT=TRUE
+    elif [ -n "${para_record_csv}" ];then
+        G_VERBOSE_OUTPUT=FALSE
+    fi
+}
+
+## Description:
+##   common framework to run test multiple times
+##   also write result into one csv file for comparing manually
+## Usage:
+##    output_test_result $test_name $result [ $measurement [ $units ] ]
+## Note:
+##    RECORD_RESULT_LOCAL: when this environment variant is set to "TRUE",
+##         the result will be recorded in a csv file in the following path:
+##              rawdata/final_result.csv
+run_test(){
+    func_parse_parameters_common "$@"
+    if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then
+        [ -f "${F_RAW_DATA_CSV}" ] && rm "${F_RAW_DATA_CSV}"
+        [ -f "${F_STATISTIC_DATA_CSV}" ] && rm "${F_STATISTIC_DATA_CSV}"
+        mkdir -p $(dirname ${F_RAW_DATA_CSV})
+    fi
+
+    loop_index=0
+    while [ ${loop_index} -lt ${G_LOOP_COUNT} ]; do
+        if [ -n "${var_test_func}" ]; then
+            ${var_test_func}
+        fi
+        loop_index=$((loop_index + 1))
+    done
+
+    if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then
+        statistic ${F_RAW_DATA_CSV} 2 |tee ${F_STATISTIC_DATA_CSV}
+        sed -i 's/=/,/' "${F_STATISTIC_DATA_CSV}"
+
+        attach_cmd="lava-test-run-attach"
+        if [ -n "$(which ${attach_cmd})" ]; then
+            if [ -f "${F_RAW_DATA_CSV}" ]; then
+                ${attach_cmd} ${F_RAW_DATA_CSV} text/plain
+            fi
+            if [ -f "${F_STATISTIC_DATA_CSV}" ]; then
+                ${attach_cmd} ${F_STATISTIC_DATA_CSV} text/plain
+            fi
+        fi
+    fi
+}
diff --git a/android/scripts/libc-bench.sh b/android/scripts/libc-bench.sh
new file mode 100755
index 0000000..961a743
--- /dev/null
+++ b/android/scripts/libc-bench.sh
@@ -0,0 +1,60 @@
+#!/system/bin/sh
+
+local_file_path="$0"
+local_file_parent=$(cd $(dirname ${local_file_path}); pwd)
+. ${local_file_parent}/common.sh
+
+test_stringbench(){
+    local cmd=$1 && shift
+    local prefix=$1 && shift
+    if [ -z "${cmd}" ]; then
+        return
+    fi
+    if [ ! -x "${cmd}" ]; then
+        return
+    fi
+    if [ -n "$prefix" ]; then
+        prefix="${prefix}_"
+    fi
+    
+    local oldkey=""
+    local newkey=""
+    local value_line=""=
+    for res_line in $(${cmd} |tr -d ' '); do
+        if echo "${res_line}"|grep -q '^b_'; then
+            if [ -n "${oldkey}" ] && [ -n "${value_line}" ]; then
+               local time_value=$(echo ${value_line}|cut -d, -f1|cut -d: -f2)
+               local virt_value=$(echo ${value_line}|cut -d, -f2|cut -d: -f2)
+               local res_value=$(echo ${value_line}|cut -d, -f3|cut -d: -f2)
+               local dirty_value=$(echo ${value_line}|cut -d, -f4|cut -d: -f2)
+               output_test_result "${prefix}${oldkey}_time" "pass" "${time_value}" "seconds"
+               output_test_result "${prefix}${oldkey}_virt" "pass" "${virt_value}" "kB"
+               output_test_result "${prefix}${oldkey}_res" "pass" "${res_value}" "kB"
+               output_test_result "${prefix}${oldkey}_dirty" "pass" "${dirty_value}" "kB"
+            fi
+            newkey=$(echo $res_line|tr -c '[:alnum:]' '_'|tr -s '_' |sed 's/_$//')
+            value_line=""
+            oldkey=${newkey}
+            continue
+        fi
+
+        if echo "${res_line}"|grep -q '^time:'; then
+            value_line="${res_line}"
+            continue
+        fi
+    done
+}
+
+test_func(){
+    stringbench="/system/xbin/libcbench"
+    stringbench64="/system/xbin/libcbench64"
+    test_stringbench "${stringbench}" "32Bit"
+    test_stringbench "${stringbench64}" "64Bit"
+}
+
+main(){
+    var_test_func="test_func"
+    run_test "$@"
+}
+
+main "$@"
diff --git a/android/scripts/stringbench.sh b/android/scripts/stringbench.sh
new file mode 100755
index 0000000..34ee40f
--- /dev/null
+++ b/android/scripts/stringbench.sh
@@ -0,0 +1,41 @@
+#!/system/bin/sh
+
+local_file_path="$0"
+local_file_parent=$(cd $(dirname ${local_file_path}); pwd)
+. ${local_file_parent}/common.sh
+
+test_stringbench(){
+    local cmd=$1 && shift
+    local prefix=$1 && shift
+    if [ -z "${cmd}" ]; then
+        return
+    fi
+    if [ ! -x "${cmd}" ]; then
+        return
+    fi
+    if [ -n "$prefix" ]; then
+        prefix="${prefix}_"
+    fi
+
+    for res_line in $(${cmd} |tr ' ' '_'); do
+        local test_case_id=$(echo $res_line|cut -d: -f1|tr -c '[:alnum:]:.' '_'|tr -s '_' |sed 's/_$//')
+        local measurement_units=$(echo $res_line|cut -d: -f2)
+        local measurement=$(echo ${measurement_units}|cut -d_ -f2)
+        local units=$(echo ${measurement_units}|cut -d_ -f2)
+        output_test_result "${prefix}${test_case_id}" "pass" "${measurement}" "seconds"
+    done
+}
+
+test_func(){
+    stringbench="/system/xbin/stringbench"
+    stringbench64="/system/xbin/stringbench64"
+    test_stringbench "${stringbench}" "32Bit"
+    test_stringbench "${stringbench64}" "64Bit"
+}
+
+main(){
+    var_test_func="test_func"
+    run_test "$@"
+}
+
+main "$@"
diff --git a/android/stringbench.yaml b/android/stringbench.yaml
new file mode 100644
index 0000000..9c4ee1a
--- /dev/null
+++ b/android/stringbench.yaml
@@ -0,0 +1,24 @@
+metadata:
+    name: stringbench
+    format: "Lava-Test-Shell Test Definition 1.0"
+    description: "Run stringbench command to benchmark the
+                  performance of string relevant feature of bionic.
+                  In the feature this test should be upstream to bionic-benchmarks.
+                  The source is here now:
+                  git://android.git.linaro.org/platform/external/stringbench"
+    maintainer:
+        - yongqin.liu@linaro.org
+    os:
+        - android
+    scope:
+        - performance
+    devices:
+        - juno
+
+params:
+    RECORD_RESULT_LOCAL: "FALSE"
+    LOOP_COUNT: "1"
+
+run:
+    steps:
+        - ./android/scripts/stringbench.sh --record-csv ${RECORD_RESULT_LOCAL} --loop-count ${LOOP_COUNT}