diff options
author | Yongqin Liu <yongqin.liu@linaro.org> | 2015-03-19 13:11:30 +0800 |
---|---|---|
committer | Yongqin Liu <yongqin.liu@linaro.org> | 2015-03-20 17:12:34 +0800 |
commit | bb1d6b1cb8cdcadd0a76d4f048d2831ef1170f9b (patch) | |
tree | 077102acc860351c445e64d3b3bdc51f55e5673b | |
parent | 11b3ba6ed690e07e3a3b9da5e9cddde58e48719d (diff) |
add flag to indicate if the we record the result in the local csv file
test locally with several apks,
and update javawhetstone and scimark for 5.1
Change-Id: Idd58bc3f2ac9f142ae662f39844c5c1cb0b7c8a6
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>
-rw-r--r-- | application-benchmark-host.yaml | 4 | ||||
-rwxr-xr-x | common/common.sh | 2 | ||||
-rw-r--r-- | common/common2.sh | 166 | ||||
-rwxr-xr-x | common/output-test-result.sh | 2 | ||||
-rw-r--r-- | common/statistic_average.sh | 51 | ||||
-rw-r--r-- | java-benchmarks-host.yaml | 3 | ||||
-rwxr-xr-x | java-benchmarks.sh | 16 | ||||
-rw-r--r-- | javawhetstone/vc.py | 5 | ||||
-rw-r--r-- | lava-android-benchmark-host.yaml | 8 | ||||
-rw-r--r-- | scimark/vc.py | 4 | ||||
-rwxr-xr-x | smartbench2012/extract_results.sh | 1 |
11 files changed, 170 insertions, 92 deletions
diff --git a/application-benchmark-host.yaml b/application-benchmark-host.yaml index 6c3f5b0..00152d3 100644 --- a/application-benchmark-host.yaml +++ b/application-benchmark-host.yaml @@ -23,7 +23,6 @@ metadata: - kvm params: - BASE_URL: "http://testdata.validation.linaro.org/apks/JavaBenchmark/pure-java-applications/" APP_CONFIG_LIST: "" LOOP_COUNT: 12 COLLECT_STREAMLINE: "false" @@ -37,8 +36,7 @@ run: - IPADDR=`awk -F '=' '{print $2}' /tmp/lava_multi_node_cache.txt` - adb connect $IPADDR - adb wait-for-device - - ./application-benchmark.sh --base-url "$BASE_URL" --loop-count "$LOOP_COUNT" --streamline "$COLLECT_STREAMLINE" $APP_CONFIG_LIST - - lava-test-run-attach rawdata.zip application/x-gzip + - ./application-benchmark.sh --loop-count "$LOOP_COUNT" --streamline "$COLLECT_STREAMLINE" $APP_CONFIG_LIST - lava-sync application-benchmark-stop-target parse: diff --git a/common/common.sh b/common/common.sh index 4fd5ed8..793f027 100755 --- a/common/common.sh +++ b/common/common.sh @@ -24,7 +24,7 @@ func_install_start_RotationAPK(){ if [ -f "${apk_path}" ]; then echo "The file(${apk_path}) already exists." else - get_file_with_base_url "${apk_name}" "${base_url}" "${D_APKS}" + get_file_with_base_url "${apk_name}" "${BASE_URL}" "${D_APKS}" fi adb shell pm list packages | grep rotation.off if [ $? -ne 0 ]; then diff --git a/common/common2.sh b/common/common2.sh index a11de4f..2d3ba65 100644 --- a/common/common2.sh +++ b/common/common2.sh @@ -2,6 +2,8 @@ local_common2_file_path="${BASH_SOURCE[0]}" local_common2_parent_dir=$(cd $(dirname ${local_common2_file_path}); pwd) +source "${local_common2_parent_dir}/statistic_average.sh" + D_ROOT=$(cd $(dirname ${local_common2_parent_dir}); pwd) D_RAWDATA="${D_ROOT}/rawdata" @@ -10,6 +12,8 @@ F_RAWDAT_ZIP="${D_ROOT}/rawdata.zip" F_LOGCAT="${D_RAWDATA}/logcat.log" F_LOGCAT_EVENTS="${D_RAWDATA}/logcat-events.log" +F_RAW_DATA_CSV="${D_RAWDATA}/final_raw_data_result.csv" +F_STATISTIC_DATA_CSV="${D_RAWDATA}/final_statistic_result.csv" D_STREAMLINE="${D_RAWDATA}/streamline" D_SCREENSHOT="${D_RAWDATA}/screenshots" @@ -17,6 +21,8 @@ COLLECT_STREAMLINE=false SERIAL="" G_APPS="" G_LOOP_COUNT=1 +[ -z "${G_RECORD_LOCAL_CSV}" ] && G_RECORD_LOCAL_CSV=TRUE +[ -z "${G_VERBOSE_OUTPUT}" ] && G_VERBOSE_OUTPUT=FALSE BASE_URL="" ## Description: @@ -378,9 +384,11 @@ func_prepare_environment(){ export ANDROID_SERIAL=${serial} fi fi + export G_RECORD_LOCAL_CSV G_VERBOSE_OUTPUT rm -fr ${D_RAWDATA} mkdir -p "${D_STREAMLINE}" "${D_SCREENSHOT}" "${D_APKS}" + mkdir -p $(dirname ${F_RAW_DATA_CSV}) func_get_all_apks "$G_APPS"|| exit 1 @@ -394,6 +402,9 @@ func_print_usage_common(){ echo " --serial: specify serial number for the device" echo " --base-url: specify the based url where the apks will be gotten from" echo " --loop-count: specify the number that how many times should be run for each application to get the average result, default is 12" + echo " --record-csv: specify if record the result in csv format file." + echo " Only record the file when TRUE is specified." + echo " --verbose-output: output the result and lava-test-case command for each test case each time it is run" echo " --streamline: specify if we need to collect the streamline data, true amd false can be specified, default is fasle" echo " APP_CONFIG_LIST: specify the configurations for each application as following format:" echo " APK_NAME,PACKAGE_NAME/APP_ACTIVITY,APP_NICKNAME" @@ -407,22 +418,56 @@ func_print_usage_common(){ func_parse_parameters_common(){ local para_loop_count="" local para_apps="" + local para_record_csv="" + local para_verbose_output="" while [ -n "$1" ]; do case "X$1" in X--base-url) BASE_URL=$2 + if [ -z "${BASE_URL}" ]; then + echo "Please specify the value for --base-url option" + exit 1 + fi + shift 2 + ;; + X--record-csv) + para_record_csv=$2 + if [ -z "${para_record_csv}" ]; then + echo "Please specify the value for --record-csv option" + exit 1 + fi + shift 2 + ;; + X--verbose-output) + para_verbose_output=$2 + if [ -z "${para_verbose_output}" ]; then + echo "Please specify the value for --verbose-output option" + exit 1 + fi shift 2 ;; X--streamline) COLLECT_STREAMLINE=$2 + if [ -z "${COLLECT_STREAMLINE}" ]; then + echo "Please specify the value for --streamline option" + exit 1 + fi shift 2 ;; X--loop-count) para_loop_count=$2 + if [ -z "${para_loop_count}" ]; then + echo "Please specify the value for --loop-count option" + exit 1 + fi shift 2 ;; X-s|X--serial) SERIAL=$2 + if [ -z "${SERIAL}" ]; then + echo "Please specify the value for --serial|-s option" + exit 1 + fi shift 2 ;; X-h|X--help) @@ -456,6 +501,18 @@ func_parse_parameters_common(){ if [ -n "${para_apps}" ]; then G_APPS="${para_apps}" fi + + if [ -n "${para_record_csv}" ] && [ "X${para_record_csv}" = "XTRUE" ];then + G_RECORD_LOCAL_CSV=TRUE + elif [ -n "${para_record_csv}" ];then + G_RECORD_LOCAL_CSV=FALSE + fi + + if [ -n "${para_verbose_output}" ] && [ "X${para_verbose_output}" = "XTRUE" ];then + G_VERBOSE_OUTPUT=TRUE + elif [ -n "${para_record_csv}" ];then + G_VERBOSE_OUTPUT=FALSE + fi } ## Description: @@ -481,7 +538,7 @@ common_main(){ exit 1 fi else - func_parse_parameters_common + func_parse_parameters_common "$@" fi if [ -n "${var_func_prepare_environment}" ]; then @@ -504,12 +561,103 @@ common_main(){ fi fi - rm -fr "${F_RAWDAT_ZIP}" - local old_pwd=$(pwd) - local d_zip_dir="$(dirname ${D_RAWDATA})" - local d_zip_name="$(basename ${D_RAWDATA})" - cd ${d_zip_dir}; - zip -r "${F_RAWDAT_ZIP}" "${d_zip_name}" - cd ${old_pwd} - echo "Please reference the file ${F_RAWDAT_ZIP} for all the raw data." + if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then + + if [ -f "${F_RAW_DATA_CSV}" ]; then + sort ${F_RAW_DATA_CSV}|tr ' ' '_'|tr -d '=' >${F_RAW_DATA_CSV}.sort + statistic ${F_RAW_DATA_CSV}.sort 2 |tee ${F_STATISTIC_DATA_CSV} + sed -i 's/=/,/' "${F_STATISTIC_DATA_CSV}" + rm -f ${F_RAW_DATA_CSV}.sort + fi + + rm -fr "${F_RAWDAT_ZIP}" + local old_pwd=$(pwd) + local d_zip_dir="$(dirname ${D_RAWDATA})" + local d_zip_name="$(basename ${D_RAWDATA})" + cd ${d_zip_dir}; + zip -r "${F_RAWDAT_ZIP}" "${d_zip_name}" + cd ${old_pwd} + echo "Please reference the file ${F_RAWDAT_ZIP} for all the raw data." + attach_for_lava "${F_RAWDAT_ZIP}" "application/x-gzip" + fi +} + +## Description: +## output the test result to console and add for lava-test-shell, +## also write into one csv file for comparing manually +## Usage: +## output_test_result $test_name $result [ $measurement [ $units ] ] +## Note: +## G_RECORD_LOCAL_CSV: when this environment variant is set to "TRUE", +## the result will be recorded in a csv file in the following path: +## rawdata/final_result.csv +## G_VERBOSE_OUTPUT: when this environment variant is set to "TRUE", and only it is TRUE, +## the verbose informain about the result will be outputed +output_test_result(){ + local test_name=$1 + local result=$2 + local measurement=$3 + local units=$4 + + if [ -z "${test_name}" ] || [ -z "$result" ]; then + return + fi + local output="" + local lava_paras="" + local output_csv="" + test_name=$(echo ${test_name}|tr ' ' '_') + + if [ -z "$units" ]; then + units="points" + fi + + if [ -z "${measurement}" ]; then + output="${test_name}=${result}" + lava_paras="${test_name} --result ${result}" + else + output="${test_name}=${measurement} ${units}" + lava_paras="${test_name} --result ${result} --measurement ${measurement} --units ${units}" + output_csv="${test_name},${measurement}" + fi + + if [ "X${G_VERBOSE_OUTPUT}" = "XTRUE" ];then + echo "${output}" + fi + + local cmd="lava-test-case" + if [ -n "$(which $cmd)" ];then + $cmd ${lava_paras} + elif [ "X${G_VERBOSE_OUTPUT}" = "XTRUE" ];then + echo "$cmd ${lava_paras}" + fi + + if [ "X${G_RECORD_LOCAL_CSV}" = "XTRUE" ]; then + if [ -n "${output_csv}" ];then + echo "${output_csv}">>${F_RAW_DATA_CSV} + fi + fi +} + +## Description: +## run lava-test-run-attach to attach the file for lava +## Usage: +## attach_for_lava "${f_path}" "$f_mime_type" +## Example: +## attach_for_lava logcat.log text/plain +## attach_for_lava rawdata.zip application/x-gzip +attach_for_lava(){ + local f_path=$1 + local f_mime_type=$2 + + if [ ! -f "${f_path}" ]; then + return + fi + if [ -z "${f_mime_type}" ]; then + f_mime_type="text/plain" + fi + local cmd="lava-test-run-attach" + + if [ -n "$(which $cmd)" ];then + ${cmd} ${f_path} ${f_mime_type} + fi } diff --git a/common/output-test-result.sh b/common/output-test-result.sh index b54349a..517273d 100755 --- a/common/output-test-result.sh +++ b/common/output-test-result.sh @@ -2,6 +2,6 @@ local_common_file_path="$0" local_common_parent_dir=$(cd $(dirname ${local_common_file_path}); pwd) -source "${local_common_parent_dir}/statistic_average.sh" +source "${local_common_parent_dir}/common2.sh" output_test_result "$@" diff --git a/common/statistic_average.sh b/common/statistic_average.sh index 8f91395..c8ac753 100644 --- a/common/statistic_average.sh +++ b/common/statistic_average.sh @@ -1,9 +1,5 @@ #!/bin/bash -local_common_file_path="${BASH_SOURCE[0]}" -local_common_parent_dir=$(cd $(dirname ${local_common_file_path}); pwd) -f_result_csv="${local_common_parent_dir}/../rawdata/final_result.csv" - ## Description: ## output the max value of the passed 2 parameters ## Usage: @@ -106,50 +102,3 @@ statistic(){ echo "$new_key=$average" fi } - -## Description: -## output the test result to console and add for lava-test-shell, -## also write into one csv file for comparing manually -## Usage: -## output_test_result $test_name $result [ $measurement [ $units ] ] -output_test_result(){ - local test_name=$1 - local result=$2 - local measurement=$3 - local units=$4 - - if [ -z "${test_name}" ] || [ -z "$result" ]; then - return - fi - local output="" - local lava_paras="" - local output_csv="" - test_name=$(echo ${test_name}|tr ' ' '_') - if [ -z "${measurement}" ]; then - output="${test_name}=${result}" - lava_paras="${test_name} --result ${result}" - else - output="${test_name}=${measurement}" - lava_paras="${test_name} --result ${result} --measurement ${measurement}" - output_csv="${test_name},${measurement}" - fi - - if [ -z "$units" ]; then - units="points" - fi - output="${output} ${units}" - lava_paras="${lava_paras} --units ${units}" - - echo "${output}" - - local cmd="lava-test-case" - if [ -n "$(which $cmd)" ];then - $cmd ${lava_paras} - else - echo "$cmd ${lava_paras}" - fi - if [ -n "${output_csv}" ];then - mkdir -p $(dirname ${f_result_csv}) - echo "${output_csv}">>${f_result_csv} - fi -} diff --git a/java-benchmarks-host.yaml b/java-benchmarks-host.yaml index 9e2cecc..8f825ae 100644 --- a/java-benchmarks-host.yaml +++ b/java-benchmarks-host.yaml @@ -37,6 +37,5 @@ run: - IPADDR=`awk -F '=' '{print $2}' /tmp/lava_multi_node_cache.txt` - adb connect $IPADDR - adb wait-for-device - - ./java-benchmarks.sh --base-url "$BASE_URL" --loop-count "$LOOP_COUNT" --streamline "$COLLECT_STREAMLINE" $APP_CONFIG_LIST - - lava-test-run-attach rawdata.zip application/x-gzip + - ./java-benchmarks.sh --loop-count "$LOOP_COUNT" --streamline "$COLLECT_STREAMLINE" ${APP_CONFIG_LIST} - lava-sync application-benchmark-stop-target diff --git a/java-benchmarks.sh b/java-benchmarks.sh index 32127c2..fc89b28 100755 --- a/java-benchmarks.sh +++ b/java-benchmarks.sh @@ -19,22 +19,14 @@ function func_post_uninstall_bench_batch(){ fi } -func_post_test_batch(){ - func_cleanup - - sort ${f_result_csv}|tr ' ' '_'|tr -d '=' >${f_result_csv}.sort - statistic "${f_result_csv}.sort" 2|tee ${f_result_csv}.ok - sed -i 's/=/,/' "${f_result_csv}.ok" -} - -function main(){ +function local_main(){ echo "test timeout: ${timeout}" parent_dir=$(cd ${parent_dir}; pwd) export parent_dir=${parent_dir} var_func_parse_parameters="" var_func_prepare_environment="func_prepare_benchmark" - var_func_post_test="func_post_test_batch" + var_func_post_test="func_cleanup" var_func_pre_install="" var_func_post_install="${post_install}" @@ -59,6 +51,4 @@ function main(){ return ${ret_value} } -rm -fr ${f_result_csv} - -main "$@" +local_main "$@" diff --git a/javawhetstone/vc.py b/javawhetstone/vc.py index f336ba8..f5e5003 100644 --- a/javawhetstone/vc.py +++ b/javawhetstone/vc.py @@ -39,8 +39,8 @@ while(not finished): if re.search('Total Elapsed Time', results.getText()): finished = True print "benchmark finished" - for line in results.getText().split("?"): - line = line.strip() + for line in results.getText().split('\n'): + line = str(line.strip()) elements = re.split(r'\s+', line) if line.startswith('MWIPS'): units = key_unit_hash['MWIPS'] @@ -52,7 +52,6 @@ while(not finished): value = elements[2] else: continue - print "%s=%s %s" % (key, value, units) call([f_output_result, key, 'pass', value, units]) except ViewNotFoundException: pass diff --git a/lava-android-benchmark-host.yaml b/lava-android-benchmark-host.yaml index 2c43287..c540446 100644 --- a/lava-android-benchmark-host.yaml +++ b/lava-android-benchmark-host.yaml @@ -31,16 +31,12 @@ run: steps: - apt-add-repository -y http://ppa.launchpad.net/nilarimogard/webupd8/ubuntu - apt-get update -y - - apt-get install -y android-tools-adb android-tools-fastboot + - apt-get install -y android-tools-adb android-tools-fastboot zip - lava-wait $TEST_NAME-send-ip - IPADDR=`awk -F '=' '{print $2}' /tmp/lava_multi_node_cache.txt` - adb connect $IPADDR - adb wait-for-device - adb shell id - cd $TEST_NAME - - for (( LOOP=1; LOOP<=$ITERATIONS; LOOP++ )); do lava-test-case $TEST_NAME-execution-$LOOP --shell ./execute.sh $IPADDR:5555; done; - - if [ -f logcat_canvas.log ]; then lava-test-run-attach logcat_canvas.log text/plain; fi - - if [ -f logcat.log ]; then lava-test-run-attach logcat.log text/plain; fi - - if [ -f logcat_textview.log ]; then lava-test-run-attach logcat_textview.log text/plain; fi - - if [ -f logcat_webview.log ]; then lava-test-run-attach logcat_webview.log text/plain; fi + - for (( LOOP=1; LOOP<=$ITERATIONS; LOOP++ )); do lava-test-case $TEST_NAME-execution-$LOOP --shell ./execute.sh --serial $IPADDR:5555; done; - lava-sync $TEST_NAME-finished diff --git a/scimark/vc.py b/scimark/vc.py index a8a7eb9..d059c43 100644 --- a/scimark/vc.py +++ b/scimark/vc.py @@ -30,12 +30,12 @@ while(not finished): if results.getText().find("Done") > 0: finished = True print "benchmark finished" - for line in results.getText().replace(": ?", ":").split("?"): + for line in results.getText().split("\n"): + line = str(line.strip()) key_val = line.split(":") if len(key_val) == 2: if key_val[0].strip() in keys: key = key_val[0].strip().replace(' ', '_').replace('(', '').replace(')', '').replace(',', '') - print "%s=%s" % (key, key_val[1].strip()) call([f_output_result, key, 'pass', key_val[1].strip(), 'Mflops']) except ViewNotFoundException: pass diff --git a/smartbench2012/extract_results.sh b/smartbench2012/extract_results.sh index 14bed84..c015ed4 100755 --- a/smartbench2012/extract_results.sh +++ b/smartbench2012/extract_results.sh @@ -2,7 +2,6 @@ local_this_parent="$(cd $(dirname $0);pwd)" source "${local_this_parent}/../common/common.sh" -source "${local_this_parent}/../common/statistic_average.sh" SCORE=`awk -F'[][]' '/SBGlobal.nProductivityIndex/{k=$2}END{print k}' ${F_LOGCAT}` UNIT="points" |