diff options
author | Yongqin Liu <yongqin.liu@linaro.org> | 2015-02-10 21:20:32 +0800 |
---|---|---|
committer | Yongqin Liu <yongqin.liu@linaro.org> | 2015-03-07 16:57:45 +0800 |
commit | 57f691897dfaff13c54cda6e21dfe91e25d86c13 (patch) | |
tree | 08dba5319a59d27c92a4fd404ac0c6964aa89a2e | |
parent | fffaf0e1a9eb7fc0fcd45864a9acd1e4966b1146 (diff) |
merge of application-benchmark test and android-apk automation test
create one framework for both application-benchmark and android-apk-automation test
Change-Id: I2d710365d6c6426dbfeed2b39f221185a508278f
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>
-rw-r--r-- | application-benchmark-host.yaml | 47 | ||||
-rw-r--r-- | application-benchmark-target.yaml | 23 | ||||
-rwxr-xr-x | application-benchmark.sh | 350 | ||||
-rwxr-xr-x | common/common.sh | 367 | ||||
-rw-r--r-- | common/common2.sh | 511 | ||||
-rw-r--r-- | common/statistic_average.sh | 104 | ||||
-rwxr-xr-x | gearses1eclair/vc.py | 5 | ||||
-rwxr-xr-x | gearses2eclair/vc.py | 6 | ||||
-rwxr-xr-x | geekbench3/execute.sh | 10 | ||||
-rwxr-xr-x | geekbench3/vc.py | 6 | ||||
-rwxr-xr-x | glbenchmark-2.5.1/execute.sh | 9 | ||||
-rwxr-xr-x | glbenchmark-2.5.1/file_transfer.sh | 6 | ||||
-rwxr-xr-x | glbenchmark-2.5.1/preparation.sh | 5 | ||||
-rwxr-xr-x | glbenchmark-2.5.1/vc.py | 29 | ||||
-rwxr-xr-x | smartbench2012/execute.sh | 2 | ||||
-rwxr-xr-x | smartbench2012/extract_results.sh | 7 | ||||
-rwxr-xr-x | smartbench2012/vc.py | 15 | ||||
-rwxr-xr-x | vellamo/execute.sh | 2 | ||||
-rwxr-xr-x | vellamo/vc.py | 24 | ||||
-rwxr-xr-x | vellamo3/execute.sh | 1 | ||||
-rwxr-xr-x | vellamo3/vc.py | 6 |
21 files changed, 1173 insertions, 362 deletions
diff --git a/application-benchmark-host.yaml b/application-benchmark-host.yaml new file mode 100644 index 0000000..cbbb1c8 --- /dev/null +++ b/application-benchmark-host.yaml @@ -0,0 +1,47 @@ +metadata: + name: application-benchmark-host + format: "Lava-Test-Shell Test Definition 1.0" + description: "Wrapper for benchmarking application, will collect information + like memory, cpu, starttime when start the activity. Some parameters + can be specified for this test: + BASE_URL: based url from where application apk files can be gotten. + default is http://testdata.validation.linaro.org/apks/JavaBenchmark/pure-java-applications/ + APP_CONFIG_LIST: specify the application to be benchmarked in the form of + APK_FILE_NAME,APP_PACKAGE/ACTIVITY_NAME,APP_NICKNAME + we can sepcified multiple with space separated. If the application to be benchmarked + is a system application, we can specified NULL in the APK_FILE_NAME placeholder + LOOP_COUNT: specify the count we want to run for benchmarking. Default is 12. + the average is calculated with maximum and minimum excluded. + COLLECT_STREAMLINE: specify if streamline data will be collected. only false and true are available." + maintainer: + - yongqin.liu@linaro.org + os: + - android + scope: + - functional + devices: + - kvm + +params: + BASE_URL: "http://testdata.validation.linaro.org/apks/JavaBenchmark/pure-java-applications/" + APP_CONFIG_LIST: "" + LOOP_COUNT: 12 + COLLECT_STREAMLINE: "false" + +run: + steps: + - apt-add-repository -y http://ppa.launchpad.net/nilarimogard/webupd8/ubuntu + - apt-get update -y + - apt-get install -y android-tools-adb android-tools-fastboot zip + - lava-wait application-benchmark-target-ready-send-ip + - IPADDR=`awk -F '=' '{print $2}' /tmp/lava_multi_node_cache.txt` + - adb connect $IPADDR + - adb wait-for-device + - ./application-benchmark.sh --base-url "$BASE_URL" --loop-count "$LOOP_COUNT" --streamline "$COLLECT_STREAMLINE" $APP_CONFIG_LIST + - lava-test-run-attach rawdata.zip application/x-gzip + - lava-sync application-benchmark-stop-target + +parse: + pattern: "(?P<test_case_id>\\S+)(?P<result>=)(?P<measurement>[\\d\\.]+)\\s+(?P<units>\\S+)" + fixupdict: + =: pass diff --git a/application-benchmark-target.yaml b/application-benchmark-target.yaml new file mode 100644 index 0000000..c9fb804 --- /dev/null +++ b/application-benchmark-target.yaml @@ -0,0 +1,23 @@ +metadata: + name: target-stop + format: "Lava-Test-Shell Test Definition 1.0" + description: "Wrapper for notifying the host side that target side is ready, + and waiting for the singal from host side to stop target." + maintainer: + - yongqin.liu@linaro.org + os: + - android + scope: + - functional + devices: + - juno + +run: + steps: + - lava-test-case step1-cat-build-info --shell cat /system/build.prop + - lava-test-case step2-get-adb --shell getprop service.adb.tcp.port + - lava-test-case step3-set-adb --shell setprop service.adb.tcp.port 5555 + - lava-test-case step4-stop-adbd --shell stop adbd + - lava-test-case step5-start-adbd --shell start adbd + - lava-test-case step6-send-ip --shell lava-send application-benchmark-target-ready-send-ip ipaddr=$(getprop dhcp.eth0.ipaddress) + - lava-test-case step7-multinode-sync --shell lava-sync application-benchmark-stop-target diff --git a/application-benchmark.sh b/application-benchmark.sh new file mode 100755 index 0000000..eabbfd5 --- /dev/null +++ b/application-benchmark.sh @@ -0,0 +1,350 @@ +#!/bin/bash + +local_this_file_path="${BASH_SOURCE[0]}" +local_this_parent_dir=$(cd $(dirname ${local_this_file_path}); pwd) +source ${local_this_parent_dir}/common/common2.sh +source ${local_this_parent_dir}/common/statistic_average.sh + +## override default value in common2.sh +G_LOOP_COUNT=12 +BASE_URL="http://testdata.validation.linaro.org/apks/JavaBenchmark/pure-java-applications" +#BASE_URL="scp://testdata//home/testdata.validation.linaro.org/apks/JavaBenchmark/pure-java-applications" +APPS="NULL,com.android.browser/.BrowserActivity,Browser" +APPS="${APPS} NULL,com.android.settings/.Settings,Settings" +APPS="${APPS} 01-3D_Volcano_Island.apk,com.omnigsoft.volcanoislandjava/.App,3D_Volcano_Island" +APPS="${APPS} 02-com.blong.jetboy_1.0.1.apk,com.blong.jetboy/.JetBoy,JetBoy" +APPS="${APPS} 03-HelloEffects.apk,com.example.android.mediafx/.HelloEffects,HelloEffects" +APPS="${APPS} 04-FREEdi_YouTube_Player_v2.2.apk,tw.com.freedi.youtube.player/.MainActivity,FREEdi_YouTube_Player" +APPS="${APPS} 17-GooglePlayBooks.apk,com.google.android.apps.books/.app.BooksActivity,GooglePlayBooks" +APPS="${APPS} 46-Zedge.apk,net.zedge.android/.activity.ControllerActivity,Zedge" +APPS="${APPS} 55-ShootBubbleDeluxe.apk,com.shootbubble.bubbledexlue/.FrozenBubble,ShootBubbleDeluxe" +G_APPS=${APPS} + +## Contants for this script +dir_rawdata="${D_RAWDATA}" +dir_apks="${D_APKS}" +f_starttime="${dir_rawdata}/activity_starttime.raw" +f_mem="${dir_rawdata}/activity_mem.raw" +f_cpu="${dir_rawdata}/activity_cpu.raw" +f_procrank="${dir_rawdata}/activity_procrank.raw" +f_stat="${dir_rawdata}/activity_stat.raw" +f_procmem="${dir_rawdata}/activity_procmem.raw" +f_maps="${dir_rawdata}/activity_maps.raw" +f_dumpsys_mem="${dir_rawdata}/activity_dumpsys_meminfo.raw" +f_smaps="${dir_rawdata}/activity_smaps.raw" + +f_res_starttime="${dir_rawdata}/activity_starttime.csv" +f_res_mem="${dir_rawdata}/activity_mem.csv" +f_res_cpu="${dir_rawdata}/activity_cpu.csv" +f_res_procrank="${dir_rawdata}/activity_procrank.csv" +f_res_dumpsys_mem="${dir_rawdata}/activity_dumpsys_meminfo.csv" +f_result="${dir_rawdata}/result.csv" + +# only called by collect_mem_raw_data +collect_raw_procmem_data(){ + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${f_procmem}" + adb shell su 0 procmem ${pid} >> "${f_procmem}" + echo "===package=${loop_app_package}, count=${loop_count} end" >> "${f_procmem}" + + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${f_procmem}_p" + adb shell su 0 procmem -p ${pid} >> "${f_procmem}_p" + echo "===package=${loop_app_package}, count=${loop_count} end" >> "${f_procmem}_p" + + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${f_procmem}_m" + adb shell su 0 procmem -m ${pid} >> "${f_procmem}_m" + echo "===package=${loop_app_package}, count=${loop_count} end" >> "${f_procmem}_m" +} + + +collect_mem_raw_data(){ + # get memory info + adb shell ps|grep "${loop_app_package}" >>"${f_mem}" + adb shell su 0 procrank|grep "${loop_app_package}" >> "${f_procrank}" + + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${f_dumpsys_mem}" + adb shell su 0 dumpsys meminfo "${loop_app_package}" >> "${f_dumpsys_mem}" + echo "===package=${loop_app_package}, count=${loop_count} end" >> "${f_dumpsys_mem}" + + local pid=$(adb shell ps|grep ${loop_app_package}|awk '{print $2}') + if [ -n "${pid}" ]; then + adb shell su 0 cat /proc/${pid}/stat >> "${f_stat}" + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${f_maps}" + adb shell su 0 cat /proc/${pid}/maps >> "${f_maps}" + echo "===package=${loop_app_package}, count=${loop_count} end" >> "${f_maps}" + + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${f_smaps}" + adb shell su 0 cat /proc/${pid}/smaps >> "${f_smaps}" + echo "===package=${loop_app_package}, count=${loop_count} end" >> "${f_smaps}" + collect_raw_procmem_data + fi +} + +func_post_install_app_bench(){ + func_post_install + # catch the cpu information before start activity + cpu_time_before=$(adb shell cat /proc/stat|grep 'cpu '|tr -d '\n') +} + +func_pre_uninstall_app_bench(){ + # get cpu information + cpu_time_after=$(adb shell cat /proc/stat|grep 'cpu '|tr -d '\n') + echo "${loop_app_package},${cpu_time_before},${cpu_time_after}" >>"${f_cpu}" + + # get activity start time information + adb logcat -d|grep "Displayed ${loop_app_start_activity}" >>"${f_starttime}" + + collect_mem_raw_data + + func_pre_uninstall +} + +func_post_uninstall_app_bench(){ + func_post_uninstall + echo "" >>"${f_starttime}" + echo "" >>"${f_mem}" + echo "" >>"${f_cpu}" +} + +format_starttime_raw_data(){ + if [ ! -f "${f_starttime}" ]; then + return + fi + sed '/^\s*$/d' "${f_starttime}" |tr -s ' '|tr -d '\r'|sed 's/^.*Displayed\ //'|sed 's/(.*$//' |sed 's/: +/,/' >"${f_res_starttime}.tmp" + for line in $(cat "${f_res_starttime}.tmp"); do + local app_pkg=$(echo $line|cut -d, -f1) + local app_time=$(echo $line|cut -d, -f2|sed 's/ms//g') + # assumed no minute here + if echo $app_time|grep -q 's'; then + local app_sec=$(echo $app_time|cut -ds -f1) + local app_msec=$(echo $app_time|cut -ds -f2) + app_time=$((app_sec * 1000 + app_msec)) + fi + echo "${app_pkg},${app_time}" >>"${f_res_starttime}" + done + rm -f "${f_res_starttime}.tmp" +} + +format_mem_raw_data(){ + if [ ! -f "${f_mem}" ]; then + return + fi + sed '/^\s*$/d' "${f_mem}" |tr -s ' '|tr -d '\r'|awk '{printf "%s,%s,%s\n", $9, $4, $5;}' >"${f_res_mem}" +} + +calculate_field_value(){ + local line_val=$1 && shift + local field_no=$1 && shift + [ -z "${line_val}" ] && return + [ -z "${field_no}" ] && return + + local val1=$(echo "${line_val}"|cut -d, -f${field_no}) + local field2_no=$(echo "${field_no}+10"|bc) + local val2=$(echo "${line_val}"|cut -d, -f${field2_no}) + local val=$(echo "${val2}-${val1}"|bc) + echo "${val}" +} + +format_cputime(){ + local f_data=$1 && shift + if ! [ -f "$f_data" ]; then + return + fi + rm -fr "${f_data}_2nd" + for line in $(cat "${f_data}"); do + if ! echo "$line"|grep -q ,; then + continue + fi + local key=$(echo $line|cut -d, -f1) + + local val_user=$(calculate_field_value "$line" 2) + local val_nice=$(calculate_field_value "$line" 3) + + local val_system=$(calculate_field_value "$line" 4) + local val_idle=$(calculate_field_value "$line" 5) + local val_io_wait=$(calculate_field_value "$line" 6) + local val_irq=$(calculate_field_value "$line" 7) + local val_softirq=$(calculate_field_value "$line" 8) + + local val_total_user=$(echo "scale=2; ${val_user}+${val_nice}"|bc) + local val_total_system=$(echo "scale=2; ${val_system}+${val_irq}+${val_softirq}"|bc) + local val_total_idle=$val_idle + local val_total_iowait=$val_io_wait + local val_total=$(echo "scale=2; ${val_total_system}+${val_total_user}+${val_total_idle}+${val_total_iowait}"|bc) + + local percent_user=$(echo "scale=2; $val_total_user*100/$val_total"|bc) + local percent_sys=$(echo "scale=2; $val_total_system*100/$val_total"|bc) + local percent_idle=$(echo "scale=2; $val_total_idle*100/$val_total"|bc) + echo "$key,$percent_user,$percent_sys,$percent_idle" >> "${f_data}_2nd" + done +} + +format_cpu_raw_data(){ + if [ ! -f "${f_cpu}" ]; then + return + fi + sed '/^\s*$/d' "${f_cpu}" |tr -d '\r'|tr -s ' '|tr ' ' ','|sed 's/cpu,//g' >"${f_res_cpu}" + format_cputime "${f_res_cpu}" +} + +format_procrank_data(){ + if [ ! -f "${f_procrank}" ]; then + return + fi + sed '/^\s*$/d' "${f_procrank}" |sed 's/^\s*//'|tr -d '\r'|awk '{printf("%s,%s,%s,%s,%s\n", $6, $2, $3, $4, $5)}'|tr -d 'K'>"${f_res_procrank}" +} + +format_procmem_data(){ + if [ ! -f "${f_procmem}" ]; then + return + fi + sed 's/^\s*//' "${f_procmem}" |tr -s ' '|tr ' ' ',' >"${f_procmem}.csv" + sed 's/^\s*//' "${f_procmem}_p" |tr -s ' '|tr ' ' ',' >"${f_procmem}_p.csv" + sed 's/^\s*//' "${f_procmem}_m" |tr -s ' '|tr ' ' ',' >"${f_procmem}_m.csv" +} + +format_dumpsys_meminfo(){ + if [ ! -f "${f_dumpsys_mem}" ]; then + return + fi + local package="" + local native="" + local dalvik="" + local sommap="" + #===package=com.android.browser, count=0 start + # Native Heap 5981 5916 0 0 7864 7183 680 + # Dalvik Heap 6839 6520 0 0 19304 11701 760 + # .so mmap 12434 212 11052 0 + # TOTAL 46946 20720 21524 0 29861 29539 15543 + for line in $(grep -e start -e TOTAL -e "Native Heap" -e "Dalvik Heap" -e ".so mmap" ${f_dumpsys_mem}|tr '=' ','|tr ' ' ','|tr -s ','|tr -d '\r'|sed 's/^,//') ; do + case "X${line}" in + "Xpackage"*) + if [ -n "${package}" ] && [ -n "${native}" ] \ + && [ -n "${dalvik}" ] && [ -n "${sommap}" ] && [ -n "${total}" ]; then + echo "${package},${native},${dalvik},${sommap},${total}" >>"${f_res_dumpsys_mem}" + fi + package=$(echo "${line}"|cut -d\, -f2) + ;; + "XNative,Heap"*) + native=$(echo "${line}"|cut -d\, -f3) + ;; + "XDalvik,Heap"*) + dalvik=$(echo "${line}"|cut -d\, -f3) + ;; + "X.so,mmap"*) + sommap=$(echo "${line}"|cut -d\, -f3) + ;; + "XTOTAL"*) + total=$(echo "${line}"|cut -d\, -f2) + ;; + "X"*) + continue + ;; + esac + done + if [ -n "${package}" ] && [ -n "${native}" ] \ + && [ -n "${dalvik}" ] && [ -n "${sommap}" ] && [ -n "${total}" ]; then + echo "${package},${native},${dalvik},${sommap},${total}" >>"${f_res_dumpsys_mem}" + fi +} + +format_raw_data(){ + rm -fr ${f_res_starttime} ${f_res_mem} ${f_res_cpu} ${f_res_procrank} ${f_res_dumpsys_mem} + + format_starttime_raw_data + format_mem_raw_data + format_cpu_raw_data + format_procrank_data + format_dumpsys_meminfo +} + +set_browser_homepage(){ + pref_file="com.android.browser_preferences.xml" + pref_dir="/data/data/com.android.browser/shared_prefs/" + pref_content='<?xml version="1.0" encoding="utf-8" standalone="yes" ?> +<map> + <boolean name="enable_hardware_accel_skia" value="false" /> + <boolean name="autofill_enabled" value="true" /> + <string name="homepage">about:blank</string> + <boolean name="last_paused" value="false" /> + <boolean name="debug_menu" value="false" /> +</map>' + + # start browser for the first time to genrate preference file + adb shell am start -W -S com.android.browser/.BrowserActivity + user_grp=$(adb shell su 0 ls -l "${pref_dir}/${pref_file}"|awk '{printf "%s:%s", $2, $3}') + func_kill_uninstall "NULL" "com.android.browser" + + echo "${pref_content}" > "${pref_file}" + adb push "${pref_file}" "/data/local/tmp/${pref_file}" + adb shell su 0 cp "/data/local/tmp/${pref_file}" "${pref_dir}/${pref_file}" + adb shell su 0 chown ${user_grp} "${pref_dir}/${pref_file}" + adb shell su 0 chmod 660 "${pref_dir}/${pref_file}" + + adb shell am start -W -S com.android.browser/.BrowserActivity + func_kill_uninstall "NULL" "com.android.browser" + adb shell am start -W -S com.android.browser/.BrowserActivity + func_kill_uninstall "NULL" "com.android.browser" +} + +func_prepare_app_bench(){ + func_prepare_environment + + if echo "$APPS"|grep -q "com.android.browser"; then + set_browser_homepage + fi + rm -fr "${f_starttime}" "${f_mem}" "${f_cpu}" "${f_procrank}" "${f_stat}" "${f_procmem}" "${f_procmem}_m" "${f_procmem}_p" +} + +statistic_data(){ + rm -fr "${f_result}" + statistic "$f_res_starttime" 2|sed "s/^/starttime_/"|sed "s/$/ ms/"| tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_mem}" 2|sed "s/^/ps_vss_/" |sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_mem}" 3|sed "s/^/ps_rss_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_procrank}" 2|sed "s/^/procrank_vss_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_procrank}" 3|sed "s/^/procrank_rss_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_procrank}" 4|sed "s/^/procrank_pss_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_procrank}" 5|sed "s/^/procrank_uss_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_dumpsys_mem}" 2|sed "s/^/dumpsys_meminfo_pss_native_heap_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_dumpsys_mem}" 3|sed "s/^/dumpsys_meminfo_pss_dalvik_heap_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_dumpsys_mem}" 4|sed "s/^/dumpsys_meminfo_pss_sommap_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_dumpsys_mem}" 5|sed "s/^/dumpsys_meminfo_pss_total_/"|sed "s/$/ KB/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_cpu}_2nd" 2|sed "s/^/cpu_user_/"|sed "s/$/ %/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_cpu}_2nd" 3|sed "s/^/cpu_sys_/"|sed "s/$/ %/"|tee -a "${f_result}" + echo "--------------------------------" + statistic "${f_res_cpu}_2nd" 4|sed "s/^/cpu_idle_/"|sed "s/$/ %/"|tee -a "${f_result}" + sed -i 's/=/,/' "${f_result}" +} + + +func_post_test_app_bench(){ + format_raw_data + statistic_data +} + +main(){ + var_func_parse_parameters="" + var_func_prepare_environment="func_prepare_app_bench" + var_func_post_test="func_post_test_app_bench" + + var_func_pre_install="" + var_func_post_install="func_post_install_app_bench" + var_func_run_test="" + var_test_command="" + var_func_pre_uninstall="func_pre_uninstall_app_bench" + var_func_post_uninstall="func_post_uninstall_app_bench" + common_main "$@" +} + +main "$@" diff --git a/common/common.sh b/common/common.sh index abc213a..059ee62 100755 --- a/common/common.sh +++ b/common/common.sh @@ -1,345 +1,104 @@ #!/bin/bash +local_common_file_path="${BASH_SOURCE[0]}" +local_common_parent_dir=$(cd $(dirname ${local_common_file_path}); pwd) +source ${local_common_parent_dir}/common2.sh + base_url="http://testdata.validation.linaro.org/apks/" -png_dir_device="/data/local/tmp/" post_install="" pre_uninstall="" -do_streamline=false ret_value=0 timeout=10m -#function install_linaro_android_jar(){ -# jar_name="linaro.android.jar" -# tgt_path="/data/local/tmp/${jar_name}" -# jar_url="http://testdata.validation.linaro.org/tools/${jar_name}" -# exist=`adb shell "ls ${tgt_path} 2>/dev/null"` -# if [ -z "${exist}" ]; then -# wget ${jar_url} -O ${jar_name} -# adb push ${jar_name} ${tgt_path} -# rm -f ${jar_name} -# fi -#} - -function delete_png_files_on_device(){ - png_dir=${1-$png_dir_device} - png_files=`adb shell "ls ${png_dir}/*.png 2>/dev/null"` - for png_f in ${png_files}; do - png_f=`echo ${png_f}|sed 's/\r//'` - adb shell rm "${png_f}" - done +f_tmp_governor="/data/local/tmp/governor.txt" +func_cleanup(){ + adb shell "cat ${f_tmp_governor} > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" + adb shell "cat ${f_tmp_governor} > /sys/devices/system/cpu/cpu1/cpufreq/scaling_governor" + adb shell rm ${f_tmp_governor} + func_kill_uninstall "RotationOff.apk" "rotation.off" } -function pull_png_files_from_device(){ - src_dir_device=${1-"${png_dir_device}"} - tgt_dir_local=${2-"${parent_dir}"} - png_files=`adb shell "ls ${png_dir}/*.png 2>/dev/null"` - for png_f in ${png_files}; do - png_f=`echo ${png_f}|sed 's/\r//'` - adb pull "${png_f}" "${tgt_dir_local}" &>/dev/null - done -} - -function init(){ - # uninstall the apk application - # don't uninstall if apk file name is empty - if [ ! -z "$apk_file_name" ]; then - echo "trying to unistall ${apk_package}" - adb uninstall "${apk_package}" - echo "unistalled ${apk_package}" - fi - # clear the logcat information - adb logcat -c - sleep 5 - - rm -fr "${parent_dir}"/*.png 2>/dev/null - delete_png_files_on_device "${png_dir_device}" - - disableRotationapk="${APKS_DIR}/RotationOff.apk" - if [ -f "{$disableRotationapk}" ]; then - echo "The file(${disableRotationapk}) already exists." +func_install_start_RotationAPK(){ + local apk_name="RotationOff.apk" + local apk_path="${D_APKS}/RotationOff.apk" + if [ -f "${apk_path}" ]; then + echo "The file(${apk_path}) already exists." else - get_file_with_base_url "RotationOff.apk" + local apk_base_url="http://testdata.validation.linaro.org/apks/" + get_file_with_base_url "${apk_name}" "${apk_base_url}" "${D_APKS}" fi adb shell pm list packages | grep rotation.off if [ $? -ne 0 ]; then - adb install "${disableRotationapk}" + adb install "${apk_path}" fi sleep 2 adb shell am start 'rotation.off/.RotationOff' sleep 2 - adb shell "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor > /data/governor.txt" - adb shell "echo performance > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" - adb shell "echo performance > /sys/devices/system/cpu/cpu1/cpufreq/scaling_governor" - adb shell logcat -c - adb shell setprop ro.debug.drawtext true - adb shell setprop ro.debug.textview true - adb shell setprop ro.debug.loadDataWithBaseURL true - logcat_file="${parent_dir}/logcat.log" - echo "---------------------------------------------------" - echo "A new test is started:`date`" |tee -a "${logcat_file}" - adb logcat >>${logcat_file} & - export LOGCAT_PID=$! -} - - -function cleanup(){ - adb shell "cat /data/governor.txt > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" - adb shell "cat /data/governor.txt > /sys/devices/system/cpu/cpu1/cpufreq/scaling_governor" - adb shell rm /data/governor.txt - adb shell setprop ro.debug.drawtext false - adb shell setprop ro.debug.textview false - adb shell setprop ro.debug.loadDataWithBaseURL false - adb uninstall rotation.off - if [ -n "${LOGCAT_PID}" ]; then - kill -9 ${LOGCAT_PID} - fi } -function export_serial(){ - serial="${1}" && shift - if [ -n "${serial}" ]; then - export ANDROID_SERIAL=${serial} - else - serial=`adb get-serialno|sed 's/\r//g'` - if [ "X${serial}" == "Xunknown" ]; then - echo "Can not get the serial number autotically," - echo "Please specify the serial number with the -s option" - exit 1 - else - export ANDROID_SERIAL=${serial} - fi - fi -} +function init(){ -function export_parent_dir(){ - old_pwd=`pwd` - cd ${parent_dir} - parent_dir=`pwd` - cd ${old_pwd} - export parent_dir=${parent_dir} -} + func_install_start_RotationAPK -function export_apks_dir(){ - #export APKS_DIR="${parent_dir}/../benchmark-apks" - export APKS_DIR="${parent_dir}" + adb shell "cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor > ${f_tmp_governor}" + adb shell "echo performance > /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor" + adb shell "echo performance > /sys/devices/system/cpu/cpu1/cpufreq/scaling_governor" } -function get_file_with_base_url(){ - file_name="${1}" && shift - - if [ -z "${file_name}" ]; then - echo "File name must be passed!" - exit 1 - fi +func_prepare_benchmark(){ + init + echo "init done" - if [ -f "${APKS_DIR}/${file_name}" ]; then - echo "The file(${APKS_DIR}/${file_name}) already exists." - return - fi - if [[ "${base_url}" =~ "scp://" ]]; then - mkdir -p "${APKS_DIR}" - apk_url="${base_url}/${file_name}" - url_no_scp=`echo ${apk_url}|sed 's/^\s*scp\:\/\///'|sed 's/\//\:\//'` - scp "${url_no_scp}" "${APKS_DIR}/${file_name}" - if [ $? -ne 0 ]; then - echo "Failed to get the apk(${file_name}) with ${base_url}" - exit 1 - fi - elif [[ "${base_url}" =~ "ssh://" ]]; then - rm -fr "${APKS_DIR}" - git clone "${base_url}" "${APKS_DIR}" - if [ $? -ne 0 ]; then - echo "Failed to get the apks with ${base_url}" - exit 1 - fi - elif [[ "${base_url}" =~ "http://" ]]; then - mkdir -p "${APKS_DIR}" - wget "${base_url}"/${file_name} -O "${APKS_DIR}"/${file_name} - if [ $? -ne 0 ]; then - echo "Failed to get the apks with ${base_url}" - exit 1 - fi - else - echo "Failed to get the file($file_name)." - echo "The schema of the ${base_url} is not supported now!" - exit 1 - fi + func_prepare_environment } -function install_run_uninstall(){ - #install the apk files - if [ ! -z "$apk_file_name" ]; then - apk_file="${APKS_DIR}/${apk_file_name}" - adb install "${apk_file}" - if [ $? -ne 0 ]; then - echo "Failed to install ${apk_file}." - exit 1 +func_run_test_bench(){ + local test_script="${D_ROOT}/${loop_app_name}/vc.py" + local ret + if [ -f "${test_script}" ]; then + local test_command="python ${test_script}" + if [ -n "${var_test_command_timeout}" ]; then + timeout ${var_test_command_timeout} ${test_command} + ret=$? + if [ $ret -eq 124 ]; then + echo "Time out to run ${test_command}: ${var_test_command_timeout}" + fi + else + ${test_command} + ret=$? fi - else - # force stop activity if apk is already installed - echo "stopping ${apk_package}" - adb shell am force-stop "${apk_package}" - sleep 5 - fi - if [ -n "${post_install}" ]; then - ${post_install} - fi - adb shell am start "${activity}" - sleep 5 - adb shell am kill-all - sleep 5 - streamline_init_capture - if [ -n "${test_method}" ]; then - timeout ${timeout} ${test_method} - ret_value=$? - fi - sleep 5 - streamline_end_capture - if [ -n "${pre_uninstall}" ]; then - ${pre_uninstall} - fi - - if [ ! -z "$apk_file_name" ]; then - adb uninstall "${apk_package}" - else - # force stop activity if apk is a stock app - echo "stopping ${apk_package}" - adb shell am force-stop "${apk_package}" sleep 5 + return $ret fi } -function collect_log(){ - sleep 5 - adb logcat -d -s "TextView" >${parent_dir}/logcat_textview.log - sleep 5 - adb logcat -d -s "Canvas" >${parent_dir}/logcat_canvas.log - sleep 5 - adb logcat -d -s "WebViewClassic.loadDataWithBaseURL" >${parent_dir}/logcat_webview.log - sleep 5 -} - -function streamline_locate(){ - which streamline >&/dev/null - return $? -} - -function streamline_init_capture(){ - if ! ${do_streamline}; then - return - fi - if ! streamline_locate; then - echo "There is no streamline command found." - echo "Please check your environment variable or install it" - return +func_post_uninstall_bench(){ + func_post_uninstall + if [ -n "${post_uninstall}" ]; then + ${post_uninstall} fi - - echo "Start Streamline Capture.. " - adb shell "rm -r /data/streamline 2>/dev/null" - adb shell mkdir /data/streamline - session_file="${parent_dir}/session.xml" - adb push $session_file /data/streamline - app_name=`basename $parent_dir` - adb shell "gatord -s /data/streamline/session.xml -o /data/streamline/${app_name}.apc &" - adb shell sleep 2 } -function streamline_end_capture(){ - if ! ${do_streamline}; then - return - fi - if ! streamline_locate; then - return - fi +function main(){ + echo "test timeout: ${timeout}" + parent_dir=$(cd ${parent_dir}; pwd) + export parent_dir=${parent_dir} - echo "End Streamline Capture.. " - ps_info=`adb shell ps -x | grep -E '\s+gatord\s+'` - ##TODO maybe have multiple lines here - pid=`echo $ps_info|cut -d \ -f 2|sed 's/\r//'` - if [ -n "${pid}" ]; then - adb shell kill $pid - fi + var_func_parse_parameters="" + var_func_prepare_environment="func_prepare_benchmark" + var_func_post_test="func_cleanup" - echo "Start Processing Streamline data." - app_name=`basename $parent_dir` - capture_dir="$parent_dir/${app_name}.apc" - rm -fr ${capture_dir} - adb pull /data/streamline/${app_name}.apc $capture_dir - if [ $? -ne 0 ]; then - echo "Failed to pull the streamline data from android!" - exit 1 - fi - streamline -analyze ${capture_dir} - if [ $? -ne 0 ]; then - echo "Failed to analyze the streamline data!" - exit 1 - fi - apd_f="${app_name}.apd" - streamline -report -function ${apd_f} |tee ${parent_dir}/streamlineReport.txt - if [ $? -ne 0 ]; then - echo "Failed to generate the streamline report!" - exit 1 - fi - ##TODO detail parse should be done in run.py - rm -fr ${capture_dir} - adb shell rm -r /data/streamline -} -function show_usage(){ - echo "`basename $0` [--base-url|-b <base-url>] [<device-serial>] [--streamline]" - echo "`basename $0` --help|-h" -} + var_func_pre_install="" + var_func_post_install="${post_install}" + var_func_run_test="func_run_test_bench" + var_test_command="" + var_test_command_timeout="${timeout}" + var_func_pre_uninstall="${pre_uninstall}" + var_func_post_uninstall="func_post_uninstall_bench" -function parse_arguments(){ - while test -n "$1"; do - case "$1" in - --help|-h) - show_usage - exit 1 - ;; - --streamline|-s) - do_streamline=true - shift 1 - ;; - "--base-url"|-b) - if [ -z "$2" ]; then - show_usage - exit 1 - else - base_url="$2" - shift 2 - fi - ;; - *) - if [ -n "${arg_serial}" ]; then - echo "Too many arguments are given!" - show_usage - exit 1 - fi - arg_serial="$1" - shift 1 - ;; - esac - done -} + G_APPS="${apk_file_name},${activity},$(basename ${parent_dir})" + BASE_URL="${base_url}" + common_main "$@" -function main(){ - echo "test timeout: ${timeout}" - arg_serial="" - parse_arguments "$@" - echo "parsing done" - export_serial "${arg_serial}" - export_parent_dir - export_apks_dir - echo "exports done" - init - echo "init done" - if [ ! -z "$apk_file_name" ]; then - get_file_with_base_url "${apk_file_name}" - fi - install_run_uninstall - echo "testing done" - pull_png_files_from_device "${png_dir_device}" ${parent_dir} - collect_log - cleanup return ${ret_value} } diff --git a/common/common2.sh b/common/common2.sh new file mode 100644 index 0000000..57962c7 --- /dev/null +++ b/common/common2.sh @@ -0,0 +1,511 @@ +#!/bin/bash + +local_common2_file_path="${BASH_SOURCE[0]}" +local_common2_parent_dir=$(cd $(dirname ${local_common2_file_path}); pwd) +D_ROOT=$(cd $(dirname ${local_common2_parent_dir}); pwd) + +D_RAWDATA="${D_ROOT}/rawdata" +D_APKS="${D_ROOT}/apks" +F_RAWDAT_ZIP="${D_ROOT}/rawdata.zip" + +F_LOGCAT="${D_RAWDATA}/logcat.log" +F_LOGCAT_EVENTS="${D_RAWDATA}/logcat-events.log" +D_STREAMLINE="${D_RAWDATA}/streamline" +D_SCREENSHOT="${D_RAWDATA}/screenshots" + +COLLECT_STREAMLINE=false +SERIAL="" +G_APPS="" +G_LOOP_COUNT=1 +BASE_URL="" + +## Description: +## get specified the apk file from remote +## Usage: +## get_file_with_base_url $file_name $base_url $target_dir +## Example: +## get_file_with_base_url "${apk_apk}" "${BASE_URL}" "${D_APKS}" +get_file_with_base_url(){ + local file_name=$1 && shift + local base_url=$1 && shift + local target_dir=$1 && shift + + if [ -z "${file_name}" ]; then + echo "The file name must be specified." + return 1 + fi + + if [ -z "${base_url}" ]; then + echo "The file name must be specified." + return 1 + fi + + if [ -z "${target_dir}" ]; then + echo "The file name must be specified." + return 1 + fi + + if [ -f "${target_dir}/${file_name}" ]; then + echo "The file($file_name) already exists." + return 0 + fi + mkdir -p "${target_dir}" + case "X${base_url}" in + "Xscp://"*) + # like scp://yongqin.liu@testdata.validation.linaro.org/home/yongqin.liu + apk_url="${base_url}/${file_name}" + url_no_scp=`echo ${apk_url}|sed 's/^\s*scp\:\/\///'|sed 's/\//\:\//'` + scp "${url_no_scp}" "${target_dir}/${file_name}" + if [ $? -ne 0 ]; then + echo "Failed to get the apk(${file_name}) with ${base_url}" + return 1 + fi + ;; + "Xssh://"*) + git clone "${base_url}" "${target_dir}" + if [ $? -ne 0 ]; then + echo "Failed to get the apks with ${base_url}" + return 1 + fi + ;; + "Xhttp://"*) + wget "${base_url}/${file_name}" -O "${target_dir}/${file_name}" + if [ $? -ne 0 ]; then + echo "Failed to get the apks with ${base_url}" + return 1 + fi + ;; + "X"*) + echo "Failed to get the file($file_name)." + echo "The schema of the ${base_url} is not supported now!" + return 1 + ;; + esac + + return 0 +} + +## Description: +## used to kill the process for the specified process name +## Usage: +## kill_process $process_name_pattern +## Example: +## kill_process "gatord" +kill_process(){ + local proc=$1 && shift + [ -z "${proc}" ] && return + + while adb shell ps | grep -q -E "\s+${proc}\s+"; do + local pid=$(adb shell ps|grep -E "\s+${proc}\s+"|awk '{print $2}') + if [ -n "${pid}" ]; then + adb shell su 0 kill -9 "${pid}" + fi + done + sleep 5 +} + +## Description: +## prepare environment for streamline and start the gatord +## to collect streamline data +## Usage: +## collect_streamline_data_before "${app_id}" "${timeout}" +## Example: +## collect_streamline_data_before "Browser_0" "10" +## Note: +## This function will do nothing if COLLECT_STREAMLINE is not set to true +collect_streamline_data_before(){ + if [ "X${COLLECT_STREAMLINE}" != "Xtrue" ]; then + return + fi + local app_name=$1 && shift + if [ -z "${app_name}" ];then + return + fi + local gatord_timeout="${1-10}" && shift + + kill_process "gatord" + adb shell su 0 rm -fr /data/local/tmp/streamline + adb shell mkdir /data/local/tmp/streamline + cat >session.xml <<__EOF__ +<?xml version="1.0" encoding="US-ASCII" ?> +<session version="1" title="${app_name}" target_path="@F" call_stack_unwinding="yes" parse_debug_info="yes" high_resolution="no" buffer_mode="streaming" sample_rate="normal" duration="${gatord_timeout}"> +</session> +__EOF__ + adb push session.xml /data/local/tmp/streamline/session.xml + echo "Gatord starts:$(date)" + adb shell "su 0 gatord -s /data/local/tmp/streamline/session.xml -o /data/local/tmp/streamline/${app_name}.apc" & + adb shell sleep 2 +} + +## Description: +## wait for the gatord finished or kill it if over the time specified in seconds +## Usage: +## wait_kill_gatord_finish "${timeout}" +## Example: +## wait_kill_gatord_finish "10" +wait_kill_gatord_finish(){ + local gatord_timeout="${1-10}" && shift + local proc="gatord" + local count=1 + while adb shell ps | grep -q -E "\s+${proc}\s+"; do + count=$((count+1)) + if [ ${count} -gt ${gatord_timeout} ]; then + local pid=$(adb shell ps|grep -E "\s+${proc}\s+"|awk '{print $2}') + if [ -n "${pid}" ]; then + adb shell su 0 kill -9 "${pid}" + sleep 2 + fi + else + sleep 1 + fi + done +} + +## Description: +## wait for the gatord daemon to finish, and +## pull the collected streamline data to the specified directory +## Usage: +## collect_streamline_data_post "${app_id}" "${target_dir}" +## Example: +## collect_streamline_data_post "Browser_0" "streamline_data" +## Note: +## This function will do nothing if COLLECT_STREAMLINE is not set to true +collect_streamline_data_post(){ + if [ "X${COLLECT_STREAMLINE}" != "Xtrue" ]; then + return + fi + local app_name=$1 && shift + if [ -z "${app_name}" ]; then + return + fi + local target_dir=$1 && shift + if [ -z "${target_dir}" ]; then + echo "Please specify the target directory where to save the data file" + return + fi + echo "Wait gatord to finish:$(date)" + wait_kill_gatord_finish + echo "Gatord findihed:$(date)" + adb shell su 0 chown -R shell:shell data/local/tmp/streamline + adb pull /data/local/tmp/streamline/${app_name}.apc ${target_dir}/${app_name}.apc + #streamline -analyze ${capture_dir} + #streamline -report -function ${apd_f} |tee ${parent_dir}/streamlineReport.txt +} + +# uninstall or kill the running package +func_kill_uninstall(){ + local app_apk=$1 && shift + local app_pkg=$1 && shift + if [ "X${app_apk}" != "XNULL" ];then + adb shell pm list packages | grep -q "${app_pkg}" + if [ $? -eq 0 ]; then + adb uninstall "${app_pkg}" + fi + else + adb shell am force-stop "${app_pkg}" + fi +} + +func_pre_install(){ + # clean logcat buffer + adb logcat -c + adb logcat -b events -c + sleep 3 + + func_kill_uninstall "${loop_app_apk}" "${loop_app_package}" +} + +func_post_install(){ + collect_streamline_data_before "${loop_app_name}_${loop_count}" + adb shell am kill-all + sleep 5 +} + +func_run_test(){ + if [ -n "${var_test_command}" ]; then + timeout ${var_test_command_timeout} ${var_test_command} + local ret=$? + if [ $ret -eq 124 ]; then + echo "Time out to run ${var_test_command}: ${var_test_command_timeout}" + fi + sleep 5 + return $ret + fi +} + +func_pre_uninstall(){ + collect_streamline_data_post "${loop_app_name}_${loop_count}" "${D_STREAMLINE}" + # capture screen shot + adb shell screencap /data/local/tmp/app_screen.png + adb pull /data/local/tmp/app_screen.png ${D_SCREENSHOT}/${loop_app_name}_${loop_count}.png +} + +collect_raw_logcat_data(){ + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${F_LOGCAT}" + adb logcat -d -v time >> "${F_LOGCAT}" + echo "===package=${loop_app_package}, count=${loop_count} end" >> "${F_LOGCAT}" + echo "===ackage=${loop_app_package}, count=${loop_count} start" >> "${F_LOGCAT_EVENTS}" + adb logcat -d -b events -v time >> "${F_LOGCAT_EVENTS}" + echo "===package=${loop_app_package}, count=${loop_count} start" >> "${F_LOGCAT_EVENTS}" +} + +func_post_uninstall(){ + collect_raw_logcat_data +} + +## Description: +## run test for the specified applications for the specified times +## Usage: +## func_loop_apps_for_times $app_list $loop_times +## app_list: space separated application list, each application should +## be specified as following format: +## apk_name,apk_package/start_activity_name,nickname +## like: +## 03-JBench.apk,it.JBench.bench/it.JBench.jbench.MainActivity,JBench +## or: +## NULL,com.android.settings/.Settings,Settings +## NULL in the apk_name place holder means no need to install the application +## loop_times: the count that the test should be run +## Example: +## func_loop_apps_for_times "03-JBench.apk,it.JBench.bench/it.JBench.jbench.MainActivity,JBench NULL,com.android.settings/.Settings,Settings" 12 +## Note: +## var_func_pre_install +## var_func_post_install +## var_func_run_test +## var_func_pre_uninstall +## var_func_post_uninstall +## variables of loop_app_apk/loop_app_start_activity/loop_app_package/loop_app_name/loop_count can be used in the above functions +func_loop_apps_for_times(){ + local loop_apps_list=$1 && shift + local loop_apps_times=$1 && shift + + for apk in ${loop_apps_list}; do + local loop_app_apk=$(echo $apk|cut -d, -f1) + local loop_app_start_activity=$(echo $apk|cut -d, -f2) + local loop_app_package=$(echo ${loop_app_start_activity}|cut -d\/ -f1) + local loop_app_name=$(echo $apk|cut -d, -f3) + local loop_count=0 + + while [ ${loop_count} -lt ${loop_apps_times} ]; do + ## steps before install the apk, + ## like clean the logcat buffer or uninstall the application first + ## in case failed to uninstall last time + if [ -n "${var_func_pre_install}" ];then + ${var_func_pre_install} + else + func_pre_install + fi + + # install apk + if [ "X${loop_app_apk}" != "XNULL" ];then + adb install -r "${D_APKS}/${loop_app_apk}" + if [ $? -ne 0 ]; then + echo "Failed to install ${loop_app_apk}." + return 1 + fi + fi + + # run steps after install but before start the activiety + # since for applications we need prepare something like configuration + # so that to make the application work + if [ -n "${var_func_post_install}" ]; then + ${var_func_post_install} + else + func_post_install + fi + + # start activity + adb shell am start -W -S ${loop_app_start_activity} + + # run test steps after started the activity + if [ -n "${var_func_run_test}" ]; then + ${var_func_run_test} + else + func_run_test + fi + + if [ -n "${var_func_pre_uninstall}" ]; then + ${var_func_pre_uninstall} + else + func_pre_uninstall + fi + + # uninstall or kill the app process + func_kill_uninstall "${loop_app_apk}" "${loop_app_package}" + + if [ -n "${var_func_post_uninstall}" ]; then + ${var_func_post_uninstall} + else + func_post_uninstall + fi + + loop_count=$((loop_count + 1 )) + done + done +} + +func_get_all_apks(){ + local loop_apps_list=$1 && shift + for apk in ${loop_apps_list}; do + local app_apk=$(echo $apk|cut -d, -f1) + app_apk=$(echo "$app_apk"|sed 's/\s*$//' |sed 's/^\s*//') + if [ -z "${app_apk}" ]; then + echo "Either the apk file name or NULL must be specified for one application" + echo "This application configuration is not valid: $apk" + return 1 + fi + if [ "X${app_apk}" = "XNULL" ]; then + continue + fi + get_file_with_base_url "${app_apk}" "${BASE_URL}" "${D_APKS}"|| return 1 + done + return 0 +} + +func_prepare_environment(){ + if [ -n "${SERIAL}" ];then + ANDROID_SERIAL=$SERIAL + export ANDROID_SERIAL + else + serial=`adb get-serialno|sed 's/\r//g'` + if [ "X${serial}" == "Xunknown" ]; then + echo "Can not get the serial number autotically," + echo "Please specify the serial number with the -s option" + exit 1 + else + export ANDROID_SERIAL=${serial} + fi + fi + + rm -fr ${D_RAWDATA} + mkdir -p "${D_STREAMLINE}" "${D_SCREENSHOT}" "${D_APKS}" + + func_get_all_apks "$G_APPS"|| exit 1 + + adb shell svc power stayon true +} + +func_print_usage_common(){ + echo "$(basename $0) --base-url BASE_URL [--serial serial_no] [--loop-count LOOP_COUNT] [--streamline true|false] APP_CONFIG_LIST ..." + echo " --serial: specify serial number for the device" + echo " --base-url: specify the based url where the apks will be gotten from" + echo " --loop-count: specify the number that how many times should be run for each application to get the average result, default is 12" + echo " --streamline: specify if we need to collect the streamline data, true amd false can be specified, default is fasle" + echo " APP_CONFIG_LIST: specify the configurations for each application as following format:" + echo " APK_NAME,PACKAGE_NAME/APP_ACTIVITY,APP_NICKNAME" + echo " APK_NAME: the apk file name which we will get from the BASE_URL specified by --base-url option." + echo " if NULL is specified, it means this APK is a system apk, no need to get from BASE_URL" + echo " APP_PACKAGE: the package name for this application" + echo " APP_ACTIVITY: the activity name will be started for this application" + echo " APP_NICKNAME: a nickname for this application, should only contain alphabet and digits" +} + +func_parse_parameters_common(){ + local para_loop_count="" + local para_apps="" + while [ -n "$1" ]; do + case "X$1" in + X--base-url) + BASE_URL=$2 + shift 2 + ;; + X--streamline) + COLLECT_STREAMLINE=$2 + shift 2 + ;; + X--loop-count) + para_loop_count=$2 + shift 2 + ;; + X-s|X--serial) + SERIAL=$2 + shift 2 + ;; + X-h|X--help) + func_print_usage_common + exit 1 + ;; + X-*) + echo "Unknown option: $1" + func_print_usage_common + exit 1 + ;; + X*) + para_apps="${para_apps} $1" + shift 1 + ;; + esac + done + + if [ -z "${BASE_URL}" ]; then + echo "BASE_URL information must be specified." + exit 1 + fi + if [ -n "${para_loop_count}" ] && echo "${para_loop_count}"| grep -q -P '^\d+$'; then + G_LOOP_COUNT=${para_loop_count} + elif [ -n "${para_loop_count}" ]; then + echo "The specified LOOP_COUNT(${para_loop_count}) is not valid" + exit 1 + fi + + para_apps=$(echo "$para_apps"|sed 's/\s*$//' |sed 's/^\s*//') + if [ -n "${para_apps}" ]; then + G_APPS="${para_apps}" + fi +} + +## Description: +## main function to run for all application tests, both normal acitivity start test +## or benchmark things tests +## Usage: +## +## Example: +## +## Note: +## all apks should be able to get with the same base url +## var_func_parse_parameters +## var_func_prepare_environment +## var_func_post_test +common_main(){ + if [ -n "${var_func_parse_parameters}" ]; then + ${var_func_parse_parameters} "$@" + if [ $? -ne 0 ]; then + echo "Failed to parse the parameters!" + if [ -n "${var_func_usage}" ]; then + ${var_func_usage} + fi + exit 1 + fi + else + func_parse_parameters_common + fi + + if [ -n "${var_func_prepare_environment}" ]; then + "${var_func_prepare_environment}" + if [ $? -ne 0 ]; then + echo "Failed to prepare the environment!" + exit 1 + fi + else + func_prepare_environment + fi + + func_loop_apps_for_times "${G_APPS}" "${G_LOOP_COUNT}" + + if [ -n "${var_func_post_test}" ]; then + "${var_func_post_test}" + if [ $? -ne 0 ]; then + echo "Failed to do actions after the test!" + exit 1 + fi + fi + + rm -fr "${F_RAWDAT_ZIP}" + local old_pwd=$(pwd) + local d_zip_dir="$(dirname ${D_RAWDATA})" + local d_zip_name="$(basename ${D_RAWDATA})" + cd ${d_zip_dir}; + zip -r "${F_RAWDAT_ZIP}" "${d_zip_name}" + cd ${old_pwd} + echo "Please reference the file ${F_RAWDAT_ZIP} for all the raw data." +} diff --git a/common/statistic_average.sh b/common/statistic_average.sh new file mode 100644 index 0000000..31ec3d9 --- /dev/null +++ b/common/statistic_average.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +## Description: +## output the max value of the passed 2 parameters +## Usage: +## f_max "${val1}" "${val2}" +## Example: +## max=$(f_max "1.5" "2.0") +f_max(){ + local val1=$1 && shift + local val2=$1 && shift + [ -z "$val1" ] && return $val2 + [ -z "$val2" ] && return $val1 + + local compare=$(echo "$val1>$val2"|bc) + if [ "X$compare" = "X1" ];then + echo $val1 + else + echo $val2 + fi +} + +## Description: +## output the min value of the passed 2 parameters +## Usage: +## f_min "${val1}" "${val2}" +## Example: +## min=$(f_min "1.5" "2.0") +f_min(){ + local val1=$1 && shift + local val2=$1 && shift + [ -z "$val1" ] && return $val1 + [ -z "$val2" ] && return $val2 + + local compare=$(echo "$val1<$val2"|bc) + if [ "X$compare" = "X1" ];then + echo $val1 + else + echo $val2 + fi +} + +## Description: +## calculate the average value for specified csv file. +## The first field of that csv file should be the key/name of that line, +## Lines have the same key should be together. +## Usage: +## statistic "${csv_file_path}" "${file_number}" +## Example: +## statistic "$f_res_starttime" 2 +## Note: +## if less than 4 samples for that key/item there, average will be calculated as total/count +## if 4 or more samples for that key/item there, average will be calculated with max and min excluded +statistic(){ + local f_data=$1 && shift + if ! [ -f "$f_data" ]; then + return + fi + local field_no=$1 && shift + if [ -z "$field_no" ]; then + field_no=2 + fi + local total=0 + local max=0 + local min=0 + local old_key="" + local new_key="" + local count=0 + for line in $(cat "${f_data}"); do + if ! echo "$line"|grep -q ,; then + continue + fi + new_key=$(echo $line|cut -d, -f1) + value=$(echo $line|cut -d, -f${field_no}) + if [ "X${new_key}" = "X${old_key}" ]; then + total=$(echo "scale=2; ${total}+${value}"|bc -s) + count=$(echo "$count + 1"|bc) + max=$(f_max "$max" "$value") + min=$(f_min "$min" "$value") + else + if [ "X${old_key}" != "X" ]; then + if [ $count -ge 4 ]; then + average=$(echo "scale=2; ($total-$max-$min)/($count-2)"|bc) + else + average=$(echo "scale=2; $total/$count"|bc) + fi + echo "$old_key=$average" + fi + total="${value}" + max="${value}" + min="${value}" + old_key="${new_key}" + count=1 + fi + done + if [ "X${new_key}" != "X" ]; then + if [ $count -ge 4 ]; then + average=$(echo "scale=2; ($total-$max-$min)/($count-2)"|bc) + else + average=$(echo "scale=2; $total/$count"|bc) + fi + echo "$new_key=$average" + fi +} diff --git a/gearses1eclair/vc.py b/gearses1eclair/vc.py index 332a175..506843e 100755 --- a/gearses1eclair/vc.py +++ b/gearses1eclair/vc.py @@ -1,16 +1,19 @@ # Author: Botao Sun <botao.sun@linaro.org> +import os import sys import time from subprocess import call +parent_dir = os.path.realpath(os.path.dirname(__file__)) + def collect_score(benchmark_name, run_result, score_number, score_unit): call(['lava-test-case', benchmark_name, '--result', run_result, '--measurement', str(score_number), '--units', score_unit]) benchmark_name = "GearsES1eclair" time.sleep(60) -call_return = call(['./get_raw_logcat.sh']) +call_return = call(['%s/get_raw_logcat.sh' % parent_dir]) if call_return != 0: print "Capture real time logcat output failed!" sys.exit(1) diff --git a/gearses2eclair/vc.py b/gearses2eclair/vc.py index bfd37b1..6447dd3 100755 --- a/gearses2eclair/vc.py +++ b/gearses2eclair/vc.py @@ -1,16 +1,20 @@ # Author: Botao Sun <botao.sun@linaro.org> +import os import sys import time from subprocess import call +parent_dir = os.path.realpath(os.path.dirname(__file__)) + + def collect_score(benchmark_name, run_result, score_number, score_unit): call(['lava-test-case', benchmark_name, '--result', run_result, '--measurement', str(score_number), '--units', score_unit]) benchmark_name = "GearsES2eclair" time.sleep(60) -call_return = call(['./get_raw_logcat.sh']) +call_return = call(['%s/get_raw_logcat.sh' % parent_dir]) if call_return != 0: print "Capture real time logcat output failed!" sys.exit(1) diff --git a/geekbench3/execute.sh b/geekbench3/execute.sh index 6c43511..86db159 100755 --- a/geekbench3/execute.sh +++ b/geekbench3/execute.sh @@ -3,16 +3,6 @@ # need to be defined for different benchmark apks activity="com.primatelabs.geekbench3/.HomeActivity" apk_file_name="com.primatelabs.geekbench3.apk" - -# The first added parameter has been reserved by Android View Client. -# In order to add customised parameter, the first one must be the serial number from ADB -if [ -z "$1" ]; then - device_serial_number=`adb get-serialno` -else - device_serial_number=$1 -fi - -test_method="python vc.py $device_serial_number $excluded_test_suite" apk_package="com.primatelabs.geekbench3" # following should no need to modify diff --git a/geekbench3/vc.py b/geekbench3/vc.py index 050e755..7c77006 100755 --- a/geekbench3/vc.py +++ b/geekbench3/vc.py @@ -5,6 +5,8 @@ import time from subprocess import call from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException +curdir = os.path.realpath(os.path.dirname(__file__)) + def collect_score(testcase, run_result): call(['lava-test-case', testcase, '--result', run_result]) @@ -36,6 +38,8 @@ kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': Fal vc = ViewClient(device, serialno, **kwargs2) try: + time.sleep(2) + vc.dump() trigger = vc.findViewByIdOrRaise(package_name + ":id/runBenchmarks") trigger.touch() print "Geekbench 3 Test Started!" @@ -82,7 +86,7 @@ else: sys.exit(1) # Get and parse the result file -call(['./get_result.sh']) +call(['%s/get_result.sh' % curdir ]) if os.path.exists(raw_output_file) == True: logfile = open(raw_output_file, "r") for line in logfile: diff --git a/glbenchmark-2.5.1/execute.sh b/glbenchmark-2.5.1/execute.sh index 9c8c824..984d527 100755 --- a/glbenchmark-2.5.1/execute.sh +++ b/glbenchmark-2.5.1/execute.sh @@ -3,15 +3,6 @@ # need to be defined for different benchmark apks activity="com.glbenchmark.glbenchmark25/com.glbenchmark.activities.GLBenchmarkDownloaderActivity" apk_file_name="GLBenchmark_2.5.1.apk" -excluded_test_suite="C24Z24MS4" -# The first added parameter has been reserved by Android View Client. -# In order to add customised parameter, the first one must be the serial number from ADB -if [ -z "$1" ]; then - device_serial_number=`adb get-serialno` -else - device_serial_number=$1 -fi -test_method="python vc.py $device_serial_number $excluded_test_suite" apk_package="com.glbenchmark.glbenchmark25" # following should no need to modify diff --git a/glbenchmark-2.5.1/file_transfer.sh b/glbenchmark-2.5.1/file_transfer.sh index 531bb56..8f344c0 100755 --- a/glbenchmark-2.5.1/file_transfer.sh +++ b/glbenchmark-2.5.1/file_transfer.sh @@ -2,6 +2,8 @@ # Author: Botao Sun <botao.sun@linaro.org> # Transfer obb and pre-config file from host to target device +local_this_parent="$(cd $(dirname $0);pwd)" + obb_basic_path="/storage/sdcard0/Android" config_basic_path="/data/data/com.glbenchmark.glbenchmark25" errorword="No such" @@ -56,5 +58,5 @@ function push_config(){ fi } -push_obb "main.1.com.glbenchmark.glbenchmark25.obb" "$obb_basic_path/obb/com.glbenchmark.glbenchmark25/main.1.com.glbenchmark.glbenchmark25.obb" -push_config "com.glbenchmark.glbenchmark25_preferences.xml" "$config_basic_path/shared_prefs/com.glbenchmark.glbenchmark25_preferences.xml" +push_obb "${local_this_parent}/main.1.com.glbenchmark.glbenchmark25.obb" "$obb_basic_path/obb/com.glbenchmark.glbenchmark25/main.1.com.glbenchmark.glbenchmark25.obb" +push_config "${local_this_parent}/com.glbenchmark.glbenchmark25_preferences.xml" "$config_basic_path/shared_prefs/com.glbenchmark.glbenchmark25_preferences.xml" diff --git a/glbenchmark-2.5.1/preparation.sh b/glbenchmark-2.5.1/preparation.sh index 2b6a616..496b3d0 100755 --- a/glbenchmark-2.5.1/preparation.sh +++ b/glbenchmark-2.5.1/preparation.sh @@ -1,8 +1,9 @@ #!/bin/bash # Author: Botao Sun <botao.sun@linaro.org> -./file_transfer.sh +local_this_parent="$(cd $(dirname $0);pwd)" +${local_this_parent}/file_transfer.sh sleep 2 -python glbenchmark_first_launch.py +python ${local_this_parent}/glbenchmark_first_launch.py sleep 2 diff --git a/glbenchmark-2.5.1/vc.py b/glbenchmark-2.5.1/vc.py index 901680a..384a6f4 100755 --- a/glbenchmark-2.5.1/vc.py +++ b/glbenchmark-2.5.1/vc.py @@ -1,5 +1,6 @@ # Author: Botao Sun <botao.sun@linaro.org> +import os import sys import time import re @@ -7,6 +8,8 @@ import xml.dom.minidom from subprocess import call from com.dtmilano.android.viewclient import ViewClient +curdir = os.path.realpath(os.path.dirname(__file__)) + def collect_score(benchmark_name, run_result, score_number, score_unit): call(['lava-test-case', benchmark_name, '--result', run_result, '--measurement', str(score_number), '--units', score_unit]) @@ -64,21 +67,15 @@ time.sleep(3) # Disable crashed test suites vc.dump(window='-1') -if len(sys.argv) > 3: - print "Parameter Invalid!\nUsage: python vc.py <serialno> <disabled_test_suite>.\nTo run all test simply use python vc.py without any parameter or just with <serialno>.\nIt currently only supports to disable 1 test suite, more flexbility may will be added in future." - sys.exit(1) -elif len(sys.argv) == 3: - print "Test suite " + sys.argv[2] + " is going to be disabled!" - crashed_test_name = sys.argv[2] - crashed_test = vc.findViewWithText(crashed_test_name) - if crashed_test != None: - crashed_test.touch() - print "Test suite " + crashed_test_name + " has been excluded!" - time.sleep(2) - else: - print "Can not find test suite " + crashed_test_name + ", please check the screen!" -elif len(sys.argv) == 2 or len(sys.argv) == 1: - pass +crashed_test_name = "C24Z24MS4" +print "Test suite " + crashed_test_name + " is going to be disabled!" +crashed_test = vc.findViewWithText(crashed_test_name) +if crashed_test != None: + crashed_test.touch() + print "Test suite " + crashed_test_name + " has been excluded!" + time.sleep(2) +else: + print "Can not find test suite " + crashed_test_name + ", please check the screen!" # Start selected test suites start_button = vc.findViewByIdOrRaise("com.glbenchmark.glbenchmark25:id/buttonStart") @@ -103,5 +100,5 @@ while (not finished): else: print "GLBenchmark Test is still in progress..." -call_return = call(['./get_result.sh']) +call_return = call(['%s/get_result.sh' % curdir]) logparser(cached_result_file) diff --git a/smartbench2012/execute.sh b/smartbench2012/execute.sh index 1508e8a..6b0025c 100755 --- a/smartbench2012/execute.sh +++ b/smartbench2012/execute.sh @@ -9,5 +9,5 @@ apk_package="com.smartbench.twelve" #following should no need to modify parent_dir=`dirname ${0}` source "${parent_dir}/../common/common.sh" -pre_uninstall="${parent_dir}/extract_results.sh" +post_uninstall="${parent_dir}/extract_results.sh" main "$@" diff --git a/smartbench2012/extract_results.sh b/smartbench2012/extract_results.sh index 4609f51..e5b7e6f 100755 --- a/smartbench2012/extract_results.sh +++ b/smartbench2012/extract_results.sh @@ -1,6 +1,9 @@ -#!/bin/sh +#!/bin/bash -SCORE=`awk -F'[][]' '/SBGlobal.nProductivityIndex/{k=$2}END{print k}' logcat.log` +local_this_parent="$(cd $(dirname $0);pwd)" +source "${local_this_parent}/../common/common.sh" + +SCORE=`awk -F'[][]' '/SBGlobal.nProductivityIndex/{k=$2}END{print k}' ${F_LOGCAT}` UNIT="points" echo "Score is: $SCORE" diff --git a/smartbench2012/vc.py b/smartbench2012/vc.py index 5dfcf49..0af611e 100755 --- a/smartbench2012/vc.py +++ b/smartbench2012/vc.py @@ -18,9 +18,16 @@ kwargs1 = {'verbose': False, 'ignoresecuredevice': False} device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} vc = ViewClient(device, serialno, **kwargs2) -vc.dump('-1') -btn_start = vc.findViewWithTextOrRaise("Run SmartBench") -btn_start.touch() +while True: + try: + vc.dump('-1') + btn_start = vc.findViewWithTextOrRaise("Run SmartBench") + btn_start.touch() + break + except RuntimeError: + pass + except ValueError: + pass finished = False btn_results = None @@ -33,5 +40,7 @@ while(not finished): finished = True except RuntimeError: pass + except ValueError: + pass print "benchmark finished" btn_results.touch() diff --git a/vellamo/execute.sh b/vellamo/execute.sh index dba503d..e75c12d 100755 --- a/vellamo/execute.sh +++ b/vellamo/execute.sh @@ -9,7 +9,7 @@ apk_package="com.quicinc.vellamo" #following should no need to modify parent_dir=`dirname ${0}` source "${parent_dir}/../common/common.sh" -#pre_uninstall="${parent_dir}/vellamo_getres.sh" +timeout=30m main "$@" diff --git a/vellamo/vc.py b/vellamo/vc.py index 5458e2d..63a90d9 100755 --- a/vellamo/vc.py +++ b/vellamo/vc.py @@ -8,6 +8,9 @@ from xml.etree import ElementTree from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException +parent_dir = os.path.realpath(os.path.dirname(__file__)) + + class AllEntities: def __getitem__(self, key): #key is your entity, you can do whatever you want with it here @@ -72,11 +75,15 @@ btn_setup_2.touch() vc.dump('-1') time.sleep(1) -#Discard low battery level dialog -btn_battery_3 = vc.findViewByIdOrRaise("android:id/button3") -btn_battery_3.touch() -vc.dump('-1') -time.sleep(1) +try: + #Discard low battery level dialog + btn_battery_3 = vc.findViewByIdOrRaise("android:id/button3") + btn_battery_3.touch() + vc.dump('-1') + time.sleep(1) +except ViewNotFoundException: + # doesn't show up on all boards + pass #Discard no network connection try: @@ -112,7 +119,7 @@ btn_setup_3.touch() #Wait while Vellamo is running benchmark finished = False while (not finished): - time.sleep(1) + time.sleep(30) try: vc.dump(window='-1') vc.findViewByIdOrRaise("com.quicinc.vellamo:id/score_view") @@ -121,10 +128,13 @@ while (not finished): pass except RuntimeError as e: print e + pass + except ValueError: + pass print "Benchmark finished" -return_value = call(['./adb_pull.sh']) +return_value = call(['%s/adb_pull.sh' % parent_dir]) if (return_value == 0): extract_scores(filename='latest_result.html') else: diff --git a/vellamo3/execute.sh b/vellamo3/execute.sh index f639500..640ba0a 100755 --- a/vellamo3/execute.sh +++ b/vellamo3/execute.sh @@ -10,4 +10,5 @@ apk_package="com.quicinc.vellamo" # following should no need to modify parent_dir=`dirname ${0}` source "${parent_dir}/../common/common.sh" +timeout=30m main "$@" diff --git a/vellamo3/vc.py b/vellamo3/vc.py index 8a47ef0..7f52805 100755 --- a/vellamo3/vc.py +++ b/vellamo3/vc.py @@ -2,13 +2,15 @@ # Author: # Milosz Wasilewski <milosz.wasilewski@linaro.org> # Botao Sun <botao.sun@linaro.org> - +import os import sys import time import json from subprocess import call from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException +parent_dir = os.path.realpath(os.path.dirname(__file__)) + # Result collection for LAVA debug_switcher = False default_unit = 'Points' @@ -129,7 +131,7 @@ for chapter in chapters: device.press("KEYCODE_BACK") device.press("KEYCODE_BACK") -return_value = call(['./get_result.sh']) +return_value = call(['%s/get_result.sh' % parent_dir]) if return_value == 0: extract_scores(filename='chapterscores.json') else: |