From eefb30d1bdeced54618188e154c369927d030506 Mon Sep 17 00:00:00 2001 From: Yongqin Liu Date: Wed, 8 Apr 2015 15:20:11 +0800 Subject: remove bad benchmark tests according to the table here: https://docs.google.com/a/linaro.org/spreadsheets/d/1Lo3R3Y4iYSNiBmEg1YYdvpTXKQ8YASsawb0flMusv_c/edit#gid=0 Change-Id: I2e82ff55f22db9ed4eb06b17e913c1916e09e69e Signed-off-by: Yongqin Liu --- andebench/execute.sh | 12 ---- andebench/session.xml | 3 - andebench/vc.py | 81 ---------------------- andebench2014/execute.sh | 15 ---- andebench2014/vc.py | 46 ------------- antutu5/execute.sh | 16 ----- antutu5/vc.py | 66 ------------------ caffeinemark/execute.sh | 12 ---- caffeinemark/session.xml | 3 - caffeinemark/vc.py | 73 -------------------- gearses1eclair/execute.sh | 12 ---- gearses1eclair/get_raw_logcat.sh | 15 ---- gearses1eclair/session.xml | 3 - gearses1eclair/vc.py | 53 -------------- geekbench2/execute.sh | 12 ---- geekbench2/session.xml | 3 - geekbench2/vc.py | 58 ---------------- nbench/execute.sh | 12 ---- nbench/session.xml | 3 - nbench/vc.py | 54 --------------- smartbench2012/execute.sh | 13 ---- smartbench2012/extract_results.sh | 10 --- smartbench2012/session.xml | 3 - smartbench2012/vc.py | 50 -------------- vellamo/adb_pull.sh | 12 ---- vellamo/execute.sh | 17 ----- vellamo/vc.py | 141 -------------------------------------- 27 files changed, 798 deletions(-) delete mode 100755 andebench/execute.sh delete mode 100644 andebench/session.xml delete mode 100644 andebench/vc.py delete mode 100755 andebench2014/execute.sh delete mode 100644 andebench2014/vc.py delete mode 100755 antutu5/execute.sh delete mode 100755 antutu5/vc.py delete mode 100755 caffeinemark/execute.sh delete mode 100644 caffeinemark/session.xml delete mode 100755 caffeinemark/vc.py delete mode 100755 gearses1eclair/execute.sh delete mode 100755 gearses1eclair/get_raw_logcat.sh delete mode 100644 gearses1eclair/session.xml delete mode 100755 gearses1eclair/vc.py delete mode 100755 geekbench2/execute.sh delete mode 100644 geekbench2/session.xml delete mode 100755 geekbench2/vc.py delete mode 100755 nbench/execute.sh delete mode 100644 nbench/session.xml delete mode 100755 nbench/vc.py delete mode 100755 smartbench2012/execute.sh delete mode 100755 smartbench2012/extract_results.sh delete mode 100644 smartbench2012/session.xml delete mode 100755 smartbench2012/vc.py delete mode 100755 vellamo/adb_pull.sh delete mode 100755 vellamo/execute.sh delete mode 100755 vellamo/vc.py diff --git a/andebench/execute.sh b/andebench/execute.sh deleted file mode 100755 index 7804acd..0000000 --- a/andebench/execute.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="com.eembc.coremark/.tabs" -apk_file_name="com.eembc.coremark-1.apk" -test_method="python vc.py" -apk_package="com.eembc.coremark" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" -main "$@" diff --git a/andebench/session.xml b/andebench/session.xml deleted file mode 100644 index 5579d1c..0000000 --- a/andebench/session.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/andebench/vc.py b/andebench/vc.py deleted file mode 100644 index 6b0ada9..0000000 --- a/andebench/vc.py +++ /dev/null @@ -1,81 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call - -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -kwargs1 = {'verbose': False, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} -vc = ViewClient(device, serialno, **kwargs2) -vc.dump('-1') -btn_start_on = vc.findViewByIdOrRaise("com.eembc.coremark:id/btn_start_on") -btn_start_on.touch() -finished = False -while(not finished): - try: - time.sleep(5) - vc.dump('-1') - results = vc.findViewByIdOrRaise("com.eembc.coremark:id/cid") - if not results.getText().find("Running") > 0: - finished = True - print "benchmark finished" - result_re = re.compile("(?P[a-zA-Z\s]+):\s(?P\d+)", re.MULTILINE) - search_results = result_re.finditer(results.getText()) - for result in search_results: - call(['lava-test-case', result.group('test_case_id'), '--result', 'pass', '--measurement', result.group('measurement'), '--units', 'Iterations/sec']) - except ViewNotFoundException: - pass - except RuntimeError: - pass - -""" -try: - progress_button = vc.findViewByIdOrRaise("com.eembc.coremark:id/btn_progress2") -except ViewNotFoundException: - pass - -try: - progress_button = vc.findViewByIdOrRaise("com.eembc.coremark:id/btn_progress1") -except ViewNotFoundException: - pass - -if not progress_button: - sys.exit(1) -progress_found = True - -while(progress_found): - try: - time.sleep(3) - vc.dump('-1') - except RuntimeError: - pass - found1 = True - found2 = True - try: - progress_button = vc.findViewByIdOrRaise("com.eembc.coremark:id/btn_progress1") - except ViewNotFoundException: - found1 = False - try: - progress_button = vc.findViewByIdOrRaise("com.eembc.coremark:id/btn_progress2") - except ViewNotFoundException: - found2 = False - - if not (found1 or found2): - progress_found = False -print "benchmark finished" - -time.sleep(3) -vc.dump('-1') -results = vc.findViewByIdOrRaise("com.eembc.coremark:id/cid") -results_text = results.getText() -#Results in Iterations/sec: -#AndEMark Native: 6335 -#AndEMark Java: 377 -result_re = re.compile("^(?P[a-zA-Z\s]+):\s(?P\d+)", re.MULTILINE) -search_results = result_re.finditer(results_text) -for result in search_results: - call(['lava-test-case', result.group('test_case_id'), '--result', 'pass', '--measurement', result.group('measurement'), '--units', 'Iterations/sec']) -""" diff --git a/andebench2014/execute.sh b/andebench2014/execute.sh deleted file mode 100755 index 1bb51ac..0000000 --- a/andebench2014/execute.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -apk_package="com.eembc.coremark" -activity="${apk_package}/.tabs" -apk_file_name="02-AndEBench2014.apk" -test_method="python vc.py" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" - -base_url="http://testdata.validation.linaro.org/apks/JavaBenchmark/non-pure-java-benchmarks/" - -main "$@" diff --git a/andebench2014/vc.py b/andebench2014/vc.py deleted file mode 100644 index d04ba33..0000000 --- a/andebench2014/vc.py +++ /dev/null @@ -1,46 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call - -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -parent_dir = os.path.realpath(os.path.dirname(__file__)) -f_output_result="%s/../common/output-test-result.sh" % parent_dir - -kwargs1 = {'verbose': False, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} -vc = ViewClient(device, serialno, **kwargs2) -vc.dump() - -btn_start_on = vc.findViewByIdOrRaise("com.eembc.coremark:id/btn_start_on") -btn_start_on.touch() -finished = False -while(not finished): - try: - time.sleep(20) - vc.dump() - btn_close = vc.findViewById("com.eembc.coremark:id/btn_close") - if btn_close: - btn_close.touch() - continue - - results = vc.findViewByIdOrRaise("com.eembc.coremark:id/cid") - if not results.getText().find("Running") > 0: - finished = True - print "benchmark finished" - result_re = re.compile("(?P[a-zA-Z\s]+):\s(?P\d+)", re.MULTILINE) - search_results = result_re.finditer(results.getText()) - for result in search_results: - test_case_id = result.group('test_case_id').strip() - measurement = result.group('measurement').strip() - print "%s=%s Iterations/sec" % (test_case_id, measurement) - call([f_output_result, test_case_id, 'pass', measurement, 'Iterations/sec']) - except ViewNotFoundException: - pass - except RuntimeError: - pass - except ValueError: - pass diff --git a/antutu5/execute.sh b/antutu5/execute.sh deleted file mode 100755 index 264f2de..0000000 --- a/antutu5/execute.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="com.antutu.ABenchMark5/.ABenchMarkStart" -apk_file_name="antutu-benchmark-v5-alpha.apk" -test_method="python vc.py" -apk_package="com.antutu.ABenchMark5" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" -main "$@" - - - - diff --git a/antutu5/vc.py b/antutu5/vc.py deleted file mode 100755 index bad6819..0000000 --- a/antutu5/vc.py +++ /dev/null @@ -1,66 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call - -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -kwargs1 = {'verbose': False, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} -vc = ViewClient(device, serialno, **kwargs2) - -#Wait while application loads -time.sleep(2) - -#Start test button -vc.dump(window='-1') -start_button = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/test_btn") -start_button.touch() - -#Start all test button -vc.dump(window='-1') -start_test_button = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/start_test_text") -start_test_button.touch() - -#Wait while antutu4 is running benchmark -finished = False -while(not finished): - time.sleep(1) - vc.dump('-1') - try: - progress_button = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/total_score_layout") - finished = True - except ViewNotFoundException: - pass -print("Benchmark Finished") - -#Get the score -vc.dump(window='-1') -multitask_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/ue_multitask_text") -runtime_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/ue_dalvik_text") -ram_operation_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/mem_text") -ram_speed_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/ram_text") -cpu_multi_integer_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/cpu_int_text") -cpu_multi_float_point_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/cpu_float_text") -cpu_single_integer_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/cpu_int_text2") -cpu_single_float_point_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/cpu_float_text2") -twod_graphics_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/gpu_2d_text") -threed_graphics_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/gpu_3d_text") -storage_io_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/io_sdw_text") -database_io_score = vc.findViewByIdOrRaise("com.antutu.ABenchMark5:id/io_db_text") -default_unit = 'Inapplicable' - -call(['lava-test-case', '"AnTuTu 5.0 Alpha UX Multitask Score"', '--result', 'pass', '--measurement', multitask_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha UX Runtime Score"', '--result', 'pass', '--measurement', runtime_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha RAM Operation Score"', '--result', 'pass', '--measurement', ram_operation_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha RAM Speed Score"', '--result', 'pass', '--measurement', ram_speed_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha Multi Core CPU Integer Score"', '--result', 'pass', '--measurement', cpu_multi_integer_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha Multi Core CPU Float-Point Score"', '--result', 'pass', '--measurement', cpu_multi_float_point_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha Single Thread CPU Integer Score"', '--result', 'pass', '--measurement', cpu_single_integer_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha Single Thread CPU Float-Point Score"', '--result', 'pass', '--measurement', cpu_single_float_point_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha GPU 2D Graphics Score"', '--result', 'pass', '--measurement', twod_graphics_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha GPU 3D Graphics Score"', '--result', 'pass', '--measurement', threed_graphics_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha IO Storage I/O Score"', '--result', 'pass', '--measurement', storage_io_score.getText(), '--units', default_unit]) -call(['lava-test-case', '"AnTuTu 5.0 Alpha IO Database I/O Score"', '--result', 'pass', '--measurement', database_io_score.getText(), '--units', default_unit]) diff --git a/caffeinemark/execute.sh b/caffeinemark/execute.sh deleted file mode 100755 index 60fb333..0000000 --- a/caffeinemark/execute.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="com.flexycore.caffeinemark/.Application" -apk_file_name="com.flexycore.caffeinemark-1.apk" -test_method="python vc.py" -apk_package="com.flexycore.caffeinemark" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" -main "$@" diff --git a/caffeinemark/session.xml b/caffeinemark/session.xml deleted file mode 100644 index 6223cb6..0000000 --- a/caffeinemark/session.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/caffeinemark/vc.py b/caffeinemark/vc.py deleted file mode 100755 index fb70f1a..0000000 --- a/caffeinemark/vc.py +++ /dev/null @@ -1,73 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call - -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -# Result collection for LAVA -debug_switcher = False -default_unit = 'points' -result = 'pass' -def collect_score(score_name, result, score, default_unit): - if debug_switcher == False: - call(['lava-test-case', score_name, '--result', result, '--measurement', score, '--unit', default_unit]) - else: - print ['lava-test-case', score_name, '--result', result, '--measurement', score, '--unit', default_unit] - -kwargs1 = {'verbose': False, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} -vc = ViewClient(device, serialno, **kwargs2) - -time.sleep(2) -vc.dump(window='-1') -start_button = vc.findViewByIdOrRaise("com.flexycore.caffeinemark:id/startButton") -start_button.touch() - -finished = False -while (not finished): - time.sleep(1) - try: - vc.dump(window='-1') - vc.findViewByIdOrRaise("com.flexycore.caffeinemark:id/testResultsCellOneTitle") - finished = True - except ViewNotFoundException: - pass - except RuntimeError as e: - print e -print "benchmark finished" - -total_score = vc.findViewByIdOrRaise("com.flexycore.caffeinemark:id/testResultEntryOverAllScore").getText() -collect_score("Caffeinemark-score", result, total_score, default_unit) - -details_button = vc.findViewByIdOrRaise("com.flexycore.caffeinemark:id/testResultsDetailsButton") -details_button.touch() - -time.sleep(2) -vc.dump(window='-1') - -sieve_name = vc.findViewByIdOrRaise("id/no_id/9").getText() -sieve_score = vc.findViewByIdOrRaise("id/no_id/10").getText() -collect_score("Caffeinemark-Sieve-score", result, sieve_score, default_unit) - -loop_name = vc.findViewByIdOrRaise("id/no_id/13").getText() -loop_score = vc.findViewByIdOrRaise("id/no_id/14").getText() -collect_score("Caffeinemark-Loop-score", result, loop_score, default_unit) - -logic_name = vc.findViewByIdOrRaise("id/no_id/17").getText() -logic_score = vc.findViewByIdOrRaise("id/no_id/18").getText() -collect_score("Caffeinemark-Collect-score", result, logic_score, default_unit) - -string_name = vc.findViewByIdOrRaise("id/no_id/21").getText() -string_score = vc.findViewByIdOrRaise("id/no_id/22").getText() -collect_score("Caffeinemark-String-score", result, string_score, default_unit) - -float_name = vc.findViewByIdOrRaise("id/no_id/25").getText() -float_score = vc.findViewByIdOrRaise("id/no_id/26").getText() -collect_score("Caffeinemark-Float-score", result, float_score, default_unit) - -method_name = vc.findViewByIdOrRaise("id/no_id/29").getText() -method_score = vc.findViewByIdOrRaise("id/no_id/30").getText() -collect_score("Caffeinemark-Method-score", result, method_score, default_unit) diff --git a/gearses1eclair/execute.sh b/gearses1eclair/execute.sh deleted file mode 100755 index 5b9436d..0000000 --- a/gearses1eclair/execute.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="com.jeffboody.GearsES1eclair/.GearsES1eclair" -apk_file_name="GearsES1eclair-20110501.apk" -test_method="python vc.py" -apk_package="com.jeffboody.GearsES1eclair" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" -main "$@" diff --git a/gearses1eclair/get_raw_logcat.sh b/gearses1eclair/get_raw_logcat.sh deleted file mode 100755 index 8d9ce3c..0000000 --- a/gearses1eclair/get_raw_logcat.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -# Author: Botao Sun - -function get_raw_logcat(){ - adb logcat > $1 2>&1 & - raw_logcat_pid=$! - echo "Raw logcat output transfer in progress..." - sleep 120 - echo "Raw logcat output transfer finished!" - if [ -n "${raw_logcat_pid}" ]; then - kill -9 ${raw_logcat_pid} - fi -} - -get_raw_logcat "logcat_gearses1eclair.txt" diff --git a/gearses1eclair/session.xml b/gearses1eclair/session.xml deleted file mode 100644 index 0f189fd..0000000 --- a/gearses1eclair/session.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/gearses1eclair/vc.py b/gearses1eclair/vc.py deleted file mode 100755 index 506843e..0000000 --- a/gearses1eclair/vc.py +++ /dev/null @@ -1,53 +0,0 @@ -# Author: Botao Sun - -import os -import sys -import time -from subprocess import call - -parent_dir = os.path.realpath(os.path.dirname(__file__)) - -def collect_score(benchmark_name, run_result, score_number, score_unit): - call(['lava-test-case', benchmark_name, '--result', run_result, '--measurement', str(score_number), '--units', score_unit]) - -benchmark_name = "GearsES1eclair" -time.sleep(60) - -call_return = call(['%s/get_raw_logcat.sh' % parent_dir]) -if call_return != 0: - print "Capture real time logcat output failed!" - sys.exit(1) - -raw_output_file = "logcat_gearses1eclair.txt" -flagwordA = "a3d_GLES_dump" -flagwordB = "fps" -result_collector = [] - -logfile = open(raw_output_file, "r") -for line in logfile: - linelist = line.strip("\n").strip("\r").split(" ") - linelist = filter(None, linelist) - for itemA in linelist: - if itemA.find(flagwordA) != -1: - for itemB in linelist: - if itemB.find(flagwordB) != -1: - print linelist - for i in range(0, len(linelist)): - grouplist = linelist[i].split("=") - if len(grouplist) == 2 and grouplist[0] == flagwordB: - result_collector.append(grouplist[1]) -logfile.close() - -print result_collector -if len(result_collector) > 0: - average_fps = sum(float(element) for element in result_collector) / len(result_collector) - score_number = average_fps - run_result = "pass" - score_unit = flagwordB - print "The average FPS in this test run is " + str(score_number) -else: - print "The collector is empty, no actual result received!" - sys.exit(1) - -# Submit the test result to LAVA -collect_score(benchmark_name, run_result, score_number, score_unit) diff --git a/geekbench2/execute.sh b/geekbench2/execute.sh deleted file mode 100755 index 3f161c5..0000000 --- a/geekbench2/execute.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="ca.primatelabs.geekbench2/.HomeActivity" -apk_file_name="ca.primatelabs.geekbench2-1.apk" -test_method="python vc.py" -apk_package="ca.primatelabs.geekbench2" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" -main "$@" diff --git a/geekbench2/session.xml b/geekbench2/session.xml deleted file mode 100644 index a3fb1b0..0000000 --- a/geekbench2/session.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/geekbench2/vc.py b/geekbench2/vc.py deleted file mode 100755 index 1e192a3..0000000 --- a/geekbench2/vc.py +++ /dev/null @@ -1,58 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call - -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -default_unit = 'points' -def get_score_with_content_desc(vc, content_desc, offset=1): - score_view = vc.findViewWithContentDescriptionOrRaise(content_desc) - score_uid = score_view.getUniqueId() - uid = int(re.search("id/no_id/(?P\d+)", score_uid).group('uid')) - score = vc.findViewByIdOrRaise("id/no_id/%s" % (uid + offset)) - call(['lava-test-case', content_desc, '--result', 'pass', '--measurement', score.getContentDescription(), '--units', default_unit]) - -kwargs1 = {'verbose': False, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} -vc = ViewClient(device, serialno, **kwargs2) - -time.sleep(2) -vc.dump(window='-1') -start_button = vc.findViewByIdOrRaise("ca.primatelabs.geekbench2:id/runBenchmarks") -start_button.touch() - -time.sleep(2) -finished = False -while (not finished): - time.sleep(1) - try: - vc.dump(window='-1') - vc.findViewByIdOrRaise("android:id/progress") - except ViewNotFoundException: - finished = True - pass - except RuntimeError as e: - print e -print "benchmark finished" - -time.sleep(3) -# need to touch the screen to update the view structure -device.touch(300,300) -time.sleep(1) -device.drag((300,1000), (300,300), 500) -time.sleep(1) -device.drag((1000,300), (300,300), 500) -time.sleep(1) -device.press("KEYCODE_MENU") -time.sleep(1) -device.press("KEYCODE_MENU") -time.sleep(1) -vc.dump(window='-1') -get_score_with_content_desc(vc, "Geekbench Score", 4) -get_score_with_content_desc(vc, "Integer") -get_score_with_content_desc(vc, "Floating Point") -get_score_with_content_desc(vc, "Memory") -get_score_with_content_desc(vc, "Stream") diff --git a/nbench/execute.sh b/nbench/execute.sh deleted file mode 100755 index cbadcbc..0000000 --- a/nbench/execute.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="com.drolez.nbench/.MainActivity" -apk_file_name="com.drolez.nbench-1.apk" -test_method="python vc.py" -apk_package="com.drolez.nbench" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" -main "$@" diff --git a/nbench/session.xml b/nbench/session.xml deleted file mode 100644 index 7f5cb6d..0000000 --- a/nbench/session.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/nbench/vc.py b/nbench/vc.py deleted file mode 100755 index 61d938b..0000000 --- a/nbench/vc.py +++ /dev/null @@ -1,54 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -default_unit = 'points' - -kwargs1 = {'verbose': False, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} -vc = ViewClient(device, serialno, **kwargs2) - -time.sleep(2) -vc.dump(window='-1') -start_button = vc.findViewByIdOrRaise("com.drolez.nbench:id/start") -start_button.touch() - -# benchmark runs for 10 minutes -finished = False -while (not finished): - time.sleep(1) - vc.dump(window='-1') - start_button = vc.findViewByIdOrRaise("com.drolez.nbench:id/start") - if not start_button.getText().startswith("Benchmark running"): - finished = True - -time.sleep(2) -vc.dump(window='-1') -memory_score = vc.findViewByIdOrRaise("com.drolez.nbench:id/TextView01v") -integer_score = vc.findViewByIdOrRaise("com.drolez.nbench:id/TextView02v") -floating_score = vc.findViewByIdOrRaise("com.drolez.nbench:id/TextView03v") -results = vc.findViewByIdOrRaise("com.drolez.nbench:id/editor") - -call(['lava-test-case', 'Nbench memory score', '--result', 'pass', '--measurement', memory_score.getText(), '--units', default_unit]) -call(['lava-test-case', 'Nbench integer score', '--result', 'pass', '--measurement', integer_score.getText(), '--units', default_unit]) -call(['lava-test-case', 'Nbench floating point score', '--result', 'pass', '--measurement', floating_score.getText(), '--units', default_unit]) - -test_ids = ['NUMERIC SORT', - 'STRING SORT', - 'BITFIELD', - 'FP EMULATION', - 'FOURIER', - 'ASSIGNMENT', - 'IDEA', - 'HUFFMAN', - 'NEURAL NET', - 'LU DECOMPOSITION'] - -results_re = re.compile("^(?P[A-Z\s]+)\s+:\s+(?P[\d\.e\+]+)", re.MULTILINE) -for result in results_re.finditer(results.getText()): - if result.group('test_case_id').strip() in test_ids: - call(['lava-test-case', result.group('test_case_id').strip(), '--result', 'pass', '--measurement', result.group('measurement'), '--units', 'Iterations/sec']) diff --git a/smartbench2012/execute.sh b/smartbench2012/execute.sh deleted file mode 100755 index fa5c96a..0000000 --- a/smartbench2012/execute.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="com.smartbench.twelve/.Smartbench2012" -apk_file_name="Smartbench2012.apk" -test_method="python vc.py" -apk_package="com.smartbench.twelve" - -#following should no need to modify -parent_dir=$(cd $(dirname ${0});pwd) -source "${parent_dir}/../common/common.sh" -post_uninstall="${parent_dir}/extract_results.sh" -main "$@" diff --git a/smartbench2012/extract_results.sh b/smartbench2012/extract_results.sh deleted file mode 100755 index c015ed4..0000000 --- a/smartbench2012/extract_results.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -local_this_parent="$(cd $(dirname $0);pwd)" -source "${local_this_parent}/../common/common.sh" - -SCORE=`awk -F'[][]' '/SBGlobal.nProductivityIndex/{k=$2}END{print k}' ${F_LOGCAT}` -UNIT="points" - -echo "Score is: $SCORE" -output_test_result SmartbenchScore pass $SCORE $UNIT diff --git a/smartbench2012/session.xml b/smartbench2012/session.xml deleted file mode 100644 index ecca46b..0000000 --- a/smartbench2012/session.xml +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/smartbench2012/vc.py b/smartbench2012/vc.py deleted file mode 100755 index 0211596..0000000 --- a/smartbench2012/vc.py +++ /dev/null @@ -1,50 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call - -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -default_unit = 'points' -def get_score_with_text(vc, text, offset=1): - score_view = vc.findViewWithTextOrRaise(text) - score_uid = score_view.getUniqueId() - uid = int(re.search("id/no_id/(?P\d+)", score_uid).group('uid')) - score = vc.findViewByIdOrRaise("id/no_id/%s" % (uid + offset)) - call(['lava-test-case', text.strip(), '--result', 'pass', '--measurement', score.getText().strip(), '--units', default_unit]) - -kwargs1 = {'verbose': False, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} -vc = ViewClient(device, serialno, **kwargs2) -while True: - try: - time.sleep(5) - vc.dump('-1') - btn_start = vc.findViewWithTextOrRaise("Run SmartBench") - btn_start.touch() - break - except ViewNotFoundException: - pass - except RuntimeError: - pass - except ValueError: - pass - -finished = False -while not finished: - try: - time.sleep(5) - vc.dump('-1') - btn_results = vc.findViewWithTextOrRaise("Display Index Scores") - btn_results.touch() - time.sleep(3) - print "benchmark finished" - finished = True - except ViewNotFoundException: - pass - except RuntimeError: - pass - except ValueError: - pass diff --git a/vellamo/adb_pull.sh b/vellamo/adb_pull.sh deleted file mode 100755 index 79cda3a..0000000 --- a/vellamo/adb_pull.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - -#Required for N7 even after rooting -adb shell chmod 777 /data/data/com.quicinc.vellamo/files/latest_result.html - -#Pull results from device -adb pull /data/data/com.quicinc.vellamo/files/latest_result.html . - -if [ $? -ne 0 ]; then - echo "Failed to get the result of vellamo test" - exit 1 -fi diff --git a/vellamo/execute.sh b/vellamo/execute.sh deleted file mode 100755 index e75c12d..0000000 --- a/vellamo/execute.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -#need to be defined for different benchmark apks -activity="com.quicinc.vellamo/.Vellamo" -apk_file_name="com.quicinc.vellamo-1.apk" -test_method="python vc.py" -apk_package="com.quicinc.vellamo" - -#following should no need to modify -parent_dir=`dirname ${0}` -source "${parent_dir}/../common/common.sh" -timeout=30m -main "$@" - - - - diff --git a/vellamo/vc.py b/vellamo/vc.py deleted file mode 100755 index 63a90d9..0000000 --- a/vellamo/vc.py +++ /dev/null @@ -1,141 +0,0 @@ -import re -import sys -import os -import time -from subprocess import call -from HTMLParser import HTMLParser -from xml.etree import ElementTree - -from com.dtmilano.android.viewclient import ViewClient, ViewNotFoundException - -parent_dir = os.path.realpath(os.path.dirname(__file__)) - - -class AllEntities: - def __getitem__(self, key): - #key is your entity, you can do whatever you want with it here - return "" - -default_unit = 'points' -def extract_scores(filename): - - testf = open(filename, 'r') - - parser = ElementTree.XMLParser() - parser.parser.UseForeignDTD(True) - parser.entity = AllEntities() - - tree = ElementTree.parse(testf, parser=parser) - - # search for h2 header - totalscore = -1 - for node in tree.iter('h2'): - if node.text.startswith("Total Score"): - totalscore = node.getchildren()[0].text - - benchmarks = [] - for node in tree.iter('div'): - isscorenode = False - if node.find("./img") is not None: - isscorenode = True - if isscorenode: - scorename = node.find("./div").text.strip() - benchmark_dict = {'name': scorename, 'values': {}} - - for subscore in node.findall(".//li"): - key, value = subscore.text.split(":") - benchmark_dict['values'].update({key: value.strip()}) - benchmarks.append(benchmark_dict) - - call(['lava-test-case', "Vellamo 1.0.6", '--result', 'pass', '--measurement', totalscore, '--units', default_unit]) - for benchmark in benchmarks: - name = benchmark['name'] - result = 'pass' - if 'failed' in benchmark['values'].keys(): - result = 'fail' - for subbenchkey, subbenchvalue in benchmark['values'].items(): - if subbenchkey != 'failed': - call(['lava-test-case', "%s %s" % (name, subbenchkey), '--result', result, '--measurement', subbenchvalue, '--units', default_unit]) - -kwargs1 = {'verbose': True, 'ignoresecuredevice': False} -device, serialno = ViewClient.connectToDeviceOrExit(**kwargs1) -kwargs2 = {'startviewserver': True, 'forceviewserveruse': False, 'autodump': False, 'ignoreuiautomatorkilled': True, 'compresseddump': False} - -vc = ViewClient(device, serialno, **kwargs2) -vc.dump('-1') - -#Accept Vellamo EULA -btn_setup_1 = vc.findViewByIdOrRaise("android:id/button1") -btn_setup_1.touch() -vc.dump('-1') - -#Edit list of websites button -btn_setup_2 = vc.findViewByIdOrRaise("android:id/button2") -btn_setup_2.touch() -vc.dump('-1') -time.sleep(1) - -try: - #Discard low battery level dialog - btn_battery_3 = vc.findViewByIdOrRaise("android:id/button3") - btn_battery_3.touch() - vc.dump('-1') - time.sleep(1) -except ViewNotFoundException: - # doesn't show up on all boards - pass - -#Discard no network connection -try: - btn_network_3 = vc.findViewByIdOrRaise("android:id/button3") - btn_network_3.touch() - vc.dump('-1') - time.sleep(1) -except ViewNotFoundException: - # doesn't show up on all boards - pass - -#Disable safeguards -btn_more = vc.findViewWithTextOrRaise("More") -btn_more.touch() -vc.dump('-1') -time.sleep(1) - -btn_safeguards = vc.findViewWithTextOrRaise("Override Safeguards") -btn_safeguards.touch() -vc.dump('-1') -time.sleep(1) - -#Start Button -btn_start_on = vc.findViewWithTextOrRaise("Start") -btn_start_on.touch() -vc.dump('-1') -time.sleep(5) - -#Discard Enable Tutorial dialog -btn_setup_3 = vc.findViewByIdOrRaise("android:id/button2") -btn_setup_3.touch() - -#Wait while Vellamo is running benchmark -finished = False -while (not finished): - time.sleep(30) - try: - vc.dump(window='-1') - vc.findViewByIdOrRaise("com.quicinc.vellamo:id/score_view") - finished = True - except ViewNotFoundException: - pass - except RuntimeError as e: - print e - pass - except ValueError: - pass - -print "Benchmark finished" - -return_value = call(['%s/adb_pull.sh' % parent_dir]) -if (return_value == 0): - extract_scores(filename='latest_result.html') -else: - sys.exit(1) -- cgit v1.2.3