android apk-automation: add some improvements

1. print trace information as well when exception raised in module implementation

2. add sleep before calling of self.vc.dump() in dump_always, since dump_always
   might be called in while loop in the module implementation, and call dump_always
   immediately without sleep would cause failures

3. add sleep after the "am start" command as a workaround for the -W option,
   in the new tag r34, this -W option seems not work as expected which makes the am start
   commnd only return after the activity displayed completely.
   sometimes this command will return before the activity displayed completely.
   but this should be just a workaround for better fault tolerance

4. improved some module implementation for better fault tolerance

Change-Id: I1f9c6f0a3c40447de53b1b1d558c1dbc2e3164db
Signed-off-by: Yongqin Liu <yongqin.liu@linaro.org>

6 files changed, 81 insertions, 75 deletions

diff --git a/automated/android/apk-automation/common/__init__.py b/automated/android/apk-automation/common/__init__.py
index fafeec9..33814f0 100755
--- a/automated/android/apk-automation/common/__init__.py
+++ b/automated/android/apk-automation/common/__init__.py
@@ -68,7 +68,7 @@ class ApkTestRunner(object):
             except Exception as e:
                 self.take_screencap()
                 self.report_result(self.config['name'], 'fail')
-                self.logger.error(e)
+                self.logger.error(e, exc_info=True)
                 sys.exit(1)
 
         self.collect_log()
@@ -158,6 +158,7 @@ class ApkTestRunner(object):
         success = False
         while not success:
             try:
+                time.sleep(5)
                 self.vc.dump()
                 success = True
             except RuntimeError:
@@ -303,6 +304,7 @@ class ApkTestRunner(object):
         # Start intent.
         self.logger.info('Starting %s' % self.config['apk_package'])
         self.call_adb("shell am start -W -S %s" % self.config['activity'])
+        time.sleep(5)
 
     def execute(self):
         raise NotImplementedError
diff --git a/automated/android/apk-automation/geekbench3.py b/automated/android/apk-automation/geekbench3.py
index 658c662..ce99533 100755
--- a/automated/android/apk-automation/geekbench3.py
+++ b/automated/android/apk-automation/geekbench3.py
@@ -32,26 +32,21 @@ class ApkRunnerImpl(ApkTestRunner):
             self.all_fail()
             sys.exit(1)
 
-        time.sleep(10)
-        self.dump_always()
-
-        try:
-            self.vc.findViewByIdOrRaise("android:id/progress")
-        except ViewNotFoundException:
-            self.logger.error("Something goes wrong! It is unusual that the test has not been started after 10+ seconds! Please manually check it!")
-            self.all_fail()
-            sys.exit(1)
-
         finished = False
         while (not finished):
-            time.sleep(45)
+            time.sleep(10)
             self.dump_always()
             flag = self.vc.findViewWithText("RESULT")
+            in_progress = self.vc.findViewById("android:id/progress")
             if flag is not None:
                 self.logger.info("Geekbench 3 Test Finished!")
                 finished = True
-            else:
+            elif in_progress:
                 self.logger.info("Geekbench 3 Test is still in progress...")
+            else:
+                self.logger.error("Something goes wrong! It is unusual that the test has not been started after 10+ seconds! Please manually check it!")
+                #self.all_fail()
+                #sys.exit(1)
 
         # Generate the .gb3 file
         self.device.press('KEYCODE_MENU')
diff --git a/automated/android/apk-automation/glbenchmark25.py b/automated/android/apk-automation/glbenchmark25.py
index 6b7299e..e5bb116 100755
--- a/automated/android/apk-automation/glbenchmark25.py
+++ b/automated/android/apk-automation/glbenchmark25.py
@@ -36,42 +36,37 @@ class ApkRunnerImpl(ApkTestRunner):
         self.call_adb("shell am start -W -S %s" % self.config['activity'])
 
     def execute(self):
-        time.sleep(2)
-
-        self.vc.dump(window='-1')
-        test_type = self.vc.findViewWithText("Performance Tests")
-        if test_type:
-            test_type.touch()
-            time.sleep(2)
-
-        # By some reason in order to select all test, a back step is required
-        self.dump_always()
-        test_selection = self.vc.findViewByIdOrRaise("com.glbenchmark.glbenchmark25:id/buttonAll")
-        self.device.press('KEYCODE_BACK')
-        time.sleep(3)
-
-        test_type.touch()
-        time.sleep(2)
-        test_selection.touch()
-        self.logger.info("All selected!")
-        time.sleep(3)
+        selected_all = False
+        while not selected_all:
+            self.dump_always()
+            select_all_btn = self.vc.findViewWithText("All")
+            display_tests_menu = self.vc.findViewWithText("Performance Tests")
+            if select_all_btn:
+                select_all_btn.touch()
+                self.logger.info("All selected!")
+                selected_all = True
+            elif display_tests_menu:
+                display_tests_menu.touch()
+                self.logger.info("Display all tests to select all")
+            else:
+                # continue
+                pass
 
         # Disable crashed test suites
-        self.vc.dump(window='-1')
+        self.dump_always()
         crashed_test_name = "C24Z24MS4"
         self.logger.info('Test suite %s is going to be disabled!' % crashed_test_name)
         crashed_test = self.vc.findViewWithText(crashed_test_name)
         if crashed_test is not None:
             crashed_test.touch()
             self.logger.info('Test suite %s has been excluded!' % crashed_test_name)
-            time.sleep(2)
         else:
             self.logger.info('Can not find test suite %s, please check the screen!' % crashed_test_name)
 
         # Start selected test suites
+        self.dump_always()
         start_button = self.vc.findViewByIdOrRaise("com.glbenchmark.glbenchmark25:id/buttonStart")
         start_button.touch()
-        time.sleep(2)
 
         finished = False
         while not finished:
diff --git a/automated/android/apk-automation/quadrantpro.py b/automated/android/apk-automation/quadrantpro.py
index 8ca7b0d..3360a2b 100755
--- a/automated/android/apk-automation/quadrantpro.py
+++ b/automated/android/apk-automation/quadrantpro.py
@@ -19,29 +19,39 @@ class ApkRunnerImpl(ApkTestRunner):
         super(ApkRunnerImpl, self).tearDown()
 
     def execute(self):
-        self.dump_always()
-        view_license_btn = self.vc.findViewWithText("View license")
-        if view_license_btn:
-            ok_button = self.vc.findViewWithTextOrRaise("OK")
-            ok_button.touch()
-
-        self.dump_always()
-        run_full_item = self.vc.findViewWithTextOrRaise(u'Run full benchmark')
-        run_full_item.touch()
+        need_continue = True
+        while need_continue:
+            self.dump_always()
+            view_license_btn = self.vc.findViewWithText("View license")
+            run_full_item = self.vc.findViewWithText(u'Run full benchmark')
+            if view_license_btn:
+                ok_button = self.vc.findViewWithTextOrRaise("OK")
+                ok_button.touch()
+            elif run_full_item:
+                run_full_item.touch()
+                need_continue = False
+            else:
+                # continue check
+                pass
 
         # Hack workaround to kill the first time start up
         # then it will work from 2nd time
         self.call_adb("shell am force-stop %s" % self.config['apk_package'])
         self.call_adb("shell am start -W -S %s" % self.config['activity'])
-        self.dump_always()
-        view_license_btn = self.vc.findViewWithText("View license")
-        if view_license_btn:
-            ok_button = self.vc.findViewWithTextOrRaise("OK")
-            ok_button.touch()
-
-        self.dump_always()
-        run_full_item = self.vc.findViewWithTextOrRaise(u'Run full benchmark')
-        run_full_item.touch()
+        need_continue = True
+        while need_continue:
+            self.dump_always()
+            view_license_btn = self.vc.findViewWithText("View license")
+            run_full_item = self.vc.findViewWithText(u'Run full benchmark')
+            if view_license_btn:
+                ok_button = self.vc.findViewWithTextOrRaise("OK")
+                ok_button.touch()
+            elif run_full_item:
+                run_full_item.touch()
+                need_continue = False
+            else:
+                # continue check
+                pass
 
         finished = False
         while not finished:
diff --git a/automated/android/apk-automation/rl-sqlite.py b/automated/android/apk-automation/rl-sqlite.py
index 48ff4a8..10929b2 100755
--- a/automated/android/apk-automation/rl-sqlite.py
+++ b/automated/android/apk-automation/rl-sqlite.py
@@ -31,7 +31,7 @@ class ApkRunnerImpl(ApkTestRunner):
                 subitem = self.vc.findViewWithText(text, ch)
                 if subitem:
                     subitem_result = self.vc.findViewByIdOrRaise("com.redlicense.benchmark.sqlite:id/test_result", ch)
-                    score = subitem_result.getText().replace("sec", "").strip()
+                    score = subitem_result.getText().replace("sec", "").replace("Running", "").strip()
                     score_in_ms = float(score) * 1000
                     self.report_result("RL-sqlite-" + text.replace(" ", "-"), 'pass', str(score_in_ms), "ms")
                     found_score_view = True
diff --git a/automated/android/apk-automation/vellamo3.py b/automated/android/apk-automation/vellamo3.py
index d9719a7..413ee61 100755
--- a/automated/android/apk-automation/vellamo3.py
+++ b/automated/android/apk-automation/vellamo3.py
@@ -42,24 +42,28 @@ class ApkRunnerImpl(ApkTestRunner):
                         break
 
     def execute(self):
-        self.dump_always()
-        # Accept Vellamo EULA
-        btn_setup_1 = self.vc.findViewByIdOrRaise("android:id/button1")
-        btn_setup_1.touch()
-
-        # Open settings
-        self.dump_always()
-        btn_settings = self.vc.findViewByIdOrRaise('com.quicinc.vellamo:id/main_toolbar_wheel')
-        btn_settings.touch()
-
-        # Disable animations
-        self.dump_always()
-        btn_animations = self.vc.findViewWithTextOrRaise(u'Make Vellamo even more beautiful')
-        btn_animations.touch()
+        need_continue = True
+        while need_continue:
+            self.dump_always()
+            btn_setup_1 = self.vc.findViewById("android:id/button1")
+            btn_settings = self.vc.findViewById('com.quicinc.vellamo:id/main_toolbar_wheel')
+            btn_animations = self.vc.findViewWithText(u'Make Vellamo even more beautiful')
+            if btn_setup_1:
+                # Accept Vellamo EULA
+                btn_setup_1.touch()
+            elif btn_settings:
+                # Open settings
+                btn_settings.touch()
+            elif btn_animations:
+                # Disable animations
+                btn_animations.touch()
+                need_continue = False
 
         # Back to the home screen
         self.device.press("KEYCODE_BACK")
 
+        self.logger.info("Benchmark started now")
+
         chapters = ['Browser', 'Multicore', 'Metal']
         for chapter in chapters:
             self.choose_chapter(chapter)
@@ -67,9 +71,9 @@ class ApkRunnerImpl(ApkTestRunner):
             # Start benchmark
             self.dump_always()
             try:
-                btn_start = self.vc.findViewById("com.quicinc.vellamo:id/main_toolbar_operation_button")
-                if btn_start:
-                    btn_start.touch()
+                gotit_button = self.vc.findViewWithText(u'GOT IT')
+                if gotit_button:
+                    gotit_button.touch()
             except ViewNotFoundException:
                 self.report_result('vellamo3-%s' % chapter, 'fail')
                 self.logger.error('Start button for chapter %s NOT found, moving to the next chapter...')
@@ -81,11 +85,10 @@ class ApkRunnerImpl(ApkTestRunner):
                 time.sleep(1)
                 try:
                     self.dump_always()
-                    goback_title = self.vc.findViewById("com.quicinc.vellamo:id/main_toolbar_goback_title")
                     goback_btn = self.vc.findViewById("com.quicinc.vellamo:id/main_toolbar_goback_button")
-                    if goback_btn or goback_title:
-                        btn_no = self.vc.findViewByIdOrRaise("com.quicinc.vellamo:id/button_no")
-                        btn_no.touch()
+                    if goback_btn:
+                        goback_btn.touch()
+                        time.sleep(5)
                         finished = True
                 except ViewNotFoundException:
                     pass
@@ -96,6 +99,7 @@ class ApkRunnerImpl(ApkTestRunner):
 
             self.logger.info("Benchmark finished: %s" % chapter)
             self.device.press("KEYCODE_BACK")
+            time.sleep(5)
             self.device.press("KEYCODE_BACK")
 
     def parseResult(self):