test-runner: add support for test plan overlay

On ERP testing, we need to skip disk partitioning test on boards that
don't have the second hard drive installed. We may also need to amend
test parameters or add new tests in the future. This patch add support
for test plan overlay to:

    * skip tests.
    * amend test parameters.
    * add new tests.

Change-Id: I9143b313b32f0c541fb08bacce80f371b1cf15e6
Signed-off-by: Chase Qi <chase.qi@linaro.org>

3 files changed, 95 insertions, 1 deletions

diff --git a/automated/README b/automated/README
index 6c05d67..2852ef6 100644
--- a/automated/README
+++ b/automated/README
@@ -45,6 +45,8 @@ single test run:
 running test plan:
     # Run a set of tests defined in agenda file.
     test-runner -p ./plans/linux-example.yaml
+    # Apply test plan overlay to skip, amend or add tests.
+    test-runner -p ./plans/linux-example.yaml -O test-plan-overlay-example.yaml
 
 Collecting result
 =================
diff --git a/automated/utils/test-runner.py b/automated/utils/test-runner.py
index 267a74a..27300dc 100755
--- a/automated/utils/test-runner.py
+++ b/automated/utils/test-runner.py
@@ -2,6 +2,7 @@
 import argparse
 import csv
 import cmd
+import copy
 import json
 import logging
 import os
@@ -73,6 +74,59 @@ class TestPlan(object):
         self.timeout = args.timeout
         self.skip_install = args.skip_install
         self.logger = logging.getLogger('RUNNER.TestPlan')
+        self.overlay = args.overlay
+
+    def apply_overlay(self, test_list):
+        fixed_test_list = copy.deepcopy(test_list)
+        logger = logging.getLogger('RUNNER.TestPlan.Overlay')
+        with open(self.overlay) as f:
+            data = yaml.load(f)
+
+        if data.get('skip'):
+            skip_tests = data['skip']
+            for test in test_list:
+                for skip_test in skip_tests:
+                    if test['path'] == skip_test['path'] and test['repository'] == skip_test['repository']:
+                        fixed_test_list.remove(test)
+                        logger.info("Skipped: {}".format(test))
+                    else:
+                        continue
+
+        if data.get('amend'):
+            amend_tests = data['amend']
+            for test in fixed_test_list:
+                for amend_test in amend_tests:
+                    if test['path'] == amend_test['path'] and test['repository'] == skip_test['repository']:
+                        if amend_test.get('parameters'):
+                            if test.get('parameters'):
+                                test['parameters'].update(amend_test['parameters'])
+                            else:
+                                test['parameters'] = amend_test['parameters']
+                            logger.info('Updated: {}'.format(test))
+                        else:
+                            logger.warning("'parameters' not found in {}, nothing to amend.".format(amend_test))
+
+        if data.get('add'):
+            add_tests = data['add']
+            unique_add_tests = []
+            for test in add_tests:
+                if test not in unique_add_tests:
+                    unique_add_tests.append(test)
+                else:
+                    logger.warning("Skipping duplicate test {}".format(test))
+
+            for test in test_list:
+                del test['uuid']
+
+            for add_test in unique_add_tests:
+                if add_test in test_list:
+                    logger.warning("{} already included in test plan, do nothing.".format(add_test))
+                else:
+                    add_test['uuid'] = str(uuid4())
+                    fixed_test_list.append(add_test)
+                    logger.info("Added: {}".format(add_test))
+
+        return fixed_test_list
 
     def test_list(self, kind="automated"):
         if self.test_def:
@@ -117,7 +171,10 @@ class TestPlan(object):
             self.logger.error('Plese specify a test or test plan.')
             sys.exit(1)
 
-        return test_list
+        if self.overlay is None:
+            return test_list
+        else:
+            return self.apply_overlay(test_list)
 
 
 class TestSetup(object):
@@ -801,6 +858,13 @@ def get_args():
     parser.add_argument('-l', '--lava_run', dest='lava_run',
                         default=False, action='store_true',
                         help='send test result to LAVA with lava-test-case.')
+    parser.add_argument('-O', '--overlay', default=None,
+                        dest='overlay', help=textwrap.dedent('''\
+                        Specify test plan ovelay file to:
+                        * skip tests
+                        * amend test parameters
+                        * add new tests
+                        '''))
     args = parser.parse_args()
     return args
 
diff --git a/plans/test-plan-overlay-example.yaml b/plans/test-plan-overlay-example.yaml
new file mode 100644
index 0000000..5dd19aa
--- /dev/null
+++ b/plans/test-plan-overlay-example.yaml
@@ -0,0 +1,28 @@
+metadata:
+  name: test-plan-overlay-example
+  description: Test plan overlay example
+  os: debian
+  devices:
+      - all
+  maintainer:
+    - first.last@linaro.org
+  format: Linaro Test Plan v2
+
+skip:
+      # Skip disk-partitioning test.
+    - path: automated/linux/disk-partitioning/disk-partitioning.yaml
+      repository: https://git.linaro.org/qa/test-definitions.git
+
+amend:
+      # Test the second network interface.
+    - path: automated/linux/network-basic/network-basic.yaml
+      repository: https://git.linaro.org/qa/test-definitions.git
+      parameters:
+          INTERFACE: enahisic2i1
+
+add:
+      # Add sysbench oltp test.
+    - path: automated/linux/sysbench/sysbench.yaml
+      repository: https://git.linaro.org/qa/test-definitions.git
+      parameters:
+          TESTS: oltp