summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorDan Rue <dan.rue@linaro.org>2017-11-28 15:09:31 +0000
committerLinaro Code Review <review@review.linaro.org>2017-11-28 15:09:31 +0000
commitf9897305f379908871f07046fe4f4629f191965e (patch)
tree35f56d758cd0370b1ec9fa8041bef4d3a3420f9c
parentdca4fb61a9dcc74945305358b2059abc9301c648 (diff)
parenta158efe3ecbcdd888271faee14ffca911b8af455 (diff)
Merge "test-runner: add support for test plan overlay"
-rw-r--r--automated/README2
-rwxr-xr-xautomated/utils/test-runner.py66
-rw-r--r--plans/test-plan-overlay-example.yaml28
3 files changed, 95 insertions, 1 deletions
diff --git a/automated/README b/automated/README
index 6c05d67..2852ef6 100644
--- a/automated/README
+++ b/automated/README
@@ -45,6 +45,8 @@ single test run:
running test plan:
# Run a set of tests defined in agenda file.
test-runner -p ./plans/linux-example.yaml
+ # Apply test plan overlay to skip, amend or add tests.
+ test-runner -p ./plans/linux-example.yaml -O test-plan-overlay-example.yaml
Collecting result
=================
diff --git a/automated/utils/test-runner.py b/automated/utils/test-runner.py
index c5e9172..d5d8e7f 100755
--- a/automated/utils/test-runner.py
+++ b/automated/utils/test-runner.py
@@ -2,6 +2,7 @@
import argparse
import csv
import cmd
+import copy
import json
import logging
import os
@@ -73,6 +74,59 @@ class TestPlan(object):
self.timeout = args.timeout
self.skip_install = args.skip_install
self.logger = logging.getLogger('RUNNER.TestPlan')
+ self.overlay = args.overlay
+
+ def apply_overlay(self, test_list):
+ fixed_test_list = copy.deepcopy(test_list)
+ logger = logging.getLogger('RUNNER.TestPlan.Overlay')
+ with open(self.overlay) as f:
+ data = yaml.load(f)
+
+ if data.get('skip'):
+ skip_tests = data['skip']
+ for test in test_list:
+ for skip_test in skip_tests:
+ if test['path'] == skip_test['path'] and test['repository'] == skip_test['repository']:
+ fixed_test_list.remove(test)
+ logger.info("Skipped: {}".format(test))
+ else:
+ continue
+
+ if data.get('amend'):
+ amend_tests = data['amend']
+ for test in fixed_test_list:
+ for amend_test in amend_tests:
+ if test['path'] == amend_test['path'] and test['repository'] == skip_test['repository']:
+ if amend_test.get('parameters'):
+ if test.get('parameters'):
+ test['parameters'].update(amend_test['parameters'])
+ else:
+ test['parameters'] = amend_test['parameters']
+ logger.info('Updated: {}'.format(test))
+ else:
+ logger.warning("'parameters' not found in {}, nothing to amend.".format(amend_test))
+
+ if data.get('add'):
+ add_tests = data['add']
+ unique_add_tests = []
+ for test in add_tests:
+ if test not in unique_add_tests:
+ unique_add_tests.append(test)
+ else:
+ logger.warning("Skipping duplicate test {}".format(test))
+
+ for test in test_list:
+ del test['uuid']
+
+ for add_test in unique_add_tests:
+ if add_test in test_list:
+ logger.warning("{} already included in test plan, do nothing.".format(add_test))
+ else:
+ add_test['uuid'] = str(uuid4())
+ fixed_test_list.append(add_test)
+ logger.info("Added: {}".format(add_test))
+
+ return fixed_test_list
def test_list(self, kind="automated"):
if self.test_def:
@@ -125,7 +179,10 @@ class TestPlan(object):
self.logger.error('Plese specify a test or test plan.')
sys.exit(1)
- return test_list
+ if self.overlay is None:
+ return test_list
+ else:
+ return self.apply_overlay(test_list)
class TestSetup(object):
@@ -809,6 +866,13 @@ def get_args():
parser.add_argument('-l', '--lava_run', dest='lava_run',
default=False, action='store_true',
help='send test result to LAVA with lava-test-case.')
+ parser.add_argument('-O', '--overlay', default=None,
+ dest='overlay', help=textwrap.dedent('''\
+ Specify test plan ovelay file to:
+ * skip tests
+ * amend test parameters
+ * add new tests
+ '''))
args = parser.parse_args()
return args
diff --git a/plans/test-plan-overlay-example.yaml b/plans/test-plan-overlay-example.yaml
new file mode 100644
index 0000000..5dd19aa
--- /dev/null
+++ b/plans/test-plan-overlay-example.yaml
@@ -0,0 +1,28 @@
+metadata:
+ name: test-plan-overlay-example
+ description: Test plan overlay example
+ os: debian
+ devices:
+ - all
+ maintainer:
+ - first.last@linaro.org
+ format: Linaro Test Plan v2
+
+skip:
+ # Skip disk-partitioning test.
+ - path: automated/linux/disk-partitioning/disk-partitioning.yaml
+ repository: https://git.linaro.org/qa/test-definitions.git
+
+amend:
+ # Test the second network interface.
+ - path: automated/linux/network-basic/network-basic.yaml
+ repository: https://git.linaro.org/qa/test-definitions.git
+ parameters:
+ INTERFACE: enahisic2i1
+
+add:
+ # Add sysbench oltp test.
+ - path: automated/linux/sysbench/sysbench.yaml
+ repository: https://git.linaro.org/qa/test-definitions.git
+ parameters:
+ TESTS: oltp