diff options
author | Dan Rue <dan.rue@linaro.org> | 2017-11-28 15:09:31 +0000 |
---|---|---|
committer | Linaro Code Review <review@review.linaro.org> | 2017-11-28 15:09:31 +0000 |
commit | f9897305f379908871f07046fe4f4629f191965e (patch) | |
tree | 35f56d758cd0370b1ec9fa8041bef4d3a3420f9c | |
parent | dca4fb61a9dcc74945305358b2059abc9301c648 (diff) | |
parent | a158efe3ecbcdd888271faee14ffca911b8af455 (diff) |
Merge "test-runner: add support for test plan overlay"
-rw-r--r-- | automated/README | 2 | ||||
-rwxr-xr-x | automated/utils/test-runner.py | 66 | ||||
-rw-r--r-- | plans/test-plan-overlay-example.yaml | 28 |
3 files changed, 95 insertions, 1 deletions
diff --git a/automated/README b/automated/README index 6c05d67..2852ef6 100644 --- a/automated/README +++ b/automated/README @@ -45,6 +45,8 @@ single test run: running test plan: # Run a set of tests defined in agenda file. test-runner -p ./plans/linux-example.yaml + # Apply test plan overlay to skip, amend or add tests. + test-runner -p ./plans/linux-example.yaml -O test-plan-overlay-example.yaml Collecting result ================= diff --git a/automated/utils/test-runner.py b/automated/utils/test-runner.py index c5e9172..d5d8e7f 100755 --- a/automated/utils/test-runner.py +++ b/automated/utils/test-runner.py @@ -2,6 +2,7 @@ import argparse import csv import cmd +import copy import json import logging import os @@ -73,6 +74,59 @@ class TestPlan(object): self.timeout = args.timeout self.skip_install = args.skip_install self.logger = logging.getLogger('RUNNER.TestPlan') + self.overlay = args.overlay + + def apply_overlay(self, test_list): + fixed_test_list = copy.deepcopy(test_list) + logger = logging.getLogger('RUNNER.TestPlan.Overlay') + with open(self.overlay) as f: + data = yaml.load(f) + + if data.get('skip'): + skip_tests = data['skip'] + for test in test_list: + for skip_test in skip_tests: + if test['path'] == skip_test['path'] and test['repository'] == skip_test['repository']: + fixed_test_list.remove(test) + logger.info("Skipped: {}".format(test)) + else: + continue + + if data.get('amend'): + amend_tests = data['amend'] + for test in fixed_test_list: + for amend_test in amend_tests: + if test['path'] == amend_test['path'] and test['repository'] == skip_test['repository']: + if amend_test.get('parameters'): + if test.get('parameters'): + test['parameters'].update(amend_test['parameters']) + else: + test['parameters'] = amend_test['parameters'] + logger.info('Updated: {}'.format(test)) + else: + logger.warning("'parameters' not found in {}, nothing to amend.".format(amend_test)) + + if data.get('add'): + add_tests = data['add'] + unique_add_tests = [] + for test in add_tests: + if test not in unique_add_tests: + unique_add_tests.append(test) + else: + logger.warning("Skipping duplicate test {}".format(test)) + + for test in test_list: + del test['uuid'] + + for add_test in unique_add_tests: + if add_test in test_list: + logger.warning("{} already included in test plan, do nothing.".format(add_test)) + else: + add_test['uuid'] = str(uuid4()) + fixed_test_list.append(add_test) + logger.info("Added: {}".format(add_test)) + + return fixed_test_list def test_list(self, kind="automated"): if self.test_def: @@ -125,7 +179,10 @@ class TestPlan(object): self.logger.error('Plese specify a test or test plan.') sys.exit(1) - return test_list + if self.overlay is None: + return test_list + else: + return self.apply_overlay(test_list) class TestSetup(object): @@ -809,6 +866,13 @@ def get_args(): parser.add_argument('-l', '--lava_run', dest='lava_run', default=False, action='store_true', help='send test result to LAVA with lava-test-case.') + parser.add_argument('-O', '--overlay', default=None, + dest='overlay', help=textwrap.dedent('''\ + Specify test plan ovelay file to: + * skip tests + * amend test parameters + * add new tests + ''')) args = parser.parse_args() return args diff --git a/plans/test-plan-overlay-example.yaml b/plans/test-plan-overlay-example.yaml new file mode 100644 index 0000000..5dd19aa --- /dev/null +++ b/plans/test-plan-overlay-example.yaml @@ -0,0 +1,28 @@ +metadata: + name: test-plan-overlay-example + description: Test plan overlay example + os: debian + devices: + - all + maintainer: + - first.last@linaro.org + format: Linaro Test Plan v2 + +skip: + # Skip disk-partitioning test. + - path: automated/linux/disk-partitioning/disk-partitioning.yaml + repository: https://git.linaro.org/qa/test-definitions.git + +amend: + # Test the second network interface. + - path: automated/linux/network-basic/network-basic.yaml + repository: https://git.linaro.org/qa/test-definitions.git + parameters: + INTERFACE: enahisic2i1 + +add: + # Add sysbench oltp test. + - path: automated/linux/sysbench/sysbench.yaml + repository: https://git.linaro.org/qa/test-definitions.git + parameters: + TESTS: oltp |