diff options
author | Aníbal Limón <anibal.limon@linaro.org> | 2018-05-09 15:27:49 -0500 |
---|---|---|
committer | Milosz Wasilewski <milosz.wasilewski@linaro.org> | 2018-05-10 08:24:57 +0000 |
commit | b02c865b64136f430c891769b53688c433515702 (patch) | |
tree | ae66b695bdd5785661929b4f9c6da490aa24e3ab | |
parent | 0223fd2db0019e820299d57069a83bf65d57a3a7 (diff) |
automated/linux/gst-validate: Add support for ignore tests
There are some gst-validate tests that fails depending on the board
so add a way to use a file stored in a git repository to ignore the
result.
The gst-validate-launcher dosen't provide a command line option to
exclude certain tests by design depends on write your own testsuite.py
module so this is the better implementation for now.
Change-Id: I8e05de42892a58c1ccb344eb72932217de23921d
Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
-rw-r--r-- | automated/linux/gst-validate/gst-validate.yaml | 23 | ||||
-rwxr-xr-x | automated/linux/gst-validate/gst_validate_lava_parse.py | 62 |
2 files changed, 76 insertions, 9 deletions
diff --git a/automated/linux/gst-validate/gst-validate.yaml b/automated/linux/gst-validate/gst-validate.yaml index 39d6066..7a3e765 100644 --- a/automated/linux/gst-validate/gst-validate.yaml +++ b/automated/linux/gst-validate/gst-validate.yaml @@ -26,17 +26,22 @@ params: MAIN_DIR: "/gst-validate" OPTIONS: "-nd -v -j 1" GST_INTEGRATION_SUITES: "http://testdata.validation.linaro.org/gst-validate/gst-integration-testsuites_1.12.tar.gz" + GST_IGNORE_TESTS_REPO: "" + GST_IGNORE_TESTS_BRANCH: "master" + GST_IGNORE_TESTS_FILE: "" run: steps: + - . ./automated/lib/sh-test-lib + - cd ./automated/linux/gst-validate - wget ${GST_INTEGRATION_SUITES} - tar -xzf $(basename ${GST_INTEGRATION_SUITES}) -C / - - gst-validate-launcher -M ${MAIN_DIR} ${OPTIONS} || true - -parse: - pattern: '(?P<test_case_id>validate\..*):\s+(?P<result>(Failed|Passed|Skipped|Timeout))' - fixupdict: - Failed: fail - Passed: pass - Skipped: skip - Timeout: fail + - IGNORE_FILE="" + - if [ ! -z "${GST_IGNORE_TESTS_REPO}" ] && [ ! -z "${GST_IGNORE_TESTS_FILE}" ]; then + - repo_path=${PWD}/$(basename ${GST_IGNORE_TESTS_REPO}) + - git clone -b ${GST_IGNORE_TESTS_BRANCH} ${GST_IGNORE_TESTS_REPO} $repo_path + - IGNORE_FILE=${repo_path}/${GST_IGNORE_TESTS_FILE} + - fi + - gst-validate-launcher -M ${MAIN_DIR} ${OPTIONS} 2>&1 | tee ./gst-validate-raw.log + - ./gst_validate_lava_parse.py ./gst-validate-raw.log ${IGNORE_FILE} > ./result.txt + - ../../utils/send-to-lava.sh ./result.txt diff --git a/automated/linux/gst-validate/gst_validate_lava_parse.py b/automated/linux/gst-validate/gst_validate_lava_parse.py new file mode 100755 index 0000000..8845fe6 --- /dev/null +++ b/automated/linux/gst-validate/gst_validate_lava_parse.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# LAVA/OE gst-validate results parse script +# +# Copyright (C) 2018, Linaro Limited. +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# +# Author: Aníbal Limón <anibal.limon@linaro.org> +# + +import json +import os +import re +import sys + + +def map_result_to_lava(result): + if result == 'Passed': + result = 'pass' + elif result == 'Failed': + result = 'fail' + elif result == 'Skipped': + result = 'skip' + elif result == 'Timeout': + result = 'fail' + + return result + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print("Usage: %s <result_file> [ignore_file]" % sys.argv[0]) + sys.exit(1) + + ignore_tests = [] + if len(sys.argv) == 3: + with open(sys.argv[2], 'r') as f: + ignore_tests = f.read().split() + + rex = re.compile('^(?P<test_case_id>validate\..*):\s+(?P<result>(Failed|Passed|Skipped|Timeout))') + with open(sys.argv[1], 'r') as f: + for line in f.readlines(): + s = rex.search(line) + if s: + test_case_id = s.group('test_case_id') + result = s.group('result') + + if test_case_id not in ignore_tests: + print("%s %s" % (test_case_id, map_result_to_lava(result))) |