diff options
| author | Aníbal Limón <anibal.limon@linaro.org> | 2018-05-11 10:43:29 -0500 |
|---|---|---|
| committer | Aníbal Limón <anibal.limon@linaro.org> | 2018-05-11 10:43:29 -0500 |
| commit | e93bc2fad7e00a83b752f581e67533ac5696cfa3 (patch) | |
| tree | 2f2dcc5c8d8acc4373dac6fd519472f248333444 | |
| parent | af55f6dea57b5a2006dfac67245346760777d047 (diff) | |
automated/linux/piglit: Add support to ignore tests from repo/file
The piglit tool support regex exclusion (-x) to avoid run certain tests
but in order to exclude a high number of tests [1] isn't usable because
needs to be passed into the command line and there are limits in the
amount of data that you can pass using cmdline.
In my use case i want to exclude 212 failing tests [1].
[1]
https://qa-reports.linaro.org/qcomlt/openembedded-rpb-rocko/build/146/testrun/1787216/
Change-Id: I4b96e3c2540a4ba14bc086b5eb7856d67bc2b8de
Signed-off-by: Aníbal Limón <anibal.limon@linaro.org>
| -rw-r--r-- | automated/linux/piglit/piglit.yaml | 13 | ||||
| -rwxr-xr-x | automated/linux/piglit/piglit_lava_parse.py | 11 |
2 files changed, 21 insertions, 3 deletions
diff --git a/automated/linux/piglit/piglit.yaml b/automated/linux/piglit/piglit.yaml index 476808c..8b87b7c 100644 --- a/automated/linux/piglit/piglit.yaml +++ b/automated/linux/piglit/piglit.yaml @@ -25,6 +25,9 @@ params: OPTIONS: "-x streaming-texture-leak -x glx" SUITE: "tests/quick.py" USE_XVFB: "False" + IGNORE_TESTS_REPO: "" + IGNORE_TESTS_BRANCH: "master" + IGNORE_TESTS_FILE: "" run: steps: @@ -39,11 +42,17 @@ run: - fi - export PIGLIT_SOURCE_DIR=${PIGLIT_SOURCE_DIR} - cd ./automated/linux/piglit + - IGNORE_FILE="" + - if [ ! -z "${IGNORE_TESTS_REPO}" ] && [ ! -z "${IGNORE_TESTS_FILE}" ]; then + - repo_path=${PWD}/$(basename ${IGNORE_TESTS_REPO}) + - git clone -b ${IGNORE_TESTS_BRANCH} ${IGNORE_TESTS_REPO} ${repo_path} + - IGNORE_FILE=${repo_path}/${IGNORE_TESTS_FILE} + - fi - piglit run ${OPTIONS} ${PIGLIT_SOURCE_DIR}/${SUITE} ./results - if [ -f ./results/results.json.bz2 ]; then - bzip2 -d ./results/results.json.bz2 - - ./piglit_lava_parse.py ./results/results.json > ./result.txt + - ./piglit_lava_parse.py ./results/results.json ${IGNORE_FILE} > ./result.txt - else - - ./piglit_lava_parse.py ./results/tests > ./result.txt + - ./piglit_lava_parse.py ./results/tests ${IGNORE_FILE} > ./result.txt - fi - ../../utils/send-to-lava.sh ./result.txt diff --git a/automated/linux/piglit/piglit_lava_parse.py b/automated/linux/piglit/piglit_lava_parse.py index 95b3636..1e780fa 100755 --- a/automated/linux/piglit/piglit_lava_parse.py +++ b/automated/linux/piglit/piglit_lava_parse.py @@ -48,9 +48,14 @@ def natural_keys(text): if __name__ == '__main__': if len(sys.argv) < 2: - print("Usage: %s <result_dir|result_file>" % sys.argv[0]) + print("Usage: %s <result_dir|result_file> [ignore_file]" % sys.argv[0]) sys.exit(1) + ignore_tests = [] + if len(sys.argv) == 3: + with open(sys.argv[2], 'r') as f: + ignore_tests = f.read().split() + if os.path.isdir(sys.argv[1]): for root, dirs, files in os.walk(sys.argv[1]): result_types = {} @@ -62,11 +67,15 @@ if __name__ == '__main__': with open(full_f, 'r') as f: piglit_results = json.loads(f.read()) for test in piglit_results.keys(): + if test in ignore_tests: + continue result = map_result_to_lava(piglit_results[test]['result']) print("%s %s" % (test, result)) else: with open(sys.argv[1], 'r') as f: piglit_results = json.loads(f.read()) for test in sorted(piglit_results['tests'].keys()): + if test in ignore_tests: + continue result = map_result_to_lava(piglit_results['tests'][test]['result']) print("%s %s" % (test, result)) |