From 9e5843ee8346e4e7ac5803c3990dc9a21bfae2aa Mon Sep 17 00:00:00 2001 From: Tom Gall Date: Wed, 21 Nov 2012 15:17:47 -0600 Subject: Fix up results reporting between the python side in glmark2test.py and from the glmark2 side. Based on scene results return a proper piglit marked dictionary --- framework/glmark2test.py | 138 +++++++++++++++++++++++++++-- tests/glmark2/src/scene-simpletriangle.cpp | 2 +- tests/glmark2/src/scene.h | 1 + tests/glmark2/src/test-collection.cpp | 21 ++++- 4 files changed, 151 insertions(+), 11 deletions(-) diff --git a/framework/glmark2test.py b/framework/glmark2test.py index ed8031f9..98a7d6e8 100644 --- a/framework/glmark2test.py +++ b/framework/glmark2test.py @@ -1,5 +1,5 @@ # -# Permission is hereby granted, free of charge, to any person +# Permission is hereby granted, free f charge, to any person # obtaining a copy of this software and associated documentation # files (the "Software"), to deal in the Software without # restriction, including without limitation the rights to use, @@ -25,6 +25,7 @@ import subprocess from core import checkDir, testBinDir, Test, TestResult from exectest import ExecTest +from exectest import PlainExecTest ############################################################################# ##### Glmark2Test: Execute a sub-test of Glean @@ -35,7 +36,7 @@ def glmark2Executable(): def glmark2ResultDir(r_dir): return os.path.join(r_dir, 'glmark2') -class Glmark2Test(ExecTest): +class Glmark2Test(PlainExecTest): globalParams = [] def __init__(self, name, resdir): @@ -49,12 +50,131 @@ class Glmark2Test(ExecTest): self.name = name def run(self, valgrind): - self.command += Glmark2Test.globalParams - return ExecTest.run(self, valgrind) + fullenv = os.environ.copy() + for e in self.env: + fullenv[e] = str(self.env[e]) + + self.command += Glmark2Test.globalParams + + if self.command is not None: + command = self.command + i = 0 + while True: + if self.skip_test: + out = "PIGLIT: {'result': 'skip'}\n" + err = "" + returncode = None + else: + (out, err, returncode) = \ + self.get_command_result(command, fullenv) + + # https://bugzilla.gnome.org/show_bug.cgi?id=680214 is + # affecting many developers. If we catch it + # happening, try just re-running the test. + if out.find("Got spurious window resize") >= 0: + i = i + 1 + if i >= 5: + break + else: + break + + # proc.communicate() returns 8-bit strings, but we need # unicode strings. In Python 2.x, this is because we + # will eventually be serializing the strings as JSON, + # and the JSON library expects unicode. In Python 3.x, + # this is because all string operations require + # unicode. So translate the strings into unicode, # assuming they are using UTF-8 encoding. + # + # If the subprocess output wasn't properly UTF-8 + # encoded, we don't want to raise an exception, so + # translate the strings using 'replace' mode, which + # replaces erroneous charcters with the Unicode + # "replacement character" (a white question mark inside + # a black diamond). + out = out.decode('utf-8', 'replace') + err = err.decode('utf-8', 'replace') + results = TestResult() + + if self.skip_test: + results['result'] = 'skip' + else: + out = self.interpretResult(out, results) + crash_codes = [ + # Unix: terminated by a signal + -5, # SIGTRAP + -6, # SIGABRT + -8, # SIGFPE (Floating point exception) + -10, # SIGUSR1 + -11, # SIGSEGV (Segmentation fault) + # Windows: + # EXCEPTION_ACCESS_VIOLATION (0xc0000005): + -1073741819, + # EXCEPTION_INT_DIVIDE_BY_ZERO (0xc0000094): + -1073741676 + ] + + if returncode in crash_codes: + results['result'] = 'crash' + elif returncode != 0: + results['note'] = 'Returncode was {0}'.format(returncode) + env = '' + for key in self.env: + env = env + key + '="' + self.env[key] + '" ' + if env: + results['environment'] = env + + results['info'] = "Returncode: {0}\n\nErrors:\n{1}\n\nOutput:\n{2}".format(returncode, err, out) + results['returncode'] = returncode + results['command'] = ' '.join(self.command) + + self.handleErr(results, err) - def interpretResult(self, out, results): - if out.find('FAIL') >= 0: - results['result'] = 'fail' else: - results['result'] = 'pass' - return out + results = TestResult() + results['result'] = 'skip' + + return results + + def get_command_result(self, command, fullenv): + try: + proc = subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=fullenv, + universal_newlines=True + ) + out, err = proc.communicate() + returncode = proc.returncode + except OSError as e: + # Different sets of tests get built under + # different build configurations. If + # a developer chooses to not build a test, + # Piglit should not report that test as having + # failed. + if e.strerror == "No such file or directory": + out = "PIGLIT: {'result': 'skip'}\n" \ + + "Test executable not found.\n" + err = "" + returncode = None + else: + raise e + return out, err, returncode + + def handleErr(self, results,err): + errors = filter(lambda s: len(s) > 0, map(lambda s: s.strip(), err.split('\n'))) + + ignored = [s for s in errors if self.isIgnored(s)] + errors = [s for s in errors if s not in ignored] + + if len(errors) > 0: + results['errors'] = errors + + if len(ignored) > 0: + results['errors_ignored'] = ignored + + def isIgnored(self,s): + if s == "libEGL warning: unsupported platform Windows": + return True + else: + return False + diff --git a/tests/glmark2/src/scene-simpletriangle.cpp b/tests/glmark2/src/scene-simpletriangle.cpp index a148d70a..c8785fc4 100644 --- a/tests/glmark2/src/scene-simpletriangle.cpp +++ b/tests/glmark2/src/scene-simpletriangle.cpp @@ -38,7 +38,7 @@ SceneSimpleTriangle::SceneSimpleTriangle(Canvas &pCanvas) : Scene(pCanvas, "simpletriangle"), radius_(0.0), orientModel_(false), orientationAngle_(0.0) { - +testResult=Scene::ValidationUnknown; duration_=30; // set for 30 seconds } diff --git a/tests/glmark2/src/scene.h b/tests/glmark2/src/scene.h index e543bf57..3930928d 100644 --- a/tests/glmark2/src/scene.h +++ b/tests/glmark2/src/scene.h @@ -293,6 +293,7 @@ public: ~SceneSimpleTriangle(); protected: + Scene::ValidationResult testResult; Program program_; Mesh mesh_; GLuint texture_; diff --git a/tests/glmark2/src/test-collection.cpp b/tests/glmark2/src/test-collection.cpp index bf960903..31775f30 100644 --- a/tests/glmark2/src/test-collection.cpp +++ b/tests/glmark2/src/test-collection.cpp @@ -25,6 +25,7 @@ #include "options.h" #include "log.h" #include "util.h" +#include "scene.h" TestCollection::~TestCollection() { @@ -120,5 +121,23 @@ TestCollection::benchmarks_contain_normal_scenes() void TestCollection::report_results() { - + Scene::ValidationResult r; + + fflush(stderr); + + for (std::vector::const_iterator iter = tests_.begin(); + iter != tests_.end(); + iter++) { + const Benchmark *bench=*iter; + switch(bench->scene().validate()) { + case Scene::ValidationSuccess: + printf("PIGLIT: {'result': 'pass' }\n"); + break; + case Scene::ValidationFailure: + case Scene::ValidationUnknown: + printf("PIGLIT: {'result': 'fail' }\n"); + break; + } + } + fflush(stdout); } -- cgit v1.2.3