framework: Use shader_test.py to drive shader tests

Add a new class to shader_test.py, ShaderTest. ShaderTest.run() parses the
[require] block of the shader test to determine which shader_runner
executable should be used to run the test.

This is needed because GLES3 shader tests must be run by
shader_runner_gles3 and the GL tests by shader_runner.

Signed-off-by: Chad Versace <chad.versace@linux.intel.com>

1 files changed, 226 insertions, 1 deletions

diff --git a/framework/shader_test.py b/framework/shader_test.py
index 1a83545d..a587f371 100644..100755
--- a/framework/shader_test.py
+++ b/framework/shader_test.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python2
+
 # Copyright (C) 2012 Intel Corporation
 #
 # Permission is hereby granted, free of charge, to any person
@@ -21,11 +23,43 @@
 # OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
 # DEALINGS IN THE SOFTWARE.
 
+import json
 import os
+import os.path
 import os.path as path
+import re
+import sys
+import textwrap
 
+from core import testBinDir, Group, Test, TestResult
 from exectest import PlainExecTest
 
+"""This module enables running shader tests.
+
+This module can be used to add shader tests to a Piglit test group or to run
+standalone shader tests on the command line. To add shader tests to a Piglit
+test group, use ``add_shader_test()`` or ``add_shader_test_dir()``. To run
+a single standalone test, execute ``shader_test.py FILENAME``.
+"""
+
+_PROGNAME = "shader_test.py"
+
+_HELP_TEXT = textwrap.dedent("""\
+    NAME
+        {progname} - run a shader test
+
+    SYNOPSIS
+        {progname} <filename> <extra_args>
+
+    DESCRIPTION
+        This script runs shader tests. Typically, the filename extension for
+        shader tests is ".shader_test". The extra_args are passed verbatim
+        to the shader_runner executable.
+    """.format(progname = _PROGNAME))
+
+def add_shader_test(group, testname, filepath):
+    group[testname] = ShaderTest([filepath, '-auto'])
+
 def add_shader_test_dir(group, dirpath, recursive=False):
     """Add all shader tests in a directory to the given group."""
     for filename in os.listdir(dirpath):
@@ -41,5 +75,196 @@ def add_shader_test_dir(group, dirpath, recursive=False):
             if ext != 'shader_test':
                 continue
             testname = filename[0:-(len(ext) + 1)] # +1 for '.'
-            group[testname] = concurrent_test('shader_runner ' + filepath)
+            add_shader_test(group, testname, filepath)
+
+class ShaderTest(PlainExecTest):
+
+    API_ERROR = 0
+    API_GL = 1
+    API_GLES2 = 2
+    API_GLES3 = 3
+
+    __has_compiled_regexes = False
+    __re_require_header = None
+    __re_gl = None
+    __re_gles2 = None
+    __re_gles3 = None
+    __re_gl_unknown = None
+
+    @classmethod
+    def __compile_regexes(cls):
+        """Compile the regular expressions needed to parse shader tests.
+
+        Hundreds, maybe thousands, of ShaderTests may be instantiated.  This
+        function compiles the regular expressions only once, at class scope,
+        and uses them for all instances.
+
+        This function is idempotent."""
+
+        if cls.__has_compiled_regexes:
+            return
+
+        common = {
+            'cmp' : r'(<|<=|=|>=|>)',
+            'gl_version' : r'(\d.\d)',
+            'gles2_version' : r'(2.\d\s+es)',
+            'gles3_version' : r'(3.\d\s+es)',
+            'comment' : r'(#.*)',
+        }
+
+        cls.__re_require_header = re.compile(r'^\s*\[require\]\s*{comment}?$'.format(**common))
+        cls.__re_end_require_block = re.compile(r'^\s*\['.format(*common))
+        cls.__re_gl = re.compile(r'^\s*GL\s*{cmp}\s*{gl_version}\s*{comment}?$'.format(**common))
+        cls.__re_gles2 = re.compile(r'^\s*GL\s*{cmp}\s*{gles2_version}\s*{comment}?$'.format(**common))
+        cls.__re_gles3 = re.compile(r'^\s*GL\s*{cmp}\s*{gles3_version}\s*{comment}?$'.format(**common))
+        cls.__re_gl_unknown = re.compile(r'^\s*GL\s*{cmp}'.format(**common))
+
+    def __init__(self, shader_runner_args, run_standalone=False):
+        """run_standalone: Run the test outside the Python framework."""
+
+        Test.__init__(self, runConcurrent=True)
+
+        assert(isinstance(shader_runner_args, list))
+        assert(isinstance(shader_runner_args[0], str))
+
+        self.__run_standalone = run_standalone
+        self.__shader_runner_args = shader_runner_args
+        self.__test_filepath = shader_runner_args[0]
+        self.__result = None
+        self.__command = None
+        self.__gl_api = None
+
+        self.env = {}
+
+    def __report_failure(self, message):
+        if self.__run_standalone:
+            print("error: " + message)
+            sys.exit(1)
+        else:
+            assert(self.__result is None)
+            self.__result = TestResult()
+            self.__result["result"] = "fail"
+            self.__result["errors"] = [message]
+
+    def __parse_test_file(self):
+        self.__set_gl_api()
+
+    def __set_gl_api(self):
+        """Set self.__gl_api by parsing the test's requirement block.
+
+        This function is idempotent."""
+
+        if self.__gl_api is not None:
+            return
+
+        cls = self.__class__
+        cls.__compile_regexes()
+
+        PARSE_FIND_REQUIRE_HEADER = 0
+        PARSE_FIND_GL_REQUIREMENT = 1
+
+        parse_state = PARSE_FIND_REQUIRE_HEADER
+
+        try:
+            with open(self.__test_filepath) as f:
+                for line in f:
+                    if parse_state == PARSE_FIND_REQUIRE_HEADER:
+                        if cls.__re_require_header.match(line) is not None:
+                            parse_state = PARSE_FIND_GL_REQUIREMENT
+                        else:
+                            continue
+                    elif parse_state == PARSE_FIND_GL_REQUIREMENT:
+                        if cls.__re_gl.match(line) is not None:
+                            self.__gl_api = ShaderTest.API_GL
+                            return
+                        elif cls.__re_gles2.match(line) is not None:
+                            self.__gl_api = ShaderTest.API_GLES2
+                            return
+                        elif cls.__re_gles3.match(line) is not None:
+                            self.__gl_api = ShaderTest.API_GLES3
+                            return
+                        elif cls.__re_gl_unknown.match(line) is not None:
+                            self.__report_failure("Failed to parse GL requirement: " + line)
+                            self.__gl_api = ShaderTest.API_ERROR
+                            return
+                        elif cls.__re_end_require_block.match(line) is not None:
+                            # Default to GL if no API is given.
+                            self.__gl_api = ShaderTest.API_GL
+                            return
+                        else:
+                            continue
+                    else:
+                        assert(False)
+
+                # Failed to parse the GL requirement.
+                self.__gl_api = ShaderTest.API_ERROR
+
+                if parse_state == PARSE_FIND_REQUIRE_HEADER:
+                    self.__report_failure("Failed to find [require] block")
+                elif parse_state == PARSE_FIND_GL_REQUIREMENT:
+                    self.__report_failure("Failed to find end of [require] block")
+                else:
+                    assert(False)
+
+        except IOError:
+            self._report_failure("Failed to read test file {0!r}".format(self.__test_filepath))
+            return
+
+    @property
+    def command(self):
+        if self.__command is not None:
+            return self.__command
+
+        self.__set_gl_api()
+
+        if self.__result is not None:
+            assert(self.__result["result"] == "fail")
+            return ["/bin/false"]
+
+        if self.__gl_api == ShaderTest.API_GL:
+            runner = "shader_runner"
+        elif self.__gl_api == ShaderTest.API_GLES2:
+            # Tentatively, let's use the gles3 shader runner to run gles2
+            # tests.
+            runner = "shader_runner_gles3"
+        elif self.__gl_api == ShaderTest.API_GLES3:
+            runner = "shader_runner_gles3"
+        else:
+            assert(False)
+
+        runner = os.path.join(testBinDir, runner)
+        self.__command = [runner] + self.__shader_runner_args
+        return self.__command
+
+    def run(self, valgrind=False):
+        """ Parse the test file's [require] block to determine which
+        executable is needed to run the test. Then run the executable on the
+        test file."""
+
+        # Parse the test file to discover any errors.
+        self.__parse_test_file()
+
+        if self.__run_standalone:
+            os.execv(self.command[0], self.command)
+        else:
+            if self.__result is not None:
+                # We've already decided the test result, most likely because
+                # parsing the test file discovered an error.
+                return self.__result
+
+            return PlainExecTest.run(self, valgrind)
+
+def _usage_error():
+    sys.stdout.write("usage error: {0}\n\n".format(_PROGNAME))
+    sys.stdout.write(_HELP_TEXT)
+    sys.exit(1)
+
+def _main(args):
+    if len(sys.argv) < 2:
+        _usage_error()
+
+    test = ShaderTest(sys.argv[1:], run_standalone=True)
+    test.run()
 
+if __name__ == "__main__":
+    _main(sys.argv)