piglt: Add support for subtests v2

Subtests allow a test to produce more than one result.  This is useful
for tests where it may be convenient to combine several somewhat
unrelated tests into a single program.  For example, a cl test like:

kernel void test(TYPE *out, TYPE *in) {
	out[0] = some_builtin_function(in[0], in[1]);
}

That uses only one kernel, but recompiles it several times with different
definitions of the macro 'TYPE' in order to test the builtin with all
the possible types.

To take advantage of the subtest feature, programs should output one
PIGLIT line per subtest as in the following example:

PIGLIT:subtest {'testA' : 'pass'}
PIGLIT:subtest {'testB' : 'fail'}

Where testA and testB are the name of the subtests.

In the result summary, this will be displayed as:

TestName                            1/2
      testA                         pass
      testB                         fail

v2:
  - Print one line for each subtest rather than printing them all
    together.

Acked-by: Chad Versace <chad.versace@linux.intel.com>

5 files changed, 80 insertions, 11 deletions

diff --git a/framework/core.py b/framework/core.py
index 73a238d0..0a8237de 100644
--- a/framework/core.py
+++ b/framework/core.py
@@ -460,7 +460,12 @@ class Test:
 
 			status(result['result'])
 
-			json_writer.write_dict_item(path, result)
+			if 'subtest' in result and len(result['subtest'].keys()) > 1:
+				for test in result['subtest'].keys():
+					result['result'] = result['subtest'][test]
+					json_writer.write_dict_item(path + '/' + test, result)
+			else:
+				json_writer.write_dict_item(path, result)
 		else:
 			status("dry-run")
 
diff --git a/framework/exectest.py b/framework/exectest.py
index aa89c661..e96b5baa 100644
--- a/framework/exectest.py
+++ b/framework/exectest.py
@@ -213,7 +213,13 @@ class PlainExecTest(ExecTest):
 
 		if len(outpiglit) > 0:
 			try:
-				results.update(eval(''.join(outpiglit), {}))
+				for piglit in outpiglit:
+					if piglit.startswith('subtest'):
+						if not results.has_key('subtest'):
+							results['subtest'] = {}
+						results['subtest'].update(eval(piglit[7:]))
+					else:
+						results.update(eval(piglit))
 				out = '\n'.join(filter(lambda s: not s.startswith('PIGLIT:'), outlines))
 			except:
 				results['result'] = 'fail'
diff --git a/framework/summary.py b/framework/summary.py
index a1933512..91e08707 100644
--- a/framework/summary.py
+++ b/framework/summary.py
@@ -106,6 +106,11 @@ results is an array of TestResult instances, one per testrun
 ##### GroupSummary: Summarize a group of tests
 #############################################################################
 class GroupSummary:
+	def createDummyGroup(self, result, test_name):
+		new_group = core.GroupResult()
+		new_group[' ' + test_name + '(All Tests)'] = result[test_name]
+		result[test_name] = new_group
+
 	def __init__(self, summary, path, name, results):
 		"""\
 summary is the root summary object
@@ -141,6 +146,25 @@ results is an array of GroupResult instances, one per testrun
 					childpath = self.path + '/' + childpath
 
 				if isinstance(result[name], core.GroupResult):
+					# This loop checks to make sure that all results
+					# with the same 'name' are of the same type.
+					# This is necessary to handle the case where
+					# a testname in an earlier result (an earlier
+					# result in this case means a result that
+					# comes before the current result in self.results)
+					# denotes a test group but in a later
+					# result it denotes a single test case, for example:
+					#
+					# result 0:
+					#	test/group/a PASS
+					#	test/group/b PASS
+					#	test/group/c PASS
+					# result 1:
+					#	test/group PASS
+					for r in self.results:
+						if r.has_key(name) and not isinstance(r[name], core.GroupResult):
+							self.createDummyGroup(r, name)
+
 					childresults = [r.get(name, core.GroupResult())
 							for r in self.results]
 
@@ -151,15 +175,40 @@ results is an array of GroupResult instances, one per testrun
 						childresults
 					)
 				else:
-					childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
-							for r in self.results]
-
-					self.children[name] = TestSummary(
-						summary,
-						childpath,
-						name,
-						childresults
-					)
+					# We need to check and handle the reversed case
+					# described in the above comment e.g.:
+					# result 0:
+					#	test/group PASS
+					# result 1:
+					#	test/group/a PASS
+					#	test/group/b PASS
+					#	test/group/c PASS
+					need_group = 0
+					for r in self.results:
+						if r.has_key(name) and not isinstance(r[name], core.TestResult):
+							need_group = 1
+					if need_group:
+						for r in self.results:
+							if r.has_key(name) and isinstance(r[name], core.TestResult):
+								self.createDummyGroup(r, name)
+						childresults = [r.get(name, core.GroupResult())
+								for r in self.results]
+
+						self.children[name] = GroupSummary(
+							summary,
+							childpath,
+							name,
+							childresults
+						)
+					else:
+						childresults = [r.get(name, core.TestResult({ 'result': 'skip' }))
+								for r in self.results]
+						self.children[name] = TestSummary(
+							summary,
+							childpath,
+							name,
+							childresults
+						)
 
 				for j in range(len(self.results)):
 					self.results[j].passvector.add(childresults[j].passvector)
diff --git a/tests/util/piglit-util.c b/tests/util/piglit-util.c
index 1fe734f7..4fd3a144 100644
--- a/tests/util/piglit-util.c
+++ b/tests/util/piglit-util.c
@@ -190,6 +190,14 @@ piglit_report_result(enum piglit_result result)
 	}
 }
 
+void
+piglit_report_subtest_result(const char *name, enum piglit_result result)
+{
+	const char *result_str = piglit_result_to_string(result);
+	printf("PIGLIT:subtest {'%s' : '%s'}\n", name, result_str);
+	fflush(stdout);
+}
+
 #ifndef HAVE_STRCHRNUL
 char *strchrnul(const char *s, int c)
 {
diff --git a/tests/util/piglit-util.h b/tests/util/piglit-util.h
index 2bbc7673..600bf92d 100644
--- a/tests/util/piglit-util.h
+++ b/tests/util/piglit-util.h
@@ -135,6 +135,7 @@ int piglit_find_line(const char *program, int position);
 void piglit_merge_result(enum piglit_result *all, enum piglit_result subtest);
 const char * piglit_result_to_string(enum piglit_result result);
 void piglit_report_result(enum piglit_result result);
+void piglit_report_subtest_result(const char *name, enum piglit_result result);
 
 #ifndef HAVE_STRCHRNUL
 char *strchrnul(const char *s, int c);