Warn if all execution tests either passed or failed.

This probably indicates that the execution engine is
not configured correctly.

Return an error if no execution test passed.

Change-Id: I4f86ddd9d783fe7a089a32f7ff623ef90b7efaf1

4 files changed, 48 insertions, 6 deletions

diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index 932ec80..2bccb7f 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -203,6 +203,9 @@ sub read_sum($)
    $res->{testcases} = \%testcases;
    my $pending_timeout=0;
 
+   $res->{EXEC}->{PASS} = 0;
+   $res->{EXEC}->{FAIL} = 0;
+
    open SUMFILE, $sum_file or die $!;
    while (<SUMFILE>)
    {
@@ -216,6 +219,11 @@ sub read_sum($)
 		$testcases{$tc}->{HAS_TIMED_OUT} = $pending_timeout;
 		$pending_timeout = 0;
 		$res->{$diag}++;
+		# Count execution tests, for a sanity check
+		if ($tc =~ /execution/)
+		{
+		    $res->{EXEC}->{$diag}++;
+		}
 	  }
 	  elsif (m/WARNING: program timed out/) 
 	  {
@@ -410,6 +418,9 @@ sub print_compare_results_summary($$)
    my $ref_ratio = 0;
    my $res_ratio = 0;
    my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold
+   my $ignore_exec = 0;
+   my $ref_has_exec_tests = 0;
+   my $res_has_exec_tests = 0;
 
    # Compute the pass ratio as a sanity check
    $ref_ratio = ($ref->{PASS} + $ref->{XFAIL}) /
@@ -459,6 +470,25 @@ sub print_compare_results_summary($$)
 	   printf "    ***** WARNING: RES PASS ratio is abnormally low *****\n";
        }
 
+       # If both PASS and FAIL EXEC tests are zero, assume there is no
+       # execution test for this tool, and do not warn.
+       $ref_has_exec_tests = ($ref->{EXEC}->{PASS} + $ref->{EXEC}->{FAIL}) != 0;
+       if ($ref_has_exec_tests)
+       {
+	   printf "    ***** ERROR: No REF execution test PASSed. Check execution engine configuration. *****\n" if ($ref->{EXEC}->{PASS} == 0);
+	   printf "    ***** WARNING: No REF execution test FAILed. Check execution engine configuration. *****\n" if ($ref->{EXEC}->{FAIL} == 0);
+       }
+
+       $res_has_exec_tests = ($res->{EXEC}->{PASS} + $res->{EXEC}->{FAIL}) != 0;
+       if ($res_has_exec_tests)
+       {
+	   printf "    ***** ERROR: No RES execution test PASSed. Check execution engine configuration. *****\n" if ($res->{EXEC}->{PASS} == 0);
+	   printf "    ***** WARNING: No RES execution test FAILed. Check execution engine configuration. *****\n" if ($res->{EXEC}->{FAIL} == 0);
+       }
+
+       # Ignore number of execution tests when computing the return
+       # value, if both REF and RES have no execution test.
+       $ignore_exec = !$ref_has_exec_tests && !$res_has_exec_tests;
    }
 
    #### REGRESSIONS ?
@@ -622,5 +652,11 @@ sub print_compare_results_summary($$)
    $return_value = 2 if (($ref_ratio == 100) || ($ref_ratio < $ratio_thresh));
    $return_value = 2 if (($res_ratio == 100) || ($res_ratio < $ratio_thresh));
 
+   # Error if no execution test passed.
+   # It is possible that no execution test fails, though.
+   $return_value = 2 if (($ignore_exec == 0)
+			 && (($ref->{EXEC}->{PASS} == 0)
+			     || ($res->{EXEC}->{PASS} == 0)));
+
    return $return_value;
 }
diff --git a/testsuite/expected-1-2.txt b/testsuite/expected-1-2.txt
index 614fcec..9aa53c4 100644
--- a/testsuite/expected-1-2.txt
+++ b/testsuite/expected-1-2.txt
@@ -5,8 +5,8 @@ CURRENT:  sum-2.txt
               `                              +---------+---------+
 o  RUN STATUS:                               |   REF   |   RES   |
   +------------------------------------------+---------+---------+
-  | Passes                            [PASS] |       6 |      10 |
-  | Unexpected fails                  [FAIL] |       6 |      10 |
+  | Passes                            [PASS] |       6 |      11 |
+  | Unexpected fails                  [FAIL] |       7 |      10 |
   | Unexpected passes                [XPASS] |       5 |       9 |
   | Expected fails                   [XFAIL] |       5 |       9 |
   | Unresolved                  [UNRESOLVED] |       5 |       1 |
@@ -14,10 +14,12 @@ o  RUN STATUS:                               |   REF   |   RES   |
   | Untested                      [UNTESTED] |       5 |       1 |
   +------------------------------------------+---------+---------+
 
-    REF PASS ratio: 0.30
-    RES PASS ratio: 0.46
+    REF PASS ratio: 0.29
+    RES PASS ratio: 0.48
     ***** WARNING: REF PASS ratio is abnormally low *****
     ***** WARNING: RES PASS ratio is abnormally low *****
+    ***** ERROR: No REF execution test PASSed. Check execution engine configuration. *****
+    ***** WARNING: No RES execution test FAILed. Check execution engine configuration. *****
 
 o  REGRESSIONS:
   +------------------------------------------+---------+
@@ -51,7 +53,7 @@ o  MINOR TO BE CHECKED:
   | XPASS now PASS            [XPASS=> PASS] |       1 |
   | XFAIL now PASS            [XFAIL=> PASS] |       1 |
   | XFAIL now XPASS           [XFAIL=>XPASS] |       1 |
-  | FAIL now PASS             [FAIL => PASS] |       1 |
+  | FAIL now PASS             [FAIL => PASS] |       2 |
   | FAIL now XPASS            [FAIL =>XPASS] |       1 |
   | New PASS                  [     => PASS] |       1 |
   | UNSUPPORTED now PASS      [UNSUP=> PASS] |       1 |
@@ -64,7 +66,7 @@ o  MINOR TO BE CHECKED:
   | UNRESOLVED now XFAIL      [UNRES=>XFAIL] |       1 |
   | UNRESOLVED disappears     [UNRES=>     ] |       1 |
   +------------------------------------------+---------+
-  | TOTAL_MINOR_TO_BE_CHECKED                |      20 |
+  | TOTAL_MINOR_TO_BE_CHECKED                |      21 |
   +------------------------------------------+---------+
 
 
diff --git a/testsuite/sum-1.txt b/testsuite/sum-1.txt
index 019b829..b83ffe0 100644
--- a/testsuite/sum-1.txt
+++ b/testsuite/sum-1.txt
@@ -42,3 +42,5 @@ UNRESOLVED: test5-unresolved-disappears
 
 PASS: test1-pass-and-fails
 FAIL: test1-pass-and-fails
+
+FAIL: test1-fail-pass execution
diff --git a/testsuite/sum-2.txt b/testsuite/sum-2.txt
index 30527ce..3df52f0 100644
--- a/testsuite/sum-2.txt
+++ b/testsuite/sum-2.txt
@@ -48,3 +48,5 @@ UNRESOLVED: test4-unresolved-appears
 
 PASS: test1-pass-and-fails
 FAIL: test1-pass-and-fails
+
+PASS: test1-fail-pass execution