From d618f660197a4091e862f068ef7026f90a5837d6 Mon Sep 17 00:00:00 2001
From: Christophe Lyon <christophe.lyon@linaro.org>
Date: Tue, 2 Feb 2016 19:07:58 +0100
Subject: Compute pass ratio as a sanity check. Warn if it is below 0.95.

Change-Id: Id8c1ad8692020f77cfefbda9311ebdba19000317
---
 compare_dg_tests.pl        | 38 ++++++++++++++++++++++++++++++++++++--
 testsuite/expected-1-2.txt |  4 ++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index c8b166c..932ec80 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -407,6 +407,23 @@ sub print_compare_results_summary($$)
    my $total_better = 0;
    my $rtotal = 0;
    my $quiet_reg = $quiet;
+   my $ref_ratio = 0;
+   my $res_ratio = 0;
+   my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold
+
+   # Compute the pass ratio as a sanity check
+   $ref_ratio = ($ref->{PASS} + $ref->{XFAIL}) /
+       ($ref->{PASS} + $ref->{XFAIL}
+	+ $ref->{XPASS} + $ref->{FAIL}
+	+ $ref->{UNRESOLVED}
+	+ $ref->{UNSUPPORTED}
+	+ $ref->{UNTESTED});
+   $res_ratio = ($res->{PASS} + $res->{XFAIL}) /
+       ($res->{PASS} + $res->{XFAIL}
+	+ $res->{XPASS} + $res->{FAIL}
+	+ $res->{UNRESOLVED}
+	+ $res->{UNSUPPORTED}
+	+ $res->{UNTESTED});
 
    if (not $quiet)
    {
@@ -430,6 +447,18 @@ sub print_compare_results_summary($$)
        printf "  | %-40s | %7d | %7d |\n", "Untested                      [UNTESTED]", $ref->{UNTESTED}, $res->{UNTESTED};
        printf "  +------------------------------------------+---------+---------+\n";
        printf "\n";
+
+       printf "    REF PASS ratio: %.2f\n", $ref_ratio;
+       printf "    RES PASS ratio: %.2f\n", $res_ratio;
+       if ($ref_ratio < $ratio_thresh)
+       {
+	   printf "    ***** WARNING: REF PASS ratio is abnormally low *****\n";
+       }
+       if ($res_ratio < $ratio_thresh)
+       {
+	   printf "    ***** WARNING: RES PASS ratio is abnormally low *****\n";
+       }
+
    }
 
    #### REGRESSIONS ?
@@ -585,8 +614,13 @@ sub print_compare_results_summary($$)
 
    $return_value = 2 if ($rtotal);
 
-   # Error if there was no PASS (eg when sth went wrong and no .sum was generated
-   $return_value = 2 if (($res->{PASS} + $res->{XPASS}) == 0);
+   # Error if there was no PASS (eg when sth went wrong and no .sum was generated)
+   $return_value = 2 if (($res->{PASS} + $res->{XFAIL}) == 0);
+
+   # Error if every test passed, or the ratio was too low. This
+   # generally means the simulator is not configured well.
+   $return_value = 2 if (($ref_ratio == 100) || ($ref_ratio < $ratio_thresh));
+   $return_value = 2 if (($res_ratio == 100) || ($res_ratio < $ratio_thresh));
 
    return $return_value;
 }
diff --git a/testsuite/expected-1-2.txt b/testsuite/expected-1-2.txt
index 133effc..614fcec 100644
--- a/testsuite/expected-1-2.txt
+++ b/testsuite/expected-1-2.txt
@@ -14,6 +14,10 @@ o  RUN STATUS:                               |   REF   |   RES   |
   | Untested                      [UNTESTED] |       5 |       1 |
   +------------------------------------------+---------+---------+
 
+    REF PASS ratio: 0.30
+    RES PASS ratio: 0.46
+    ***** WARNING: REF PASS ratio is abnormally low *****
+    ***** WARNING: RES PASS ratio is abnormally low *****
 
 o  REGRESSIONS:
   +------------------------------------------+---------+
-- 
cgit v1.2.3