diff options
| author | Christophe Lyon <christophe.lyon@linaro.org> | 2016-02-02 19:07:58 +0100 |
|---|---|---|
| committer | Christophe Lyon <christophe.lyon@linaro.org> | 2016-02-02 21:35:37 +0100 |
| commit | d618f660197a4091e862f068ef7026f90a5837d6 (patch) | |
| tree | fa760029e8fd8a144bf3ac50a480c4de18061aec | |
| parent | fd5316d639f291be590d8716863348948dabb017 (diff) | |
Compute pass ratio as a sanity check.
Warn if it is below 0.95.
Change-Id: Id8c1ad8692020f77cfefbda9311ebdba19000317
| -rwxr-xr-x | compare_dg_tests.pl | 38 | ||||
| -rw-r--r-- | testsuite/expected-1-2.txt | 4 |
2 files changed, 40 insertions, 2 deletions
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl index c8b166c..932ec80 100755 --- a/compare_dg_tests.pl +++ b/compare_dg_tests.pl @@ -407,6 +407,23 @@ sub print_compare_results_summary($$) my $total_better = 0; my $rtotal = 0; my $quiet_reg = $quiet; + my $ref_ratio = 0; + my $res_ratio = 0; + my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold + + # Compute the pass ratio as a sanity check + $ref_ratio = ($ref->{PASS} + $ref->{XFAIL}) / + ($ref->{PASS} + $ref->{XFAIL} + + $ref->{XPASS} + $ref->{FAIL} + + $ref->{UNRESOLVED} + + $ref->{UNSUPPORTED} + + $ref->{UNTESTED}); + $res_ratio = ($res->{PASS} + $res->{XFAIL}) / + ($res->{PASS} + $res->{XFAIL} + + $res->{XPASS} + $res->{FAIL} + + $res->{UNRESOLVED} + + $res->{UNSUPPORTED} + + $res->{UNTESTED}); if (not $quiet) { @@ -430,6 +447,18 @@ sub print_compare_results_summary($$) printf " | %-40s | %7d | %7d |\n", "Untested [UNTESTED]", $ref->{UNTESTED}, $res->{UNTESTED}; printf " +------------------------------------------+---------+---------+\n"; printf "\n"; + + printf " REF PASS ratio: %.2f\n", $ref_ratio; + printf " RES PASS ratio: %.2f\n", $res_ratio; + if ($ref_ratio < $ratio_thresh) + { + printf " ***** WARNING: REF PASS ratio is abnormally low *****\n"; + } + if ($res_ratio < $ratio_thresh) + { + printf " ***** WARNING: RES PASS ratio is abnormally low *****\n"; + } + } #### REGRESSIONS ? @@ -585,8 +614,13 @@ sub print_compare_results_summary($$) $return_value = 2 if ($rtotal); - # Error if there was no PASS (eg when sth went wrong and no .sum was generated - $return_value = 2 if (($res->{PASS} + $res->{XPASS}) == 0); + # Error if there was no PASS (eg when sth went wrong and no .sum was generated) + $return_value = 2 if (($res->{PASS} + $res->{XFAIL}) == 0); + + # Error if every test passed, or the ratio was too low. This + # generally means the simulator is not configured well. + $return_value = 2 if (($ref_ratio == 100) || ($ref_ratio < $ratio_thresh)); + $return_value = 2 if (($res_ratio == 100) || ($res_ratio < $ratio_thresh)); return $return_value; } diff --git a/testsuite/expected-1-2.txt b/testsuite/expected-1-2.txt index 133effc..614fcec 100644 --- a/testsuite/expected-1-2.txt +++ b/testsuite/expected-1-2.txt @@ -14,6 +14,10 @@ o RUN STATUS: | REF | RES | | Untested [UNTESTED] | 5 | 1 | +------------------------------------------+---------+---------+ + REF PASS ratio: 0.30 + RES PASS ratio: 0.46 + ***** WARNING: REF PASS ratio is abnormally low ***** + ***** WARNING: RES PASS ratio is abnormally low ***** o REGRESSIONS: +------------------------------------------+---------+ |