diff options
author | Christophe Lyon <christophe.lyon@linaro.org> | 2016-04-05 17:38:07 +0200 |
---|---|---|
committer | Christophe Lyon <christophe.lyon@linaro.org> | 2016-04-05 17:38:07 +0200 |
commit | 4e0282674eab6aa32985105278cdc93f8b298a57 (patch) | |
tree | 058fc7f3318828627d807944b329debc0a9dc26a | |
parent | 9e7524088987b1fc67c79446506e862a77a84cc4 (diff) |
Print new UNTESTED, UNRESOLVED and UNSUPPORTED tests in the report.new-unresolved
Such new tests used to be omitted from the report, now:
- UNRESOLVED are shown as regressions
- UNTESTED and UNSUPPORTED are shown as "to be checked"
Change-Id: Ic715b7a1e52069746ade6cec376f02a737cce0a5
-rwxr-xr-x | compare_dg_tests.pl | 20 | ||||
-rw-r--r-- | testsuite/expected-1-2.txt | 7 |
2 files changed, 25 insertions, 2 deletions
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl index cc45605..713303d 100755 --- a/compare_dg_tests.pl +++ b/compare_dg_tests.pl @@ -48,6 +48,8 @@ my $UNTESTED_DISAPPEARS = "UNTESTED disappears [UNTES=> ]"; my $UNRESOLVED_PASS = "UNRESOLVED now PASS [UNRES=> PASS]"; my $UNRESOLVED_XFAIL = "UNRESOLVED now XFAIL [UNRES=>XFAIL]"; my $UNRESOLVED_DISAPPEARS = "UNRESOLVED disappears [UNRES=> ]"; +my $UNSUPPORTED_APPEARS = "UNSUPPORTED appears [ =>UNSUP]"; +my $UNTESTED_APPEARS = "UNTESTED appears [ =>UNTES]"; # REGRESSIONS my $PASS_NOW_XPASS = "PASS now XPASS [PASS =>XPASS]"; @@ -56,6 +58,7 @@ my $PASS_DISAPPEARS = "PASS disappears [PASS => ]"; my $XPASS_DISAPPEARS = "XPASS disappears [XPASS=> ]"; my $FAIL_APPEARS = "FAIL appears [ => FAIL]"; my $XPASS_APPEARS = "XPASS appears [ =>XPASS]"; +my $UNRESOLVED_APPEARS = "UNRESOLVED appears [ =>UNRES]"; my $XPASS_NOW_FAIL = "XPASS now FAIL [XPASS=> FAIL]"; my $XFAIL_NOW_FAIL = "XFAIL now FAIL [XFAIL=> FAIL]"; my $XFAIL_DISAPPEARS = "XFAIL disappears [XFAIL=> ]"; @@ -113,6 +116,10 @@ my @handler_list = ( {was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL}, {was=>"UNRESOLVED", is=>"NO_EXIST", cat=>$UNRESOLVED_DISAPPEARS}, + {was=>"NO_EXIST", is=>"UNRESOLVED", cat=>$UNRESOLVED_APPEARS}, + {was=>"NO_EXIST", is=>"UNTESTED", cat=>$UNTESTED_APPEARS}, + {was=>"NO_EXIST", is=>"UNSUPPORTED", cat=>$UNSUPPORTED_APPEARS}, + # {was=>"NO_EXIST", is=>"NO_EXIST", handler=>\&handle_not_yet_supported} ); @@ -333,16 +340,19 @@ sub compare_results($$) @{$res->{$UNSUPPORTED_FAIL}} = (); @{$res->{$UNSUPPORTED_XFAIL}} = (); @{$res->{$UNSUPPORTED_DISAPPEARS}} = (); + @{$res->{$UNSUPPORTED_APPEARS}} = (); @{$res->{$UNTESTED_PASS}} = (); @{$res->{$UNTESTED_XPASS}} = (); @{$res->{$UNTESTED_FAIL}} = (); @{$res->{$UNTESTED_XFAIL}} = (); @{$res->{$UNTESTED_DISAPPEARS}} = (); + @{$res->{$UNTESTED_APPEARS}} = (); @{$res->{$UNRESOLVED_PASS}} = (); @{$res->{$UNRESOLVED_XPASS}} = (); @{$res->{$UNRESOLVED_FAIL}} = (); @{$res->{$UNRESOLVED_XFAIL}} = (); @{$res->{$UNRESOLVED_DISAPPEARS}} = (); + @{$res->{$UNRESOLVED_APPEARS}} = (); @{$res->{$UNHANDLED_CASES}} = (); @{$res->{$UNSTABLE_CASES}} = (); @@ -373,6 +383,7 @@ sub compare_results($$) if ($ref->{testcases}->{$key}->{$diag_diag->{was}} != $res->{testcases}->{$key}->{$diag_diag->{was}} and $res->{testcases}->{$key}->{$diag_diag->{is}}) { + print "$key ref was:$ref->{testcases}->{$key}->{$diag_diag->{was}}, res was:$res->{testcases}->{$key}->{$diag_diag->{was}}, res is:$res->{testcases}->{$key}->{$diag_diag->{is}} \n" if ($debug); # If testcase is listed as 'unstable' mark it as # such and skip other processing. { @@ -507,6 +518,7 @@ sub print_compare_results_summary($$) +scalar(@{$res->{$UNTESTED_XPASS}}) +scalar(@{$res->{$UNRESOLVED_FAIL}}) +scalar(@{$res->{$UNRESOLVED_XPASS}}) + +scalar(@{$res->{$UNRESOLVED_APPEARS}}) +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}); $quiet_reg=1 if ($short and not $rtotal); @@ -530,6 +542,7 @@ sub print_compare_results_summary($$) printf " | %-40s | %7d |\n", $UNTESTED_XPASS, scalar(@{$res->{$UNTESTED_XPASS}}) if (scalar(@{$res->{$UNTESTED_XPASS}})); printf " | %-40s | %7d |\n", $UNRESOLVED_FAIL, scalar(@{$res->{$UNRESOLVED_FAIL}}) if (scalar(@{$res->{$UNRESOLVED_FAIL}})); printf " | %-40s | %7d |\n", $UNRESOLVED_XPASS, scalar(@{$res->{$UNRESOLVED_XPASS}}) if (scalar(@{$res->{$UNRESOLVED_XPASS}})); + printf " | %-40s | %7d |\n", $UNRESOLVED_APPEARS, scalar(@{$res->{$UNRESOLVED_APPEARS}}) if (scalar(@{$res->{$UNRESOLVED_APPEARS}})); printf " | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}})); printf " +------------------------------------------+---------+\n"; printf " | %-40s | %7d |\n", "TOTAL_REGRESSIONS", $rtotal; @@ -553,6 +566,7 @@ sub print_compare_results_summary($$) print_tclist($UNTESTED_XPASS, @{$res->{$UNTESTED_XPASS}}); print_tclist($UNRESOLVED_FAIL, @{$res->{$UNRESOLVED_FAIL}}); print_tclist($UNRESOLVED_XPASS, @{$res->{$UNRESOLVED_XPASS}}); + print_tclist($UNRESOLVED_APPEARS, @{$res->{$UNRESOLVED_APPEARS}}); print_tclist($PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}}); } printf "$col_reset\n"; @@ -575,9 +589,11 @@ sub print_compare_results_summary($$) scalar(@{$res->{$UNSUPPORTED_PASS}})+ scalar(@{$res->{$UNSUPPORTED_XFAIL}})+ scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}})+ + scalar(@{$res->{$UNSUPPORTED_APPEARS}})+ scalar(@{$res->{$UNTESTED_PASS}})+ scalar(@{$res->{$UNTESTED_XFAIL}})+ scalar(@{$res->{$UNTESTED_DISAPPEARS}})+ + scalar(@{$res->{$UNTESTED_APPEARS}})+ scalar(@{$res->{$UNRESOLVED_PASS}})+ scalar(@{$res->{$UNRESOLVED_XFAIL}})+ scalar(@{$res->{$UNRESOLVED_DISAPPEARS}})+ @@ -599,9 +615,11 @@ sub print_compare_results_summary($$) printf " | %-40s | %7d |\n", $UNSUPPORTED_PASS, scalar(@{$res->{$UNSUPPORTED_PASS}}) if (scalar(@{$res->{$UNSUPPORTED_PASS}})); printf " | %-40s | %7d |\n", $UNSUPPORTED_XFAIL, scalar(@{$res->{$UNSUPPORTED_XFAIL}}) if (scalar(@{$res->{$UNSUPPORTED_XFAIL}})); printf " | %-40s | %7d |\n", $UNSUPPORTED_DISAPPEARS, scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}})); + printf " | %-40s | %7d |\n", $UNSUPPORTED_APPEARS, scalar(@{$res->{$UNSUPPORTED_APPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_APPEARS}})); printf " | %-40s | %7d |\n", $UNTESTED_PASS, scalar(@{$res->{$UNTESTED_PASS}}) if (scalar(@{$res->{$UNTESTED_PASS}})); printf " | %-40s | %7d |\n", $UNTESTED_XFAIL, scalar(@{$res->{$UNTESTED_XFAIL}}) if (scalar(@{$res->{$UNTESTED_XFAIL}})); printf " | %-40s | %7d |\n", $UNTESTED_DISAPPEARS, scalar(@{$res->{$UNTESTED_DISAPPEARS}}) if (scalar(@{$res->{$UNTESTED_DISAPPEARS}})); + printf " | %-40s | %7d |\n", $UNTESTED_APPEARS, scalar(@{$res->{$UNTESTED_APPEARS}}) if (scalar(@{$res->{$UNTESTED_APPEARS}})); printf " | %-40s | %7d |\n", $UNRESOLVED_PASS, scalar(@{$res->{$UNRESOLVED_PASS}}) if (scalar(@{$res->{$UNRESOLVED_PASS}})); printf " | %-40s | %7d |\n", $UNRESOLVED_XFAIL, scalar(@{$res->{$UNRESOLVED_XFAIL}}) if (scalar(@{$res->{$UNRESOLVED_XFAIL}})); printf " | %-40s | %7d |\n", $UNRESOLVED_DISAPPEARS, scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}) if (scalar(@{$res->{$UNRESOLVED_DISAPPEARS}})); @@ -628,9 +646,11 @@ sub print_compare_results_summary($$) print_tclist($UNSUPPORTED_PASS, @{$res->{$UNSUPPORTED_PASS}}); print_tclist($UNSUPPORTED_XFAIL, @{$res->{$UNSUPPORTED_XFAIL}}); print_tclist($UNSUPPORTED_DISAPPEARS, @{$res->{$UNSUPPORTED_DISAPPEARS}}); + print_tclist($UNSUPPORTED_APPEARS, @{$res->{$UNSUPPORTED_APPEARS}}); print_tclist($UNTESTED_PASS, @{$res->{$UNTESTED_PASS}}); print_tclist($UNTESTED_XFAIL, @{$res->{$UNTESTED_XFAIL}}); print_tclist($UNTESTED_DISAPPEARS, @{$res->{$UNTESTED_DISAPPEARS}}); + print_tclist($UNTESTED_APPEARS, @{$res->{$UNTESTED_APPEARS}}); print_tclist($UNRESOLVED_PASS, @{$res->{$UNRESOLVED_PASS}}); print_tclist($UNRESOLVED_XFAIL, @{$res->{$UNRESOLVED_XFAIL}}); print_tclist($UNRESOLVED_DISAPPEARS, @{$res->{$UNRESOLVED_DISAPPEARS}}); diff --git a/testsuite/expected-1-2.txt b/testsuite/expected-1-2.txt index 2b8b883..7b49705 100644 --- a/testsuite/expected-1-2.txt +++ b/testsuite/expected-1-2.txt @@ -38,8 +38,9 @@ o REGRESSIONS: | UNTESTED now XPASS [UNTES=>XPASS] | 1 | | UNRESOLVED now FAIL [UNRES=> FAIL] | 1 | | UNRESOLVED now XPASS [UNRES=>XPASS] | 1 | + | UNRESOLVED appears [ =>UNRES] | 1 | +------------------------------------------+---------+ - | TOTAL_REGRESSIONS | 15 | + | TOTAL_REGRESSIONS | 16 | +------------------------------------------+---------+ @@ -59,14 +60,16 @@ o MINOR TO BE CHECKED: | UNSUPPORTED now PASS [UNSUP=> PASS] | 1 | | UNSUPPORTED now XFAIL [UNSUP=>XFAIL] | 1 | | UNSUPPORTED disappears [UNSUP=> ] | 1 | + | UNSUPPORTED appears [ =>UNSUP] | 1 | | UNTESTED now PASS [UNTES=> PASS] | 1 | | UNTESTED now XFAIL [UNTES=>XFAIL] | 1 | | UNTESTED disappears [UNTES=> ] | 1 | + | UNTESTED appears [ =>UNTES] | 1 | | UNRESOLVED now PASS [UNRES=> PASS] | 1 | | UNRESOLVED now XFAIL [UNRES=>XFAIL] | 1 | | UNRESOLVED disappears [UNRES=> ] | 1 | +------------------------------------------+---------+ - | TOTAL_MINOR_TO_BE_CHECKED | 21 | + | TOTAL_MINOR_TO_BE_CHECKED | 23 | +------------------------------------------+---------+ |