diff options
Diffstat (limited to 'compare_dg_tests.pl')
| -rwxr-xr-x | compare_dg_tests.pl | 198 |
1 files changed, 121 insertions, 77 deletions
diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl index 290c9a1..05acb44 100755 --- a/compare_dg_tests.pl +++ b/compare_dg_tests.pl @@ -94,80 +94,88 @@ my $ERROR_APPEARS = "ERROR appears [ =>ERROR]"; my $UNHANDLED_CASES = "Unhandled cases [ ..??.. ]"; my $UNSTABLE_CASES = "Unstable cases, ignored [~RANDOM ]"; - +my $HWDEP_CASES = "HW dependent cases [HW-DEPENDENT]"; + +# Handle status transitions: +# 'was' is the reference status +# 'is' is the current status +# 'cat' is the short name for the transition type +# 'handler' is a function handling the transition, if non-trivial +# report_hwdep is a boolean controlling whether to report the +# transition even if flagged as hardware-dependent my @handler_list = ( - {was=>"PASS", is=>"PASS", cat=>$PASS_PASS}, - {was=>"PASS", is=>"XPASS", cat=>$PASS_XPASS}, - {was=>"PASS", is=>"FAIL", handler=>\&handle_pass_fail}, - {was=>"PASS", is=>"XFAIL", cat=>$PASS_XFAIL}, - {was=>"PASS", is=>"UNSUPPORTED",cat=>$PASS_UNSUPPORTED}, - {was=>"PASS", is=>"UNTESTED", cat=>$PASS_UNTESTED}, - {was=>"PASS", is=>"UNRESOLVED",cat=>$PASS_UNRESOLVED}, - {was=>"PASS", is=>"NO_EXIST", cat=>$PASS_DISAPPEARS}, - {was=>"NO_EXIST", is=>"PASS", cat=>$PASS_APPEARS}, - - {was=>"XPASS", is=>"PASS", cat=>$XPASS_PASS}, - {was=>"XPASS", is=>"XPASS", cat=>$XPASS_XPASS}, - {was=>"XPASS", is=>"FAIL", cat=>$XPASS_FAIL}, - {was=>"XPASS", is=>"XFAIL", cat=>$XPASS_XFAIL}, - {was=>"XPASS", is=>"UNSUPPORTED",cat=>$XPASS_UNSUPPORTED}, - {was=>"XPASS", is=>"UNTESTED", cat=>$XPASS_UNTESTED}, - {was=>"XPASS", is=>"UNRESOLVED",cat=>$XPASS_UNRESOLVED}, - {was=>"XPASS", is=>"NO_EXIST", cat=>$XPASS_DISAPPEARS}, - {was=>"NO_EXIST", is=>"XPASS", cat=>$XPASS_APPEARS}, - - {was=>"FAIL", is=>"PASS", cat=>$FAIL_PASS}, - {was=>"FAIL", is=>"XPASS", cat=>$FAIL_XPASS}, - {was=>"FAIL", is=>"FAIL", cat=>$FAIL_FAIL}, - {was=>"FAIL", is=>"XFAIL", cat=>$FAIL_XFAIL}, - {was=>"FAIL", is=>"UNSUPPORTED",cat=>$FAIL_UNSUPPORTED}, - {was=>"FAIL", is=>"UNTESTED", cat=>$FAIL_UNTESTED}, - {was=>"FAIL", is=>"UNRESOLVED",cat=>$FAIL_UNRESOLVED}, - {was=>"FAIL", is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS}, - {was=>"NO_EXIST", is=>"FAIL", cat=>$FAIL_APPEARS}, - - {was=>"XFAIL", is=>"PASS", cat=>$XFAIL_PASS}, - {was=>"XFAIL", is=>"XPASS", cat=>$XFAIL_XPASS}, - {was=>"XFAIL", is=>"FAIL", cat=>$XFAIL_FAIL}, - {was=>"XFAIL", is=>"XFAIL", cat=>$XFAIL_XFAIL}, - {was=>"XFAIL", is=>"UNSUPPORTED",cat=>$XFAIL_UNSUPPORTED}, - {was=>"XFAIL", is=>"UNTESTED", cat=>$XFAIL_UNTESTED}, - {was=>"XFAIL", is=>"UNRESOLVED",cat=>$XFAIL_UNRESOLVED}, - {was=>"XFAIL", is=>"NO_EXIST", cat=>$XFAIL_DISAPPEARS}, - {was=>"NO_EXIST", is=>"XFAIL", cat=>$XFAIL_APPEARS}, - - {was=>"UNSUPPORTED", is=>"PASS", cat=>$UNSUPPORTED_PASS}, - {was=>"UNSUPPORTED", is=>"XPASS", cat=>$UNSUPPORTED_XPASS}, - {was=>"UNSUPPORTED", is=>"FAIL", cat=>$UNSUPPORTED_FAIL}, - {was=>"UNSUPPORTED", is=>"XFAIL", cat=>$UNSUPPORTED_XFAIL}, - {was=>"UNSUPPORTED", is=>"UNSUPPORTED",cat=>$UNSUPPORTED_UNSUPPORTED}, - {was=>"UNSUPPORTED", is=>"UNTESTED", cat=>$UNSUPPORTED_UNTESTED}, - {was=>"UNSUPPORTED", is=>"UNRESOLVED",cat=>$UNSUPPORTED_UNRESOLVED}, - {was=>"UNSUPPORTED", is=>"NO_EXIST", cat=>$UNSUPPORTED_DISAPPEARS}, - {was=>"NO_EXIST", is=>"UNSUPPORTED", cat=>$UNSUPPORTED_APPEARS}, - - {was=>"UNTESTED", is=>"PASS", cat=>$UNTESTED_PASS}, - {was=>"UNTESTED", is=>"XPASS", cat=>$UNTESTED_XPASS}, - {was=>"UNTESTED", is=>"FAIL", cat=>$UNTESTED_FAIL}, - {was=>"UNTESTED", is=>"XFAIL", cat=>$UNTESTED_XFAIL}, - {was=>"UNTESTED", is=>"UNSUPPORTED",cat=>$UNTESTED_UNSUPPORTED}, - {was=>"UNTESTED", is=>"UNTESTED", cat=>$UNTESTED_UNTESTED}, - {was=>"UNTESTED", is=>"UNRESOLVED",cat=>$UNTESTED_UNRESOLVED}, - {was=>"UNTESTED", is=>"NO_EXIST", cat=>$UNTESTED_DISAPPEARS}, - {was=>"NO_EXIST", is=>"UNTESTED", cat=>$UNTESTED_APPEARS}, - - {was=>"UNRESOLVED", is=>"PASS", cat=>$UNRESOLVED_PASS}, - {was=>"UNRESOLVED", is=>"XPASS", cat=>$UNRESOLVED_XPASS}, - {was=>"UNRESOLVED", is=>"FAIL", cat=>$UNRESOLVED_FAIL}, - {was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL}, - {was=>"UNRESOLVED", is=>"UNSUPPORTED",cat=>$UNRESOLVED_UNSUPPORTED}, - {was=>"UNRESOLVED", is=>"UNTESTED", cat=>$UNRESOLVED_UNTESTED}, - {was=>"UNRESOLVED", is=>"UNRESOLVED",cat=>$UNRESOLVED_UNRESOLVED}, - {was=>"UNRESOLVED", is=>"NO_EXIST", cat=>$UNRESOLVED_DISAPPEARS}, - {was=>"NO_EXIST", is=>"UNRESOLVED", cat=>$UNRESOLVED_APPEARS}, - - {was=>"ERROR", is=>"NO_EXIST", cat=>$ERROR_DISAPPEARS}, - {was=>"NO_EXIST", is=>"ERROR", cat=>$ERROR_APPEARS}, + {was=>"PASS", is=>"PASS", cat=>$PASS_PASS, report_hwdep=>"false"}, + {was=>"PASS", is=>"XPASS", cat=>$PASS_XPASS, report_hwdep=>"true"}, + {was=>"PASS", is=>"FAIL", handler=>\&handle_pass_fail, report_hwdep=>"true"}, + {was=>"PASS", is=>"XFAIL", cat=>$PASS_XFAIL, report_hwdep=>"true"}, + {was=>"PASS", is=>"UNSUPPORTED",cat=>$PASS_UNSUPPORTED, report_hwdep=>"false"}, + {was=>"PASS", is=>"UNTESTED", cat=>$PASS_UNTESTED, report_hwdep=>"true"}, + {was=>"PASS", is=>"UNRESOLVED",cat=>$PASS_UNRESOLVED, report_hwdep=>"true"}, + {was=>"PASS", is=>"NO_EXIST", cat=>$PASS_DISAPPEARS, report_hwdep=>"false"}, + {was=>"NO_EXIST", is=>"PASS", cat=>$PASS_APPEARS, report_hwdep=>"false"}, + + {was=>"XPASS", is=>"PASS", cat=>$XPASS_PASS, report_hwdep=>"true"}, + {was=>"XPASS", is=>"XPASS", cat=>$XPASS_XPASS, report_hwdep=>"true"}, + {was=>"XPASS", is=>"FAIL", cat=>$XPASS_FAIL, report_hwdep=>"true"}, + {was=>"XPASS", is=>"XFAIL", cat=>$XPASS_XFAIL, report_hwdep=>"true"}, + {was=>"XPASS", is=>"UNSUPPORTED",cat=>$XPASS_UNSUPPORTED, report_hwdep=>"true"}, + {was=>"XPASS", is=>"UNTESTED", cat=>$XPASS_UNTESTED, report_hwdep=>"true"}, + {was=>"XPASS", is=>"UNRESOLVED",cat=>$XPASS_UNRESOLVED, report_hwdep=>"true"}, + {was=>"XPASS", is=>"NO_EXIST", cat=>$XPASS_DISAPPEARS, report_hwdep=>"true"}, + {was=>"NO_EXIST", is=>"XPASS", cat=>$XPASS_APPEARS, report_hwdep=>"true"}, + + {was=>"FAIL", is=>"PASS", cat=>$FAIL_PASS, report_hwdep=>"true"}, + {was=>"FAIL", is=>"XPASS", cat=>$FAIL_XPASS, report_hwdep=>"true"}, + {was=>"FAIL", is=>"FAIL", cat=>$FAIL_FAIL, report_hwdep=>"true"}, + {was=>"FAIL", is=>"XFAIL", cat=>$FAIL_XFAIL, report_hwdep=>"true"}, + {was=>"FAIL", is=>"UNSUPPORTED",cat=>$FAIL_UNSUPPORTED, report_hwdep=>"true"}, + {was=>"FAIL", is=>"UNTESTED", cat=>$FAIL_UNTESTED, report_hwdep=>"true"}, + {was=>"FAIL", is=>"UNRESOLVED",cat=>$FAIL_UNRESOLVED, report_hwdep=>"true"}, + {was=>"FAIL", is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS, report_hwdep=>"true"}, + {was=>"NO_EXIST", is=>"FAIL", cat=>$FAIL_APPEARS, report_hwdep=>"true"}, + + {was=>"XFAIL", is=>"PASS", cat=>$XFAIL_PASS, report_hwdep=>"true"}, + {was=>"XFAIL", is=>"XPASS", cat=>$XFAIL_XPASS, report_hwdep=>"true"}, + {was=>"XFAIL", is=>"FAIL", cat=>$XFAIL_FAIL, report_hwdep=>"true"}, + {was=>"XFAIL", is=>"XFAIL", cat=>$XFAIL_XFAIL, report_hwdep=>"true"}, + {was=>"XFAIL", is=>"UNSUPPORTED",cat=>$XFAIL_UNSUPPORTED, report_hwdep=>"true"}, + {was=>"XFAIL", is=>"UNTESTED", cat=>$XFAIL_UNTESTED, report_hwdep=>"true"}, + {was=>"XFAIL", is=>"UNRESOLVED",cat=>$XFAIL_UNRESOLVED, report_hwdep=>"true"}, + {was=>"XFAIL", is=>"NO_EXIST", cat=>$XFAIL_DISAPPEARS, report_hwdep=>"true"}, + {was=>"NO_EXIST", is=>"XFAIL", cat=>$XFAIL_APPEARS, report_hwdep=>"true"}, + + {was=>"UNSUPPORTED", is=>"PASS", cat=>$UNSUPPORTED_PASS, report_hwdep=>"true"}, + {was=>"UNSUPPORTED", is=>"XPASS", cat=>$UNSUPPORTED_XPASS, report_hwdep=>"true"}, + {was=>"UNSUPPORTED", is=>"FAIL", cat=>$UNSUPPORTED_FAIL, report_hwdep=>"true"}, + {was=>"UNSUPPORTED", is=>"XFAIL", cat=>$UNSUPPORTED_XFAIL, report_hwdep=>"true"}, + {was=>"UNSUPPORTED", is=>"UNSUPPORTED",cat=>$UNSUPPORTED_UNSUPPORTED, report_hwdep=>"true"}, + {was=>"UNSUPPORTED", is=>"UNTESTED", cat=>$UNSUPPORTED_UNTESTED, report_hwdep=>"true"}, + {was=>"UNSUPPORTED", is=>"UNRESOLVED",cat=>$UNSUPPORTED_UNRESOLVED, report_hwdep=>"true"}, + {was=>"UNSUPPORTED", is=>"NO_EXIST", cat=>$UNSUPPORTED_DISAPPEARS, report_hwdep=>"true"}, + {was=>"NO_EXIST", is=>"UNSUPPORTED", cat=>$UNSUPPORTED_APPEARS, report_hwdep=>"true"}, + + {was=>"UNTESTED", is=>"PASS", cat=>$UNTESTED_PASS, report_hwdep=>"true"}, + {was=>"UNTESTED", is=>"XPASS", cat=>$UNTESTED_XPASS, report_hwdep=>"true"}, + {was=>"UNTESTED", is=>"FAIL", cat=>$UNTESTED_FAIL, report_hwdep=>"true"}, + {was=>"UNTESTED", is=>"XFAIL", cat=>$UNTESTED_XFAIL, report_hwdep=>"true"}, + {was=>"UNTESTED", is=>"UNSUPPORTED",cat=>$UNTESTED_UNSUPPORTED,report_hwdep=>"true"}, + {was=>"UNTESTED", is=>"UNTESTED", cat=>$UNTESTED_UNTESTED, report_hwdep=>"true"}, + {was=>"UNTESTED", is=>"UNRESOLVED",cat=>$UNTESTED_UNRESOLVED, report_hwdep=>"true"}, + {was=>"UNTESTED", is=>"NO_EXIST", cat=>$UNTESTED_DISAPPEARS, report_hwdep=>"true"}, + {was=>"NO_EXIST", is=>"UNTESTED", cat=>$UNTESTED_APPEARS, report_hwdep=>"true"}, + + {was=>"UNRESOLVED", is=>"PASS", cat=>$UNRESOLVED_PASS, report_hwdep=>"true"}, + {was=>"UNRESOLVED", is=>"XPASS", cat=>$UNRESOLVED_XPASS, report_hwdep=>"true"}, + {was=>"UNRESOLVED", is=>"FAIL", cat=>$UNRESOLVED_FAIL, report_hwdep=>"true"}, + {was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL, report_hwdep=>"true"}, + {was=>"UNRESOLVED", is=>"UNSUPPORTED",cat=>$UNRESOLVED_UNSUPPORTED, report_hwdep=>"true"}, + {was=>"UNRESOLVED", is=>"UNTESTED", cat=>$UNRESOLVED_UNTESTED, report_hwdep=>"true"}, + {was=>"UNRESOLVED", is=>"UNRESOLVED",cat=>$UNRESOLVED_UNRESOLVED, report_hwdep=>"true"}, + {was=>"UNRESOLVED", is=>"NO_EXIST", cat=>$UNRESOLVED_DISAPPEARS, report_hwdep=>"true"}, + {was=>"NO_EXIST", is=>"UNRESOLVED", cat=>$UNRESOLVED_APPEARS, report_hwdep=>"true"}, + + {was=>"ERROR", is=>"NO_EXIST", cat=>$ERROR_DISAPPEARS, report_hwdep=>"true"}, + {was=>"NO_EXIST", is=>"ERROR", cat=>$ERROR_APPEARS, report_hwdep=>"true"}, # {was=>"NO_EXIST", is=>"NO_EXIST", handler=>\&handle_not_yet_supported} ); @@ -185,6 +193,11 @@ my ($ref_file_name, $res_file_name); my $nounstable=0; my $unstablefile=0; my @unstable_markers=(); + +my $no_hwdep=0; # Ignore hw-dependent flag (ie. consider all tests as *not* hw-dependent +my $hwdep_file=0; # File containing the list of hw-dependent tests +my @hwdep_markers=(); # Optional markers when parsing hwdep_file + my $ratio_thresh = 0.95; # Warn if pass ratio is below this threshold GetOptions ("l" => \$long, @@ -197,7 +210,10 @@ GetOptions ("l" => \$long, "pass-thresh=f" => \$ratio_thresh, "no-unstable" => \$nounstable, "unstable-tests=s" => \$unstablefile, - "unstable-marker=s" => \@unstable_markers); + "unstable-marker=s" => \@unstable_markers, + "no-hwdep" => \$no_hwdep, + "hwdep-tests=s" => \$hwdep_file, + "hwdep-marker=s" => \@hwdep_markers); $ref_file_name = $ARGV[0] if ($#ARGV == 1); $res_file_name = $ARGV[1] if ($#ARGV == 1); @@ -224,6 +240,11 @@ my @unstablelist = (); @unstablelist = read_unstable($unstablefile) if ($unstablefile ne 0); +# Import list of hw-dependent files, if provided +my @hwdep_list = (); + +@hwdep_list = read_unstable($hwdep_file) if ($hwdep_file ne 0); + compare_results($ref, $res); my $final_result = print_compare_results_summary($ref, $res); @@ -312,7 +333,7 @@ sub read_sum($) return $res; } -# Parse list on unstable tests +# Parse list of unstable/hw-dependent tests sub read_unstable($) { my ($unstable_file) = @_; @@ -363,7 +384,17 @@ sub handle_pass_fail($$$$) } else { - push @{$res->{$PASS_FAIL}}, $tc; + # This transition is flagged as hw-dependent if: + # - the name is in the list + # - the transition should not be reported (but be flagged as hw-dependent instead) + # - --no-hwdep was not provided + if ((grep { (index $tc,$_)!=-1} @hwdep_list) + && ($diag_diag->{report_hwdep} eq "false") + && ($no_hwdep == 0)) { + push @{$res->{$HWDEP_CASES}}, $tc; + } else { + push @{$res->{$PASS_FAIL}}, $tc; + } } } @@ -447,6 +478,7 @@ sub compare_results($$) @{$res->{$UNHANDLED_CASES}} = (); @{$res->{$UNSTABLE_CASES}} = (); + @{$res->{$HWDEP_CASES}} = (); #### MERGE REF AND RES foreach my $key (sort (keys %{$res->{testcases}})) @@ -494,7 +526,17 @@ sub compare_results($$) } else { - push @{$res->{$diag_diag->{cat}}}, $key; + # This transition is flagged as hw-dependent if: + # - the name is in the list + # - the transition should not be reported (but be flagged as hw-dependent instead) + # - --no-hwdep was not provided + if ((grep { (index $key,$_)!=-1} @hwdep_list) + && ($diag_diag->{report_hwdep} eq "false") + && ($no_hwdep == 0)) { + push @{$res->{$HWDEP_CASES}}, $key; + } else { + push @{$res->{$diag_diag->{cat}}}, $key; + } } } } @@ -797,6 +839,7 @@ sub print_compare_results_summary($$) printf " | %-40s | %7d |\n", $ERROR_DISAPPEARS, scalar(@{$res->{$ERROR_DISAPPEARS}}) if (scalar(@{$res->{$ERROR_DISAPPEARS}})); printf " | %-40s | %7d |\n", $UNHANDLED_CASES, scalar(@{$res->{$UNHANDLED_CASES}}) if (scalar(@{$res->{$UNHANDLED_CASES}})); printf " | %-40s | %7d |\n", $UNSTABLE_CASES, scalar(@{$res->{$UNSTABLE_CASES}}) if (scalar(@{$res->{$UNSTABLE_CASES}})); + printf " | %-40s | %7d |\n", $HWDEP_CASES, scalar(@{$res->{$HWDEP_CASES}}) if (scalar(@{$res->{$HWDEP_CASES}})); printf " +------------------------------------------+---------+\n"; printf " | %-40s | %7d |\n", "TOTAL_MINOR_TO_BE_CHECKED", $total_better + scalar(@{$res->{$UNSTABLE_CASES}});; printf " +------------------------------------------+---------+\n"; @@ -845,6 +888,7 @@ sub print_compare_results_summary($$) print_tclist($res, $ERROR_DISAPPEARS, @{$res->{$ERROR_DISAPPEARS}}); print_tclist($res, $UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}}); print_tclist($res, $UNSTABLE_CASES, @{$res->{$UNSTABLE_CASES}}); + print_tclist($res, $HWDEP_CASES, @{$res->{$HWDEP_CASES}}); } printf "$col_reset\n"; } |