compare_dg_tests.pl: Add support for all possible status changes.

The ouput is improved to report more combinations, that were
previously silently ommitted.
Other transitions are now reported correctly.

compare_tests now removes srcdir path in testcase names
to avoid reporting differences.

Finally, a new testsuite directory contains a Makefile,
2 sample inputs and 1 expected result.

Change-Id: I67e454b198d4d80c899bd4da47924350549bae63

6 files changed, 368 insertions, 43 deletions

diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
index 53ab851..c8b166c 100755
--- a/compare_dg_tests.pl
+++ b/compare_dg_tests.pl
@@ -18,53 +18,101 @@ sub usage();
 sub print_compare_results_summary($$);
 sub nothing($$$$);
 
-  # OK
+# OK
 my $STILL_PASSES          = "Still passes              [PASS => PASS]";
 my $STILL_FAILS           = "Still fails               [FAIL => FAIL]";
+my $STILL_XPASS           = "Still xpass               [XPASS=>XPASS]";
+my $STILL_XFAIL           = "Still xfail               [XFAIL=>XFAIL]";
 
 # TO BE CHECKED
-my $XFAIL_APPEARS         = "Xfail appears             [PASS =>XFAIL]";
+my $PASS_NOW_XFAIL        = "PASS now XFAIL            [PASS =>XFAIL]";
+my $XFAIL_APPEARS         = "XFAIL appears             [     =>XFAIL]";
+my $FAIL_NOW_XFAIL        = "FAIL now XFAIL            [FAIL =>XFAIL]";
+my $XPASS_NOW_XFAIL       = "XPASS now XFAIL           [XPASS=>XFAIL]";
 my $PASSED_NOW_TIMEOUTS   = "Timeout                   [PASS =>T.OUT]";
-my $FAIL_DISAPPEARS	  = "Fail disappears           [FAIL =>     ]";
-my $XFAIL_NOW_PASSES      = "Expected fail passes      [XFAIL=>XPASS]";
-my $FAIL_NOW_PASSES       = "Fail now passes           [FAIL => PASS]";
-my $NEW_PASSES		  = "New pass                  [     => PASS]";
+my $FAIL_DISAPPEARS	  = "FAIL disappears           [FAIL =>     ]";
+my $XPASS_NOW_PASSES      = "XPASS now PASS            [XPASS=> PASS]";
+my $XFAIL_NOW_PASSES      = "XFAIL now PASS            [XFAIL=> PASS]";
+my $XFAIL_NOW_XPASS       = "XFAIL now XPASS           [XFAIL=>XPASS]";
+my $FAIL_NOW_PASSES       = "FAIL now PASS             [FAIL => PASS]";
+my $FAIL_NOW_XPASS        = "FAIL now XPASS            [FAIL =>XPASS]";
+my $NEW_PASSES		  = "New PASS                  [     => PASS]";
 my $UNHANDLED_CASES	  = "Unhandled cases           [   ..??..   ]";
 my $UNSTABLE_CASES	  = "Unstable cases, ignored   [~RANDOM     ]";
-
-# ERRORS
-my $PASSED_NOW_FAILS      = "Passed now fails          [PASS => FAIL]";
-my $PASS_DISAPPEARS       = "Pass disappears           [PASS =>     ]";
-my $FAIL_APPEARS          = "Fail appears              [     => FAIL]";
+my $UNSUPPORTED_XPASS     = "UNSUPPORTED now XPASS     [UNSUP=>XPASS]";
+my $UNSUPPORTED_XFAIL     = "UNSUPPORTED now XFAIL     [UNSUP=>XFAIL]";
+my $UNSUPPORTED_DISAPPEARS= "UNSUPPORTED disappears    [UNSUP=>     ]";
+my $UNTESTED_PASS         = "UNTESTED now PASS         [UNTES=> PASS]";
+my $UNTESTED_XFAIL        = "UNTESTED now XFAIL        [UNTES=>XFAIL]";
+my $UNTESTED_DISAPPEARS   = "UNTESTED disappears       [UNTES=>     ]";
+my $UNRESOLVED_PASS       = "UNRESOLVED now PASS       [UNRES=> PASS]";
+my $UNRESOLVED_XFAIL      = "UNRESOLVED now XFAIL      [UNRES=>XFAIL]";
+my $UNRESOLVED_DISAPPEARS = "UNRESOLVED disappears     [UNRES=>     ]";
+
+# REGRESSIONS
+my $PASS_NOW_XPASS        = "PASS now XPASS            [PASS =>XPASS]";
+my $PASSED_NOW_FAILS      = "PASS now FAIL             [PASS => FAIL]";
+my $PASS_DISAPPEARS       = "PASS disappears           [PASS =>     ]";
+my $XPASS_DISAPPEARS      = "XPASS disappears          [XPASS=>     ]";
+my $FAIL_APPEARS          = "FAIL appears              [     => FAIL]";
+my $XPASS_APPEARS         = "XPASS appears             [     =>XPASS]";
+my $XPASS_NOW_FAIL        = "XPASS now FAIL            [XPASS=> FAIL]";
+my $XFAIL_NOW_FAIL        = "XFAIL now FAIL            [XFAIL=> FAIL]";
+my $XFAIL_DISAPPEARS	  = "XFAIL disappears          [XFAIL=>     ]";
+
+my $UNSUPPORTED_PASS      = "UNSUPPORTED now PASS      [UNSUP=> PASS]";
+my $UNSUPPORTED_FAIL      = "UNSUPPORTED now FAIL      [UNSUP=> FAIL]";
+my $UNTESTED_FAIL         = "UNTESTED now FAIL         [UNTES=> FAIL]";
+my $UNTESTED_XPASS        = "UNTESTED now XPASS        [UNTES=>XPASS]";
+my $UNRESOLVED_FAIL       = "UNRESOLVED now FAIL       [UNRES=> FAIL]";
+my $UNRESOLVED_XPASS      = "UNRESOLVED now XPASS      [UNRES=>XPASS]";
 
 my @handler_list = (
   {was=>"PASS",     is=>"PASS",     cat=>$STILL_PASSES},
-  {was=>"PASS",     is=>"XPASS",    cat=>$STILL_PASSES},
-  {was=>"XPASS",    is=>"PASS",     cat=>$STILL_PASSES},
-  {was=>"XPASS",    is=>"XPASS",    cat=>$STILL_PASSES},
+  {was=>"PASS",     is=>"XPASS",    cat=>$PASS_NOW_XPASS},
+  {was=>"XPASS",    is=>"PASS",     cat=>$XPASS_NOW_PASSES},
+  {was=>"XPASS",    is=>"XPASS",    cat=>$STILL_XPASS},
   {was=>"FAIL",     is=>"FAIL",     cat=>$STILL_FAILS},
-  {was=>"FAIL",     is=>"XFAIL",    cat=>$STILL_FAILS},
-  {was=>"XFAIL",    is=>"FAIL",     cat=>$STILL_FAILS},
-  {was=>"XFAIL",    is=>"XFAIL",    cat=>$STILL_FAILS},
+  {was=>"FAIL",     is=>"XFAIL",    cat=>$FAIL_NOW_XFAIL},
+  {was=>"XFAIL",    is=>"FAIL",     cat=>$XFAIL_NOW_FAIL},
+  {was=>"XFAIL",    is=>"XFAIL",    cat=>$STILL_XFAIL},
 
-  {was=>"XPASS",    is=>"XFAIL",    cat=>$XFAIL_APPEARS},
-  {was=>"PASS",     is=>"XFAIL",    cat=>$XFAIL_APPEARS},
+  {was=>"XPASS",    is=>"XFAIL",    cat=>$XPASS_NOW_XFAIL},
+  {was=>"PASS",     is=>"XFAIL",    cat=>$PASS_NOW_XFAIL},
   {was=>"FAIL",     is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS},
-  {was=>"XFAIL",    is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS},
+  {was=>"XFAIL",    is=>"NO_EXIST", cat=>$XFAIL_DISAPPEARS},
   {was=>"XFAIL",    is=>"PASS",     cat=>$XFAIL_NOW_PASSES},
-  {was=>"XFAIL",    is=>"XPASS",    cat=>$XFAIL_NOW_PASSES},
+  {was=>"XFAIL",    is=>"XPASS",    cat=>$XFAIL_NOW_XPASS},
   {was=>"FAIL",     is=>"PASS",     cat=>$FAIL_NOW_PASSES},
-  {was=>"FAIL",     is=>"XPASS",    cat=>$FAIL_NOW_PASSES},
+  {was=>"FAIL",     is=>"XPASS",    cat=>$FAIL_NOW_XPASS},
   {was=>"NO_EXIST", is=>"PASS",     cat=>$NEW_PASSES},
-  {was=>"NO_EXIST", is=>"XPASS",    cat=>$NEW_PASSES},
+  {was=>"NO_EXIST", is=>"XPASS",    cat=>$XPASS_APPEARS},
 
   {was=>"PASS",     is=>"FAIL",     handler=>\&handle_pass_fail},
-  {was=>"XPASS",    is=>"FAIL",     handler=>\&handle_pass_fail},
+  {was=>"XPASS",    is=>"FAIL",     cat=>$XPASS_NOW_FAIL},
   {was=>"PASS",     is=>"NO_EXIST", cat=>$PASS_DISAPPEARS},
-  {was=>"XPASS",    is=>"NO_EXIST", cat=>$PASS_DISAPPEARS},
+  {was=>"XPASS",    is=>"NO_EXIST", cat=>$XPASS_DISAPPEARS},
   {was=>"NO_EXIST", is=>"FAIL",     cat=>$FAIL_APPEARS},
   {was=>"NO_EXIST", is=>"XFAIL",    cat=>$XFAIL_APPEARS},
 
+  {was=>"UNSUPPORTED", is=>"PASS", cat=>$UNSUPPORTED_PASS},
+  {was=>"UNSUPPORTED", is=>"XPASS", cat=>$UNSUPPORTED_XPASS},
+  {was=>"UNSUPPORTED", is=>"FAIL", cat=>$UNSUPPORTED_FAIL},
+  {was=>"UNSUPPORTED", is=>"XFAIL", cat=>$UNSUPPORTED_XFAIL},
+  {was=>"UNSUPPORTED", is=>"NO_EXIST", cat=>$UNSUPPORTED_DISAPPEARS},
+
+  {was=>"UNTESTED", is=>"PASS", cat=>$UNTESTED_PASS},
+  {was=>"UNTESTED", is=>"XPASS", cat=>$UNTESTED_XPASS},
+  {was=>"UNTESTED", is=>"FAIL", cat=>$UNTESTED_FAIL},
+  {was=>"UNTESTED", is=>"XFAIL", cat=>$UNTESTED_XFAIL},
+  {was=>"UNTESTED", is=>"NO_EXIST", cat=>$UNTESTED_DISAPPEARS},
+
+  {was=>"UNRESOLVED", is=>"PASS", cat=>$UNRESOLVED_PASS},
+  {was=>"UNRESOLVED", is=>"XPASS", cat=>$UNRESOLVED_XPASS},
+  {was=>"UNRESOLVED", is=>"FAIL", cat=>$UNRESOLVED_FAIL},
+  {was=>"UNRESOLVED", is=>"XFAIL", cat=>$UNRESOLVED_XFAIL},
+  {was=>"UNRESOLVED", is=>"NO_EXIST", cat=>$UNRESOLVED_DISAPPEARS},
+
 #  {was=>"NO_EXIST", is=>"NO_EXIST", handler=>\&handle_not_yet_supported}
 );
 
@@ -249,15 +297,44 @@ sub compare_results($$)
 
    @{$res->{$STILL_PASSES}} = ();
    @{$res->{$STILL_FAILS}} = ();
+   @{$res->{$STILL_XPASS}} = ();
    @{$res->{$PASSED_NOW_FAILS}} = ();
+   @{$res->{$XPASS_NOW_FAIL}} = ();
    @{$res->{$PASS_DISAPPEARS}} = ();
+   @{$res->{$XPASS_DISAPPEARS}} = ();
    @{$res->{$FAIL_APPEARS}} = ();
+   @{$res->{$XPASS_APPEARS}} = ();
    @{$res->{$NEW_PASSES}} = ();
    @{$res->{$FAIL_DISAPPEARS}} = ();
+   @{$res->{$XFAIL_DISAPPEARS}} = ();
    @{$res->{$XFAIL_APPEARS}} = ();
+   @{$res->{$FAIL_NOW_XFAIL}} = ();
+   @{$res->{$PASS_NOW_XFAIL}} = ();
+   @{$res->{$XPASS_NOW_XFAIL}} = ();
+   @{$res->{$XPASS_APPEARS}} = ();
+   @{$res->{$PASS_NOW_XPASS}} = ();
+   @{$res->{$XPASS_NOW_PASSES}} = ();
    @{$res->{$XFAIL_NOW_PASSES}} = ();
+   @{$res->{$XFAIL_NOW_XPASS}} = ();
+   @{$res->{$XFAIL_NOW_FAIL}} = ();
    @{$res->{$FAIL_NOW_PASSES}} = ();
+   @{$res->{$FAIL_NOW_XPASS}} = ();
    @{$res->{$PASSED_NOW_TIMEOUTS}} = ();
+   @{$res->{$UNSUPPORTED_PASS}} = ();
+   @{$res->{$UNSUPPORTED_XPASS}} = ();
+   @{$res->{$UNSUPPORTED_FAIL}} = ();
+   @{$res->{$UNSUPPORTED_XFAIL}} = ();
+   @{$res->{$UNSUPPORTED_DISAPPEARS}} = ();
+   @{$res->{$UNTESTED_PASS}} = ();
+   @{$res->{$UNTESTED_XPASS}} = ();
+   @{$res->{$UNTESTED_FAIL}} = ();
+   @{$res->{$UNTESTED_XFAIL}} = ();
+   @{$res->{$UNTESTED_DISAPPEARS}} = ();
+   @{$res->{$UNRESOLVED_PASS}} = ();
+   @{$res->{$UNRESOLVED_XPASS}} = ();
+   @{$res->{$UNRESOLVED_FAIL}} = ();
+   @{$res->{$UNRESOLVED_XFAIL}} = ();
+   @{$res->{$UNRESOLVED_DISAPPEARS}} = ();
    @{$res->{$UNHANDLED_CASES}} = ();
    @{$res->{$UNSTABLE_CASES}} = ();
 
@@ -342,33 +419,59 @@ sub print_compare_results_summary($$)
    if (not $quiet and not $short)
    {
        printf "              `                              +---------+---------+\n";
-       printf "o  RUN STATUS :                              |   REF   |   RES   |\n";
+       printf "o  RUN STATUS:                               |   REF   |   RES   |\n";
        printf "  +------------------------------------------+---------+---------+\n";
-       printf "  | %-40s | %7d | %7d |\n", "Passes                      [PASS+XPASS]", $ref->{PASS} + $ref->{XPASS}, $res->{PASS} + $res->{XPASS};
+       printf "  | %-40s | %7d | %7d |\n", "Passes                            [PASS]", $ref->{PASS}, $res->{PASS};
        printf "  | %-40s | %7d | %7d |\n", "Unexpected fails                  [FAIL]", $ref->{FAIL}, $res->{FAIL};
+       printf "  | %-40s | %7d | %7d |\n", "Unexpected passes                [XPASS]", $ref->{XPASS}, $res->{XPASS};
        printf "  | %-40s | %7d | %7d |\n", "Expected fails                   [XFAIL]", $ref->{XFAIL}, $res->{XFAIL};
        printf "  | %-40s | %7d | %7d |\n", "Unresolved                  [UNRESOLVED]", $ref->{UNRESOLVED}, $res->{UNRESOLVED};
-       printf "  | %-40s | %7d | %7d |\n", "Unsupported       [UNTESTED+UNSUPPORTED]", $ref->{UNTESTED}+$ref->{UNSUPPORTED}, $res->{UNTESTED}+$res->{UNSUPPORTED};
+       printf "  | %-40s | %7d | %7d |\n", "Unsupported                [UNSUPPORTED]", $ref->{UNSUPPORTED}, $res->{UNSUPPORTED};
+       printf "  | %-40s | %7d | %7d |\n", "Untested                      [UNTESTED]", $ref->{UNTESTED}, $res->{UNTESTED};
        printf "  +------------------------------------------+---------+---------+\n";
        printf "\n";
    }
 
    #### REGRESSIONS ?
-   $quiet_reg=1 if ($short and not scalar(@{$res->{$PASSED_NOW_FAILS}})+scalar(@{$res->{$PASS_DISAPPEARS}})+scalar(@{$res->{$FAIL_APPEARS}})+scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
+   $rtotal = scalar(@{$res->{$PASSED_NOW_FAILS}})
+       +scalar(@{$res->{$XPASS_NOW_FAIL}})
+       +scalar(@{$res->{$PASS_DISAPPEARS}})
+       +scalar(@{$res->{$XPASS_DISAPPEARS}})
+       +scalar(@{$res->{$FAIL_APPEARS}})
+       +scalar(@{$res->{$XPASS_APPEARS}})
+       +scalar(@{$res->{$PASS_NOW_XPASS}})
+       +scalar(@{$res->{$XFAIL_NOW_FAIL}})
+       +scalar(@{$res->{$XFAIL_DISAPPEARS}})
+       +scalar(@{$res->{$UNSUPPORTED_FAIL}})
+       +scalar(@{$res->{$UNSUPPORTED_XPASS}})
+       +scalar(@{$res->{$UNTESTED_FAIL}})
+       +scalar(@{$res->{$UNTESTED_XPASS}})
+       +scalar(@{$res->{$UNRESOLVED_FAIL}})
+       +scalar(@{$res->{$UNRESOLVED_XPASS}})
+       +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}});
+
+   $quiet_reg=1 if ($short and not $rtotal);
 
    if (not $quiet_reg)
    {
-       $rtotal = scalar(@{$res->{$PASSED_NOW_FAILS}})
-	   +scalar(@{$res->{$PASS_DISAPPEARS}})
-	   +scalar(@{$res->{$FAIL_APPEARS}})
-	   +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}});
-
-       printf "\n$col_red"."o  REGRESSIONS : \n";
+       printf "\n$col_red"."o  REGRESSIONS:\n";
        printf "  +------------------------------------------+---------+\n";
        printf "  | %-40s | %7d |\n", $PASSED_NOW_FAILS, scalar(@{$res->{$PASSED_NOW_FAILS}}) if (scalar(@{$res->{$PASSED_NOW_FAILS}}));
-       printf "  | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
+       printf "  | %-40s | %7d |\n", $XPASS_NOW_FAIL, scalar(@{$res->{$XPASS_NOW_FAIL}}) if (scalar(@{$res->{$XPASS_NOW_FAIL}}));
        printf "  | %-40s | %7d |\n", $PASS_DISAPPEARS, scalar(@{$res->{$PASS_DISAPPEARS}}) if (scalar(@{$res->{$PASS_DISAPPEARS}}));
+       printf "  | %-40s | %7d |\n", $XPASS_DISAPPEARS, scalar(@{$res->{$XPASS_DISAPPEARS}}) if (scalar(@{$res->{$XPASS_DISAPPEARS}}));
        printf "  | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}}));
+       printf "  | %-40s | %7d |\n", $XPASS_APPEARS, scalar(@{$res->{$XPASS_APPEARS}}) if (scalar(@{$res->{$XPASS_APPEARS}}));
+       printf "  | %-40s | %7d |\n", $PASS_NOW_XPASS, scalar(@{$res->{$PASS_NOW_XPASS}}) if (scalar(@{$res->{$PASS_NOW_XPASS}}));
+       printf "  | %-40s | %7d |\n", $XFAIL_NOW_FAIL, scalar(@{$res->{$XFAIL_NOW_FAIL}}) if (scalar(@{$res->{$XFAIL_NOW_FAIL}}));
+       printf "  | %-40s | %7d |\n", $XFAIL_DISAPPEARS, scalar(@{$res->{$XFAIL_DISAPPEARS}}) if (scalar(@{$res->{$XFAIL_DISAPPEARS}}));
+       printf "  | %-40s | %7d |\n", $UNSUPPORTED_FAIL, scalar(@{$res->{$UNSUPPORTED_FAIL}}) if (scalar(@{$res->{$UNSUPPORTED_FAIL}}));
+       printf "  | %-40s | %7d |\n", $UNSUPPORTED_XPASS, scalar(@{$res->{$UNSUPPORTED_XPASS}}) if (scalar(@{$res->{$UNSUPPORTED_XPASS}}));
+       printf "  | %-40s | %7d |\n", $UNTESTED_FAIL, scalar(@{$res->{$UNTESTED_FAIL}}) if (scalar(@{$res->{$UNTESTED_FAIL}}));
+       printf "  | %-40s | %7d |\n", $UNTESTED_XPASS, scalar(@{$res->{$UNTESTED_XPASS}}) if (scalar(@{$res->{$UNTESTED_XPASS}}));
+       printf "  | %-40s | %7d |\n", $UNRESOLVED_FAIL, scalar(@{$res->{$UNRESOLVED_FAIL}}) if (scalar(@{$res->{$UNRESOLVED_FAIL}}));
+       printf "  | %-40s | %7d |\n", $UNRESOLVED_XPASS, scalar(@{$res->{$UNRESOLVED_XPASS}}) if (scalar(@{$res->{$UNRESOLVED_XPASS}}));
+       printf "  | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
        printf "  +------------------------------------------+---------+\n";
        printf "  | %-40s | %7d |\n", "TOTAL_REGRESSIONS", $rtotal;
        printf "  +------------------------------------------+---------+\n";
@@ -377,9 +480,21 @@ sub print_compare_results_summary($$)
        if ($long)
        {
 	      print_tclist($PASSED_NOW_FAILS, @{$res->{$PASSED_NOW_FAILS}});
-	      print_tclist($PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}});
+	      print_tclist($XPASS_NOW_FAIL, @{$res->{$XPASS_NOW_FAIL}});
 	      print_tclist($PASS_DISAPPEARS, @{$res->{$PASS_DISAPPEARS}});
+	      print_tclist($XPASS_DISAPPEARS, @{$res->{$XPASS_DISAPPEARS}});
 	      print_tclist($FAIL_APPEARS, @{$res->{$FAIL_APPEARS}});
+	      print_tclist($XPASS_APPEARS, @{$res->{$XPASS_APPEARS}});
+	      print_tclist($PASS_NOW_XPASS, @{$res->{$PASS_NOW_XPASS}});
+	      print_tclist($XFAIL_NOW_FAIL, @{$res->{$XFAIL_NOW_FAIL}});
+	      print_tclist($XFAIL_DISAPPEARS, @{$res->{$XFAIL_DISAPPEARS}});
+	      print_tclist($UNSUPPORTED_FAIL, @{$res->{$UNSUPPORTED_FAIL}});
+	      print_tclist($UNSUPPORTED_XPASS, @{$res->{$UNSUPPORTED_XPASS}});
+	      print_tclist($UNTESTED_FAIL, @{$res->{$UNTESTED_FAIL}});
+	      print_tclist($UNTESTED_XPASS, @{$res->{$UNTESTED_XPASS}});
+	      print_tclist($UNRESOLVED_FAIL, @{$res->{$UNRESOLVED_FAIL}});
+	      print_tclist($UNRESOLVED_XPASS, @{$res->{$UNRESOLVED_XPASS}});
+	      print_tclist($PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}});
        }
        printf "$col_reset\n";
    }
@@ -387,22 +502,50 @@ sub print_compare_results_summary($$)
    #### MINOR TO BE CHECKED ?
    if (not $quiet and not $short)
    {
-       $total_better = scalar(@{$res->{$XFAIL_NOW_PASSES}})+
+       $total_better = scalar(@{$res->{$XPASS_NOW_PASSES}})+
+	   scalar(@{$res->{$XFAIL_NOW_PASSES}})+
+	   scalar(@{$res->{$XFAIL_NOW_XPASS}})+
 	   scalar(@{$res->{$FAIL_NOW_PASSES}})+
+	   scalar(@{$res->{$FAIL_NOW_XPASS}})+
 	   scalar(@{$res->{$NEW_PASSES}})+
 	   scalar(@{$res->{$FAIL_DISAPPEARS}})+
 	   scalar(@{$res->{$XFAIL_APPEARS}})+
+	   scalar(@{$res->{$FAIL_NOW_XFAIL}})+
+	   scalar(@{$res->{$PASS_NOW_XFAIL}})+
+	   scalar(@{$res->{$XPASS_NOW_XFAIL}})+
+	   scalar(@{$res->{$UNSUPPORTED_PASS}})+
+	   scalar(@{$res->{$UNSUPPORTED_XFAIL}})+
+	   scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}})+
+	   scalar(@{$res->{$UNTESTED_PASS}})+
+	   scalar(@{$res->{$UNTESTED_XFAIL}})+
+	   scalar(@{$res->{$UNTESTED_DISAPPEARS}})+
+	   scalar(@{$res->{$UNRESOLVED_PASS}})+
+	   scalar(@{$res->{$UNRESOLVED_XFAIL}})+
+	   scalar(@{$res->{$UNRESOLVED_DISAPPEARS}})+
 	   scalar(@{$res->{$UNHANDLED_CASES}});
 
-       printf "$col_pink"."o  MINOR TO BE CHECKED : \n";
+       printf "$col_pink"."o  MINOR TO BE CHECKED:\n";
        printf "  +------------------------------------------+---------+\n";
        printf "  | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}}));
-
+       printf "  | %-40s | %7d |\n", $FAIL_NOW_XFAIL, scalar(@{$res->{$FAIL_NOW_XFAIL}}) if (scalar(@{$res->{$FAIL_NOW_XFAIL}}));
+       printf "  | %-40s | %7d |\n", $PASS_NOW_XFAIL, scalar(@{$res->{$PASS_NOW_XFAIL}}) if (scalar(@{$res->{$PASS_NOW_XFAIL}}));
+       printf "  | %-40s | %7d |\n", $XPASS_NOW_XFAIL, scalar(@{$res->{$XPASS_NOW_XFAIL}}) if (scalar(@{$res->{$XPASS_NOW_XFAIL}}));
        printf "  | %-40s | %7d |\n", $FAIL_DISAPPEARS, scalar(@{$res->{$FAIL_DISAPPEARS}}) if (scalar(@{$res->{$FAIL_DISAPPEARS}}));
-
+       printf "  | %-40s | %7d |\n", $XPASS_NOW_PASSES, scalar(@{$res->{$XPASS_NOW_PASSES}}) if (scalar(@{$res->{$XPASS_NOW_PASSES}}));
        printf "  | %-40s | %7d |\n", $XFAIL_NOW_PASSES, scalar(@{$res->{$XFAIL_NOW_PASSES}}) if (scalar(@{$res->{$XFAIL_NOW_PASSES}}));
+       printf "  | %-40s | %7d |\n", $XFAIL_NOW_XPASS, scalar(@{$res->{$XFAIL_NOW_XPASS}}) if (scalar(@{$res->{$XFAIL_NOW_XPASS}}));
        printf "  | %-40s | %7d |\n", $FAIL_NOW_PASSES, scalar(@{$res->{$FAIL_NOW_PASSES}}) if (scalar(@{$res->{$FAIL_NOW_PASSES}}));
+       printf "  | %-40s | %7d |\n", $FAIL_NOW_XPASS, scalar(@{$res->{$FAIL_NOW_XPASS}}) if (scalar(@{$res->{$FAIL_NOW_XPASS}}));
        printf "  | %-40s | %7d |\n", $NEW_PASSES, scalar(@{$res->{$NEW_PASSES}}) if (scalar(@{$res->{$NEW_PASSES}}));
+       printf "  | %-40s | %7d |\n", $UNSUPPORTED_PASS, scalar(@{$res->{$UNSUPPORTED_PASS}}) if (scalar(@{$res->{$UNSUPPORTED_PASS}}));
+       printf "  | %-40s | %7d |\n", $UNSUPPORTED_XFAIL, scalar(@{$res->{$UNSUPPORTED_XFAIL}}) if (scalar(@{$res->{$UNSUPPORTED_XFAIL}}));
+       printf "  | %-40s | %7d |\n", $UNSUPPORTED_DISAPPEARS, scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}) if (scalar(@{$res->{$UNSUPPORTED_DISAPPEARS}}));
+       printf "  | %-40s | %7d |\n", $UNTESTED_PASS, scalar(@{$res->{$UNTESTED_PASS}}) if (scalar(@{$res->{$UNTESTED_PASS}}));
+       printf "  | %-40s | %7d |\n", $UNTESTED_XFAIL, scalar(@{$res->{$UNTESTED_XFAIL}}) if (scalar(@{$res->{$UNTESTED_XFAIL}}));
+       printf "  | %-40s | %7d |\n", $UNTESTED_DISAPPEARS, scalar(@{$res->{$UNTESTED_DISAPPEARS}}) if (scalar(@{$res->{$UNTESTED_DISAPPEARS}}));
+       printf "  | %-40s | %7d |\n", $UNRESOLVED_PASS, scalar(@{$res->{$UNRESOLVED_PASS}}) if (scalar(@{$res->{$UNRESOLVED_PASS}}));
+       printf "  | %-40s | %7d |\n", $UNRESOLVED_XFAIL, scalar(@{$res->{$UNRESOLVED_XFAIL}}) if (scalar(@{$res->{$UNRESOLVED_XFAIL}}));
+       printf "  | %-40s | %7d |\n", $UNRESOLVED_DISAPPEARS, scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}) if (scalar(@{$res->{$UNRESOLVED_DISAPPEARS}}));
        printf "  | %-40s | %7d |\n", $UNHANDLED_CASES, scalar(@{$res->{$UNHANDLED_CASES}}) if (scalar(@{$res->{$UNHANDLED_CASES}}));
        printf "  | %-40s | %7d |\n", $UNSTABLE_CASES, scalar(@{$res->{$UNSTABLE_CASES}}) if (scalar(@{$res->{$UNSTABLE_CASES}}));
        printf "  +------------------------------------------+---------+\n";
@@ -412,11 +555,26 @@ sub print_compare_results_summary($$)
 
        if ($long)
        {
+	      print_tclist($XPASS_NOW_PASSES, @{$res->{$XPASS_NOW_PASSES}});
 	      print_tclist($XFAIL_NOW_PASSES, @{$res->{$XFAIL_NOW_PASSES}});
+	      print_tclist($XFAIL_NOW_XPASS, @{$res->{$XFAIL_NOW_XPASS}});
 	      print_tclist($FAIL_NOW_PASSES, @{$res->{$FAIL_NOW_PASSES}});
+	      print_tclist($FAIL_NOW_XPASS, @{$res->{$FAIL_NOW_XPASS}});
 	      print_tclist($FAIL_DISAPPEARS, @{$res->{$FAIL_DISAPPEARS}});
 	      print_tclist($XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}});
+	      print_tclist($FAIL_NOW_XFAIL, @{$res->{$FAIL_NOW_XFAIL}});
+	      print_tclist($PASS_NOW_XFAIL, @{$res->{$PASS_NOW_XFAIL}});
+	      print_tclist($XPASS_NOW_XFAIL, @{$res->{$XPASS_NOW_XFAIL}});
 	      print_tclist($UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}});
+	      print_tclist($UNSUPPORTED_PASS, @{$res->{$UNSUPPORTED_PASS}});
+	      print_tclist($UNSUPPORTED_XFAIL, @{$res->{$UNSUPPORTED_XFAIL}});
+	      print_tclist($UNSUPPORTED_DISAPPEARS, @{$res->{$UNSUPPORTED_DISAPPEARS}});
+	      print_tclist($UNTESTED_PASS, @{$res->{$UNTESTED_PASS}});
+	      print_tclist($UNTESTED_XFAIL, @{$res->{$UNTESTED_XFAIL}});
+	      print_tclist($UNTESTED_DISAPPEARS, @{$res->{$UNTESTED_DISAPPEARS}});
+	      print_tclist($UNRESOLVED_PASS, @{$res->{$UNRESOLVED_PASS}});
+	      print_tclist($UNRESOLVED_XFAIL, @{$res->{$UNRESOLVED_XFAIL}});
+	      print_tclist($UNRESOLVED_DISAPPEARS, @{$res->{$UNRESOLVED_DISAPPEARS}});
 	      print_tclist($UNSTABLE_CASES, @{$res->{$UNSTABLE_CASES}});
 	      print_tclist($NEW_PASSES, @{$res->{$NEW_PASSES}});
        }
diff --git a/compare_tests b/compare_tests
index 74da3be..797af74 100755
--- a/compare_tests
+++ b/compare_tests
@@ -99,11 +99,13 @@ if [ -d "$1" -a -d "$2" ] ; then
 	cat $lst5
 	( for fname in `cat $lst5`; do
 	    bname=`basename $fname .sum.xz`
-	    xzcat $1/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/'
+	    xzcat $1/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
+		| sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
 	  done ) >$sum1
 	( for fname in `cat $lst5`; do
 	    bname=`basename $fname .sum.xz`
-	    xzcat $2/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/'
+	    xzcat $2/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/' \
+		| sed -e "s|^\([A-Z]*: \).*/home/.*/testsuite/|\1|"
 	  done ) >$sum2
 	if [ "x${target}" != "x" ] ; then
 	    unstable_target="--unstable-marker ${target}"
diff --git a/testsuite/Makefile b/testsuite/Makefile
new file mode 100644
index 0000000..cabb3b6
--- /dev/null
+++ b/testsuite/Makefile
@@ -0,0 +1,5 @@
+check:
+	LC_ALL=C ../compare_dg_tests.pl sum-1.txt sum-2.txt > cmp-1-2.txt ; diff cmp-1-2.txt expected-1-2.txt
+
+clean:
+	rm -f cmp-1-2.txt
diff --git a/testsuite/expected-1-2.txt b/testsuite/expected-1-2.txt
new file mode 100644
index 0000000..133effc
--- /dev/null
+++ b/testsuite/expected-1-2.txt
@@ -0,0 +1,66 @@
+Comparing:
+REFERENCE:sum-1.txt
+CURRENT:  sum-2.txt
+
+              `                              +---------+---------+
+o  RUN STATUS:                               |   REF   |   RES   |
+  +------------------------------------------+---------+---------+
+  | Passes                            [PASS] |       6 |      10 |
+  | Unexpected fails                  [FAIL] |       6 |      10 |
+  | Unexpected passes                [XPASS] |       5 |       9 |
+  | Expected fails                   [XFAIL] |       5 |       9 |
+  | Unresolved                  [UNRESOLVED] |       5 |       1 |
+  | Unsupported                [UNSUPPORTED] |       5 |       1 |
+  | Untested                      [UNTESTED] |       5 |       1 |
+  +------------------------------------------+---------+---------+
+
+
+o  REGRESSIONS:
+  +------------------------------------------+---------+
+  | PASS now FAIL             [PASS => FAIL] |       1 |
+  | XPASS now FAIL            [XPASS=> FAIL] |       1 |
+  | PASS disappears           [PASS =>     ] |       1 |
+  | XPASS disappears          [XPASS=>     ] |       1 |
+  | FAIL appears              [     => FAIL] |       1 |
+  | XPASS appears             [     =>XPASS] |       1 |
+  | PASS now XPASS            [PASS =>XPASS] |       1 |
+  | XFAIL now FAIL            [XFAIL=> FAIL] |       1 |
+  | XFAIL disappears          [XFAIL=>     ] |       1 |
+  | UNSUPPORTED now FAIL      [UNSUP=> FAIL] |       1 |
+  | UNSUPPORTED now XPASS     [UNSUP=>XPASS] |       1 |
+  | UNTESTED now FAIL         [UNTES=> FAIL] |       1 |
+  | UNTESTED now XPASS        [UNTES=>XPASS] |       1 |
+  | UNRESOLVED now FAIL       [UNRES=> FAIL] |       1 |
+  | UNRESOLVED now XPASS      [UNRES=>XPASS] |       1 |
+  +------------------------------------------+---------+
+  | TOTAL_REGRESSIONS                        |      15 |
+  +------------------------------------------+---------+
+
+
+o  MINOR TO BE CHECKED:
+  +------------------------------------------+---------+
+  | XFAIL appears             [     =>XFAIL] |       1 |
+  | FAIL now XFAIL            [FAIL =>XFAIL] |       1 |
+  | PASS now XFAIL            [PASS =>XFAIL] |       1 |
+  | XPASS now XFAIL           [XPASS=>XFAIL] |       1 |
+  | FAIL disappears           [FAIL =>     ] |       1 |
+  | XPASS now PASS            [XPASS=> PASS] |       1 |
+  | XFAIL now PASS            [XFAIL=> PASS] |       1 |
+  | XFAIL now XPASS           [XFAIL=>XPASS] |       1 |
+  | FAIL now PASS             [FAIL => PASS] |       1 |
+  | FAIL now XPASS            [FAIL =>XPASS] |       1 |
+  | New PASS                  [     => PASS] |       1 |
+  | UNSUPPORTED now PASS      [UNSUP=> PASS] |       1 |
+  | UNSUPPORTED now XFAIL     [UNSUP=>XFAIL] |       1 |
+  | UNSUPPORTED disappears    [UNSUP=>     ] |       1 |
+  | UNTESTED now PASS         [UNTES=> PASS] |       1 |
+  | UNTESTED now XFAIL        [UNTES=>XFAIL] |       1 |
+  | UNTESTED disappears       [UNTES=>     ] |       1 |
+  | UNRESOLVED now PASS       [UNRES=> PASS] |       1 |
+  | UNRESOLVED now XFAIL      [UNRES=>XFAIL] |       1 |
+  | UNRESOLVED disappears     [UNRES=>     ] |       1 |
+  +------------------------------------------+---------+
+  | TOTAL_MINOR_TO_BE_CHECKED                |      20 |
+  +------------------------------------------+---------+
+
+
diff --git a/testsuite/sum-1.txt b/testsuite/sum-1.txt
new file mode 100644
index 0000000..019b829
--- /dev/null
+++ b/testsuite/sum-1.txt
@@ -0,0 +1,44 @@
+PASS: test1-pass-pass
+PASS: test2-pass-xpass
+PASS: test3-pass-fail
+PASS: test4-pass-xfail
+PASS: test5-pass-disappears
+
+XPASS: test1-xpass-pass
+XPASS: test2-xpass-xpass
+XPASS: test3-xpass-fail
+XPASS: test4-xpass-xfail
+XPASS: test5-xpass-disappears
+
+FAIL: test1-fail-pass
+FAIL: test2-fail-xpass
+FAIL: test3-fail-fail
+FAIL: test4-fail-xfail
+FAIL: test5-fail-disappears
+
+XFAIL: test1-xfail-pass
+XFAIL: test2-xfail-xpass
+XFAIL: test3-xfail-fail
+XFAIL: test4-xfail-xfail
+XFAIL: test5-xfail-disappears
+
+UNSUPPORTED: test1-unsupported-pass
+UNSUPPORTED: test2-unsupported-xpass
+UNSUPPORTED: test3-unsupported-fail
+UNSUPPORTED: test4-unsupported-xfail
+UNSUPPORTED: test5-unsupported-disappears
+
+UNTESTED: test1-untested-pass
+UNTESTED: test2-untested-xpass
+UNTESTED: test3-untested-fail
+UNTESTED: test4-untested-xfail
+UNTESTED: test5-untested-disappears
+
+UNRESOLVED: test1-unresolved-pass
+UNRESOLVED: test2-unresolved-xpass
+UNRESOLVED: test3-unresolved-fail
+UNRESOLVED: test4-unresolved-xfail
+UNRESOLVED: test5-unresolved-disappears
+
+PASS: test1-pass-and-fails
+FAIL: test1-pass-and-fails
diff --git a/testsuite/sum-2.txt b/testsuite/sum-2.txt
new file mode 100644
index 0000000..30527ce
--- /dev/null
+++ b/testsuite/sum-2.txt
@@ -0,0 +1,50 @@
+PASS: test1-pass-pass
+XPASS: test2-pass-xpass
+FAIL: test3-pass-fail
+XFAIL: test4-pass-xfail
+
+PASS: test1-xpass-pass
+XPASS: test2-xpass-xpass
+FAIL: test3-xpass-fail
+XFAIL: test4-xpass-xfail
+
+PASS: test1-fail-pass
+XPASS: test2-fail-xpass
+FAIL: test3-fail-fail
+XFAIL: test4-fail-xfail
+
+PASS: test1-xfail-pass
+XPASS: test2-xfail-xpass
+FAIL: test3-xfail-fail
+XFAIL: test4-xfail-xfail
+
+PASS: test1-pass-appears
+XPASS: test2-xpass-appears
+FAIL: test3-fail-appears
+XFAIL: test4-xfail-appears
+
+PASS: test1-unsupported-pass
+XPASS: test2-unsupported-xpass
+FAIL: test3-unsupported-fail
+XFAIL: test4-unsupported-xfail
+
+PASS: test1-untested-pass
+XPASS: test2-untested-xpass
+FAIL: test3-untested-fail
+XFAIL: test4-untested-xfail
+
+PASS: test1-unresolved-pass
+XPASS: test2-unresolved-xpass
+FAIL: test3-unresolved-fail
+XFAIL: test4-unresolved-xfail
+
+PASS: test1-pass-appears
+XPASS: test2-xpass-appears
+FAIL: test3-fail-appears
+XFAIL: test4-xfail-appears
+UNSUPPORTED: test4-unsupported-appears
+UNTESTED: test4-untested-appears
+UNRESOLVED: test4-unresolved-appears
+
+PASS: test1-pass-and-fails
+FAIL: test1-pass-and-fails