Initial version of the comparison scripts and list of unstable tests.

4 files changed, 903 insertions, 0 deletions

diff --git a/compare_dg_tests.pl b/compare_dg_tests.pl
new file mode 100755
index 0000000..6eb4fd1
--- /dev/null
+++ b/compare_dg_tests.pl
@@ -0,0 +1,435 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+use File::Glob;
+use Getopt::Long;
+use Term::ANSIColor qw(:constants);
+use File::Basename;
+use Cwd;
+
+my $app = $0;
+
+sub read_sum($);
+sub read_unstable($);
+sub dump_result($);
+sub compare_results($$);
+sub usage();
+sub print_compare_results_summary($$);
+sub nothing($$$$);
+
+  # OK
+my $STILL_PASSES          = "Still passes              [PASS => PASS]";
+my $STILL_FAILS           = "Still fails               [FAIL => FAIL]";
+
+# TO BE CHECKED
+my $XFAIL_APPEARS         = "Xfail appears             [PASS =>XFAIL]";
+my $PASSED_NOW_TIMEOUTS   = "Timeout                   [PASS =>T.OUT]";
+my $FAIL_DISAPPEARS	  = "Fail disappears           [FAIL =>     ]";
+my $XFAIL_NOW_PASSES      = "Expected fail passes      [XFAIL=>XPASS]";
+my $FAIL_NOW_PASSES       = "Fail now passes           [FAIL => PASS]";
+my $NEW_PASSES		  = "New pass                  [     => PASS]";
+my $UNHANDLED_CASES	  = "Unhandled cases           [   ..??..   ]";
+my $UNSTABLE_CASES	  = "Unstable cases, ignored   [~RANDOM     ]";
+
+# ERRORS
+my $PASSED_NOW_FAILS      = "Passed now fails          [PASS => FAIL]";
+my $PASS_DISAPPEARS       = "Pass disappears           [PASS =>     ]";
+my $FAIL_APPEARS          = "Fail appears              [     => FAIL]";
+
+my @handler_list = (
+  {was=>"PASS",     is=>"PASS",     cat=>$STILL_PASSES},
+  {was=>"PASS",     is=>"XPASS",    cat=>$STILL_PASSES},
+  {was=>"XPASS",    is=>"PASS",     cat=>$STILL_PASSES},
+  {was=>"XPASS",    is=>"XPASS",    cat=>$STILL_PASSES},
+  {was=>"FAIL",     is=>"FAIL",     cat=>$STILL_FAILS},
+  {was=>"FAIL",     is=>"XFAIL",    cat=>$STILL_FAILS},
+  {was=>"XFAIL",    is=>"FAIL",     cat=>$STILL_FAILS},
+  {was=>"XFAIL",    is=>"XFAIL",    cat=>$STILL_FAILS},
+
+  {was=>"XPASS",    is=>"XFAIL",    cat=>$XFAIL_APPEARS},
+  {was=>"PASS",     is=>"XFAIL",    cat=>$XFAIL_APPEARS},
+  {was=>"FAIL",     is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS},
+  {was=>"XFAIL",    is=>"NO_EXIST", cat=>$FAIL_DISAPPEARS},
+  {was=>"XFAIL",    is=>"PASS",     cat=>$XFAIL_NOW_PASSES},
+  {was=>"XFAIL",    is=>"XPASS",    cat=>$XFAIL_NOW_PASSES},
+  {was=>"FAIL",     is=>"PASS",     cat=>$FAIL_NOW_PASSES},
+  {was=>"FAIL",     is=>"XPASS",    cat=>$FAIL_NOW_PASSES},
+  {was=>"NO_EXIST", is=>"PASS",     cat=>$NEW_PASSES},
+  {was=>"NO_EXIST", is=>"XPASS",    cat=>$NEW_PASSES},
+
+  {was=>"PASS",     is=>"FAIL",     handler=>\&handle_pass_fail},
+  {was=>"XPASS",    is=>"FAIL",     handler=>\&handle_pass_fail},
+  {was=>"PASS",     is=>"NO_EXIST", cat=>$PASS_DISAPPEARS},
+  {was=>"XPASS",    is=>"NO_EXIST", cat=>$PASS_DISAPPEARS},
+  {was=>"NO_EXIST", is=>"FAIL",     cat=>$FAIL_APPEARS},
+  {was=>"NO_EXIST", is=>"XFAIL",    cat=>$XFAIL_APPEARS},
+
+#  {was=>"NO_EXIST", is=>"NO_EXIST", handler=>\&handle_not_yet_supported}
+);
+
+######################################################
+# TREAT ARGUMENTS
+
+my $verbose=0;
+my $quiet=0;
+my $long=0;
+my $short=0;
+my $debug=0;
+my ($testroot, $basename);
+my ($ref_file_name, $res_file_name);
+my $nounstable=0;
+my $unstablefile=0;
+my @unstable_markers=();
+
+GetOptions ("l"           => \$long,
+            "s"           => \$short,
+            "q"           => \$quiet,
+            "v"           => \$verbose,
+            "dbg"         => \$debug,
+            "testroot=s"  => \$testroot,
+            "basename=s"  => \$basename,
+            "no-unstable"  => \$nounstable,
+            "unstable-tests=s" => \$unstablefile,
+            "unstable-marker=s"    => \@unstable_markers);
+
+$ref_file_name = $ARGV[0] if ($#ARGV == 1);
+$res_file_name = $ARGV[1] if ($#ARGV == 1);
+
+$ref_file_name = $testroot."/expected_results/".$basename if ($testroot and $basename);
+$res_file_name = $testroot."/testing/run/".$basename if ($testroot and $basename);
+&usage if (not $ref_file_name or not $res_file_name);
+
+my ($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset)
+    = ("\033[31;1m","\033[31;3m","\033[32;1m","\033[32;3m","\033[35;1m","\033[35;2m","\033[0m");
+($col_boldred, $col_red, $col_boldgreen, $col_green, $col_boldpink, $col_pink, $col_reset)
+    = ("","","","","","","") if (not I_am_interactive());
+
+######################################################
+# MAIN PROGRAM
+# print "comparing $ref_file_name $res_file_name\n";
+
+# If none of the 2 .sum exists, nothing to compare: exit early.
+exit 0 if ( (! -e $ref_file_name) && (! -e $res_file_name ));
+
+my $ref = read_sum($ref_file_name) ;
+my $res = read_sum($res_file_name) ;
+my @unstablelist = ();
+
+@unstablelist = read_unstable($unstablefile) if ($unstablefile ne 0);
+
+compare_results($ref, $res);
+
+my $final_result = print_compare_results_summary($ref, $res);
+
+exit $final_result;
+
+######################################################
+# UTILITIES
+
+sub empty_result()
+{
+   my %empty_result;# = {PASS=>0, FAIL=>0, XPASS=>0, XFAIL=>0, UNSUPPORTED=>0, UNTESTED=>0, UNRESOLVED=>0};
+   $empty_result{PASS}=$empty_result{FAIL}=$empty_result{XPASS}=$empty_result{XFAIL}=0;
+   $empty_result{UNSUPPORTED}=$empty_result{UNTESTED}=$empty_result{UNRESOLVED}=$empty_result{NO_EXIST}=0;
+   return \%empty_result;
+}
+sub I_am_interactive {
+  return -t STDIN && -t STDOUT;
+}
+sub usage()
+{
+	print "Usage : $app <ref_file.sum> <result_file.sum>\n";
+	exit 1;
+}
+
+
+######################################################
+# PARSING
+sub read_sum($)
+{
+   my ($sum_file) = @_;
+   my $res = empty_result();
+   my %testcases;
+   my %unsupported;
+   $res->{testcases} = \%testcases;
+   my $pending_timeout=0;
+
+   open SUMFILE, $sum_file or die $!;
+   while (<SUMFILE>)
+   {
+	  if (m/^(PASS|XPASS|FAIL|XFAIL|UNSUPPORTED|UNTESTED|UNRESOLVED): (.*)/)
+	  {
+	  	my ($diag,$tc) = ($1,$2);
+		my %tcresult;
+		$tc =~ s/==[0-9]+== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./==<pid>== Shadow memory range interleaves with an existing memory mapping. ASan cannot proceed correctly./;
+		$testcases{$tc} = empty_result() if (not exists $testcases{$tc});
+		$testcases{$tc}->{$diag}++;
+		$testcases{$tc}->{HAS_TIMED_OUT} = $pending_timeout;
+		$pending_timeout = 0;
+		$res->{$diag}++;
+	  }
+	  elsif (m/WARNING: program timed out/) 
+	  {
+	  	$pending_timeout = 1;
+	  }
+	  elsif (m/^(# of expected passes|# of unexpected failures|# of expected failures|# of known failures|# of unsupported tests|# of untested testcases)\s+(.*)/)
+	  {
+	  	 $res->{"summary - "."$1"} = $2;
+	  }
+	  elsif (m/^\/.*\/([^\/]+)\s+version\s+(.*)/)
+	  {
+	  	 $res->{tool} = $1;
+	  	 $res->{version} = $2;
+		 $res->{version} =~ s/ [-(].*//;
+	  }
+   }
+   close SUMFILE;
+   return $res;
+}
+
+# Parse list on unstable tests
+sub read_unstable($)
+{
+   my ($unstable_file) = @_;
+   my @unstable_tests = ();
+
+   open UNSTABLEFILE, $unstable_file or die $!;
+   while (<UNSTABLEFILE>)
+   {
+       # Skip lines starting with '#', or with spaces only
+       if ((/^#/) || (/^[ 	]*$/))
+       {
+       }
+       else
+       {
+	   chomp;
+
+	   my $test = $_;
+
+	   # Check if line is of type: target:testname
+	   if (/^(.*):/)
+	   {
+	       foreach my $unstable_marker (@unstable_markers)
+	       {
+		   if ($unstable_marker eq $1) {
+		       # If target matches the one supplied as script
+		       # argument, add the testname to the list
+		       $test =~ s/.*://;
+		       push @unstable_tests, $test;
+		   }
+	       }
+	   } else {
+	       push @unstable_tests, $test;
+	   }
+       }
+   }
+   close UNSTABLEFILE;
+   return @unstable_tests;
+}
+
+######################################################
+# DIFFING
+sub handle_pass_fail($$$$)
+{
+   my ($ref, $res, $diag_diag, $tc) = @_;
+   if ($res->{testcases}->{$tc}->{HAS_TIMED_OUT})
+   {
+     push @{$res->{$PASSED_NOW_TIMEOUTS}}, $tc;
+   }
+   else
+   {
+     push @{$res->{$PASSED_NOW_FAILS}}, $tc;
+   }
+}
+
+sub compare_results($$)
+{
+   my ($ref, $res) = @_;
+
+   @{$res->{$STILL_PASSES}} = ();
+   @{$res->{$STILL_FAILS}} = ();
+   @{$res->{$PASSED_NOW_FAILS}} = ();
+   @{$res->{$PASS_DISAPPEARS}} = ();
+   @{$res->{$FAIL_APPEARS}} = ();
+   @{$res->{$NEW_PASSES}} = ();
+   @{$res->{$FAIL_DISAPPEARS}} = ();
+   @{$res->{$XFAIL_APPEARS}} = ();
+   @{$res->{$XFAIL_NOW_PASSES}} = ();
+   @{$res->{$FAIL_NOW_PASSES}} = ();
+   @{$res->{$PASSED_NOW_TIMEOUTS}} = ();
+   @{$res->{$UNHANDLED_CASES}} = ();
+   @{$res->{$UNSTABLE_CASES}} = ();
+
+   #### MERGE REF AND RES
+   foreach my $key (sort (keys %{$res->{testcases}}))
+   {
+		if (not exists $ref->{testcases}->{$key}) {
+		   $ref->{testcases}->{$key} = empty_result();
+		   $ref->{testcases}->{$key}->{NO_EXIST} = 1;
+		}
+   }
+   foreach my $key (keys %{$ref->{testcases}})
+   {
+   		if (not exists $res->{testcases}->{$key})
+		{
+		   $res->{testcases}->{$key} = empty_result();
+		   $res->{testcases}->{$key}->{NO_EXIST} = 1;
+		}
+   }
+
+   #### ACTIONS FOR EACH CASES
+   my %unstable_found;
+
+   foreach my $key (sort (keys %{$ref->{testcases}}))
+   {
+       foreach my $diag_diag (@handler_list)
+       {
+	   if ($ref->{testcases}->{$key}->{$diag_diag->{was}} != $res->{testcases}->{$key}->{$diag_diag->{was}}
+	       and $res->{testcases}->{$key}->{$diag_diag->{is}})
+	   {
+	       # If testcase is listed as 'unstable' mark it as
+	       # such and skip other processing.
+	       {
+		   if (grep { (index $key,$_)!=-1} @unstablelist)
+		   {
+		       print "[unstable] $key\n" if ($debug);
+		       $unstable_found{$key}=1;
+		   }
+		   else {
+		       print "[$diag_diag->{was} => $diag_diag->{is}] $key\n" if ($debug);
+		       if ($diag_diag->{handler})
+		       {
+			   $diag_diag->{handler} ($ref, $res, $diag_diag, $key);
+		       }
+		       else
+		       {
+			   push @{$res->{$diag_diag->{cat}}}, $key;
+		       }
+		   }
+	       }
+	   }
+       }
+   }
+   push @{$res->{$UNSTABLE_CASES}}, (sort (keys (%unstable_found)))  if ($nounstable == 0);
+
+}
+
+######################################################
+# PRINTING
+sub print_tclist($@)
+{
+   my ($cat, @tclist) = @_;
+   print "  - ".$cat.":\n\n  ". join("\n  ",@tclist) . "\n\n" if (scalar(@tclist));
+}
+
+sub print_compare_results_summary($$)
+{
+   my ($ref, $res) = @_;
+   my $return_value=0;
+   my $total = 0;
+   my $rtotal = 0;
+   my $quiet_reg = $quiet;
+
+   if (not $quiet)
+   {
+       printf "Comparing:\n";
+       printf "REFERENCE:$ref_file_name\n";
+       printf "CURRENT:  $res_file_name\n\n";
+   }
+
+   #### TESTS STATUS
+   if (not $quiet and not $short)
+   {
+       printf "              `                              +---------+---------+\n";
+       printf "o  RUN STATUS :                              |   REF   |   RES   |\n";
+       printf "  +------------------------------------------+---------+---------+\n";
+       printf "  | %-40s | %7d | %7d |\n", "Passes                      [PASS+XPASS]", $ref->{PASS} + $ref->{XPASS}, $res->{PASS} + $res->{XPASS};
+       printf "  | %-40s | %7d | %7d |\n", "Unexpected fails                  [FAIL]", $ref->{FAIL}, $res->{FAIL};
+       printf "  | %-40s | %7d | %7d |\n", "Expected fails                   [XFAIL]", $ref->{XFAIL}, $res->{XFAIL};
+       printf "  | %-40s | %7d | %7d |\n", "Unresolved                  [UNRESOLVED]", $ref->{UNRESOLVED}, $res->{UNRESOLVED};
+       printf "  | %-40s | %7d | %7d |\n", "Unsupported       [UNTESTED+UNSUPPORTED]", $ref->{UNTESTED}+$ref->{UNSUPPORTED}, $res->{UNTESTED}+$res->{UNSUPPORTED};
+       printf "  +------------------------------------------+---------+---------+\n";
+       printf "\n";
+   }
+
+   #### REGRESSIONS ?
+   $quiet_reg=1 if ($short and not scalar(@{$res->{$PASSED_NOW_FAILS}})+scalar(@{$res->{$PASS_DISAPPEARS}})+scalar(@{$res->{$FAIL_APPEARS}})+scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
+
+   if (not $quiet_reg)
+   {
+       $rtotal = scalar(@{$res->{$PASSED_NOW_FAILS}})
+	   +scalar(@{$res->{$PASS_DISAPPEARS}})
+	   +scalar(@{$res->{$FAIL_APPEARS}})
+	   +scalar(@{$res->{$PASSED_NOW_TIMEOUTS}});
+
+       printf "\n$col_red"."o  REGRESSIONS : \n";
+       printf "  +------------------------------------------+---------+\n";
+       printf "  | %-40s | %7d |\n", $PASSED_NOW_FAILS, scalar(@{$res->{$PASSED_NOW_FAILS}}) if (scalar(@{$res->{$PASSED_NOW_FAILS}}));
+       printf "  | %-40s | %7d |\n", $PASSED_NOW_TIMEOUTS, scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}) if (scalar(@{$res->{$PASSED_NOW_TIMEOUTS}}));
+       printf "  | %-40s | %7d |\n", $PASS_DISAPPEARS, scalar(@{$res->{$PASS_DISAPPEARS}}) if (scalar(@{$res->{$PASS_DISAPPEARS}}));
+       printf "  | %-40s | %7d |\n", $FAIL_APPEARS, scalar(@{$res->{$FAIL_APPEARS}}) if (scalar(@{$res->{$FAIL_APPEARS}}));
+       printf "  +------------------------------------------+---------+\n";
+       printf "  | %-40s | %7d |\n", "TOTAL_REGRESSIONS", $rtotal;
+       printf "  +------------------------------------------+---------+\n";
+       printf "\n";
+
+       if ($long)
+       {
+	      print_tclist($PASSED_NOW_FAILS, @{$res->{$PASSED_NOW_FAILS}});
+	      print_tclist($PASSED_NOW_TIMEOUTS, @{$res->{$PASSED_NOW_TIMEOUTS}});
+	      print_tclist($PASS_DISAPPEARS, @{$res->{$PASS_DISAPPEARS}});
+	      print_tclist($FAIL_APPEARS, @{$res->{$FAIL_APPEARS}});
+       }
+       printf "$col_reset\n";
+   }
+
+   #### MINOR TO BE CHECKED ?
+   if (not $quiet and not $short)
+   {
+       $total = scalar(@{$res->{$XFAIL_NOW_PASSES}})+
+	   scalar(@{$res->{$FAIL_NOW_PASSES}})+
+	   scalar(@{$res->{$NEW_PASSES}})+
+	   scalar(@{$res->{$FAIL_DISAPPEARS}})+
+	   scalar(@{$res->{$XFAIL_APPEARS}})+
+	   scalar(@{$res->{$UNHANDLED_CASES}})+
+	   scalar(@{$res->{$UNSTABLE_CASES}});
+
+       printf "$col_pink"."o  MINOR TO BE CHECKED : \n";
+       printf "  +------------------------------------------+---------+\n";
+       printf "  | %-40s | %7d |\n", $XFAIL_APPEARS, scalar(@{$res->{$XFAIL_APPEARS}}) if (scalar(@{$res->{$XFAIL_APPEARS}}));
+
+       printf "  | %-40s | %7d |\n", $FAIL_DISAPPEARS, scalar(@{$res->{$FAIL_DISAPPEARS}}) if (scalar(@{$res->{$FAIL_DISAPPEARS}}));
+
+       printf "  | %-40s | %7d |\n", $XFAIL_NOW_PASSES, scalar(@{$res->{$XFAIL_NOW_PASSES}}) if (scalar(@{$res->{$XFAIL_NOW_PASSES}}));
+       printf "  | %-40s | %7d |\n", $FAIL_NOW_PASSES, scalar(@{$res->{$FAIL_NOW_PASSES}}) if (scalar(@{$res->{$FAIL_NOW_PASSES}}));
+       printf "  | %-40s | %7d |\n", $NEW_PASSES, scalar(@{$res->{$NEW_PASSES}}) if (scalar(@{$res->{$NEW_PASSES}}));
+       printf "  | %-40s | %7d |\n", $UNHANDLED_CASES, scalar(@{$res->{$UNHANDLED_CASES}}) if (scalar(@{$res->{$UNHANDLED_CASES}}));
+       printf "  | %-40s | %7d |\n", $UNSTABLE_CASES, scalar(@{$res->{$UNSTABLE_CASES}}) if (scalar(@{$res->{$UNSTABLE_CASES}}));
+       printf "  +------------------------------------------+---------+\n";
+       printf "  | %-40s | %7d |\n", "TOTAL_MINOR_TO_BE_CHECKED", $total;
+       printf "  +------------------------------------------+---------+\n";
+       printf "\n";
+
+       if ($long)
+       {
+	      print_tclist($XFAIL_NOW_PASSES, @{$res->{$XFAIL_NOW_PASSES}});
+	      print_tclist($FAIL_NOW_PASSES, @{$res->{$FAIL_NOW_PASSES}});
+	      print_tclist($FAIL_DISAPPEARS, @{$res->{$FAIL_DISAPPEARS}});
+	      print_tclist($XFAIL_APPEARS, @{$res->{$XFAIL_APPEARS}});
+	      print_tclist($UNHANDLED_CASES, @{$res->{$UNHANDLED_CASES}});
+	      print_tclist($UNSTABLE_CASES, @{$res->{$UNSTABLE_CASES}});
+	      print_tclist($NEW_PASSES, @{$res->{$NEW_PASSES}});
+       }
+       printf "$col_reset\n";
+   }
+
+   $return_value = 1 if ($total);
+
+   $return_value = 2 if ($rtotal);
+
+   # Error if there was no PASS (eg when sth went wrong and no .sum was generated
+   $return_value = 2 if (($res->{PASS} + $res->{XPASS}) == 0);
+
+   return $return_value;
+}
diff --git a/compare_jobs.sh b/compare_jobs.sh
new file mode 100755
index 0000000..6e8b383
--- /dev/null
+++ b/compare_jobs.sh
@@ -0,0 +1,211 @@
+#!/bin/bash
+
+mydir="`dirname $0`"
+status=0
+
+if [ $# != 2 ]
+then
+    echo "Usage: $0 ref_logs new_logs"
+    exit 1
+fi
+
+ref_logs=$1
+new_logs=$2
+
+tmptargets=/tmp/targets.$$
+trap "rm -f ${tmptargets}" 0 1 2 3 5 9 13 15
+
+rm -f ${tmptargets}
+
+function xml_report_print_row
+{
+    local target=${1?}
+    local failed=${2?}
+    local log_url=BUILD_URL/artifact/artifacts/logs/diff-${target}.txt
+    local color='#00FF00'
+    $failed && color='#FF0000'
+    local message=PASSED
+    $failed && message=FAILED
+    cat <<EOF
+<tr>
+  <td>${target}</td>
+  <td fontattribute="bold" bgcolor="${color}">${message}</td>
+  <td><![CDATA[<a href="$log_url">log for ${target}</a>]]></td>
+</tr>
+EOF
+}
+
+function html_report_print_row
+{
+    local target=${1?}
+    local failed=${2?}
+    local log_url=diff-${target}.txt
+    local color='#00FF00'
+    $failed && color='#FF0000'
+    local message=PASSED
+    $failed && message=FAILED
+    cat <<EOF
+  <tr>
+    <td>${target}</td>
+    <td style="background-color:$color"><a href="$log_url"><b>${message}</b></a></td>
+  </tr>
+EOF
+}
+
+function xml_report_print_header
+{
+    cat <<EOF
+<section name="Results comparison ${ref_logs} vs ${new_logs}"><table>
+  <tr>
+  <td fontattribute="bold" width="120" align="center">Target</td>
+  <td fontattribute="bold" width="120" align="center">Status</td>
+  <td fontattribute="bold" width="120" align="center">Log</td>
+</tr>
+EOF
+}
+
+function html_report_print_header
+{
+    cat <<EOF
+<h1>Results comparison ${ref_logs} vs ${new_logs}</h1>
+<table border="1">
+  <tr>
+    <td><b>Target</b></td>
+    <td><b>Status</b></td>
+  </tr>
+EOF
+}
+
+function xml_report_print_footer
+{
+    cat <<EOF
+</table></section>
+EOF
+}
+
+function html_report_print_footer
+{
+    cat <<EOF
+</table>
+EOF
+}
+
+function xml_log_print_field
+{
+    local target=${1?}
+    local log=${2?}
+    cat <<EOF
+  <field name="${target}">
+    <![CDATA[
+EOF
+cat $log
+cat <<EOF
+    ]]></field>
+EOF
+}
+
+function html_log_print_field
+{
+    local target=${1?}
+    local log=${2?}
+    cat <<EOF
+  <p>${target}</p>
+EOF
+cat $log
+cat <<EOF
+EOF
+}
+
+function xml_log_print_header
+{
+    cat <<EOF
+<section name="Logs">
+EOF
+}
+
+function html_log_print_header
+{
+    cat <<EOF
+<h1>Logs</h1>
+EOF
+}
+
+function xml_log_print_footer
+{
+    cat <<EOF
+</section>
+EOF
+}
+
+function html_log_print_footer
+{
+    cat <<EOF
+EOF
+}
+
+# For the time being, we expect different jobs to store their results
+# in similar directories.
+
+# Build list of all build-targets validated for ${ref_logs}
+for dir in `find ${ref_logs}/ -mindepth 1 -maxdepth 1 -type d`
+do
+    basename ${dir} >> ${tmptargets}
+done
+
+# Build list of all build-targets validated for ${new_logs}
+for dir in `find ${new_logs}/ -mindepth 1 -maxdepth 1 -type d`
+do
+    basename ${dir} >> ${tmptargets}
+done
+
+if [ -s ${tmptargets} ]; then
+    buildtargets=`sort -u ${tmptargets}`
+fi
+rm -f ${tmptargets}
+
+XML_REPORT=${mydir}/report0.xml
+HTML_REPORT=${mydir}/report0.html
+rm -f ${XML_REPORT} ${XML_REPORT}.part
+rm -f ${HTML_REPORT} ${HTML_REPORT}.part
+XML_LOG=${mydir}/report1.xml
+HTML_LOG=${mydir}/report1.html
+rm -f ${XML_LOG} ${XML_LOG}.part
+rm -f ${HTML_LOG} ${HTML_LOG}.part
+
+xml_report_print_header > ${XML_REPORT}.part
+html_report_print_header > ${HTML_REPORT}.part
+xml_log_print_header > ${XML_LOG}.part
+html_log_print_header > ${HTML_LOG}.part
+
+for buildtarget in ${buildtargets}
+do
+    ref="${ref_logs}/${buildtarget}"
+    build="${new_logs}/${buildtarget}"
+    echo "REF = "${ref}
+    echo "BUILD = "${build}
+    failed=false
+    mylog=${mydir}/diff-${buildtarget}.txt
+    target=`echo ${buildtarget} | cut -d. -f2`
+    printf "\t# ============================================================== #\n" > ${mylog}
+    printf "\t#\t\t*** ${buildtarget} ***\n" >> ${mylog}
+    printf "\t# ============================================================== #\n\n" >> ${mylog}
+    [ -d "${build}" -a -d "${ref}" ] && ${mydir}/compare_tests -target ${target} \
+	${ref} ${build} >> ${mylog} || failed=true
+
+    ${failed} && status=1
+    xml_report_print_row "${buildtarget}" "${failed}" >> $XML_REPORT.part
+    html_report_print_row "${buildtarget}" "${failed}" >> $HTML_REPORT.part
+    xml_log_print_field "${buildtarget}" ${mylog} >> $XML_LOG.part
+    html_log_print_field "${buildtarget}" ${mylog} >> $HTML_LOG.part
+done
+
+xml_report_print_footer >> ${XML_REPORT}.part
+html_report_print_footer >> ${HTML_REPORT}.part
+xml_log_print_footer >> ${XML_LOG}.part
+html_log_print_footer >> ${HTML_LOG}.part
+mv ${XML_REPORT}.part ${XML_REPORT}
+mv ${HTML_REPORT}.part ${HTML_REPORT}
+mv ${XML_LOG}.part ${XML_LOG}
+mv ${HTML_LOG}.part ${HTML_LOG}
+
+exit ${status}
diff --git a/compare_tests b/compare_tests
new file mode 100755
index 0000000..c2bcb2a
--- /dev/null
+++ b/compare_tests
@@ -0,0 +1,137 @@
+#!/bin/sh
+# This script automatically test the given tool with the tool's test cases,
+# reporting anything of interest.
+
+# Written by Mike Stump <mrs@cygnus.com>
+# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>
+# Modified by Yvan Roux <yvan.roux@linaro.org> and
+# Christophe Lyon <christophe.lyon@linaro.org>
+usage()
+{
+	if [ -n "$1" ] ; then
+		echo "$0: Error: $1" >&2
+		echo >&2
+	fi
+	cat >&2 <<EOUSAGE
+Usage: $0 [-strict] [-target target-triplet] PREVIOUS CURRENT
+
+Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
+
+	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
+
+	Unless -strict is given, these discrepancies are not counted as errors:
+		missing/extra .sum files when comparing directories
+		tests that failed in PREVIOUS but pass in CURRENT
+		tests that were not in PREVIOUS but appear in CURRENT
+		tests in PREVIOUS that are missing in CURRENT
+
+        -target enables to provide the target name to use when parsing
+         the file containing the list of unstable tests.
+
+	Exit with the following values:
+		0 if there is nothing of interest
+		1 if there are errors when comparing single test case files
+		N for the number of errors found when comparing directories
+EOUSAGE
+	exit 2
+}
+
+export LC_ALL=C
+
+me="`which $0`"
+my_path="`dirname ${me}`"
+
+tool=gxx
+
+tmp1=/tmp/$tool-testing.$$a
+tmp2=/tmp/$tool-testing.$$b
+now_s=/tmp/$tool-testing.$$d
+before_s=/tmp/$tool-testing.$$e
+lst1=/tmp/$tool-lst1.$$
+lst2=/tmp/$tool-lst2.$$
+lst3=/tmp/$tool-lst3.$$
+lst4=/tmp/$tool-lst4.$$
+lst5=/tmp/$tool-lst5.$$
+sum1=/tmp/$tool-sum1.$$
+sum2=/tmp/$tool-sum2.$$
+tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
+
+[ "$1" = "-strict" ] && strict=$1 && shift
+[ "$1" = "-target" ] && target=$2 && shift 2
+[ "$1" = "-?" ] && usage
+[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
+
+trap "rm -f $tmps" 0 1 2 3 5 9 13 15
+exit_status=0
+
+if [ -d "$1" -a -d "$2" ] ; then
+	find "$1/" \( -name '*.sum.xz' ! -name '*go.sum.xz' ! -name 'libgo-all.sum.xz' \)>$lst1
+	find "$2/" \( -name '*.sum.xz' ! -name '*go.sum.xz' ! -name 'libgo-all.sum.xz' \)>$lst2
+	echo "# Comparing directories"
+	echo "# REFERENCE: $1"
+	echo "# CURRENT:   $2"
+	echo
+	# remove leading directory components to compare
+	sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
+	sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
+	comm -23 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra sum files in Dir1=$1"
+		sed -e "s|^|< $1/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -13 $lst3 $lst4 >$lst5
+	if [ -s $lst5 ] ; then
+		echo "# Extra sum files in Dir2=$2"
+		sed -e "s|^|> $2/|" $lst5
+		echo
+		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+	fi
+	comm -12 $lst3 $lst4 | sort -u >$lst5
+	if [ ! -s $lst5 ] ; then
+		echo "# No common sum files"
+		exit_status=`expr $exit_status + 1`
+		exit $exit_status
+	fi
+	cmnsums=`cat $lst5 | wc -l`
+	echo "# Comparing $cmnsums common sum files:"
+	cat $lst5
+	( for fname in `cat $lst5`; do
+	    bname=`basename $fname .sum.xz`
+	    xzcat $1/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/'
+	  done ) >$sum1
+	( for fname in `cat $lst5`; do
+	    bname=`basename $fname .sum.xz`
+	    xzcat $2/$fname | sed -e "s/^\([A-Z]*: \)/\1 $bname:/" | sed -r 's:of file /home.*/gcc/:of file :;s:==[0-9]+==:==X==:;s/output pattern test,.*$/output pattern XXX/'
+	  done ) >$sum2
+	if [ "x${target}" != "x" ] ; then
+	    unstable_target="--unstable-marker ${target}"
+	fi
+	for sum in $sum1 $sum2; do
+	    board="$(grep "Running target " $sum | head -n 1 | sed -e "s/Running target //")"
+	    if [ x"$board" != x"" ]; then
+		unstable_target="$unstable_target --unstable-marker $board"
+	    fi
+	done
+	rm -rf ${my_path}/gcc-unstable-tests
+	git clone http://git.linaro.org/toolchain/gcc-unstable-tests.git ${my_path}/gcc-unstable-tests || exit 1
+	${CONFIG_SHELL-/usr/bin/perl} ${my_path}/compare_dg_tests.pl -l --unstable-test=${my_path}/gcc-unstable-tests/unstable-tests.txt ${unstable_target} $sum1 $sum2
+	ret=$?
+	if [ $ret -eq 2 ]; then
+		exit_status=`expr $exit_status + 1`
+		echo "# Regressions found"
+	else
+	    if [ $ret -eq 1 ]; then
+		echo "# Improvements found"
+	    fi
+	fi
+	if [ $exit_status -ne 0 ]; then
+		echo "# Regressions in $cmnsums common sum files found"
+	else
+		echo "# No regression found in $cmnsums common sum files"
+	fi
+	exit $exit_status
+elif [ -d "$1" -o -d "$2" ] ; then
+	usage "Must specify either two directories or two files"
+fi
diff --git a/unstable-tests.txt b/unstable-tests.txt
new file mode 100644
index 0000000..dc8e877
--- /dev/null
+++ b/unstable-tests.txt
@@ -0,0 +1,120 @@
+# List of unstable tests.
+# Syntax:
+# target:test name
+#
+# gcc
+aarch64-linux-gnu:gcc.c-torture/execute/20000801-2.c execution, -O2 -flto -fno-use-linker-plugin -flto-partition=none
+aarch64-linux-gnu:gcc.c-torture/execute/20000819-1.c execution,  -Og -g
+aarch64-linux-gnu:gcc.c-torture/execute/20010711-1.c execution,  -O3 -fomit-frame-pointer
+aarch64-linux-gnu:gcc.c-torture/execute/20051215-1.c execution,  -Og -g
+aarch64-linux-gnu:gcc.c-torture/execute/920604-1.c execution,  -O0
+aarch64-linux-gnu:gcc.c-torture/execute/920625-1.c execution,  -O2 -flto -fuse-linker-plugin -fno-fat-lto-objects
+aarch64-linux-gnu:gcc.c-torture/execute/921124-1.c execution,  -O2 -flto -fno-use-linker-plugin -flto-partition=none
+aarch64-linux-gnu:gcc.c-torture/execute/930111-1.c execution,  -O0
+aarch64-linux-gnu:gcc.c-torture/execute/941101-1.c execution,  -O3 -g
+aarch64-linux-gnu:gcc.c-torture/execute/990525-1.c execution,  -O2
+aarch64-linux-gnu:gcc.c-torture/execute/anon-1.c execution,  -O2
+aarch64-linux-gnu:gcc.c-torture/execute/anon-1.c execution,  -O3 -fomit-frame-pointer
+aarch64-linux-gnu:gcc.c-torture/execute/builtins/pr22237.c execution,  -O1
+aarch64-linux-gnu:gcc.c-torture/execute/memset-1.c execution,  -Os
+aarch64-linux-gnu:gcc.c-torture/execute/pr28865.c execution,  -O1
+aarch64-linux-gnu:gcc.c-torture/execute/pr33779-1.c execution,  -Os
+aarch64-linux-gnu:gcc.c-torture/execute/pr40668.c execution,  -O2
+aarch64-linux-gnu:gcc.c-torture/execute/pr40668.c execution,  -O3 -fomit-frame-pointer
+aarch64-linux-gnu:gcc.dg/atomic/stdatomic-vm.c  -O3 -fomit-frame-pointer  execution test
+gcc.dg/atomic/c11-atomic-exec-5.c
+arm-linux-gnueabihf:gcc.dg/cpp/_Pragma3.c
+aarch64-linux-gnu:gcc.dg/torture/vec-cvt-1.c  -O3 -fomit-frame-pointer -funroll-loops  execution test
+aarch64-linux-gnu:gcc.target/aarch64/aapcs64/test_19.c execution,  -O2
+arm-sim:gcc.dg/di-sync-multithread.c execution test
+
+# g++
+aarch64-linux-gnu:g++.dg/opt/temp2.C -std=c++1y execution test
+aarch64-linux-gnu:g++.old-deja/g++.eh/new2.C -std=gnu++11 execution test
+aarch64-linux-gnu:g++.old-deja/g++.jason/temporary.C -std=c++1y execution test
+aarch64-linux-gnu:g++.old-deja/g++.law/refs4.C -std=c++11 execution test
+aarch64-linux-gnu:g++.old-deja/g++.mike/p3708a.C -std=c++1y execution test
+
+# gfortran
+aarch64-linux-gnu:gfortran.dg/backslash_1.f90  -O3 -fomit-frame-pointer -funroll-all-loops -finline-functions  execution test
+gnu:gfortran.dg/convert_implied_open.f90
+arm-linux-gnueabihf:gfortran.dg/backspace_9.f  -O3 -fomit-frame-pointer -funroll-all-loops -finline-functions  execution test
+aarch64-linux-gnu:gfortran.dg/finalize_15.f90  -O3 -fomit-frame-pointer  execution test
+gfortran.dg/minlocval_3.f90
+arm-linux-gnueabihf:gfortran.dg/streamio_1.f90  -O3 -g  execution test
+aarch64-linux-gnu:gfortran.dg/tiny_2.f90  -Os  execution test
+aarch64-linux-gnu:gfortran.dg/unf_io_convert_3.f90  -O3 -g  execution test
+gfortran.dg/tl_editing.f90
+gfortran.dg/streamio_14.f90
+# This is https://gcc.gnu.org/bugzilla/show_bug.cgi?id=64921
+gfortran.dg/class_allocate_18.f90   -O0  execution test
+gfortran.dg/class_allocate_18.f90   -O1  execution test
+gfortran.dg/class_allocate_18.f90   -O2  execution test
+gfortran.dg/class_allocate_18.f90   -O3 -fomit-frame-pointer  execution test
+gfortran.dg/class_allocate_18.f90   -O3 -fomit-frame-pointer -funroll-all-loops -finline-functions  execution test
+gfortran.dg/class_allocate_18.f90   -O3 -fomit-frame-pointer -funroll-loops  execution test
+gfortran.dg/class_allocate_18.f90   -Os  execution test
+
+#libgomp
+aarch64-linux-gnu:libgomp.fortran/allocatable10.f90  -O3 -fomit-frame-pointer -funroll-loops  execution test
+aarch64-linux-gnu:libgomp.fortran/allocatable7.f90  -O2  execution test
+aarch64-linux-gnu:libgomp.fortran/omp_atomic1.f90  -O3 -fomit-frame-pointer -funroll-all-loops -finline-functions  execution test
+aarch64-linux-gnu:libgomp.fortran/reference1.f90  -O2  execution test
+aarch64-linux-gnu:libgomp.fortran/simd7.f90  -O3 -fomit-frame-pointer -funroll-loops  execution test
+aarch64-linux-gnu:libgomp.fortran/task3.f90  -O2  execution test
+
+# libstdc++
+arm-sim:20_util/shared_ptr/thread/mutex_weaktoshared.cc execution test
+aarch64-linux-gnu:21_strings/basic_string/operations/find/wchar_t/3.cc execution test
+arm-sim:21_strings/basic_string/pthread33394.cc execution test
+arm-sim:21_strings/basic_string/pthread4.cc execution test
+aarch64-linux-gnu:22_locale/num_get/get/wchar_t/2.cc execution test
+aarch64-linux-gnu:22_locale/numpunct/members/pod/1.cc execution test
+23_containers/deque/cons/2.cc execution test
+arm-sim:23_containers/list/pthread5.cc execution test
+arm-sim:23_containers/map/pthread6.cc execution test
+aarch64-linux-gnu:26_numerics/random/lognormal_distribution/operators/serialize.cc execution test
+26_numerics/complex/inserters_extractors/char/1.cc execution test
+arm-sim:27_io/basic_filebuf/seekoff/char/1-io.cc execution test
+aarch64-linux-gnu:27_io/basic_filebuf/sgetc/char/1-out.cc execution test
+27_io/basic_filebuf/open/char/2.cc execution test
+arm-sim:27_io/basic_filebuf/sbumpc/char/1-in.cc execution test
+arm-sim:27_io/basic_filebuf/sbumpc/char/1-io.cc execution test
+arm-sim:27_io/basic_filebuf/sbumpc/char/2-in.cc execution test
+arm-sim:27_io/basic_filebuf/sbumpc/char/2-io.cc execution test
+arm-sim:27_io/basic_filebuf/seekpos/char/2-io.cc execution test
+arm-sim:27_io/basic_filebuf/sgetc/char/1-io.cc execution test
+arm-sim:27_io/basic_filebuf/sgetc/char/2-in.cc execution test
+arm-sim:27_io/basic_filebuf/sgetc/char/2-io.cc execution test
+arm-sim:27_io/basic_filebuf/snextc/char/1-in.cc execution test
+arm-sim:27_io/basic_filebuf/snextc/char/1-io.cc execution test
+arm-sim:27_io/basic_filebuf/snextc/char/2-in.cc execution test
+arm-sim:27_io/basic_filebuf/snextc/char/2-io.cc execution test
+arm-sim:27_io/basic_filebuf/sputbackc/char/1-in.cc execution test
+arm-sim:27_io/basic_filebuf/sputbackc/char/2-in.cc execution test
+arm-sim:27_io/basic_filebuf/sungetc/char/1-in.cc execution test
+arm-sim:27_io/basic_filebuf/sungetc/char/2-in.cc execution test
+arm-sim:27_io/basic_ostringstream/pthread3.cc execution test
+27_io/basic_istream/seekg/char/fstream.cc execution test
+27_io/basic_istream/tellg/char/fstream.cc execution test
+arm-sim:27_io/basic_ofstream/pthread2.cc execution test
+arm-linux-gnueabi:27_io/basic_ostream/inserters_other/wchar_t/1.cc execution test
+arm-sim:30_threads/call_once/39909.cc execution test
+arm-sim:30_threads/condition_variable/54185.cc execution test
+arm-sim:30_threads/condition_variable_any/50862.cc execution test
+arm-sim:30_threads/lock/2.cc execution test
+arm-sim:30_threads/promise/60966.cc execution test
+aarch64-linux-gnu:experimental/string_view/operations/find/wchar_t/2.cc execution test
+aarch64-linux-gnu:ext/slist/check_construct_destroy.cc execution test
+aarch64-linux-gnu:ext/random/simd_fast_mersenne_twister_engine/cons/default.cc execution test
+aarch64-linux-gnu:tmpdir-g++.dg-struct-layout-1/t011 cp_compat_x_tst.o-cp_compat_y_tst.o execute
+aarch64-linux-gnu:tr1/4_metaprogramming/is_enum/24808.cc execution test
+aarch64-linux-gnu:tr1/5_numerical_facilities/special_functions/07_conf_hyperg/check_value.cc execution test
+aarch64-linux-gnu:tr1/5_numerical_facilities/special_functions/14_ellint_3/check_value.cc execution test
+aarch64-linux-gnu:tr1/5_numerical_facilities/special_functions/17_hyperg/check_nan.cc execution test
+
+# asan
+aarch64-linux-gnu:c-c++-common/asan/swapcontext-test-1.c
+arm-linux-gnueabi:c-c++-common/asan/swapcontext-test-1.c
+arm-linux-gnueabihf:c-c++-common/asan/swapcontext-test-1.c
+arm-sim:c-c++-common/asan/heap-overflow-1.c