compare_tests: Fix usage and exit codes.

compare_jobs.sh: Handle the new return codes.

compare_test has now 5 exit codes:
0: no change
1: improvements
2: regressions
3: no logs in common, cannot compare
4: extra logs in either previous or current

Exit codes 2, 3 and 4 are reported as failures by compare_jobs.sh,
with different error messages in the report table.

Change-Id: I7313ed38cd4cbb5ab1c6795391a6977f3576562a

2 files changed, 28 insertions, 20 deletions

diff --git a/compare_jobs.sh b/compare_jobs.sh
index 3684717..61cf545 100755
--- a/compare_jobs.sh
+++ b/compare_jobs.sh
@@ -193,11 +193,11 @@ do
     ret=$?
     if [ ! -d "${ref}" ]; then
 	printf "\t# REF RESULTS NOT PRESENT: BUILD FAILED\n" >> ${mylog}
-	ret=3
+	ret=5
     fi
     if [ ! -d "${build}" ]; then
 	printf "\t# BUILD RESULTS NOT PRESENT: BUILD FAILED\n" >> ${mylog}
-	ret=3
+	ret=5
     fi
     case $ret in
 	0)  # No change
@@ -211,10 +211,21 @@ do
 	    message=FAILED
 	    failed=true
 	    ;;
-	3) # Build failed
+	3) # No common logs
+	    color=red
+	    message=NO-COMMON-LOGS
+	    failed=true
+	    ;;
+	4) # Extra logs
+	    color=red
+	    message=EXTRA-LOGS
+	    failed=true
+	    ;;
+	5) # Build failed
 	    color=darkred
 	    message=BUILDFAILED
 	    failed=true
+	    ;;
     esac
 
     ${failed} && status=1
diff --git a/compare_tests b/compare_tests
index 74da3be..092760d 100755
--- a/compare_tests
+++ b/compare_tests
@@ -13,25 +13,23 @@ usage()
 		echo >&2
 	fi
 	cat >&2 <<EOUSAGE
-Usage: $0 [-strict] [-target target-triplet] PREVIOUS CURRENT
+Usage: $0 [-target target-triplet] PREVIOUS CURRENT
 
 Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.
 
-	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.
-
-	Unless -strict is given, these discrepancies are not counted as errors:
-		missing/extra .sum files when comparing directories
-		tests that failed in PREVIOUS but pass in CURRENT
-		tests that were not in PREVIOUS but appear in CURRENT
-		tests in PREVIOUS that are missing in CURRENT
+	PREVIOUS and CURRENT must be directories, this tool finds and
+	compares any *.sum files they contain.
 
         -target enables to provide the target name to use when parsing
          the file containing the list of unstable tests.
 
 	Exit with the following values:
 		0 if there is nothing of interest
-		1 if there are errors when comparing single test case files
-		N for the number of errors found when comparing directories
+		1 if there are improvements
+		2 if the are regressions or new errors
+		3 if there were build errors (no common logs)
+		4 if there are extra .sum files in either PREVIOUS or
+		  CURRENT
 EOUSAGE
 	exit 2
 }
@@ -56,7 +54,6 @@ sum1=/tmp/$tool-sum1.$$
 sum2=/tmp/$tool-sum2.$$
 tmps="$tmp1 $tmp2 $now_s $before_s $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"
 
-[ "$1" = "-strict" ] && strict=$1 && shift
 [ "$1" = "-target" ] && target=$2 && shift 2
 [ "$1" = "-?" ] && usage
 [ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"
@@ -79,19 +76,19 @@ if [ -d "$1" -a -d "$2" ] ; then
 		echo "# Extra sum files in Dir1=$1"
 		sed -e "s|^|< $1/|" $lst5
 		echo
-		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+		exit_status=4
 	fi
 	comm -13 $lst3 $lst4 >$lst5
 	if [ -s $lst5 ] ; then
 		echo "# Extra sum files in Dir2=$2"
 		sed -e "s|^|> $2/|" $lst5
 		echo
-		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
+		exit_status=4
 	fi
 	comm -12 $lst3 $lst4 | sort -u >$lst5
 	if [ ! -s $lst5 ] ; then
 		echo "# No common sum files"
-		exit_status=`expr $exit_status + 1`
+		exit_status=3
 		exit $exit_status
 	fi
 	cmnsums=`cat $lst5 | wc -l`
@@ -118,11 +115,11 @@ if [ -d "$1" -a -d "$2" ] ; then
 	ret=$?
 	case $ret in
 	    2)
-		exit_status=`expr $exit_status + 2`
+		[ $exit_status -eq 0 ] && exit_status=2
 		echo "# Regressions found"
 		;;
 	    1)
-		exit_status=`expr $exit_status + 1`
+		[ $exit_status -eq 0 ] && exit_status=1
 		echo "# Improvements found"
 		;;
 	esac
@@ -133,5 +130,5 @@ if [ -d "$1" -a -d "$2" ] ; then
 	fi
 	exit $exit_status
 elif [ -d "$1" -o -d "$2" ] ; then
-	usage "Must specify either two directories or two files"
+	usage "Must specify two directories"
 fi