contrib/compare_tests

#!/bin/sh
# This script automatically test the given tool with the tool's test cases,
# reporting anything of interest.

# Written by Mike Stump <mrs@cygnus.com>
# Subdir comparison added by Quentin Neill <quentin.neill@amd.com>

usage()
{
	if [ -n "$1" ] ; then
		echo "$0: Error: $1" >&2
		echo >&2
	fi
	cat >&2 <<EOUSAGE
Usage: $0 [-strict] PREVIOUS CURRENT

Compare the PREVIOUS and CURRENT test case .sum files, reporting anything of interest.

	If PREVIOUS and CURRENT are directories, find and compare any *.sum files.

	Unless -strict is given, these discrepancies are not counted as errors:
		missing/extra .sum files when comparing directories
		tests that failed in PREVIOUS but pass in CURRENT
		tests that were not in PREVIOUS but appear in CURRENT
		tests in PREVIOUS that are missing in CURRENT

	Exit with the following values:
		0 if there is nothing of interest
		1 if there are errors when comparing single test case files
		N for the number of errors found when comparing directories
EOUSAGE
	exit 2
}

export LC_ALL=C

tool=gxx

TMPDIR=${TMPDIR:-/tmp}
tmp1=$TMPDIR/$tool-testing.$$a
tmp2=$TMPDIR/$tool-testing.$$b
now_s=$TMPDIR/$tool-testing.$$d
before_s=$TMPDIR/$tool-testing.$$e
now_u=$TMPDIR/$tool-uniq.$$d
before_u=$TMPDIR/$tool-uniq.$$e
lst1=$TMPDIR/$tool-lst1.$$
lst2=$TMPDIR/$tool-lst2.$$
lst3=$TMPDIR/$tool-lst3.$$
lst4=$TMPDIR/$tool-lst4.$$
lst5=$TMPDIR/$tool-lst5.$$
sum1=$TMPDIR/$tool-sum1.$$
sum2=$TMPDIR/$tool-sum2.$$
tmps="$tmp1 $tmp2 $now_s $before_s $now_u $before_u $lst1 $lst2 $lst3 $lst4 $lst5 $sum1 $sum2"

[ "$1" = "-strict" ] && strict=$1 && shift
[ "$1" = "-?" ] && usage
[ "$2" = "" ] && usage "Must specify both PREVIOUS and CURRENT"

trap "rm -f $tmps" 0 1 2 3 5 9 13 15
exit_status=0

if [ -d "$1" -a -d "$2" ] ; then
	find "$1/" -name '*.sum' >$lst1
	find "$2/" -name '*.sum' >$lst2
	echo "# Comparing directories"
	echo "## Dir1=$1: `cat $lst1 | wc -l` sum files"
	echo "## Dir2=$2: `cat $lst2 | wc -l` sum files"
	echo
	# remove leading directory components to compare
	sed -e "s|^$1[/]*||" $lst1 | sort >$lst3
	sed -e "s|^$2[/]*||" $lst2 | sort >$lst4
	comm -23 $lst3 $lst4 >$lst5
	if [ -s $lst5 ] ; then
		echo "# Extra sum files in Dir1=$1"
		sed -e "s|^|< $1/|" $lst5
		echo
		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
	fi
	comm -13 $lst3 $lst4 >$lst5
	if [ -s $lst5 ] ; then
		echo "# Extra sum files in Dir2=$2"
		sed -e "s|^|> $2/|" $lst5
		echo
		[ -n "$strict" ] && exit_status=`expr $exit_status + 1`
	fi
	comm -12 $lst3 $lst4 | sort -u >$lst5
	if [ ! -s $lst5 ] ; then
		echo "# No common sum files"
		exit_status=`expr $exit_status + 1`
		exit $exit_status
	fi
	cmnsums=`cat $lst5 | wc -l`
	echo "# Comparing $cmnsums common sum files"
	( for fname in `cat $lst5`; do cat $1/$fname; done ) >$sum1
	( for fname in `cat $lst5`; do cat $2/$fname; done ) >$sum2
	echo "## ${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2"
	${CONFIG_SHELL-/bin/sh} $0 $strict $sum1 $sum2
	ret=$?
	if [ $ret -ne 0 ]; then
		exit_status=`expr $exit_status + 1`
		echo "## Differences found"
	fi
	if [ $exit_status -ne 0 ]; then
		echo "# $exit_status differences in $cmnsums common sum files found"
	else
		echo "# No differences found in $cmnsums common sum files"
	fi
	exit $exit_status
elif [ -d "$1" -o -d "$2" ] ; then
	usage "Must specify either two directories or two files"
fi

sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$1" | awk '/^[	 ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ 	]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ 	]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp1
sed 's/^XFAIL/FAIL/; s/^ERROR/FAIL/; s/^XPASS/PASS/' < "$2" | awk '/^[	 ]*=== [^ ]* tests ===$/ {tool = $2} /^Running target / {target = $3} { if (tool != "") { sub(/:[ 	]/, "&"tool": " ); }; if (target != "unix") { sub(/:[ 	]/, "&"target": " ); }; print $0; }' | cut -c1-2000 >$tmp2

before=$tmp1
now=$tmp2


if sort -k 2 </dev/null >/dev/null 2>&1; then
  skip1='-k 2'
else
  skip1='+1'
fi

sort -t ':' $skip1 "$now" > "$now_s"
sort -t ':' $skip1 "$before" > "$before_s"

# Report non-unique test names and print the diff between the two
# lists if they are different.
#
# If the results include several configurations (schedule of
# variations), we report duplicates several times with different
# target/tool prefixes because at this stage we do not know if the
# target and/or tool prefix was inserted.
# If we used the input files (so generally several times the same
# results in one section per target), we would incorreclty detect
# duplicates (as many as targets)
EGREP="grep -E"

if ! echo PASS | $EGREP '^(PASS|FAIL)' >/dev/null 2>&1; then
    EGREP="egrep"
fi

$EGREP '^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED|UNSUPPORTED|UNTESTED|ERROR):' "$now_s"    | uniq -cd > "$now_u"
$EGREP '^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED|UNSUPPORTED|UNTESTED|ERROR):' "$before_s" | uniq -cd > "$before_u"

same_uniq=" now"
cmp -s "$before_u" "$now_u" && same_uniq=""

if [ -s "$now_u" ]; then
    echo "Non-unique test names$same_uniq: (Eeek!)"
    cat "$now_u"
    echo
    exit_status=1
fi

if [ -s "$before_u" -a "x$same_uniq" != "x" ]; then
    echo "Changes to non-unique test names:"
    diff -u "$before_u" "$now_u" | $EGREP '^[-\\+] '
    echo
    exit_status=1
fi

grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	num=`cat $tmp2 | wc -l`
	echo "Tests that now fail, but worked before ($num tests):"
	echo
	cat $tmp2
	echo
	exit_status=1
fi

grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -12 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	num=`cat $tmp2 | wc -l`
	echo "Tests that now work, but didn't before ($num tests):"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

grep '^FAIL' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	num=`cat $tmp2 | wc -l`
	echo "New tests that FAIL ($num tests):"
	echo
	cat $tmp2
	echo
	exit_status=1
fi

grep '^PASS' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -23 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	num=`cat $tmp2 | wc -l`
	echo "New tests that PASS ($num tests):"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^PASS' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	num=`cat $tmp2 | wc -l`
	echo "Old tests that passed, that have disappeared ($num tests): (Eeek!)"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[ 	]//' >$tmp1
grep '^FAIL' "$before_s" | sed 's/^[^:]*:[ 	]//' | comm -13 $tmp1 - >$tmp2

grep -s . $tmp2 >/dev/null
if [ $? = 0 ]; then
	num=`cat $tmp2 | wc -l`
	echo "Old tests that failed, that have disappeared ($num tests): (Eeek!)"
	echo
	cat $tmp2
	[ -n "$strict" ] && echo "Strict test fails" && exit_status=1
	echo
fi

exit $exit_status