| #!/bin/bash |
| # Copyright (C) 2006, 2008 Free Software Foundation |
| # |
| # Analyze changes in GCC DejaGNU test logs for binutils, gcc, gdb, etc. |
| # Original version written in 2005 by James Lemke <jwlemke@wasabisystems.com>. |
| # |
| # See usage() below. |
| |
| usage () { |
| cat <<EOF >&2 |
| Usage: |
| dg-cmp-results.sh [-v] [-v] [-v] <variant-name> <old-file> <new-file> |
| <variant-name> names the desired variant, "/" must be written as "\/". |
| Use the empty string ("") for the first variant in each file. |
| Output is to stdout. |
| Non-verbose output is degradation info like PASS->FAIL. |
| -v adds improvement info like FAIL->PASS. |
| -v -v adds info like tests that are no longer run. |
| -v -v -v adds info for tests that have not changed status. |
| -v -v -v -v is used for debugging. |
| EOF |
| } |
| |
| verbose=0 |
| while test "$1" = "-v"; do |
| verbose=`expr $verbose + 1` |
| shift |
| done |
| |
| if test $# -ne 3 ; then |
| usage |
| exit 1 |
| fi |
| |
| if test ! -f "$2"; then |
| echo "unable to open $2" >&2 |
| exit 1 |
| fi |
| |
| if test ! -f "$3"; then |
| echo "unable to open $3" >&2 |
| exit 1 |
| fi |
| |
| # Command differences for various platforms. |
| case `uname -s` in |
| Darwin|NetBSD) |
| E=-E # sed |
| ;; |
| *) |
| E=-r # sed |
| ;; |
| esac |
| |
| # sections are identified by separator lines beginning with '\t\t==='. |
| # section 0 identifies run date, target, and host. |
| # section 1 and subsequent contain test data for a target variant. |
| # -skip to /^Running target/ and use that line to identify the variant. |
| # -subsequent lines contain the result data. They begin with: |
| # '(PASS|FAIL|XFAIL|XPASS|UNTESTED|UNSUPPORTED|UNRESOLVED):' |
| VARIANT="$1" |
| OFILE="$2" |
| OBASE=`basename "$2"` |
| NFILE="$3" |
| NBASE=`basename "$3"` |
| |
| echo "dg-cmp-results.sh: Verbosity is ${verbose}, Variant is \"${VARIANT}\"" |
| echo |
| |
| header="^Running target $VARIANT" |
| |
| temp=`grep "$header" $OFILE` |
| if test -z "$temp"; then |
| echo "Error: variant \"$VARIANT\" not found in $OFILE." |
| exit 1 |
| fi |
| temp=`grep "$header" $NFILE` |
| if test -z "$temp"; then |
| echo "Error: variant \"$VARIANT\" not found in $NFILE." |
| exit 1 |
| fi |
| unset temp |
| |
| # Copy out the old file's section 0. |
| echo "Older log file: $OFILE" |
| sed $E -e '/^[[:space:]]+===/,$d' $OFILE |
| |
| # Copy out the new file's section 0. |
| echo "Newer log file: $NFILE" |
| sed $E -e '/^[[:space:]]+===/,$d' $NFILE |
| |
| # Create a temporary file from the old file's interesting section. |
| sed $E -e "1,/$header/d" \ |
| -e '/^[[:space:]]+===/,$d' \ |
| -e '/^[A-Z]+:/!d' \ |
| -e '/^(WARNING|ERROR):/d' \ |
| -e 's/\r$//' \ |
| -e 's/^/O:/' \ |
| $OFILE | |
| sort -s -t : -k 3b - \ |
| >/tmp/o$$-$OBASE |
| |
| # Create a temporary file from the new file's interesting section. |
| sed $E -e "1,/$header/d" \ |
| -e '/^[[:space:]]+===/,$d' \ |
| -e '/^[A-Z]+:/!d' \ |
| -e '/^(WARNING|ERROR):/d' \ |
| -e 's/\r$//' \ |
| -e 's/^/N:/' \ |
| $NFILE | |
| sort -s -t : -k 3b - \ |
| >/tmp/n$$-$NBASE |
| |
| # Merge the two files, then compare adjacent lines. |
| # Comparison is complicated by tests that may be run multiple times. |
| # If that case, we assume that the order is the same in both files. |
| cat <<EOF >compare-$$.awk |
| BEGIN { |
| FS = ":" |
| queue1 = 1; queueN = 0; status[queue1] = ""; name[queue1] = "" |
| verbose = verbose + 0 # Make sure it's defined. |
| } |
| |
| # FIFO circular queue |
| function push(st, nm) { |
| queueN += 1; status[queueN] = st; name[queueN] = nm |
| } |
| function peek() { |
| result = 0 |
| if (queueN >= queue1) result = queue1 |
| return result |
| } |
| function drop() { |
| queue1 += 1 |
| if (queue1 > queueN) { queue1 = 1; queueN = 0; } |
| } |
| |
| function compare(st, nm) { |
| old = peek() |
| if (old == 0) { |
| # This new test wasn't run last time. |
| if (verbose >= 2) printf("NA->%s:%s\n", st, nm) |
| } |
| else { |
| # Compare this new test to the first queued old one. |
| if (verbose >= 4) { |
| printf("Comparing two lines:\n O:%s:%s\n N:%s:%s\n", |
| status[old], name[old], st, nm) |
| } |
| if (name[old] != nm) { |
| # The old test wasn't run this time and |
| # the new test wasn't run last time. |
| if (verbose >= 2) { |
| printf("%s->NA:%s\n", status[old], name[old]) |
| if (nm != "") printf("NA->%s:%s\n", st, nm) |
| } |
| drop() |
| } |
| else { |
| notable = 0 |
| if (status[old] == st) { |
| # Status of this test has not changed. |
| if (verbose >= 3) printf("%s:%s\n", st, nm) |
| } |
| else if(status[old] == "PASS" && st == "XFAIL") { |
| if (verbose >= 1) notable = 1 |
| } |
| else if(status[old] == "PASS" || st == "FAIL") { |
| # Test did pass but doesn't now |
| # or didn't fail but does now. |
| notable = 1 |
| } |
| else if(st == "PASS") { |
| # Test didn't pass but does now. |
| if (verbose >= 1) notable = 1 |
| } |
| else if(verbose >= 2) { |
| # Miscellaneous status change. |
| notable = 1 |
| } |
| if (notable > 0) printf("%s->%s:%s\n", status[old], st, nm) |
| drop() |
| } |
| } |
| } |
| |
| /^O:/ { |
| while (old = peek()) { |
| if (name[old] == \$3) break; |
| # The queued test is no longer run. |
| compare("", ""); |
| } |
| # Save this test for later comparison. |
| push(\$2, \$3) |
| } |
| |
| /^N:/ { |
| compare(\$2, \$3) |
| } |
| |
| END { |
| while (old = peek()) compare("", "") |
| } |
| EOF |
| sort -m -s -t : -k 3b /tmp/o$$-$OBASE /tmp/n$$-$NBASE | |
| awk -v verbose=$verbose -f compare-$$.awk /dev/stdin |
| |
| # Delete the temporary files. |
| rm -f compare-$$.awk /tmp/o$$-$OBASE /tmp/n$$-$NBASE |
| |
| exit 0 |