contrib/compare_tests

   1 #!/bin/sh
   2 # This script automatically test the given tool with the tool's test cases,
   3 # reporting anything of interest.
   4
   5 # exits with 0 if there is nothing of interest
   6 # exits with 1 if there is something interesting
   7 # exits with 2 if an error occurred
   8
   9 # Give two .sum files to compare them
  10
  11 # Written by Mike Stump <mrs@cygnus.com>
  12
  13 tool=gxx
  14
  15 tmp1=/tmp/$tool-testing.$$a
  16 tmp2=/tmp/$tool-testing.$$b
  17 now_s=/tmp/$tool-testing.$$d
  18 before_s=/tmp/$tool-testing.$$e
  19
  20 if [ "$2" = "" ]; then
  21         echo "Usage: $0 previous current" >&2
  22         exit 2
  23 fi
  24
  25 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" >$tmp1
  26 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" >$tmp2
  27
  28 before=$tmp1
  29 now=$tmp2
  30
  31 exit_status=0
  32 trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
  33
  34 sort -t ':' +1 "$now" > "$now_s"
  35 sort -t ':' +1 "$before" > "$before_s"
  36
  37 grep '^FAIL:' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
  38 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
  39
  40 grep -s . $tmp2 >/dev/null
  41 if [ $? = 0 ]; then
  42         echo "Tests that now fail, but worked before:"
  43         echo
  44         cat $tmp2
  45         echo
  46         exit_status=1
  47 fi
  48
  49 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
  50 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -12 $tmp1 - >$tmp2
  51
  52 grep -s . $tmp2 >/dev/null
  53 if [ $? = 0 ]; then
  54         echo "Tests that now work, but didn't before:"
  55         echo
  56         cat $tmp2
  57         echo
  58 fi
  59
  60 grep '^FAIL' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
  61 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
  62
  63 grep -s . $tmp2 >/dev/null
  64 if [ $? = 0 ]; then
  65         echo "New tests that FAIL:"
  66         echo
  67         cat $tmp2
  68         echo
  69         exit_status=1
  70 fi
  71
  72 grep '^PASS' "$now_s" | sed 's/^[^:]*:[         ]//' >$tmp1
  73 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^[^:]*:[     ]//' | comm -23 $tmp1 - >$tmp2
  74
  75 grep -s . $tmp2 >/dev/null
  76 if [ $? = 0 ]; then
  77         echo "New tests that PASS:"
  78         echo
  79         cat $tmp2
  80         echo
  81 fi
  82
  83 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
  84 grep '^PASS' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
  85
  86 grep -s . $tmp2 >/dev/null
  87 if [ $? = 0 ]; then
  88         echo "Old tests that passed, that have disappeared: (Eeek!)"
  89         echo
  90         cat $tmp2
  91         echo
  92 fi
  93
  94 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^[^:]*:[        ]//' >$tmp1
  95 grep '^FAIL' "$before_s" | sed 's/^[^:]*:[      ]//' | comm -13 $tmp1 - >$tmp2
  96
  97 grep -s . $tmp2 >/dev/null
  98 if [ $? = 0 ]; then
  99         echo "Old tests that failed, that have disappeared: (Eeek!)"
 100         echo
 101         cat $tmp2
 102         echo
 103 fi
 104
 105 exit $exit_status