contrib/compare_tests

   1 #!/bin/sh
   2 # This script automatically test the given tool with the tool's test cases,
   3 # reporting anything of interest.
   4
   5 # exits with 1 if there is nothing of interest
   6 # exits with 0 if there is something interesting
   7 # exits with 2 if an error occurred
   8
   9 # Give two .sum files to compare them
  10
  11 # Written by Mike Stump <mrs@cygnus.com>
  12
  13 tmp1=/tmp/$tool-testing.$$a
  14 tmp2=/tmp/$tool-testing.$$b
  15 now_s=/tmp/$tool-testing.$$d
  16 before_s=/tmp/$tool-testing.$$e
  17
  18 if [ "$2" = "" ]; then
  19         echo "Usage: $0 previous current" >&2
  20         exit 2
  21 fi
  22
  23 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$1" >$tmp1
  24 sed 's/^XFAIL/FAIL/; s/^XPASS/PASS/' < "$2" >$tmp2
  25
  26 before=$tmp1
  27 now=$tmp2
  28
  29 trap "rm -f $tmp1 $tmp2 $now_s $before_s" 0 1 2 3 5 9 13 15
  30
  31 sort +0.4 "$now" > "$now_s"
  32 sort +0.4 "$before" > "$before_s"
  33
  34 grep '^FAIL' "$now_s" | sed 's/^....:   //' >$tmp1
  35 grep '^PASS' "$before_s" | sed 's/^....:        //' | comm -12 $tmp1 - >$tmp2
  36
  37 grep -s . $tmp2 >/dev/null
  38 if [ $? = 0 ]; then
  39         echo "Tests that now fail, but worked before:"
  40         echo
  41         cat $tmp2
  42         echo
  43 fi
  44
  45 grep '^PASS' "$now_s" | sed 's/^....:   //' >$tmp1
  46 grep '^FAIL' "$before_s" | sed 's/^....:        //' | comm -12 $tmp1 - >$tmp2
  47
  48 grep -s . $tmp2 >/dev/null
  49 if [ $? = 0 ]; then
  50         echo "Tests that now work, but didn't before:"
  51         echo
  52         cat $tmp2
  53         echo
  54 fi
  55
  56 grep '^FAIL' "$now_s" | sed 's/^....:   //' >$tmp1
  57 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^....:       //' | comm -23 $tmp1 - >$tmp2
  58
  59 grep -s . $tmp2 >/dev/null
  60 if [ $? = 0 ]; then
  61         echo "New tests that FAIL:"
  62         echo
  63         cat $tmp2
  64         echo
  65 fi
  66
  67 grep '^PASS' "$now_s" | sed 's/^....:   //' >$tmp1
  68 grep '^[PF]A[SI][SL]' "$before_s" | sed 's/^....:       //' | comm -23 $tmp1 - >$tmp2
  69
  70 grep -s . $tmp2 >/dev/null
  71 if [ $? = 0 ]; then
  72         echo "New tests that PASS:"
  73         echo
  74         cat $tmp2
  75         echo
  76 fi
  77
  78 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^....:  //' >$tmp1
  79 grep '^PASS' "$before_s" | sed 's/^....:        //' | comm -13 $tmp1 - >$tmp2
  80
  81 grep -s . $tmp2 >/dev/null
  82 if [ $? = 0 ]; then
  83         echo "Old tests that passed, that have disappeared: (Eeek!)"
  84         echo
  85         cat $tmp2
  86         echo
  87 fi
  88
  89 grep '^[PF]A[SI][SL]' "$now_s" | sed 's/^....:  //' >$tmp1
  90 grep '^FAIL' "$before_s" | sed 's/^....:        //' | comm -13 $tmp1 - >$tmp2
  91
  92 grep -s . $tmp2 >/dev/null
  93 if [ $? = 0 ]; then
  94         echo "Old tests that failed, that have disappeared: (Eeek!)"
  95         echo
  96         cat $tmp2
  97         echo
  98 fi