avremu/tests/test-suite

#!/bin/bash

#set -x

default_path=".."
tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
prog_name=`basename $0`

# counts:
#	- tests that have not been converted to test-suite format
#	- tests that passed
#	- tests that failed
#	- tests that failed but are known to fail
unhandled_tests=0
ok_tests=0
ko_tests=0
known_ko_tests=0

# defaults to not verbose
[ -z "$V" ] && V=0

##
# get_value(key, file) - gets the value of a (key, value) pair in file.
#
# returns 0 on success, 1 if the file does not have the key
get_value()
{
	last_result=`grep $1: $2 | sed -e "s/^.*$1:\(.*\)$/\1/"`
	[ -z "$last_result" ] && return 1
	return 0
}

##
# get_tag(key, file) - does file has the tag key in it ?
#
# returns 0 if present, 1 otherwise
get_tag()
{
	last_result=`grep $1 $2`
	return $?
}

##
# verbose(string) - prints string if we are in verbose mode
verbose()
{
	[ "$V" -eq "1" ] && echo "        $1"
	return 0
}

##
# error(string[, die]) - prints an error and exits with value die if given
error()
{
	echo "   error: $1"
	[ -n "$2" ] && exit $2
	return 0
}

do_usage()
{
echo "$prog_name - a tiny automatic testing script"
echo "Usage: $prog_name [command] [command arguments]"
echo
echo "commands:"
echo "    none                       runs the whole test suite"
echo "    single file                runs the test in 'file'"
echo
echo "    help                       prints usage"
}

##
# do_test(file) - tries to validate a test case
#
# it "parses" file, looking for check-* tags and tries to validate
# the test against an expected result
# returns:
#	- 0 if the test passed,
#	- 1 if it failed,
#	- 2 if it is not a "test-suite" test.
do_test()
{
	test_failed=0
	file="$1"

	# can this test be handled by test-suite ?
	# (it has to have a check-name key in it)
	get_value "check-name" $file
	if [ "$?" -eq 1 ]; then
		echo "warning: test '$file' unhandled"
		unhandled_tests=`expr $unhandled_tests + 1`
		return 2
	fi
	test_name=$last_result

	echo -n "TEST    $test_name ($file)"

    cp HEADER "$file".tex
    get_value compiler-opts "$file"
    echo "\avrloadc[-Os -mmcu=atmega8 $last_result]{$file}" >> "$file".tex

	awk '/check-start/,/check-end/ {print}' $file \
		| egrep -v 'check-(start|end)' >> "$file".tex
    cat FOOTER >> "$file".tex


	# grab the actual output & exit value
	pdflatex -halt-on-error -shell-escape "$file".tex 1> $file.output 2> $file.error
	actual_exit_value=$?
    rm -f *.log *.aux 

	if [ "$actual_exit_value" -ne 0 ]; then
        echo
		error "   Actual exit value does not match the expected one."
		error "   expected 0, got $actual_exit_value"
        if [ x"$V" = x"1" ]; then
            cat $file.output | sed 's/^/   /'
        else
            tail -n 10 $file.output | sed 's/^/   /'
        fi
		test_failed=1
    else
        get_value BREAK "$file".output
        echo " ($last_result)" | sed 's/ instructions//' | tr -d "\n"
        echo 
	fi

	if [ "$test_failed" -eq "1" ]; then
		ko_tests=`expr $ko_tests + 1`
		get_tag "check-known-to-fail" $file
		if [ "$?" -eq "0" ]; then
			echo "info: test '$file' is known to fail"
			known_ko_tests=`expr $known_ko_tests + 1`
		fi
		return 1
	else
		ok_tests=`expr $ok_tests + 1`
		return 0
	fi
}

pwait() {
    # This functions blocks until less than $1 subprocesses are running
    jobs="$1"
    [ -z "$jobs" ] && jobs=5

    while [ $(jobs -r | wc -l) -gt "$jobs" ]; do
        sleep 0.5
    done
}

do_test_suite()
{
	for i in $tests_list; do
        if get_value "check-long" "$i" > /dev/null; then
            echo "IGNORED    $i (long running)"
            continue
        fi
		do_test "$i"
	done

	# prints some numbers
	tests_nr=`expr $ok_tests + $ko_tests`
	echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
	echo " ($known_ko_tests of them are known to fail)"
	if [ "$unhandled_tests" -ne "0" ]; then
		echo "$unhandled_tests tests could not be handled by $prog_name"
	fi
}

##
# arg_file(filename) - checks if filename exists
arg_file()
{
	[ -z "$1" ] && {
		do_usage
		exit 1
	}
	[ -e "$1" ] || {
		error "Can't open file $1"
		exit 1
	}
	return 0
}

case "$1" in
	'')
		do_test_suite
		if test `expr $ko_tests - $known_ko_tests` -gt 0; then
		    for f in *.error.diff *.output.diff; do
		        if test -s "$f"; then
		            echo "Contents of $f:"
		            cat "$f"
		            echo "==="
		        fi
		    done
		    exit 1
		fi
		;;
	single)
		arg_file "$2"
		do_test "$2"
		case "$?" in
			0) echo "$2 passed !";;
			1) echo "$2 failed !";;
			2) echo "$2 can't be handled by $prog_name";;
		esac
		;;
	format)
		arg_file "$2"
		do_format "$2" "$3" "$4"
		;;
	help | *)
		do_usage
		exit 1
		;;
esac

exit 0