test-suite: a tiny test automation script

This patch introduces test-suite, a simple script that makes test cases
verification easier. Test cases in the validation directory are annotated
and this script parses them to check if the actual result is the one
expected by the test writer.

Signed-off-by: Damien Lespiau <damien.lespiau@gmail.com>

3 files changed, 264 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
index e22a8c6..faa046b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,3 +32,8 @@ patches-*
 patches
 series
 
+# test-suite
+*.diff
+*.got
+*.expected
+
diff --git a/Makefile b/Makefile
index 039fe38..6bfb2ea 100644
--- a/Makefile
+++ b/Makefile
@@ -168,3 +168,10 @@ dist:
 		exit 1 ; \
 	fi
 	git archive --format=tar --prefix=sparse-$(VERSION)/ HEAD^{tree} | gzip -9 > sparse-$(VERSION).tar.gz
+
+check: all
+	$(Q)cd validation && ./test-suite
+
+clean-check:
+	find validation/ -name "*.c.[egd]*" -exec rm {} \;
+
diff --git a/validation/test-suite b/validation/test-suite
new file mode 100755
index 0000000..6b9d1b7
--- /dev/null
+++ b/validation/test-suite
@@ -0,0 +1,252 @@
+#!/bin/sh
+
+#set -x
+
+default_path=".."
+default_cmd="sparse \$file"
+tests_list=`find . -name '*.c' | sed -e 's#^\./\(.*\)#\1#' | sort`
+prog_name=`basename $0`
+
+# counts:
+#	- tests that have not been converted to test-suite format
+#	- tests that passed
+#	- tests that failed
+#	- tests that failed but are known to fail
+unhandled_tests=0
+ok_tests=0
+ko_tests=0
+known_ko_tests=0
+
+# defaults to not verbose
+[ -z "$V" ] && V=0
+
+##
+# get_value(key, file) - gets the value of a (key, value) pair in file.
+#
+# returns 0 on success, 1 if the file does not have the key
+get_value()
+{
+	last_result=`grep $1: $2 | sed -e "s/^.*$1:\(.*\)$/\1/"`
+	[ -z "$last_result" ] && return 1
+	return 0
+}
+
+##
+# get_tag(key, file) - does file has the tag key in it ?
+#
+# returns 0 if present, 1 otherwise
+get_tag()
+{
+	last_result=`grep $1 $2`
+	return $?
+}
+
+##
+# verbose(string) - prints string if we are in verbose mode
+verbose()
+{
+	[ "$V" -eq "1" ] && echo "        $1"
+	return 0
+}
+
+##
+# error(string[, die]) - prints an error and exits with value die if given
+error()
+{
+	echo "error: $1"
+	[ -n "$2" ] && exit $2
+	return 0
+}
+
+do_usage()
+{
+echo "$prog_name - a tiny automatic testing script"
+echo "Usage: $prog_name [command] [command arguments]"
+echo
+echo "commands:"
+echo "    none                       runs the whole test suite"
+echo "    single file                runs the test in 'file'"
+echo "    format file [name [cmd]]   helps writing a new test case using cmd"
+echo
+echo "    help                       prints usage"
+}
+
+##
+# do_test(file) - tries to validate a test case
+#
+# it "parses" file, looking for check-* tags and tries to validate
+# the test against an expected result
+# returns:
+#	- 0 if the test passed,
+#	- 1 if it failed,
+#	- 2 if it is not a "test-suite" test.
+do_test()
+{
+	test_failed=0
+	file="$1"
+
+	# can this test be handled by test-suite ?
+	# (it has to have a check-name key in it)
+	get_value "check-name" $file
+	if [ "$?" -eq 1 ]; then
+		unhandled_tests=`expr $unhandled_tests + 1`
+		return 2
+	fi
+	test_name=$last_result
+
+	echo "     TEST    $test_name ($file)"
+
+	# does the test provide a specific command ?
+	cmd=`eval echo $default_path/$default_cmd`
+	get_value "check-command" $file
+	if [ "$?" -eq "0" ]; then
+		last_result=`echo $last_result | sed -e 's/^ *//'`
+		cmd=`eval echo $default_path/$last_result`
+	fi
+	verbose "Using command       : $cmd"
+
+	# grab the expected exit value
+	get_value "check-exit-value" $file
+	if [ "$?" -eq "0" ]; then
+		expected_exit_value=`echo $last_result | tr -d ' '`
+	else
+		expected_exit_value=0
+	fi
+	verbose "Expecting exit value: $expected_exit_value"
+
+	# grab the expected output
+	sed -n '/check-output-start/,/check-output-end/p' $file \
+		| grep -v check-output > "$file".expected
+
+	# grab the actual output & exit value
+	$cmd 1> $file.got 2>&1
+	actual_exit_value=$?
+
+	diff -u "$file".expected "$file".got > "$file".diff
+	if [ "$?" -ne "0" ]; then
+		error "actual output does not match the expected one."
+		error  "see $file.* for further investigation."
+		test_failed=1
+	fi
+
+	if [ "$actual_exit_value" -ne "$expected_exit_value" ]; then
+		error "Actual exit value does not match the expected one."
+		error "expected $expected_exit_value, got $actual_exit_value."
+		test_failed=1
+	fi
+
+	if [ "$test_failed" -eq "1" ]; then
+		ko_tests=`expr $ko_tests + 1`
+		get_tag "check-known-to-fail" $file
+		[ "$?" -eq "0" ] && known_ko_tests=`expr $known_ko_tests + 1`
+		return 1
+	else
+		ok_tests=`expr $ok_tests + 1`
+		return 0
+	fi
+}
+
+do_test_suite()
+{
+	for i in $tests_list; do
+		do_test "$i"
+	done
+
+	# prints some numbers
+	tests_nr=`expr $ok_tests + $ko_tests`
+	echo -n "Out of $tests_nr tests, $ok_tests passed, $ko_tests failed"
+	echo " ($known_ko_tests of them are known to fail)"
+	if [ "$unhandled_tests" -ne "0" ]; then
+		echo "$unhandled_tests tests could not be handled by $prog_name"
+	fi
+}
+
+##
+# do_format(file[, name[, cmd]]) - helps a test writer to format test-suite tags
+do_format()
+{
+	if [ -z "$2" ]; then
+		fname="$1"
+		fcmd=$default_cmd
+	elif [ -z "$3" ]; then
+		fname="$2"
+		fcmd=$default_cmd
+	else
+		fname="$2"
+		fcmd="$3"
+	fi
+	file="$1"
+	cmd=`eval echo $default_path/$fcmd`
+	$cmd 1> $file.got 2>&1
+	fexit_value=$?
+	#foutput=`sed -e "s/^\(.*\)/ * check-output:\1/" $file.got`
+	foutput=`cat $file.got`
+	if [ -z "$foutput" ]; then
+	format=`cat <<_EOF
+/*
+ * check-name: $fname
+ *
+ * check-command: $fcmd
+ * check-exit-value: $fexit_value
+ */
+_EOF
+`
+	else
+	format=`cat <<_EOF
+/*
+ * check-name: $fname
+ *
+ * check-command: $fcmd
+ * check-exit-value: $fexit_value
+ *
+ * check-output-start
+$foutput
+ * check-output-end
+ */
+_EOF
+`
+	fi
+	echo "$format"
+	return 0
+}
+
+##
+# arg_file(filename) - checks if filename exists
+arg_file()
+{
+	[ -z "$1" ] && {
+		do_usage
+		exit 1
+	}
+	[ -e "$1" ] || {
+		error "Can't open file $1"
+		exit 1
+	}
+	return 0
+}
+
+case "$1" in
+	'')
+		do_test_suite
+		;;
+	single)
+		arg_file "$2"
+		do_test "$2"
+		case "$?" in
+			0) echo "$2 passed !";;
+			1) echo "$2 failed !";;
+			2) echo "$2 can't be handled by $prog_name";;
+		esac
+		;;
+	format)
+		arg_file "$2"
+		do_format "$2" "$3" "$4"
+		;;
+	help | *)
+		do_usage
+		exit 1
+		;;
+esac
+
+exit 0
+