Re: [PATCH 0/N] test-suite improvement - PASS/FAIL: initial patch

[Tomas Dohnalek <tdohnale at redhat dot com>]

----- Original Message -----
> On Mon, 17 Sep 2012, Tomas Dohnalek wrote:
> 
> > Ok, but then the behaviour of running simple `make check' (no -k)
> > will be
> > changed, is that ok? I guess that it is probably desired to run the
> > whole
> > test-suite even when error occurs, but maybe someone has some
> > objections?
> 
> Good point - I suppose you need to check MAKEFLAGS in the test rule
> and
> emulate -k handling there, rather than just unconditionally ignoring
> errors from $(MAKE) tests, so that errors from the inner make are
> properly
> ignored or not depending on whether -k is passed.
> 
> Ideally I think the whole testsuite should run in any case (meaning
> passing down -k whether or not the user specified it) and "make
> check"
> should exit with a nonzero status if any test had an *unexpected*
> failure
> (including failure of any of the commands involved in building an
> test, or
> the various separate makefile rules that do bits of setup for running
> tests) - printing on stdout/stderr a summary of the unexpected
> failures.
> But that would be for a later stage, since you'd need to develop a
> way to
> track results of test compilation / setup as well as the tests
> themselves,
> and a way for expectedness of failures to be more directly visible to
> the
> test harness.  (As is, expected failures simply use "-" in their
> makefile
> rules directly, e.g. in posix/Makefile for $(objpfx)annexc.out.)
> 
> --
> Joseph S. Myers
> joseph@codesourcery.com
> 


Hi,
I did little changes, so now it kind of emulates `-k' handling. But so far if it fails during compilation or if uncovered test case by PASS/FAIL line fails, the summary file won't be created anyway. I didn't want to edit the compilation...

But if you are saying, that in future it should run whole testsuite in any case, why don't use the simple `-$(MAKE) tests'? Now it changes behaviour but as you said, it is desired.

Thanks
Tomas

---
	* Makerules: New variable `tests-collect-results', changed recipies of
	rules `check' and `xcheck', covered abi-checks by PASS/FAIL line.
	by PASS/FAIL line.
	* Rules: New variables `evaluate-test', `evaluate-test-keep-rc' 
	and `test-name', covered all ordinary tests by PASS/FAIL line.
	* scripts/evaluate-test.sh: New file.

diff --git a/Makerules b/Makerules
index 1281b94..69e3e64 100644
--- a/Makerules
+++ b/Makerules
@@ -1121,13 +1121,22 @@ endef
 ALL_BUILD_CFLAGS = $(BUILD_CFLAGS) $(BUILD_CPPFLAGS) -D_GNU_SOURCE \
 		   -DIS_IN_build -include $(common-objpfx)config.h
 
+# Output for summary with testsuite results
+tests-summary = $(common-objpfx)tests.sum
+define tests-collect-results
+	-cat $(common-objpfx)*/*.test-result > $(tests-summary)
+endef
 # Support the GNU standard name for this target.
 .PHONY: check
-check: tests
+check:
+	$(MAKE) tests
+	$(tests-collect-results)
 # Special target to run tests which cannot be run unconditionally.
 # Maintainers should use this target.
 .PHONY: xcheck
-xcheck: xtests
+xcheck:
+	$(MAKE) xtests
+	$(tests-collect-results)
 
 all-nonlib = $(strip $(tests) $(xtests) $(test-srcs) $(test-extras) $(others))
 ifneq (,$(all-nonlib))
@@ -1166,7 +1175,8 @@ check-abi-%: $(common-objpfx)config.make %.abilist $(objpfx)%.symlist
 check-abi-%: $(common-objpfx)config.make %.abilist $(common-objpfx)%.symlist
 	$(check-abi)
 define check-abi
-	diff -p -U 0 $(filter %.abilist,$^) $(filter %.symlist,$^)
+	diff -p -U 0 $(filter %.abilist,$^) $(filter %.symlist,$^); \
+	    $(evaluate-test)
 endef
 
 update-abi-%: $(objpfx)%.symlist %.abilist
@@ -1275,7 +1285,10 @@ do-tests-clean:
 	-rm -f $(addprefix $(objpfx),$(addsuffix .out,$(tests) $(xtests) \
 						      $(test-srcs)) \
 				     $(addsuffix -bp.out,$(tests) $(xtests) \
-							 $(test-srcs)))
+							 $(test-srcs)) \
+				     $(addsuffix .test-result,$(tests) $(xtests) \
+						      $(test-srcs) $(extra-libs)))
+	-rm -f $(tests-summary)
 
 # Remove the object files.
 common-mostlyclean:
@@ -1289,10 +1302,13 @@ common-mostlyclean:
 				     $(addsuffix .out,$(tests) $(xtests) \
 						      $(test-srcs)) \
 				     $(addsuffix -bp.out,$(tests) $(xtests) \
-							 $(test-srcs)))
+							 $(test-srcs)) \
+				     $(addsuffix .test-result,$(tests) $(xtests) \
+						      $(test-srcs) $(extra-libs)))
 	-rm -f $(addprefix $(objpfx),$(extra-objs) $(extra-test-objs) \
 				     $(install-lib) $(install-lib.so) \
 				     $(install-lib.so:%.so=%_pic.a))
+	-rm -f $(tests-summary)
 	-rm -f core
 	-rm -f $(objpfx)rtld-*.os
 	$(rmobjs)
diff --git a/Rules b/Rules
index 17d938e..c86e619 100644
--- a/Rules
+++ b/Rules
@@ -128,6 +128,18 @@ binaries-shared-tests = $(filter-out $(binaries-pie) $(binaries-static), \
 binaries-shared-notests = $(filter-out $(binaries-pie) $(binaries-static), \
 				       $(binaries-all-notests))
 
+test-name = $(subdir)/$(*F)
+
+ifneq "" "$(findstring k,$(MAKEFLAGS))"
+evaluate-test-keep-rc=false
+else
+evaluate-test-keep-rc=true
+endif
+
+evaluate-test = $(..)scripts/evaluate-test.sh $$? $(test-name) \
+				    $(evaluate-test-keep-rc) > $(objpfx)$(*F).test-result
+
+
 ifneq "$(strip $(binaries-shared-notests))" ""
 $(addprefix $(objpfx),$(binaries-shared-notests)): %: %.o \
   $(sort $(filter $(common-objpfx)lib%,$(link-libc))) \
@@ -178,11 +190,11 @@ ifneq "$(strip $(tests) $(xtests) $(test-srcs))" ""
 make-test-out = GCONV_PATH=$(common-objpfx)iconvdata LC_ALL=C \
 		$($*-ENV) $(built-program-cmd) $($*-ARGS)
 $(objpfx)%-bp.out: %.input $(objpfx)%-bp
-	$(make-test-out) > $@ < $(word 1,$^)
+	$(make-test-out) > $@ < $(word 1,$^); $(evaluate-test)
 $(objpfx)%.out: %.input $(objpfx)%
-	$(make-test-out) > $@ < $(word 1,$^)
+	$(make-test-out) > $@ < $(word 1,$^); $(evaluate-test)
 $(objpfx)%.out: /dev/null $(objpfx)%	# Make it 2nd arg for canned sequence.
-	$(make-test-out) > $@
+	$(make-test-out) > $@; $(evaluate-test)
 
 endif	# tests
 
diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
new file mode 100755
index 0000000..ca012d7
--- /dev/null
+++ b/scripts/evaluate-test.sh
@@ -0,0 +1,38 @@
+#!/bin/sh
+# Copyright (C) 2012 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+#
+# This script is used to evaluate return code of a single testcase
+# and produce appropriate output.
+# usage: evaluate-test.sh rc test_name keep_rc
+
+rc=$1
+test_name=$2
+keep_rc=$3
+
+if [ $rc -eq 0 ]; then
+  result="PASS"
+else
+  result="FAIL"
+fi
+
+if ! $keep_rc; then
+  rc=0
+fi
+
+echo "$result: $test_name"
+exit $rc