[RFC][PATCH] Rerun benchmarks only if *.out file does not exist.

[Steve Ellcey <sellcey at cavium dot com>]

While doing some benchmarking on glibc I noticed a difference between
'make check' and 'make bench'.  If you rerun 'make check' after having
run it once already, it will only rerun tests where the *.out file does
not exist.  This is useful (as explained in the glibc Wiki) because you
can rerun one test by removing just that .out file and running 'make check'.

With 'make bench' it reruns all the benchmarks each time you do 'make bench'
even if the .out files exist.  I was thinking that it would be nice to
have this follow the 'make check' behaviour, then if you only want to retest
one test (because you are doing performance changes on one function), you
can just remove that .out file and run 'make bench'.  Right now there does
not seem to be a way to run one performance benchmark.

Two questions: One, does this sound useful to other people?  And two, what
is wrong with my patch?  This patch mostly works but the bench-memcpy,
bench-memcpy-large, and bench-memcpy-random tests get rerun everytime even
if the .out files for those tests already exist.  I don't know why.  I am
guessing is thas something to do with their common name prefix and the string
manipulations done to test names in the Makefile but memset does not seem
to have that problem.

Steve Ellcey
sellcey@cavium.com


2017-09-22  Steve Ellcey  <sellcey@cavium.com>

	* benchtests/Makefile (bench-set): Only run if *.out file
	does not exist.
	(bench-malloc): Likewise.
	(bench-func): Likewise.

diff --git a/benchtests/Makefile b/benchtests/Makefile
index 3acc39c..66b080e 100644
--- a/benchtests/Makefile
+++ b/benchtests/Makefile
@@ -168,15 +168,19 @@ endif
 
 bench-set: $(binaries-benchset)
 	for run in $^; do \
-	  echo "Running $${run}"; \
-	  $(run-bench) > $${run}.out; \
+	  if test ! -f $${run}.out; then \
+	    echo "Running $${run}"; \
+	    $(run-bench) > $${run}.out; \
+	  fi \
 	done
 
 bench-malloc: $(binaries-bench-malloc)
 	run=$(objpfx)bench-malloc-thread; \
 	for thr in 1 8 16 32; do \
-	  echo "Running $${run} $${thr}"; \
-	  $(run-bench) $${thr} > $${run}-$${thr}.out; \
+	  if test ! -f $${run}-$${thr}.out; then \
+	    echo "Running $${run} $${thr}"; \
+	    $(run-bench) $${thr} > $${run}-$${thr}.out; \
+	  fi \
 	done
 
 # Build and execute the benchmark functions.  This target generates JSON
@@ -184,25 +188,23 @@ bench-malloc: $(binaries-bench-malloc)
 # so one could even execute them individually and process it using any JSON
 # capable language or tool.
 bench-func: $(binaries-bench)
-	{ timing_type=$$($(timing-type)); \
-	echo "{\"timing_type\": \"$${timing_type}\","; \
-	echo " \"functions\": {"; \
-	for run in $^; do \
-	  if ! [ "x$${run}" = "x$<" ]; then \
-	    echo ","; \
-	  fi; \
-	  echo "Running $${run}" >&2; \
-	  $(run-bench) $(DETAILED_OPT); \
-	done; \
-	echo; \
-	echo " }"; \
-	echo "}"; } > $(objpfx)bench.out-tmp; \
-	if [ -f $(objpfx)bench.out ]; then \
-	  mv -f $(objpfx)bench.out $(objpfx)bench.out.old; \
-	fi; \
-	mv -f $(objpfx)bench.out-tmp $(objpfx)bench.out
-	$(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
-		scripts/benchout.schema.json
+	if test ! -f $(objpfx)bench.out; then \
+	  { timing_type=$$($(timing-type)); \
+	  echo "{\"timing_type\": \"$${timing_type}\","; \
+	  echo " \"functions\": {"; \
+	  for run in $^; do \
+	    if ! [ "x$${run}" = "x$<" ]; then \
+	      echo ","; \
+	    fi; \
+	    echo "Running $${run}" >&2; \
+	    $(run-bench) $(DETAILED_OPT); \
+	  done; \
+	  echo; \
+	  echo " }"; \
+	  echo "}"; } > $(objpfx)bench.out; \
+	  $(PYTHON) scripts/validate_benchout.py $(objpfx)bench.out \
+		scripts/benchout.schema.json; \
+	fi
 
 $(timing-type) $(binaries-bench) $(binaries-benchset) \
 	$(binaries-bench-malloc): %: %.o $(objpfx)json-lib.o \