| ## automake - create Makefile.in from Makefile.am |
| ## Copyright (C) 2001-2015 Free Software Foundation, Inc. |
| |
| ## This program is free software; you can redistribute it and/or modify |
| ## it under the terms of the GNU General Public License as published by |
| ## the Free Software Foundation; either version 2, or (at your option) |
| ## any later version. |
| |
| ## This program is distributed in the hope that it will be useful, |
| ## but WITHOUT ANY WARRANTY; without even the implied warranty of |
| ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| ## GNU General Public License for more details. |
| |
| ## You should have received a copy of the GNU General Public License |
| ## along with this program. If not, see <http://www.gnu.org/licenses/>. |
| |
| ## New parallel test driver. |
| ## |
| ## The first version of the code here was adapted from check.mk, which was |
| ## originally written at EPITA/LRDE, further developed at Gostai, then made |
| ## its way from GNU coreutils to end up, largely rewritten, in Automake. |
| ## The current version is an heavy rewrite of that, to allow for support |
| ## of more test metadata, and the use of custom test drivers and protocols |
| ## (among them, TAP). |
| |
| ## Used by (at least) 'check-typos.am'. |
| am.conf.using-parallel-tests := yes |
| |
| am.test-suite.is-xfail = \ |
| $(if $(filter-out $(am.test-suite.xfail-test-bases), \ |
| $(patsubst $(srcdir)/%,%,$(1))),no,yes) |
| |
| am.test-suite.runtest = \ |
| $(am.test-suite.tty-colors); \ |
| if test '$(TEST_SUITE_LOG)' = '$*.log'; then \ |
| echo "fatal: $*.log: depends on itself (check TESTS content)" >&2; \ |
| exit 1; \ |
| fi; \ |
| srcdir=$(srcdir); export srcdir; \ |
| ## Creates the directory for the log file if needed. Avoid extra forks. |
| test x$(@D) = x. || test -d $(@D) || $(MKDIR_P) $(@D) || exit $$?; \ |
| ## We need to invoke the test in way that won't cause a PATH search. |
| ## Quotes around '$<' are required to avoid extra errors when a circular |
| ## dependency is detected (e.g., because $(TEST_SUITE_LOG) is in |
| ## $(am.test-suite.test-logs)), because in that case '$<' expands to empty and an |
| ## unquote usage of it could cause syntax errors in the shell. |
| case '$<' in */*) tst='$<';; *) tst=./'$<';; esac; \ |
| ## Executes the developer-defined and user-defined test |
| ## setups (if any), in that order. |
| $(AM_TESTS_ENVIRONMENT) $(TESTS_ENVIRONMENT) \ |
| $($(1)LOG_DRIVER) \ |
| --test-name '$(patsubst $(srcdir)/%,%,$<)' \ |
| --log-file $*.log \ |
| --trs-file $*.trs \ |
| --color-tests "$$am__color_tests" \ |
| --enable-hard-errors $(if $(DISABLE_HARD_ERRORS),no,yes) \ |
| --expect-failure $(call am.test-suite.is-xfail,$*) \ |
| $(AM_$(1)LOG_DRIVER_FLAGS) \ |
| $($(1)LOG_DRIVER_FLAGS) \ |
| -- \ |
| $($(1)LOG_COMPILER) \ |
| $(AM_$(1)LOG_FLAGS) \ |
| $($(1)LOG_FLAGS) \ |
| "$$tst" \ |
| $(AM_TESTS_FD_REDIRECT) |
| |
| define am.test-suite.handle-suffix.helper |
| ifeq ($$(call am.vars.is-undef,$(2)LOG_DRIVER),yes) |
| $(2)LOG_DRIVER = $(SHELL) $(am.conf.aux-dir)/test-driver |
| endif |
| %.log %.trs: %$1 $$($(2)LOG_DEPENDENCIES) |
| @$$(call am.test-suite.runtest,$2) |
| ifdef EXEEXT |
| %.log %.trs: %$1$(EXEEXT) $$($(2)LOG_DEPENDENCIES) |
| @$$(call am.test-suite.runtest,$2) |
| endif |
| endef |
| |
| define am.test-suite.handle-suffix |
| $(call $0.helper,$1,$(if $1,$(call am.util.toupper,$(patsubst .%,%_,$1)))) |
| endef |
| |
| ifeq ($(call am.vars.is-undef,TEST_EXTENSIONS),yes) |
| TEST_EXTENSIONS := .test |
| endif |
| |
| $(foreach e,$(filter-out .%,$(TEST_EXTENSIONS)),\ |
| $(call am.error,invalid test extension: '$e')) |
| $(foreach e,$(TEST_EXTENSIONS), \ |
| $(eval $(call am.test-suite.handle-suffix,$e))) |
| # It is *imperative* that the "empty" suffix goes last. Otherwise, a |
| # declaration like "TESTS = all.test" would cause GNU make to mistakenly |
| # try to build the 'all.log' and 'all.trs' files from a non-existent |
| # 'all' program (because the Makefile contains an explicit 'all' target, |
| # albeit .PHONY), rather than from the 'all.test' script, thus causing |
| # all sort of mishaps and confusion. |
| $(eval $(call am.test-suite.handle-suffix)) |
| |
| # The names of the given tests scripts with any possible registered |
| # test extension removed, as well as any leading '$(srcdir)' component |
| # (if any) stripped. |
| # The stripping of $(srcdir) is required to support explicit use of |
| # $(srcdir) in TESTS entries. That might actually be very useful in |
| # practice, for example in usages like this: |
| # TESTS = $(wildcard $(srcdir)/t[0-9][0-9]*.sh) |
| # where removing the $(srcdir) from the $(wildcard) invocation would |
| # cause the idiom to break in VPATH builds. |
| define am.test-suite.get-test-bases |
| $(patsubst $(srcdir)/%,%,$(strip \ |
| $(call am.util.strip-suffixes, $(TEST_EXTENSIONS), \ |
| $(if $(EXEEXT),$(patsubst %$(EXEEXT),%,$1),$1)))) |
| endef |
| |
| am.test-suite.rx.recheck = ^[ ]*:recheck:[ ]* |
| am.test-suite.rx.global-result = ^[ ]*:global-test-result:[ ]* |
| am.test-suite.rx.result = ^[ ]*:test-result:[ ]* |
| am.test-suite.rx.copy-in-global-log = ^[ ]*:copy-in-global-log:[ ]* |
| |
| # Some awk code fragments used by one another and eventually by the |
| # 'check' and 'recheck' recipes. |
| # Note that in those scripts we are careful to close all the '.trs' and |
| # '.log' files once we are done with them. This is done to avoid leaking |
| # open file descriptors, which could cause serious problems when there |
| # are many tests and thus lots of '.log' and '.trs' files to open (yes, |
| # there would be problems even on Linux). |
| |
| am.test-suite.awk-functions = \ |
| function error(msg) \ |
| { \ |
| print msg | "cat >&2"; \ |
| exit_status = 1; \ |
| } \ |
| function input_error(file) \ |
| { \ |
| error("awk" ": cannot read \"" file "\""); \ |
| ## Never leak file descriptors, not even on errors. |
| close ($$0 ".trs"); close ($$0 ".log"); \ |
| } |
| |
| # Loop on the lines in the current '.trs' or '.log' file, |
| # punting on I/O errors. |
| am.test-suite.awk-io-loop/BEGIN = \ |
| while ((rc = (getline line < ($$0 ".$1"))) != 0) { \ |
| if (rc < 0) { input_error($$0 ".$1"); next; } |
| am.test-suite.awk-io-loop/END = \ |
| }; close ($$0 ".$1"); |
| |
| # A command that, given a newline-separated list of test names on the |
| # standard input, print the name of the tests that are to be re-run |
| # upon "make recheck". |
| am.test-suite.list-tests-to-recheck = $(AWK) '{ \ |
| ## By default, we assume the test is to be re-run. |
| recheck = 1; \ |
| while ((rc = (getline line < ($$0 ".trs"))) != 0) \ |
| { \ |
| if (rc < 0) \ |
| { \ |
| ## |
| ## If we have encountered an I/O error here, there are three possibilities: |
| ## |
| ## [1] The '.log' file exists, but the '.trs' does not; in this case, |
| ## we "gracefully" recover by assuming the corresponding test is |
| ## to be re-run (which will re-create the missing '.trs' file). |
| ## |
| ## [2] Both the '.log' and '.trs' files are missing; this means that |
| ## the corresponding test has not been run, and is thus *not* to |
| ## be re-run. |
| ## |
| ## [3] We have encountered some corner-case problem (e.g., a '.log' or |
| ## '.trs' files somehow made unreadable, or issues with a bad NFS |
| ## connection, or whatever); we do not handle such corner cases. |
| ## |
| if ((getline line2 < ($$0 ".log")) < 0) \ |
| recheck = 0; \ |
| break; \ |
| } \ |
| else if (line ~ /$(am.test-suite.rx.recheck)[nN][Oo]/) \ |
| ## A directive explicitly specifying the test is *not* to be re-run. |
| { \ |
| recheck = 0; \ |
| break; \ |
| } \ |
| else if (line ~ /$(am.test-suite.rx.recheck)[yY][eE][sS]/) \ |
| { \ |
| ## A directive explicitly specifying the test *is* to be re-run. |
| break; \ |
| } \ |
| ## else continue with the next iteration. |
| }; \ |
| if (recheck) \ |
| print $$0; \ |
| ## Never leak file descriptors. |
| close ($$0 ".trs"); close ($$0 ".log"); \ |
| }' |
| |
| # A command that, given a newline-separated list of test names on the |
| # standard input, output a shell code snippet setting variables that |
| # count occurrences of each test result (PASS, FAIL, etc) declared in |
| # the '.trs' files of that given tests. For example, the count of |
| # PASSes will be saved in the '$am_PASS' variable, the count of SKIPs |
| # in the '$am_SKIP' variable, and so on. |
| am.test-suite.count-results = $(AWK) ' \ |
| $(am.test-suite.awk-functions) \ |
| BEGIN { exit_status = 0; } \ |
| { \ |
| $(call am.test-suite.awk-io-loop/BEGIN,trs) \ |
| if (line ~ /$(am.test-suite.rx.result)/) \ |
| { \ |
| sub("$(am.test-suite.rx.result)", "", line); \ |
| sub("[: ].*$$", "", line); \ |
| counts[line]++;\ |
| } \ |
| $(call am.test-suite.awk-io-loop/END,trs) \ |
| } \ |
| END { \ |
| if (exit_status != 0) \ |
| error("fatal: making $@: I/O error reading test results"); \ |
| else \ |
| { \ |
| global_count = 0; \ |
| for (k in counts) \ |
| { \ |
| print "am_" k "=" counts[k]; \ |
| global_count += counts[k]; \ |
| } \ |
| } \ |
| print "am_ALL=" global_count; \ |
| exit(exit_status); \ |
| }' |
| |
| # A command that, given a newline-separated list of test names on the |
| # standard input, create the global log from their .trs and .log files. |
| am.test-suite.create-global-log = $(AWK) ' \ |
| $(am.test-suite.awk-functions) \ |
| function rst_section(header) \ |
| { \ |
| print header; \ |
| len = length(header); \ |
| for (i = 1; i <= len; i = i + 1) \ |
| printf "="; \ |
| printf "\n\n"; \ |
| } \ |
| BEGIN { exit_status = 0; } \ |
| { \ |
| ## By default, we assume the test log is to be copied in the global log, |
| ## and that its result is simply "RUN" (i.e., we still do not know what |
| ## it outcome was, but we know that at least it has run). |
| copy_in_global_log = 1; \ |
| global_test_result = "RUN"; \ |
| $(call am.test-suite.awk-io-loop/BEGIN,trs) \ |
| if (line ~ /$(am.test-suite.rx.global-result)/) \ |
| { \ |
| sub("$(am.test-suite.rx.global-result)", "", line); \ |
| sub("[ ]*$$", "", line); \ |
| global_test_result = line; \ |
| } \ |
| else if (line ~ /$(am.test-suite.rx.copy-in-global-log)[nN][oO]/) \ |
| copy_in_global_log = 0; \ |
| $(call am.test-suite.awk-io-loop/END,trs) \ |
| if (copy_in_global_log) \ |
| { \ |
| rst_section(global_test_result ": " $$0); \ |
| $(call am.test-suite.awk-io-loop/BEGIN,log) \ |
| print line; \ |
| $(call am.test-suite.awk-io-loop/END,log) \ |
| printf "\n"; \ |
| }; \ |
| } \ |
| END { \ |
| if (exit_status != 0) \ |
| error("fatal: making $@: I/O error reading test results"); \ |
| exit(exit_status); \ |
| }' |
| |
| # Restructured Text title. |
| am.test-suite.rst-title = { sed 's/.*/ & /;h;s/./=/g;p;x;s/ *$$//;p;g' && echo; } |
| |
| # These support runtime overriding of $(TESTS) and $(XFAIL_TESTS). |
| # The first one must be left overridable (hence the definition with '?=', |
| # because the 'recheck' target need to override it (and in a tricky way). |
| am.test-suite.test-bases ?= \ |
| $(call am.memoize,am.test-suite.test-bases,$(call am.test-suite.get-test-bases,$(TESTS))) |
| am.test-suite.xfail-test-bases = \ |
| $(call am.memoize,am.test-suite.xfail-test-bases,$(call am.test-suite.get-test-bases,$(XFAIL_TESTS))) |
| |
| # The $(strip) is to work around the GNU make 3.80 bug where trailing |
| # whitespace in "TESTS = foo.test $(empty)" causes $(TESTS_LOGS) to |
| # erroneously expand to "foo.log .log". |
| am.test-suite.test-results = \ |
| $(call am.memoize,am.test-suite.test-results,$(addsuffix .trs,$(strip $(am.test-suite.test-bases)))) |
| am.test-suite.test-logs = \ |
| $(call am.memoize,am.test-suite.test-logs,$(addsuffix .log,$(strip $(am.test-suite.test-bases)))) |
| |
| am.clean.mostly.f += $(am.test-suite.test-results) $(am.test-suite.test-logs) |
| |
| # $(TEST_LOGS) is a published interface. |
| TEST_LOGS = $(am.test-suite.test-logs) |
| |
| am.test-suite.workdir = $(am.dir)/test-harness |
| |
| am.test-suite.append-to-list-of-bases = \ |
| @lst='$1'; for x in $$lst; do echo $$x; done \ |
| >> $(am.test-suite.workdir)/bases$(am.chars.newline) |
| |
| define am.setup-test-harness-workdir |
| @rm -rf $(am.test-suite.workdir) |
| @$(MKDIR_P) $(am.test-suite.workdir) |
| @touch $(am.test-suite.workdir)/bases |
| $(call am.xargs-map,am.test-suite.append-to-list-of-bases, \ |
| $(am.test-suite.test-bases)) |
| @workdir='$(am.test-suite.workdir)' \ |
| && sed 's/$$/.log/' $$workdir/bases > $$workdir/logs \ |
| && sed 's/$$/.trs/' $$workdir/bases > $$workdir/trs |
| endef |
| |
| ifeq ($(call am.vars.is-undef,TEST_SUITE_LOG),yes) |
| TEST_SUITE_LOG = test-suite.log |
| endif |
| |
| $(TEST_SUITE_LOG): $(am.test-suite.test-logs) $(am.test-suite.test-results) |
| $(am.setup-test-harness-workdir) |
| @set +e; $(am.test-suite.tty-colors); \ |
| fatal () { echo "fatal: making $@: $$*" >&2; exit 1; }; \ |
| workdir='$(am.test-suite.workdir)'; \ |
| ## Prepare data for the test suite summary. These do not take into account |
| ## unreadable test results, but they'll be appropriately updated later if |
| ## needed. |
| am_PASS=0 am_FAIL=0 am_SKIP=0 am_XPASS=0 am_XFAIL=0 am_ERROR=0; \ |
| count_test_results_command=`\ |
| $(am.test-suite.count-results) <$$workdir/bases` \ |
| && eval "$$count_test_results_command" \ |
| || fatal "unknown error reading test results"; \ |
| ## Whether the testsuite was successful or not. |
| if test `expr $$am_FAIL + $$am_XPASS + $$am_ERROR` -eq 0; then \ |
| success=true; \ |
| else \ |
| success=false; \ |
| fi; \ |
| ## Make $br a line of exactly 76 '=' characters, that will be used to |
| ## enclose the testsuite summary report when displayed on the console. |
| br='==================='; br=$$br$$br$$br$$br; \ |
| ## When writing the test summary to the console, we want to color a line |
| ## reporting the count of some result *only* if at least one test |
| ## experienced such a result. This function is handy in this regard. |
| display_result_count () \ |
| { \ |
| if test x"$$1" = x"--maybe-color"; then \ |
| maybe_colorize=yes; \ |
| elif test x"$$1" = x"--no-color"; then \ |
| maybe_colorize=no; \ |
| else \ |
| echo "$@: invalid 'display_result_count' usage" >&2; \ |
| exit 4; \ |
| fi; \ |
| shift; \ |
| desc=$$1 count=$$2; \ |
| if test $$maybe_colorize = yes && test $$count -gt 0; then \ |
| color_start=$$3 color_end=$$std; \ |
| else \ |
| color_start= color_end=; \ |
| fi; \ |
| echo "$${color_start}# $$desc $$count$${color_end}"; \ |
| }; \ |
| ## A shell function that creates the testsuite summary. We need it |
| ## because we have to create *two* summaries, one for test-suite.log, |
| ## and a possibly-colorized one for console output. |
| create_testsuite_report () \ |
| { \ |
| opts=$$*; \ |
| display_result_count $$opts "TOTAL:" $$am_ALL "$$brg"; \ |
| display_result_count $$opts "PASS: " $$am_PASS "$$grn"; \ |
| display_result_count $$opts "SKIP: " $$am_SKIP "$$blu"; \ |
| display_result_count $$opts "XFAIL:" $$am_XFAIL "$$lgn"; \ |
| display_result_count $$opts "FAIL: " $$am_FAIL "$$red"; \ |
| display_result_count $$opts "XPASS:" $$am_XPASS "$$red"; \ |
| display_result_count $$opts "ERROR:" $$am_ERROR "$$mgn"; \ |
| }; \ |
| ## Write "global" testsuite log. |
| if { \ |
| st=0; \ |
| echo "$(PACKAGE_STRING): $(subdir)/$(TEST_SUITE_LOG)" | \ |
| $(am.test-suite.rst-title); \ |
| create_testsuite_report --no-color; \ |
| echo; \ |
| echo ".. contents:: :depth: 2"; \ |
| echo; \ |
| $(am.test-suite.create-global-log) <$$workdir/bases; \ |
| } >$(TEST_SUITE_LOG).tmp; then \ |
| mv -f $(TEST_SUITE_LOG).tmp $(TEST_SUITE_LOG); \ |
| else \ |
| ## The awk program in $(am.test-suite.create-global-log) should have already |
| ## emitted a proper error message about I/O error, no need to repeat it. |
| rm -f $(TEST_SUITE_LOG).tmp; exit 1; \ |
| fi; \ |
| ## Emit the test summary on the console. |
| if $$success; then \ |
| col="$$grn"; \ |
| else \ |
| col="$$red"; \ |
| test x"$$VERBOSE" = x || cat $(TEST_SUITE_LOG); \ |
| fi; \ |
| ## Multi line coloring is problematic with "less -R", so we really need |
| ## to color each line individually. |
| echo "$${col}$$br$${std}"; \ |
| echo "$${col}Testsuite summary for $(PACKAGE_STRING)$${std}"; \ |
| echo "$${col}$$br$${std}"; \ |
| ## This is expected to go to the console, so it might have to be colorized. |
| create_testsuite_report --maybe-color; \ |
| echo "$$col$$br$$std"; \ |
| if $$success; then :; else \ |
| echo "$${col}See $(subdir)/$(TEST_SUITE_LOG)$${std}"; \ |
| if test -n "$(PACKAGE_BUGREPORT)"; then \ |
| echo "$${col}Please report to $(PACKAGE_BUGREPORT)$${std}"; \ |
| fi; \ |
| echo "$$col$$br$$std"; \ |
| fi; \ |
| $$success || exit 1 |
| |
| am.clean.mostly.f += $(TEST_SUITE_LOG) |
| |
| ## ------------------------------------------ ## |
| ## Running all tests, or rechecking failures. ## |
| ## ------------------------------------------ ## |
| |
| check-TESTS: |
| ifneq ($(AM_LAZY_CHECK),yes) |
| @$(call am.clean-cmd.f, \ |
| $(am.test-suite.test-results) $(am.test-suite.test-logs)) |
| endif |
| ## We always have to remove TEST_SUITE_LOG, to ensure its rule is run |
| ## in any case even in lazy mode: otherwise, if no test needs rerunning, |
| ## or a prior run plus reruns all happen within the same timestamp (can |
| ## happen with a prior "make TESTS=<subset>"), then we get no log output. |
| ## OTOH, this means that, in the rule for '$(TEST_SUITE_LOG)', we |
| ## cannot use '$?' to compute the set of lazily rerun tests, lest |
| ## we rely on .PHONY to work portably. |
| @rm -f $(TEST_SUITE_LOG) |
| $(MAKE) $(TEST_SUITE_LOG) |
| .PHONY: check-TESTS |
| |
| # Recheck must depend on $(check_SCRIPTS), $(check_PROGRAMS), etc. |
| # It must also depend on the 'all' target. See automake bug#11252. |
| recheck: all $(am.test-suite.deps) |
| +$(am.setup-test-harness-workdir) |
| ## See comments above in the check-TESTS recipe for why remove |
| ## $(TEST_SUITE_LOG) here. |
| @test -z "$(TEST_SUITE_LOG)" || rm -f $(TEST_SUITE_LOG) |
| @bases=`$(am.test-suite.list-tests-to-recheck) \ |
| <$(am.test-suite.workdir)/bases` || exit 1; \ |
| ## Remove newlines and normalize whitespace. |
| bases=`echo $$bases`; \ |
| ## Re-run the relevant tests, without hitting command-line length limits. |
| echo am.test-suite.test-bases="$$bases" | \ |
| $(MAKE) -f- -f$(firstword $(MAKEFILE_LIST)) \ |
| $(TEST_SUITE_LOG) .am/doing-recheck=yes |
| .PHONY: recheck |
| |
| # One tricky requirement of the "recheck" target is that, in case (say) |
| # the test is a compiled program whose compilation fails, we must ensure |
| # that any '.log' and '.trs' file referring to such test are preserved, |
| # so that future "make recheck" invocations will still try to re-compile |
| # and re-run it (automake bug#11791). This indirection is aimed at |
| # satisfying such a requirement. |
| ifeq ($(.am/doing-recheck),yes) |
| $(am.test-suite.test-logs) $(am.test-suite.test-results): .am/nil |
| endif |
| |
| AM_RECURSIVE_TARGETS += check recheck |
