From f4365790daa1d1400c7f0fe73ac9a6d25f0c6d0a Mon Sep 17 00:00:00 2001 From: Justus Winter Date: Mon, 20 Mar 2017 10:30:08 +0100 Subject: [PATCH] tests: Make it possible to run all tests using our infrastructure. * Makefile.am (TESTS_ENVIRONMENT): New variable. (check-all): New phony target to run all tests. * tests/gpgme/gpgme-defs.scm (have-gpgme?): New function that tests whether the GPGME test suite is available instead of exiting the process. * tests/gpgscm/init.scm (export): New macro. * tests/gpgscm/tests.scm (run-tests): New function. (load-tests): Likewise. * tests/gpgme/run-tests.scm: Simplify and move the parsing of the list of tests to 'all-tests.scm'. * tests/gpgsm/run-tests.scm: Likewise. * tests/migrations/run-tests.scm: Likewise. * tests/openpgp/run-tests.scm: Likewise. * tests/gpgme/Makefile.am: To select the tests to run, use the variable 'TESTS'. This harmonizes the interface with the automake test suite. * tests/gpgsm/Makefile.am: Likewise. * tests/migrations/Makefile.am: Likewise. * tests/openpgp/Makefile.am: Likewise. * tests/openpgp/README: Likewise. * agent/all-tests.scm: New file. * common/all-tests.scm: Likewise. * g10/all-tests.scm: Likewise. * g13/all-tests.scm: Likewise. * tests/gpgme/all-tests.scm: Likewise. * tests/gpgsm/all-tests.scm: Likewise. * tests/migrations/all-tests.scm: Likewise. * tests/openpgp/all-tests.scm: Likewise. * tests/run-tests.scm: Likewise. -- This change allows us to run all tests in parallel and write one XML report capturing the results of every test. It also lays the foundation to parametrize test suites. Signed-off-by: Justus Winter --- Makefile.am | 14 ++++++ agent/Makefile.am | 2 +- agent/all-tests.scm | 35 ++++++++++++++ common/Makefile.am | 3 +- common/all-tests.scm | 45 ++++++++++++++++++ g10/Makefile.am | 3 +- g10/all-tests.scm | 35 ++++++++++++++ g13/Makefile.am | 2 +- g13/all-tests.scm | 35 ++++++++++++++ tests/Makefile.am | 3 +- tests/gpgme/Makefile.am | 4 +- tests/gpgme/all-tests.scm | 86 ++++++++++++++++++++++++++++++++++ tests/gpgme/gpgme-defs.scm | 19 +++++--- tests/gpgme/run-tests.scm | 68 +-------------------------- tests/gpgscm/init.scm | 5 ++ tests/gpgscm/tests.scm | 14 ++++++ tests/gpgsm/Makefile.am | 4 +- tests/gpgsm/all-tests.scm | 43 +++++++++++++++++ tests/gpgsm/run-tests.scm | 27 ++++++----- tests/migrations/Makefile.am | 5 +- tests/migrations/all-tests.scm | 35 ++++++++++++++ tests/migrations/run-tests.scm | 17 ++++--- tests/openpgp/Makefile.am | 4 +- tests/openpgp/README | 4 +- tests/openpgp/all-tests.scm | 43 +++++++++++++++++ tests/openpgp/run-tests.scm | 27 ++++++----- tests/run-tests.scm | 44 +++++++++++++++++ 27 files changed, 502 insertions(+), 124 deletions(-) create mode 100644 agent/all-tests.scm create mode 100644 common/all-tests.scm create mode 100644 g10/all-tests.scm create mode 100644 g13/all-tests.scm create mode 100644 tests/gpgme/all-tests.scm create mode 100644 tests/gpgsm/all-tests.scm create mode 100644 tests/migrations/all-tests.scm create mode 100644 tests/openpgp/all-tests.scm create mode 100644 tests/run-tests.scm diff --git a/Makefile.am b/Makefile.am index 7d023c1af..e1f9a2748 100644 --- a/Makefile.am +++ b/Makefile.am @@ -141,3 +141,17 @@ gen-ChangeLog: stowinstall: $(MAKE) $(AM_MAKEFLAGS) install prefix=/usr/local/stow/gnupg + +TESTS_ENVIRONMENT = \ + LC_ALL=C \ + EXEEXT=$(EXEEXT) \ + PATH=$(abs_top_builddir)/tests/gpgscm:$(PATH) \ + abs_top_srcdir=$(abs_top_srcdir) \ + objdir=$(abs_top_builddir) \ + GPGSCM_PATH=$(abs_top_srcdir)/tests/gpgscm + +.PHONY: check-all +check-all: + $(TESTS_ENVIRONMENT) \ + $(abs_top_builddir)/tests/gpgscm/gpgscm \ + $(abs_srcdir)/tests/run-tests.scm $(TESTFLAGS) $(TESTS) diff --git a/agent/Makefile.am b/agent/Makefile.am index 668de2a29..ce29462b2 100644 --- a/agent/Makefile.am +++ b/agent/Makefile.am @@ -25,7 +25,7 @@ libexec_PROGRAMS += gpg-preset-passphrase endif noinst_PROGRAMS = $(TESTS) -EXTRA_DIST = ChangeLog-2011 gpg-agent-w32info.rc +EXTRA_DIST = ChangeLog-2011 gpg-agent-w32info.rc all-tests.scm AM_CPPFLAGS = diff --git a/agent/all-tests.scm b/agent/all-tests.scm new file mode 100644 index 000000000..6449ebb97 --- /dev/null +++ b/agent/all-tests.scm @@ -0,0 +1,35 @@ +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; Parse the Makefile.am to find all tests. + + (load (with-path "makefile.scm")) + + (define (expander filename port key) + (parse-makefile port key)) + + (define (parse filename key) + (parse-makefile-expand filename expander key)) + + (map (lambda (name) + (test::binary #f + (path-join "agent" name) + (path-join (getenv "objdir") "agent" name))) + (parse-makefile-expand (in-srcdir "agent" "Makefile.am") + (lambda (filename port key) (parse-makefile port key)) + "TESTS"))) diff --git a/common/Makefile.am b/common/Makefile.am index 68b87104a..83d82ac1f 100644 --- a/common/Makefile.am +++ b/common/Makefile.am @@ -20,7 +20,8 @@ EXTRA_DIST = mkstrtable.awk exaudit.awk exstatus.awk ChangeLog-2011 \ audit-events.h status-codes.h ChangeLog.jnlib \ - ChangeLog-2011.include w32info-rc.h.in gnupg.ico + ChangeLog-2011.include w32info-rc.h.in gnupg.ico \ + all-tests.scm noinst_LIBRARIES = libcommon.a libcommonpth.a libgpgrl.a if !HAVE_W32CE_SYSTEM diff --git a/common/all-tests.scm b/common/all-tests.scm new file mode 100644 index 000000000..54f1153a5 --- /dev/null +++ b/common/all-tests.scm @@ -0,0 +1,45 @@ +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; XXX: Currently, the makefile parser does not understand this + ;; Makefile.am, so we hardcode the list of tests here. + (map (lambda (name) + (test::binary #f + (path-join "common" name) + (path-join (getenv "objdir") "common" name))) + (list "t-stringhelp" + "t-timestuff" + "t-convert" + "t-percent" + "t-gettime" + "t-sysutils" + "t-sexputil" + "t-session-env" + "t-openpgp-oid" + "t-ssh-utils" + "t-mapstrings" + "t-zb32" + "t-mbox-util" + "t-iobuf" + "t-strlist" + "t-name-value" + "t-ccparray" + "t-recsel" + "t-exechelp" + "t-exectool" + ))) diff --git a/g10/Makefile.am b/g10/Makefile.am index e6a173da8..330d6c51e 100644 --- a/g10/Makefile.am +++ b/g10/Makefile.am @@ -21,7 +21,8 @@ EXTRA_DIST = distsigkey.gpg \ ChangeLog-2011 gpg-w32info.rc \ gpg.w32-manifest.in test.c t-keydb-keyring.kbx \ - t-keydb-get-keyblock.gpg t-stutter-data.asc + t-keydb-get-keyblock.gpg t-stutter-data.asc \ + all-tests.scm AM_CPPFLAGS = diff --git a/g10/all-tests.scm b/g10/all-tests.scm new file mode 100644 index 000000000..982220b28 --- /dev/null +++ b/g10/all-tests.scm @@ -0,0 +1,35 @@ +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; Parse the Makefile.am to find all tests. + + (load (with-path "makefile.scm")) + + (define (expander filename port key) + (parse-makefile port key)) + + (define (parse filename key) + (parse-makefile-expand filename expander key)) + + (map (lambda (name) + (test::binary #f + (path-join "g10" name) + (path-join (getenv "objdir") "g10" name))) + (parse-makefile-expand (in-srcdir "g10" "Makefile.am") + (lambda (filename port key) (parse-makefile port key)) + "module_tests"))) diff --git a/g13/Makefile.am b/g13/Makefile.am index dfacc993b..1aac1b379 100644 --- a/g13/Makefile.am +++ b/g13/Makefile.am @@ -18,7 +18,7 @@ ## Process this file with automake to produce Makefile.in -EXTRA_DIST = ChangeLog-2011 +EXTRA_DIST = ChangeLog-2011 all-tests.scm bin_PROGRAMS = g13 sbin_PROGRAMS = g13-syshelp diff --git a/g13/all-tests.scm b/g13/all-tests.scm new file mode 100644 index 000000000..69b1f24d4 --- /dev/null +++ b/g13/all-tests.scm @@ -0,0 +1,35 @@ +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; Parse the Makefile.am to find all tests. + + (load (with-path "makefile.scm")) + + (define (expander filename port key) + (parse-makefile port key)) + + (define (parse filename key) + (parse-makefile-expand filename expander key)) + + (map (lambda (name) + (test::binary #f + (path-join "g13" name) + (path-join (getenv "objdir") "g13" name))) + (parse-makefile-expand (in-srcdir "g13" "Makefile.am") + (lambda (filename port key) (parse-makefile port key)) + "module_tests"))) diff --git a/tests/Makefile.am b/tests/Makefile.am index bb75c97be..b9be6aaa6 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -44,7 +44,8 @@ EXTRA_DIST = runtest inittests $(testscripts) ChangeLog-2011 \ samplekeys/32100C27173EF6E9C4E9A25D3D69F86D37A4F939.key \ samplekeys/cert_g10code_pete1.pem \ samplekeys/cert_g10code_test1.pem \ - samplekeys/cert_g10code_theo1.pem + samplekeys/cert_g10code_theo1.pem \ + run-tests.scm # We used to run $(testscripts) here but these asschk scripts are not # completely reliable in all environments and thus we better disable diff --git a/tests/gpgme/Makefile.am b/tests/gpgme/Makefile.am index 37485e741..f1c19eb16 100644 --- a/tests/gpgme/Makefile.am +++ b/tests/gpgme/Makefile.am @@ -46,9 +46,9 @@ check: xcheck .PHONY: xcheck xcheck: $(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \ - $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS) + $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS) -EXTRA_DIST = gpgme-defs.scm run-tests.scm setup.scm wrap.scm +EXTRA_DIST = gpgme-defs.scm run-tests.scm setup.scm wrap.scm all-tests.scm CLEANFILES = *.log report.xml diff --git a/tests/gpgme/all-tests.scm b/tests/gpgme/all-tests.scm new file mode 100644 index 000000000..f72f8af97 --- /dev/null +++ b/tests/gpgme/all-tests.scm @@ -0,0 +1,86 @@ +;; Copyright (C) 2016 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; Parse GPGME's makefiles to find all tests. + + (load (in-srcdir "tests" "gpgme" "gpgme-defs.scm")) + (load (with-path "makefile.scm")) + + (define (expander filename port key) + ;;(interactive-repl (current-environment)) + (cond + ((string=? key "tests_unix") + (if *win32* + (parse-makefile port key) ;; Use win32 definition. + (begin + (parse-makefile port key) ;; Skip win32 definition. + (parse-makefile port key)))) + (else + (parse-makefile port key)))) + + (define (parse filename key) + (parse-makefile-expand filename expander key)) + + (define setup-c + (make-environment-cache + (test::scm + #f + (path-join "tests" "gpgme" "setup.scm" "tests" "gpg") + (in-srcdir "tests" "gpgme" "setup.scm") + "--" "tests" "gpg"))) + (define setup-py + (make-environment-cache + (test::scm + #f + (path-join "tests" "gpgme" "setup.scm" "lang" "python" "tests") + (in-srcdir "tests" "gpgme" "setup.scm") + "--" "lang" "python" "tests"))) + + (define (compiled? name) + (not (or (string-suffix? name ".py") + (string-suffix? name ".test")))) + (define :path car) + (define :key cadr) + (define :setup caddr) + + (if (have-gpgme?) + (apply append + (map (lambda (cmpnts) + (define (find-test name) + (apply path-join + `(,(if (compiled? name) + gpgme-builddir + gpgme-srcdir) ,@(:path cmpnts) ,(qualify name)))) + (let ((makefile (apply path-join `(,gpgme-srcdir ,@(:path cmpnts) + "Makefile.am")))) + (map (lambda (name) + (apply test::scm + `(,(:setup cmpnts) + ,(apply path-join + `("tests" "gpgme" ,@(:path cmpnts) ,name)) + ,(in-srcdir "tests" "gpgme" "wrap.scm") + --executable + ,(find-test name) + -- ,@(:path cmpnts)))) + (parse makefile (:key cmpnts))))) + `((("tests" "gpg") "c_tests" ,setup-c) + ,@(if (run-python-tests?) + `((("lang" "python" "tests") "py_tests" ,setup-py)) + '()) + (("lang" "qt" "tests") "TESTS" ,setup-c)))) + '())) diff --git a/tests/gpgme/gpgme-defs.scm b/tests/gpgme/gpgme-defs.scm index be6b0f191..e24db254e 100644 --- a/tests/gpgme/gpgme-defs.scm +++ b/tests/gpgme/gpgme-defs.scm @@ -20,21 +20,26 @@ (load (in-srcdir "tests" "openpgp" "defs.scm")) (define gpgme-srcdir (getenv "XTEST_GPGME_SRCDIR")) -(when (string=? "" gpgme-srcdir) - (info - "SKIP: Environment variable 'XTEST_GPGME_SRCDIR' not set. Please" - "point it to a recent GPGME source tree to run the GPGME test suite.") - (exit 0)) (define (in-gpgme-srcdir . names) (canonical-path (apply path-join (cons gpgme-srcdir names)))) (define gpgme-builddir (getenv "XTEST_GPGME_BUILDDIR")) -(when (string=? "" gpgme-builddir) + +(define (have-gpgme?) + (cond + ((string=? "" gpgme-srcdir) + (info + "SKIP: Environment variable 'XTEST_GPGME_SRCDIR' not set. Please" + "point it to a recent GPGME source tree to run the GPGME test suite.") + #f) + ((string=? "" gpgme-builddir) (info "SKIP: Environment variable 'XTEST_GPGME_BUILDDIR' not set. Please" "point it to a recent GPGME build tree to run the GPGME test suite.") - (exit 0)) + #f) + (else + #t))) ;; Make sure that GPGME picks up our gpgconf. This makes GPGME use ;; and thus executes the tests with GnuPG components from the build diff --git a/tests/gpgme/run-tests.scm b/tests/gpgme/run-tests.scm index df5f5482b..e81c9e904 100644 --- a/tests/gpgme/run-tests.scm +++ b/tests/gpgme/run-tests.scm @@ -17,70 +17,4 @@ ;; You should have received a copy of the GNU General Public License ;; along with this program; if not, see . -(load (in-srcdir "tests" "gpgme" "gpgme-defs.scm")) - -(info "Running GPGME's test suite...") - -(define (gpgme-makefile-expand filename port key) - ;;(interactive-repl (current-environment)) - (cond - ((string=? key "tests_unix") - (if *win32* - (parse-makefile port key) ;; Use win32 definition. - (begin - (parse-makefile port key) ;; Skip win32 definition. - (parse-makefile port key)))) - (else - (parse-makefile port key)))) - -(define (all-tests filename key) - (parse-makefile-expand filename gpgme-makefile-expand key)) - -(let* ((runner (if (member "--parallel" *args*) - run-tests-parallel - run-tests-sequential)) - (setup-c (make-environment-cache - (test::scm - #f - (path-join "tests" "gpgme" "setup.scm" "tests" "gpg") - (in-srcdir "tests" "gpgme" "setup.scm") - "--" "tests" "gpg"))) - (setup-py (make-environment-cache - (test::scm - #f - (path-join "tests" "gpgme" "setup.scm" "lang" "python" "tests") - (in-srcdir "tests" "gpgme" "setup.scm") - "--" "lang" "python" "tests"))) - (tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))) - (runner - (apply - append - (map (lambda (cmpnts) - (define (compiled? name) - (not (or (string-suffix? name ".py") - (string-suffix? name ".test")))) - (define :path car) - (define :key cadr) - (define :setup caddr) - (define (find-test name) - (apply path-join - `(,(if (compiled? name) - gpgme-builddir - gpgme-srcdir) ,@(:path cmpnts) ,(qualify name)))) - (let ((makefile (apply path-join `(,gpgme-srcdir ,@(:path cmpnts) - "Makefile.am")))) - (map (lambda (name) - (apply test::scm - `(,(:setup cmpnts) - ,(apply path-join - `("tests" "gpgme" ,@(:path cmpnts) ,name)) - ,(in-srcdir "tests" "gpgme" "wrap.scm") - --executable - ,(find-test name) - -- ,@(:path cmpnts)))) - (if (null? tests) (all-tests makefile (:key cmpnts)) tests)))) - `((("tests" "gpg") "c_tests" ,setup-c) - ,@(if (run-python-tests?) - `((("lang" "python" "tests") "py_tests" ,setup-py)) - '()) - (("lang" "qt" "tests") "TESTS" ,setup-c)))))) +(run-tests (load-tests "tests" "gpgme")) diff --git a/tests/gpgscm/init.scm b/tests/gpgscm/init.scm index af386201e..3769ed00d 100644 --- a/tests/gpgscm/init.scm +++ b/tests/gpgscm/init.scm @@ -701,6 +701,11 @@ ,@(cdr form) (current-environment)))) +(define-macro (export name . expressions) + `(define ,name + (begin + ,@expressions))) + ;;;;; I/O (define (input-output-port? p) diff --git a/tests/gpgscm/tests.scm b/tests/gpgscm/tests.scm index 490f95a36..eee8ce56f 100644 --- a/tests/gpgscm/tests.scm +++ b/tests/gpgscm/tests.scm @@ -226,6 +226,7 @@ (define (dirname path) (let ((i (string-rindex path #\/))) (if i (substring path 0 i) "."))) +(assert (string=? "foo/bar" (dirname "foo/bar/baz"))) ;; Helper for (pipe). (define :read-end car) @@ -739,6 +740,19 @@ (loop (pool::add (test::run-sync)) (cdr tests')))))) +;; Run tests either in sequence or in parallel, depending on the +;; number of tests and the command line flags. +(define (run-tests tests) + (if (and (flag "--parallel" *args*) + (> (length tests) 1)) + (run-tests-parallel tests) + (run-tests-sequential tests))) + +;; Load all tests from the given path. +(define (load-tests . path) + (load (apply in-srcdir `(,@path "all-tests.scm"))) + all-tests) + ;; Helper to create environment caches from test functions. SETUP ;; must be a test implementing the producer side cache protocol. ;; Returns a promise containing the arguments that must be passed to a diff --git a/tests/gpgsm/Makefile.am b/tests/gpgsm/Makefile.am index 892d3bc8b..e54db7842 100644 --- a/tests/gpgsm/Makefile.am +++ b/tests/gpgsm/Makefile.am @@ -54,7 +54,7 @@ check: xcheck .PHONY: xcheck xcheck: $(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \ - $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS) + $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS) KEYS = 32100C27173EF6E9C4E9A25D3D69F86D37A4F939 CERTS = cert_g10code_test1.der \ @@ -66,7 +66,7 @@ TEST_FILES = plain-1.cms.asc \ plain-large.cms.asc EXTRA_DIST = $(XTESTS) $(KEYS) $(CERTS) $(TEST_FILES) \ - gpgsm-defs.scm run-tests.scm setup.scm + gpgsm-defs.scm run-tests.scm setup.scm all-tests.scm CLEANFILES = *.log report.xml diff --git a/tests/gpgsm/all-tests.scm b/tests/gpgsm/all-tests.scm new file mode 100644 index 000000000..1baa92468 --- /dev/null +++ b/tests/gpgsm/all-tests.scm @@ -0,0 +1,43 @@ +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; Parse the Makefile.am to find all tests. + + (load (with-path "makefile.scm")) + + (define (expander filename port key) + (parse-makefile port key)) + + (define (parse filename key) + (parse-makefile-expand filename expander key)) + + (define setup + (make-environment-cache + (test::scm + #f + (path-join "tests" "gpgsm" "setup.scm") + (in-srcdir "tests" "gpgsm" "setup.scm") + "--" "tests" "gpg"))) + + (map (lambda (name) + (test::scm setup + (path-join "tests" "gpgsm" name) + (in-srcdir "tests" "gpgsm" name))) + (parse-makefile-expand (in-srcdir "tests" "gpgsm" "Makefile.am") + (lambda (filename port key) (parse-makefile port key)) + "XTESTS"))) diff --git a/tests/gpgsm/run-tests.scm b/tests/gpgsm/run-tests.scm index bf129a1cb..6b460b165 100644 --- a/tests/gpgsm/run-tests.scm +++ b/tests/gpgsm/run-tests.scm @@ -23,16 +23,17 @@ "tests/gpgsm.") (exit 2))) -(let* ((tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*)) - (setup (make-environment-cache (test::scm - #f - (path-join "tests" "gpgsm" "setup.scm") - (in-srcdir "tests" "gpgsm" "setup.scm")))) - (runner (if (and (member "--parallel" *args*) - (> (length tests) 1)) - run-tests-parallel - run-tests-sequential))) - (runner (map (lambda (name) - (test::scm setup - (path-join "tests" "gpgsm" name) - (in-srcdir "tests" "gpgsm" name))) tests))) +(define tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*)) + +(define setup + (make-environment-cache (test::scm + #f + (path-join "tests" "gpgsm" "setup.scm") + (in-srcdir "tests" "gpgsm" "setup.scm")))) + +(run-tests (if (null? tests) + (load-tests "tests" "gpgsm") + (map (lambda (name) + (test::scm setup + (path-join "tests" "gpgsm" name) + (in-srcdir "tests" "gpgsm" name))) tests))) diff --git a/tests/migrations/Makefile.am b/tests/migrations/Makefile.am index 398b15c80..d90c9c7f9 100644 --- a/tests/migrations/Makefile.am +++ b/tests/migrations/Makefile.am @@ -54,9 +54,10 @@ check: xcheck .PHONY: xcheck xcheck: $(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \ - $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS) + $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS) -EXTRA_DIST = common.scm run-tests.scm setup.scm $(XTESTS) $(TEST_FILES) +EXTRA_DIST = common.scm run-tests.scm setup.scm all-tests.scm \ + $(XTESTS) $(TEST_FILES) CLEANFILES = *.log report.xml diff --git a/tests/migrations/all-tests.scm b/tests/migrations/all-tests.scm new file mode 100644 index 000000000..421f69679 --- /dev/null +++ b/tests/migrations/all-tests.scm @@ -0,0 +1,35 @@ +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; Parse the Makefile.am to find all tests. + + (load (with-path "makefile.scm")) + + (define (expander filename port key) + (parse-makefile port key)) + + (define (parse filename key) + (parse-makefile-expand filename expander key)) + + (map (lambda (name) + (test::scm #f + (path-join "tests" "migrations" name) + (in-srcdir "tests" "migrations" name))) + (parse-makefile-expand (in-srcdir "tests" "migrations" "Makefile.am") + (lambda (filename port key) (parse-makefile port key)) + "XTESTS"))) diff --git a/tests/migrations/run-tests.scm b/tests/migrations/run-tests.scm index 1e4bb704b..f44334c7d 100644 --- a/tests/migrations/run-tests.scm +++ b/tests/migrations/run-tests.scm @@ -17,12 +17,11 @@ ;; You should have received a copy of the GNU General Public License ;; along with this program; if not, see . -(let* ((tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*)) - (runner (if (and (member "--parallel" *args*) - (> (length tests) 1)) - run-tests-parallel - run-tests-sequential))) - (runner (map (lambda (name) - (test::scm #f - (path-join "tests" "migrations" name) - (in-srcdir "tests" "migrations" name))) tests))) +(define tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*)) + +(run-tests (if (null? tests) + (load-tests "tests" "migrations") + (map (lambda (name) + (test::scm #f + (path-join "tests" "migrations" name) + (in-srcdir "tests" "migrations" name))) tests))) diff --git a/tests/openpgp/Makefile.am b/tests/openpgp/Makefile.am index d99c3d9c5..bf9673f74 100644 --- a/tests/openpgp/Makefile.am +++ b/tests/openpgp/Makefile.am @@ -109,7 +109,7 @@ check: xcheck .PHONY: xcheck xcheck: $(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \ - $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS) + $(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS) TEST_FILES = pubring.asc secring.asc plain-1o.asc plain-2o.asc plain-3o.asc \ plain-1.asc plain-2.asc plain-3.asc plain-1-pgp.asc \ @@ -250,7 +250,7 @@ sample_msgs = samplemsgs/clearsig-1-key-1.asc \ EXTRA_DIST = defs.scm $(XTESTS) $(TEST_FILES) \ mkdemodirs signdemokey $(priv_keys) $(sample_keys) \ $(sample_msgs) ChangeLog-2011 run-tests.scm \ - setup.scm shell.scm + setup.scm shell.scm all-tests.scm CLEANFILES = prepared.stamp x y yy z out err $(data_files) \ plain-1 plain-2 plain-3 trustdb.gpg *.lock .\#lk* \ diff --git a/tests/openpgp/README b/tests/openpgp/README index b9d560797..42e78ae2d 100644 --- a/tests/openpgp/README +++ b/tests/openpgp/README @@ -7,7 +7,7 @@ From your build directory, run to run all tests or - obj $ make -C tests/openpgp check XTESTS=your-test.scm + obj $ make -C tests/openpgp check TESTS=your-test.scm to run a specific test (or any number of tests separated by spaces). @@ -89,7 +89,7 @@ the inner progress indicator will be abbreviated using '.'. Say you are working on a new test called 'your-test.scm', you can run it on its own using - obj $ make -C tests/openpgp check XTESTS=your-test.scm + obj $ make -C tests/openpgp check TESTS=your-test.scm but something isn't working as expected. There are several little gadgets that might help. The first one is 'trace', a function that diff --git a/tests/openpgp/all-tests.scm b/tests/openpgp/all-tests.scm new file mode 100644 index 000000000..4b14c4e04 --- /dev/null +++ b/tests/openpgp/all-tests.scm @@ -0,0 +1,43 @@ +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(export all-tests + ;; Parse the Makefile.am to find all tests. + + (load (with-path "makefile.scm")) + + (define (expander filename port key) + (parse-makefile port key)) + + (define (parse filename key) + (parse-makefile-expand filename expander key)) + + (define setup + (make-environment-cache + (test::scm + #f + (path-join "tests" "openpgp" "setup.scm") + (in-srcdir "tests" "openpgp" "setup.scm") + "--" "tests" "gpg"))) + + (map (lambda (name) + (test::scm setup + (path-join "tests" "openpgp" name) + (in-srcdir "tests" "openpgp" name))) + (parse-makefile-expand (in-srcdir "tests" "openpgp" "Makefile.am") + (lambda (filename port key) (parse-makefile port key)) + "XTESTS"))) diff --git a/tests/openpgp/run-tests.scm b/tests/openpgp/run-tests.scm index d443d1fb8..d4914bda7 100644 --- a/tests/openpgp/run-tests.scm +++ b/tests/openpgp/run-tests.scm @@ -26,16 +26,17 @@ ;; Set objdir so that the tests can locate built programs. (setenv "objdir" (getcwd) #f) -(let* ((tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*)) - (setup (make-environment-cache (test::scm - #f - (path-join "tests" "openpgp" "setup.scm") - (in-srcdir "tests" "openpgp" "setup.scm")))) - (runner (if (and (member "--parallel" *args*) - (> (length tests) 1)) - run-tests-parallel - run-tests-sequential))) - (runner (map (lambda (name) - (test::scm setup - (path-join "tests" "openpgp" name) - (in-srcdir "tests" "openpgp" name))) tests))) +(define setup + (make-environment-cache (test::scm + #f + (path-join "tests" "openpgp" "setup.scm") + (in-srcdir "tests" "openpgp" "setup.scm")))) + +(define tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*)) + +(run-tests (if (null? tests) + (load-tests "tests" "openpgp") + (map (lambda (name) + (test::scm setup + (path-join "tests" "openpgp" name) + (in-srcdir "tests" "openpgp" name))) tests))) diff --git a/tests/run-tests.scm b/tests/run-tests.scm new file mode 100644 index 000000000..d3ebba07e --- /dev/null +++ b/tests/run-tests.scm @@ -0,0 +1,44 @@ +#!/usr/bin/env gpgscm + +;; Copyright (C) 2017 g10 Code GmbH +;; +;; This file is part of GnuPG. +;; +;; GnuPG is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3 of the License, or +;; (at your option) any later version. +;; +;; GnuPG is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, see . + +(info "Running all tests...") + +(define (load-tests-with-log . path) + (map (lambda (test) + (test:::set! 'log-file-name + (apply path-join `(,@path + ,(string-append (basename test::name) + ".log"))))) + (apply load-tests path))) + +(let ((prefix (flag "--prefix" *args*)) + (all-tests (append + (load-tests-with-log "common") + (load-tests-with-log "g10") + (load-tests-with-log "g13") + (load-tests-with-log "agent") + (load-tests-with-log "tests" "openpgp") + (load-tests-with-log "tests" "migrations") + (load-tests-with-log "tests" "gpgsm") + (load-tests-with-log "tests" "gpgme")))) + (run-tests (if prefix + (filter + (lambda (t) (string-prefix? t::name (apply path-join prefix))) + all-tests) + all-tests)))