tests: Make it possible to run all tests using our infrastructure.

* Makefile.am (TESTS_ENVIRONMENT): New variable.
(check-all): New phony target to run all tests.
* tests/gpgme/gpgme-defs.scm (have-gpgme?): New function that tests
whether the GPGME test suite is available instead of exiting the
process.
* tests/gpgscm/init.scm (export): New macro.
* tests/gpgscm/tests.scm (run-tests): New function.
(load-tests): Likewise.
* tests/gpgme/run-tests.scm: Simplify and move the parsing of the list
of tests to 'all-tests.scm'.
* tests/gpgsm/run-tests.scm: Likewise.
* tests/migrations/run-tests.scm: Likewise.
* tests/openpgp/run-tests.scm: Likewise.
* tests/gpgme/Makefile.am: To select the tests to run, use the
variable 'TESTS'.  This harmonizes the interface with the automake
test suite.
* tests/gpgsm/Makefile.am: Likewise.
* tests/migrations/Makefile.am: Likewise.
* tests/openpgp/Makefile.am: Likewise.
* tests/openpgp/README: Likewise.
* agent/all-tests.scm: New file.
* common/all-tests.scm: Likewise.
* g10/all-tests.scm: Likewise.
* g13/all-tests.scm: Likewise.
* tests/gpgme/all-tests.scm: Likewise.
* tests/gpgsm/all-tests.scm: Likewise.
* tests/migrations/all-tests.scm: Likewise.
* tests/openpgp/all-tests.scm: Likewise.
* tests/run-tests.scm: Likewise.
--

This change allows us to run all tests in parallel and write one XML
report capturing the results of every test.  It also lays the
foundation to parametrize test suites.

Signed-off-by: Justus Winter <justus@g10code.com>
This commit is contained in:
Justus Winter 2017-03-20 10:30:08 +01:00
parent 78d6a25a2d
commit f4365790da
No known key found for this signature in database
GPG Key ID: DD1A52F9DA8C9020
27 changed files with 502 additions and 124 deletions

View File

@ -141,3 +141,17 @@ gen-ChangeLog:
stowinstall:
$(MAKE) $(AM_MAKEFLAGS) install prefix=/usr/local/stow/gnupg
TESTS_ENVIRONMENT = \
LC_ALL=C \
EXEEXT=$(EXEEXT) \
PATH=$(abs_top_builddir)/tests/gpgscm:$(PATH) \
abs_top_srcdir=$(abs_top_srcdir) \
objdir=$(abs_top_builddir) \
GPGSCM_PATH=$(abs_top_srcdir)/tests/gpgscm
.PHONY: check-all
check-all:
$(TESTS_ENVIRONMENT) \
$(abs_top_builddir)/tests/gpgscm/gpgscm \
$(abs_srcdir)/tests/run-tests.scm $(TESTFLAGS) $(TESTS)

View File

@ -25,7 +25,7 @@ libexec_PROGRAMS += gpg-preset-passphrase
endif
noinst_PROGRAMS = $(TESTS)
EXTRA_DIST = ChangeLog-2011 gpg-agent-w32info.rc
EXTRA_DIST = ChangeLog-2011 gpg-agent-w32info.rc all-tests.scm
AM_CPPFLAGS =

35
agent/all-tests.scm Normal file
View File

@ -0,0 +1,35 @@
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; Parse the Makefile.am to find all tests.
(load (with-path "makefile.scm"))
(define (expander filename port key)
(parse-makefile port key))
(define (parse filename key)
(parse-makefile-expand filename expander key))
(map (lambda (name)
(test::binary #f
(path-join "agent" name)
(path-join (getenv "objdir") "agent" name)))
(parse-makefile-expand (in-srcdir "agent" "Makefile.am")
(lambda (filename port key) (parse-makefile port key))
"TESTS")))

View File

@ -20,7 +20,8 @@
EXTRA_DIST = mkstrtable.awk exaudit.awk exstatus.awk ChangeLog-2011 \
audit-events.h status-codes.h ChangeLog.jnlib \
ChangeLog-2011.include w32info-rc.h.in gnupg.ico
ChangeLog-2011.include w32info-rc.h.in gnupg.ico \
all-tests.scm
noinst_LIBRARIES = libcommon.a libcommonpth.a libgpgrl.a
if !HAVE_W32CE_SYSTEM

45
common/all-tests.scm Normal file
View File

@ -0,0 +1,45 @@
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; XXX: Currently, the makefile parser does not understand this
;; Makefile.am, so we hardcode the list of tests here.
(map (lambda (name)
(test::binary #f
(path-join "common" name)
(path-join (getenv "objdir") "common" name)))
(list "t-stringhelp"
"t-timestuff"
"t-convert"
"t-percent"
"t-gettime"
"t-sysutils"
"t-sexputil"
"t-session-env"
"t-openpgp-oid"
"t-ssh-utils"
"t-mapstrings"
"t-zb32"
"t-mbox-util"
"t-iobuf"
"t-strlist"
"t-name-value"
"t-ccparray"
"t-recsel"
"t-exechelp"
"t-exectool"
)))

View File

@ -21,7 +21,8 @@
EXTRA_DIST = distsigkey.gpg \
ChangeLog-2011 gpg-w32info.rc \
gpg.w32-manifest.in test.c t-keydb-keyring.kbx \
t-keydb-get-keyblock.gpg t-stutter-data.asc
t-keydb-get-keyblock.gpg t-stutter-data.asc \
all-tests.scm
AM_CPPFLAGS =

35
g10/all-tests.scm Normal file
View File

@ -0,0 +1,35 @@
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; Parse the Makefile.am to find all tests.
(load (with-path "makefile.scm"))
(define (expander filename port key)
(parse-makefile port key))
(define (parse filename key)
(parse-makefile-expand filename expander key))
(map (lambda (name)
(test::binary #f
(path-join "g10" name)
(path-join (getenv "objdir") "g10" name)))
(parse-makefile-expand (in-srcdir "g10" "Makefile.am")
(lambda (filename port key) (parse-makefile port key))
"module_tests")))

View File

@ -18,7 +18,7 @@
## Process this file with automake to produce Makefile.in
EXTRA_DIST = ChangeLog-2011
EXTRA_DIST = ChangeLog-2011 all-tests.scm
bin_PROGRAMS = g13
sbin_PROGRAMS = g13-syshelp

35
g13/all-tests.scm Normal file
View File

@ -0,0 +1,35 @@
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; Parse the Makefile.am to find all tests.
(load (with-path "makefile.scm"))
(define (expander filename port key)
(parse-makefile port key))
(define (parse filename key)
(parse-makefile-expand filename expander key))
(map (lambda (name)
(test::binary #f
(path-join "g13" name)
(path-join (getenv "objdir") "g13" name)))
(parse-makefile-expand (in-srcdir "g13" "Makefile.am")
(lambda (filename port key) (parse-makefile port key))
"module_tests")))

View File

@ -44,7 +44,8 @@ EXTRA_DIST = runtest inittests $(testscripts) ChangeLog-2011 \
samplekeys/32100C27173EF6E9C4E9A25D3D69F86D37A4F939.key \
samplekeys/cert_g10code_pete1.pem \
samplekeys/cert_g10code_test1.pem \
samplekeys/cert_g10code_theo1.pem
samplekeys/cert_g10code_theo1.pem \
run-tests.scm
# We used to run $(testscripts) here but these asschk scripts are not
# completely reliable in all environments and thus we better disable

View File

@ -46,9 +46,9 @@ check: xcheck
.PHONY: xcheck
xcheck:
$(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS)
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS)
EXTRA_DIST = gpgme-defs.scm run-tests.scm setup.scm wrap.scm
EXTRA_DIST = gpgme-defs.scm run-tests.scm setup.scm wrap.scm all-tests.scm
CLEANFILES = *.log report.xml

86
tests/gpgme/all-tests.scm Normal file
View File

@ -0,0 +1,86 @@
;; Copyright (C) 2016 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; Parse GPGME's makefiles to find all tests.
(load (in-srcdir "tests" "gpgme" "gpgme-defs.scm"))
(load (with-path "makefile.scm"))
(define (expander filename port key)
;;(interactive-repl (current-environment))
(cond
((string=? key "tests_unix")
(if *win32*
(parse-makefile port key) ;; Use win32 definition.
(begin
(parse-makefile port key) ;; Skip win32 definition.
(parse-makefile port key))))
(else
(parse-makefile port key))))
(define (parse filename key)
(parse-makefile-expand filename expander key))
(define setup-c
(make-environment-cache
(test::scm
#f
(path-join "tests" "gpgme" "setup.scm" "tests" "gpg")
(in-srcdir "tests" "gpgme" "setup.scm")
"--" "tests" "gpg")))
(define setup-py
(make-environment-cache
(test::scm
#f
(path-join "tests" "gpgme" "setup.scm" "lang" "python" "tests")
(in-srcdir "tests" "gpgme" "setup.scm")
"--" "lang" "python" "tests")))
(define (compiled? name)
(not (or (string-suffix? name ".py")
(string-suffix? name ".test"))))
(define :path car)
(define :key cadr)
(define :setup caddr)
(if (have-gpgme?)
(apply append
(map (lambda (cmpnts)
(define (find-test name)
(apply path-join
`(,(if (compiled? name)
gpgme-builddir
gpgme-srcdir) ,@(:path cmpnts) ,(qualify name))))
(let ((makefile (apply path-join `(,gpgme-srcdir ,@(:path cmpnts)
"Makefile.am"))))
(map (lambda (name)
(apply test::scm
`(,(:setup cmpnts)
,(apply path-join
`("tests" "gpgme" ,@(:path cmpnts) ,name))
,(in-srcdir "tests" "gpgme" "wrap.scm")
--executable
,(find-test name)
-- ,@(:path cmpnts))))
(parse makefile (:key cmpnts)))))
`((("tests" "gpg") "c_tests" ,setup-c)
,@(if (run-python-tests?)
`((("lang" "python" "tests") "py_tests" ,setup-py))
'())
(("lang" "qt" "tests") "TESTS" ,setup-c))))
'()))

View File

@ -20,21 +20,26 @@
(load (in-srcdir "tests" "openpgp" "defs.scm"))
(define gpgme-srcdir (getenv "XTEST_GPGME_SRCDIR"))
(when (string=? "" gpgme-srcdir)
(info
"SKIP: Environment variable 'XTEST_GPGME_SRCDIR' not set. Please"
"point it to a recent GPGME source tree to run the GPGME test suite.")
(exit 0))
(define (in-gpgme-srcdir . names)
(canonical-path (apply path-join (cons gpgme-srcdir names))))
(define gpgme-builddir (getenv "XTEST_GPGME_BUILDDIR"))
(when (string=? "" gpgme-builddir)
(define (have-gpgme?)
(cond
((string=? "" gpgme-srcdir)
(info
"SKIP: Environment variable 'XTEST_GPGME_SRCDIR' not set. Please"
"point it to a recent GPGME source tree to run the GPGME test suite.")
#f)
((string=? "" gpgme-builddir)
(info
"SKIP: Environment variable 'XTEST_GPGME_BUILDDIR' not set. Please"
"point it to a recent GPGME build tree to run the GPGME test suite.")
(exit 0))
#f)
(else
#t)))
;; Make sure that GPGME picks up our gpgconf. This makes GPGME use
;; and thus executes the tests with GnuPG components from the build

View File

@ -17,70 +17,4 @@
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(load (in-srcdir "tests" "gpgme" "gpgme-defs.scm"))
(info "Running GPGME's test suite...")
(define (gpgme-makefile-expand filename port key)
;;(interactive-repl (current-environment))
(cond
((string=? key "tests_unix")
(if *win32*
(parse-makefile port key) ;; Use win32 definition.
(begin
(parse-makefile port key) ;; Skip win32 definition.
(parse-makefile port key))))
(else
(parse-makefile port key))))
(define (all-tests filename key)
(parse-makefile-expand filename gpgme-makefile-expand key))
(let* ((runner (if (member "--parallel" *args*)
run-tests-parallel
run-tests-sequential))
(setup-c (make-environment-cache
(test::scm
#f
(path-join "tests" "gpgme" "setup.scm" "tests" "gpg")
(in-srcdir "tests" "gpgme" "setup.scm")
"--" "tests" "gpg")))
(setup-py (make-environment-cache
(test::scm
#f
(path-join "tests" "gpgme" "setup.scm" "lang" "python" "tests")
(in-srcdir "tests" "gpgme" "setup.scm")
"--" "lang" "python" "tests")))
(tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*)))
(runner
(apply
append
(map (lambda (cmpnts)
(define (compiled? name)
(not (or (string-suffix? name ".py")
(string-suffix? name ".test"))))
(define :path car)
(define :key cadr)
(define :setup caddr)
(define (find-test name)
(apply path-join
`(,(if (compiled? name)
gpgme-builddir
gpgme-srcdir) ,@(:path cmpnts) ,(qualify name))))
(let ((makefile (apply path-join `(,gpgme-srcdir ,@(:path cmpnts)
"Makefile.am"))))
(map (lambda (name)
(apply test::scm
`(,(:setup cmpnts)
,(apply path-join
`("tests" "gpgme" ,@(:path cmpnts) ,name))
,(in-srcdir "tests" "gpgme" "wrap.scm")
--executable
,(find-test name)
-- ,@(:path cmpnts))))
(if (null? tests) (all-tests makefile (:key cmpnts)) tests))))
`((("tests" "gpg") "c_tests" ,setup-c)
,@(if (run-python-tests?)
`((("lang" "python" "tests") "py_tests" ,setup-py))
'())
(("lang" "qt" "tests") "TESTS" ,setup-c))))))
(run-tests (load-tests "tests" "gpgme"))

View File

@ -701,6 +701,11 @@
,@(cdr form)
(current-environment))))
(define-macro (export name . expressions)
`(define ,name
(begin
,@expressions)))
;;;;; I/O
(define (input-output-port? p)

View File

@ -226,6 +226,7 @@
(define (dirname path)
(let ((i (string-rindex path #\/)))
(if i (substring path 0 i) ".")))
(assert (string=? "foo/bar" (dirname "foo/bar/baz")))
;; Helper for (pipe).
(define :read-end car)
@ -739,6 +740,19 @@
(loop (pool::add (test::run-sync))
(cdr tests'))))))
;; Run tests either in sequence or in parallel, depending on the
;; number of tests and the command line flags.
(define (run-tests tests)
(if (and (flag "--parallel" *args*)
(> (length tests) 1))
(run-tests-parallel tests)
(run-tests-sequential tests)))
;; Load all tests from the given path.
(define (load-tests . path)
(load (apply in-srcdir `(,@path "all-tests.scm")))
all-tests)
;; Helper to create environment caches from test functions. SETUP
;; must be a test implementing the producer side cache protocol.
;; Returns a promise containing the arguments that must be passed to a

View File

@ -54,7 +54,7 @@ check: xcheck
.PHONY: xcheck
xcheck:
$(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS)
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS)
KEYS = 32100C27173EF6E9C4E9A25D3D69F86D37A4F939
CERTS = cert_g10code_test1.der \
@ -66,7 +66,7 @@ TEST_FILES = plain-1.cms.asc \
plain-large.cms.asc
EXTRA_DIST = $(XTESTS) $(KEYS) $(CERTS) $(TEST_FILES) \
gpgsm-defs.scm run-tests.scm setup.scm
gpgsm-defs.scm run-tests.scm setup.scm all-tests.scm
CLEANFILES = *.log report.xml

43
tests/gpgsm/all-tests.scm Normal file
View File

@ -0,0 +1,43 @@
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; Parse the Makefile.am to find all tests.
(load (with-path "makefile.scm"))
(define (expander filename port key)
(parse-makefile port key))
(define (parse filename key)
(parse-makefile-expand filename expander key))
(define setup
(make-environment-cache
(test::scm
#f
(path-join "tests" "gpgsm" "setup.scm")
(in-srcdir "tests" "gpgsm" "setup.scm")
"--" "tests" "gpg")))
(map (lambda (name)
(test::scm setup
(path-join "tests" "gpgsm" name)
(in-srcdir "tests" "gpgsm" name)))
(parse-makefile-expand (in-srcdir "tests" "gpgsm" "Makefile.am")
(lambda (filename port key) (parse-makefile port key))
"XTESTS")))

View File

@ -23,16 +23,17 @@
"tests/gpgsm.")
(exit 2)))
(let* ((tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
(setup (make-environment-cache (test::scm
#f
(path-join "tests" "gpgsm" "setup.scm")
(in-srcdir "tests" "gpgsm" "setup.scm"))))
(runner (if (and (member "--parallel" *args*)
(> (length tests) 1))
run-tests-parallel
run-tests-sequential)))
(runner (map (lambda (name)
(test::scm setup
(path-join "tests" "gpgsm" name)
(in-srcdir "tests" "gpgsm" name))) tests)))
(define tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
(define setup
(make-environment-cache (test::scm
#f
(path-join "tests" "gpgsm" "setup.scm")
(in-srcdir "tests" "gpgsm" "setup.scm"))))
(run-tests (if (null? tests)
(load-tests "tests" "gpgsm")
(map (lambda (name)
(test::scm setup
(path-join "tests" "gpgsm" name)
(in-srcdir "tests" "gpgsm" name))) tests)))

View File

@ -54,9 +54,10 @@ check: xcheck
.PHONY: xcheck
xcheck:
$(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS)
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS)
EXTRA_DIST = common.scm run-tests.scm setup.scm $(XTESTS) $(TEST_FILES)
EXTRA_DIST = common.scm run-tests.scm setup.scm all-tests.scm \
$(XTESTS) $(TEST_FILES)
CLEANFILES = *.log report.xml

View File

@ -0,0 +1,35 @@
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; Parse the Makefile.am to find all tests.
(load (with-path "makefile.scm"))
(define (expander filename port key)
(parse-makefile port key))
(define (parse filename key)
(parse-makefile-expand filename expander key))
(map (lambda (name)
(test::scm #f
(path-join "tests" "migrations" name)
(in-srcdir "tests" "migrations" name)))
(parse-makefile-expand (in-srcdir "tests" "migrations" "Makefile.am")
(lambda (filename port key) (parse-makefile port key))
"XTESTS")))

View File

@ -17,12 +17,11 @@
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(let* ((tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
(runner (if (and (member "--parallel" *args*)
(> (length tests) 1))
run-tests-parallel
run-tests-sequential)))
(runner (map (lambda (name)
(test::scm #f
(path-join "tests" "migrations" name)
(in-srcdir "tests" "migrations" name))) tests)))
(define tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
(run-tests (if (null? tests)
(load-tests "tests" "migrations")
(map (lambda (name)
(test::scm #f
(path-join "tests" "migrations" name)
(in-srcdir "tests" "migrations" name))) tests)))

View File

@ -109,7 +109,7 @@ check: xcheck
.PHONY: xcheck
xcheck:
$(TESTS_ENVIRONMENT) $(abs_top_builddir)/tests/gpgscm/gpgscm \
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(XTESTS)
$(abs_srcdir)/run-tests.scm $(TESTFLAGS) $(TESTS)
TEST_FILES = pubring.asc secring.asc plain-1o.asc plain-2o.asc plain-3o.asc \
plain-1.asc plain-2.asc plain-3.asc plain-1-pgp.asc \
@ -250,7 +250,7 @@ sample_msgs = samplemsgs/clearsig-1-key-1.asc \
EXTRA_DIST = defs.scm $(XTESTS) $(TEST_FILES) \
mkdemodirs signdemokey $(priv_keys) $(sample_keys) \
$(sample_msgs) ChangeLog-2011 run-tests.scm \
setup.scm shell.scm
setup.scm shell.scm all-tests.scm
CLEANFILES = prepared.stamp x y yy z out err $(data_files) \
plain-1 plain-2 plain-3 trustdb.gpg *.lock .\#lk* \

View File

@ -7,7 +7,7 @@ From your build directory, run
to run all tests or
obj $ make -C tests/openpgp check XTESTS=your-test.scm
obj $ make -C tests/openpgp check TESTS=your-test.scm
to run a specific test (or any number of tests separated by spaces).
@ -89,7 +89,7 @@ the inner progress indicator will be abbreviated using '.'.
Say you are working on a new test called 'your-test.scm', you can run
it on its own using
obj $ make -C tests/openpgp check XTESTS=your-test.scm
obj $ make -C tests/openpgp check TESTS=your-test.scm
but something isn't working as expected. There are several little
gadgets that might help. The first one is 'trace', a function that

View File

@ -0,0 +1,43 @@
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(export all-tests
;; Parse the Makefile.am to find all tests.
(load (with-path "makefile.scm"))
(define (expander filename port key)
(parse-makefile port key))
(define (parse filename key)
(parse-makefile-expand filename expander key))
(define setup
(make-environment-cache
(test::scm
#f
(path-join "tests" "openpgp" "setup.scm")
(in-srcdir "tests" "openpgp" "setup.scm")
"--" "tests" "gpg")))
(map (lambda (name)
(test::scm setup
(path-join "tests" "openpgp" name)
(in-srcdir "tests" "openpgp" name)))
(parse-makefile-expand (in-srcdir "tests" "openpgp" "Makefile.am")
(lambda (filename port key) (parse-makefile port key))
"XTESTS")))

View File

@ -26,16 +26,17 @@
;; Set objdir so that the tests can locate built programs.
(setenv "objdir" (getcwd) #f)
(let* ((tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
(setup (make-environment-cache (test::scm
#f
(path-join "tests" "openpgp" "setup.scm")
(in-srcdir "tests" "openpgp" "setup.scm"))))
(runner (if (and (member "--parallel" *args*)
(> (length tests) 1))
run-tests-parallel
run-tests-sequential)))
(runner (map (lambda (name)
(test::scm setup
(path-join "tests" "openpgp" name)
(in-srcdir "tests" "openpgp" name))) tests)))
(define setup
(make-environment-cache (test::scm
#f
(path-join "tests" "openpgp" "setup.scm")
(in-srcdir "tests" "openpgp" "setup.scm"))))
(define tests (filter (lambda (arg) (not (string-prefix? arg "--"))) *args*))
(run-tests (if (null? tests)
(load-tests "tests" "openpgp")
(map (lambda (name)
(test::scm setup
(path-join "tests" "openpgp" name)
(in-srcdir "tests" "openpgp" name))) tests)))

44
tests/run-tests.scm Normal file
View File

@ -0,0 +1,44 @@
#!/usr/bin/env gpgscm
;; Copyright (C) 2017 g10 Code GmbH
;;
;; This file is part of GnuPG.
;;
;; GnuPG is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation; either version 3 of the License, or
;; (at your option) any later version.
;;
;; GnuPG is distributed in the hope that it will be useful,
;; but WITHOUT ANY WARRANTY; without even the implied warranty of
;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
;; GNU General Public License for more details.
;;
;; You should have received a copy of the GNU General Public License
;; along with this program; if not, see <http://www.gnu.org/licenses/>.
(info "Running all tests...")
(define (load-tests-with-log . path)
(map (lambda (test)
(test:::set! 'log-file-name
(apply path-join `(,@path
,(string-append (basename test::name)
".log")))))
(apply load-tests path)))
(let ((prefix (flag "--prefix" *args*))
(all-tests (append
(load-tests-with-log "common")
(load-tests-with-log "g10")
(load-tests-with-log "g13")
(load-tests-with-log "agent")
(load-tests-with-log "tests" "openpgp")
(load-tests-with-log "tests" "migrations")
(load-tests-with-log "tests" "gpgsm")
(load-tests-with-log "tests" "gpgme"))))
(run-tests (if prefix
(filter
(lambda (t) (string-prefix? t::name (apply path-join prefix)))
all-tests)
all-tests)))