diff --git a/tests/gpgscm/tests.scm b/tests/gpgscm/tests.scm
index 7b88e0ebb..71ca3692a 100644
--- a/tests/gpgscm/tests.scm
+++ b/tests/gpgscm/tests.scm
@@ -234,7 +234,9 @@
     `((lambda (,(caaadr form))
 	(let ((,result-sym
 	       ,(if (= 1 (length (cadr form)))
-		    `(begin ,@(cddr form))
+		    `(catch (begin (close ,(caaadr form))
+				   (apply throw *error*))
+			    ,@(cddr form))
 		    `(letfd ,(cdadr form) ,@(cddr form)))))
 	  (close ,(caaadr form))
 	  ,result-sym)) ,@(cdaadr form))))
@@ -243,7 +245,9 @@
   (let ((result-sym (gensym)) (cwd-sym (gensym)))
     `(let* ((,cwd-sym (getcwd))
 	    (_ (if ,(cadr form) (chdir ,(cadr form))))
-	    (,result-sym (begin ,@(cddr form))))
+	    (,result-sym (catch (begin (chdir ,cwd-sym)
+				       (apply throw *error*))
+				,@(cddr form))))
        (chdir ,cwd-sym)
        ,result-sym)))
 
@@ -264,7 +268,10 @@
     `(let* ((,cwd-sym (getcwd))
 	    (,tmp-sym (mkdtemp))
 	    (_ (chdir ,tmp-sym))
-	    (,result-sym (begin ,@(cdr form))))
+	    (,result-sym (catch (begin (chdir ,cwd-sym)
+				       (unlink-recursively ,tmp-sym)
+				       (apply throw *error*))
+				,@(cdr form))))
        (chdir ,cwd-sym)
        (unlink-recursively ,tmp-sym)
        ,result-sym)))
@@ -293,7 +300,9 @@
     `((lambda (,(caadr form))
 	(let ((,result-sym
 	       ,(if (= 1 (length (cadr form)))
-		    `(begin ,@(cddr form))
+		    `(catch (begin (remove-temporary-file ,(caadr form))
+				   (apply throw *error*))
+			    ,@(cddr form))
 		    `(lettmp ,(cdadr form) ,@(cddr form)))))
 	  (remove-temporary-file ,(caadr form))
 	  ,result-sym)) (make-temporary-file ,(symbol->string (caadr form))))))