From d49a945b12d98fadd0d37f4e50b5e02799e16305 Mon Sep 17 00:00:00 2001
From: Werner Koch <wk@gnupg.org>
Date: Thu, 24 Sep 2020 16:47:10 +0200
Subject: [PATCH] gpg: New experimental import option "bulk-import"

* g10/options.h (IMPORT_BULK): New.
* g10/import.c (parse_import_options): Add "bulk-import".
* g10/call-keyboxd.c (in_transaction): New var.
(gpg_keyboxd_deinit_session_data): Run a commit if in bulk import
mode.
(create_new_context): Run a begin transaction if in bulk import mode.
--

Initial tests with this option are not very promising.  Importing
about 3000 real world keys with --use-keyboxd and full logging took:

   real    33m31.724s
   user    19m54.265s
   sys     2m49.662s

With bulk-import this saves a mere 12%:

   real    29m36.542s
   user    19m3.391s
   sys     2m46.728s

Signed-off-by: Werner Koch <wk@gnupg.org>
---
 doc/gpg.texi       |  4 ++++
 g10/call-keyboxd.c | 33 +++++++++++++++++++++++++++++++++
 g10/import.c       | 10 ++++++----
 g10/options.h      |  1 +
 4 files changed, 44 insertions(+), 4 deletions(-)

diff --git a/doc/gpg.texi b/doc/gpg.texi
index 30d29f170..97990a5a9 100644
--- a/doc/gpg.texi
+++ b/doc/gpg.texi
@@ -2449,6 +2449,10 @@ opposite meaning. The options are:
   keys.  For example, this reorders signatures, and strips duplicate
   signatures.  Defaults to yes.
 
+  @item bulk-import
+  When used with --use-keyboxd do the import within a single
+  transaction.  This is an experimental feature.
+
   @item import-minimal
   Import the smallest key possible. This removes all signatures except
   the most recent self-signature on each user ID. This option is the
diff --git a/g10/call-keyboxd.c b/g10/call-keyboxd.c
index bb9901ab9..9042c833c 100644
--- a/g10/call-keyboxd.c
+++ b/g10/call-keyboxd.c
@@ -70,9 +70,13 @@ struct keyboxd_local_s
 
   /* Flag indicating that a search reset is required.  */
   unsigned int need_search_reset : 1;
+
 };
 
 
+/* Flag indicating that for example bulk import is enabled.  */
+static unsigned int in_transaction;
+
 
 
 
@@ -81,6 +85,7 @@ void
 gpg_keyboxd_deinit_session_data (ctrl_t ctrl)
 {
   keyboxd_local_t kbl;
+  gpg_error_t err;
 
   while ((kbl = ctrl->keyboxd_local))
     {
@@ -91,6 +96,20 @@ gpg_keyboxd_deinit_session_data (ctrl_t ctrl)
         {
           kbx_client_data_release (kbl->kcd);
           kbl->kcd = NULL;
+          if (kbl->ctx && in_transaction)
+            {
+              /* This is our hack to commit the changes done during a
+               * bulk import.  If we won't do that the loss of the
+               * connection would trigger a rollback in keyboxd.  Note
+               * that transactions are not associated with a
+               * connection. */
+              err = assuan_transact (kbl->ctx, "TRANSACTION commit",
+                                     NULL, NULL, NULL, NULL, NULL, NULL);
+              if (err)
+                log_error ("error commiting last transaction: %s\n",
+                            gpg_strerror (err));
+              in_transaction = 0;
+            }
           assuan_release (kbl->ctx);
           kbl->ctx = NULL;
         }
@@ -139,6 +158,20 @@ create_new_context (ctrl_t ctrl, assuan_context_t *r_ctx)
   else if (!err && !(err = warn_version_mismatch (ctx, KEYBOXD_NAME)))
     {
       /* Place to emit global options.  */
+
+      if ((opt.import_options & IMPORT_BULK) && !in_transaction)
+        {
+          err = assuan_transact (ctx, "TRANSACTION begin",
+                                 NULL, NULL, NULL, NULL, NULL, NULL);
+          if (err)
+            {
+              log_error ("error enabling bulk import option: %s\n",
+                         gpg_strerror (err));
+            }
+          else
+            in_transaction = 1;
+        }
+
     }
 
   if (err)
diff --git a/g10/import.c b/g10/import.c
index 4ff334cc7..80388835e 100644
--- a/g10/import.c
+++ b/g10/import.c
@@ -178,9 +178,15 @@ parse_import_options(char *str,unsigned int *options,int noisy)
       {"fast-import",IMPORT_FAST,NULL,
        N_("do not update the trustdb after import")},
 
+      {"bulk-import",IMPORT_BULK, NULL,
+       N_("enable bulk import mode")},
+
       {"import-show",IMPORT_SHOW,NULL,
        N_("show key during import")},
 
+      {"show-only", (IMPORT_SHOW | IMPORT_DRY_RUN), NULL,
+       N_("show key but do not actually import") },
+
       {"merge-only",IMPORT_MERGE_ONLY,NULL,
        N_("only accept updates to existing keys")},
 
@@ -206,10 +212,6 @@ parse_import_options(char *str,unsigned int *options,int noisy)
       {"repair-keys", IMPORT_REPAIR_KEYS, NULL,
        N_("repair keys on import")},
 
-      /* No description to avoid string change: Fixme for 2.3 */
-      {"show-only", (IMPORT_SHOW | IMPORT_DRY_RUN), NULL,
-       NULL},
-
       /* Hidden options which are enabled by default and are provided
        * in case of problems with the respective implementation.  */
       {"collapse-uids", IMPORT_COLLAPSE_UIDS, NULL, NULL},
diff --git a/g10/options.h b/g10/options.h
index b2bd543e6..dfa94d4e2 100644
--- a/g10/options.h
+++ b/g10/options.h
@@ -371,6 +371,7 @@ EXTERN_UNLESS_MAIN_MODULE int memory_stat_debug_mode;
 #define IMPORT_SELF_SIGS_ONLY            (1<<14)
 #define IMPORT_COLLAPSE_UIDS             (1<<15)
 #define IMPORT_COLLAPSE_SUBKEYS          (1<<16)
+#define IMPORT_BULK                      (1<<17)
 
 #define EXPORT_LOCAL_SIGS                (1<<0)
 #define EXPORT_ATTRIBUTES                (1<<1)