kbx: Add framework for the SEARCH command

* kbx/backend-kbx.c: New.
* kbx/backend-support.c: New.
* kbx/backend.h: New.
* kbx/frontend.c: New.
* kbx/frontend.h: New.
* kbx/kbxserver.c: Implement SEARCH and NEXT command.
* kbx/keybox-search-desc.h (enum pubkey_types): New.
* kbx/keybox-search.c (keybox_get_data): New.
* kbx/keyboxd.c (main): Add a standard resource.

Signed-off-by: Werner Koch <wk@gnupg.org>

doc/DETAILS

@@ -1137,6 +1137,17 @@ pkd:0:1024:B665B1435F4C2 .... FF26ABB:
 *** BEGIN_STREAM, END_STREAM
     Used to issued by the experimental pipemode.
 
+** Inter-component codes
+   Status codes are also used between the components of the GnuPG
+   system via the Assuan S lines.  Some of them are documented here:
+
+*** PUBKEY_TYPE <n>
+    The type of the public key in the following D-lines or communicated
+    via a pipe.  <n> is the value of =enum pubkey_types=.
+
+
+
+
 
 * Format of the --attribute-fd output
 

kbx/Makefile.am

@@ -73,8 +73,12 @@ kbxutil_LDADD   = $(common_libs) \
 
 
 keyboxd_SOURCES = \
-	keyboxd.c keyboxd.h \
+	keyboxd.c keyboxd.h   \
-	kbxserver.c
+	kbxserver.c           \
+	frontend.c frontend.h \
+	backend.h backend-support.c \
+	backend-kbx.c \
+	$(common_sources)
 
 keyboxd_CFLAGS = $(AM_CFLAGS) $(LIBASSUAN_CFLAGS) $(NPTH_CFLAGS) \
 		$(INCICONV)

kbx/backend-kbx.c

@@ -0,0 +1,292 @@
+/* backend-kbx.c - Keybox format backend for keyboxd
+ * Copyright (C) 2019 g10 Code GmbH
+ *
+ * This file is part of GnuPG.
+ *
+ * GnuPG is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GnuPG is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ * SPDX-License-Identifier: GPL-3.0+
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+
+#include "keyboxd.h"
+#include "../common/i18n.h"
+#include "backend.h"
+#include "keybox.h"
+
+
+/* Our definition of the backend handle.  */
+struct backend_handle_s
+{
+  enum database_types db_type; /* Always DB_TYPE_KBX.  */
+  unsigned int backend_id;     /* Id of this backend.  */
+
+  void *token;  /* Used to create a new KEYBOX_HANDLE.  */
+  char filename[1];
+};
+
+
+/* Check that the file FILENAME is a valid keybox file which can be
+ * used here.  Common return codes:
+ *
+ * 0              := Valid keybox file
+ * GPG_ERR_ENOENT := No such file
+ * GPG_ERR_NO_OBJ := File exists with size zero.
+ * GPG_ERR_INV_OBJ:= File exists but is not a keybox file.
+ */
+static gpg_error_t
+check_kbx_file_magic (const char *filename)
+{
+  gpg_error_t err;
+  u32 magic;
+  unsigned char verbuf[4];
+  estream_t fp;
+
+  fp = es_fopen (filename, "rb");
+  if (!fp)
+    return gpg_error_from_syserror ();
+
+  err = gpg_error (GPG_ERR_INV_OBJ);
+  if (es_fread (&magic, 4, 1, fp) == 1 )
+    {
+      if (es_fread (&verbuf, 4, 1, fp) == 1
+          && verbuf[0] == 1
+          && es_fread (&magic, 4, 1, fp) == 1
+          && !memcmp (&magic, "KBXf", 4))
+        {
+          err = 0;
+        }
+    }
+  else /* Maybe empty: Let's create it. */
+    err = gpg_error (GPG_ERR_NO_OBJ);
+
+  es_fclose (fp);
+  return err;
+}
+
+
+/* Create new keybox file.  This can also be used if the keybox
+ * already exists but has a length of zero.  Do not use it in any
+ * other cases.  */
+static gpg_error_t
+create_keybox (const char *filename)
+{
+  gpg_error_t err;
+  dotlock_t lockhd = NULL;
+  estream_t fp;
+
+  /* To avoid races with other temporary instances of keyboxd trying
+   * to create or update the keybox, we do the next stuff in a locked
+   * state. */
+  lockhd = dotlock_create (filename, 0);
+  if (!lockhd)
+    {
+      err = gpg_error_from_syserror ();
+      /* A reason for this to fail is that the directory is not
+       * writable. However, this whole locking stuff does not make
+       * sense if this is the case. An empty non-writable directory
+       * with no keybox is not really useful at all. */
+      if (opt.verbose)
+        log_info ("can't allocate lock for '%s': %s\n",
+                  filename, gpg_strerror (err));
+      return err;
+    }
+
+  if ( dotlock_take (lockhd, -1) )
+    {
+      err = gpg_error_from_syserror ();
+      /* This is something bad.  Probably a stale lockfile.  */
+      log_info ("can't lock '%s': %s\n", filename, gpg_strerror (err));
+      goto leave;
+    }
+
+  /* Make sure that at least one record is in a new keybox file, so
+   * that the detection magic will work the next time it is used.
+   * We always set the OpenPGP blobs maybe availabale flag.   */
+  fp = es_fopen (filename, "w+b,mode=-rw-------");
+  if (!fp)
+    {
+      err = gpg_error_from_syserror ();
+      log_error (_("error creating keybox '%s': %s\n"),
+                 filename, gpg_strerror (err));
+      goto leave;
+    }
+  err = _keybox_write_header_blob (NULL, fp, 1);
+  es_fclose (fp);
+  if (err)
+    {
+      log_error (_("error creating keybox '%s': %s\n"),
+                 filename, gpg_strerror (err));
+      goto leave;
+    }
+
+  if (!opt.quiet)
+    log_info (_("keybox '%s' created\n"), filename);
+  err = 0;
+
+ leave:
+  if (lockhd)
+    {
+      dotlock_release (lockhd);
+      dotlock_destroy (lockhd);
+    }
+  return err;
+}
+
+
+
+/* Install a new resource and return a handle for that backend.  */
+gpg_error_t
+be_kbx_add_resource (ctrl_t ctrl, backend_handle_t *r_hd,
+                     const char *filename, int readonly)
+{
+  gpg_error_t err;
+  backend_handle_t hd;
+  void *token;
+
+  (void)ctrl;
+
+  *r_hd = NULL;
+  hd = xtrycalloc (1, sizeof *hd + strlen (filename));
+  if (!hd)
+    return gpg_error_from_syserror ();
+  hd->db_type = DB_TYPE_KBX;
+  strcpy (hd->filename, filename);
+
+  err = check_kbx_file_magic (filename);
+  switch (gpg_err_code (err))
+    {
+    case 0:
+      break;
+    case GPG_ERR_ENOENT:
+    case GPG_ERR_NO_OBJ:
+      if (readonly)
+        {
+          err = gpg_error (GPG_ERR_ENOENT);
+          goto leave;
+        }
+      err = create_keybox (filename);
+      if (err)
+        goto leave;
+      break;
+    default:
+      goto leave;
+    }
+
+  err = keybox_register_file (filename, 0, &token);
+  if (err)
+    goto leave;
+
+  hd->backend_id = be_new_backend_id ();
+  hd->token = token;
+
+  *r_hd = hd;
+  hd = NULL;
+
+ leave:
+  xfree (hd);
+  return 0;
+}
+
+
+/* Release the backend handle HD and all its resources.  HD is not
+ * valid after a call to this function.  */
+void
+be_kbx_release_resource (ctrl_t ctrl, backend_handle_t hd)
+{
+  (void)ctrl;
+
+  if (!hd)
+    return;
+  hd->db_type = DB_TYPE_NONE;
+
+  xfree (hd);
+}
+
+
+void
+be_kbx_release_kbx_hd (KEYBOX_HANDLE kbx_hd)
+{
+  keybox_release (kbx_hd);
+}
+
+
+/* Search for the keys described by (DESC,NDESC) and return them to
+ * the caller.  BACKEND_HD is the handle for this backend and REQUEST
+ * is the current database request object.  */
+gpg_error_t
+be_kbx_search (ctrl_t ctrl, backend_handle_t backend_hd, db_request_t request,
+               KEYDB_SEARCH_DESC *desc, unsigned int ndesc)
+{
+  gpg_error_t err;
+  db_request_part_t part;
+  size_t descindex;
+  unsigned long skipped_long_blobs;
+
+  log_assert (backend_hd && backend_hd->db_type == DB_TYPE_KBX);
+  log_assert (request);
+
+  /* Find the specific request part or allocate it.  */
+  for (part = request->part; part; part = part->next)
+    if (part->backend_id == backend_hd->backend_id)
+      break;
+  if (!part)
+    {
+      part = xtrycalloc (1, sizeof *part);
+      if (!part)
+        {
+          err = gpg_error_from_syserror ();
+          goto leave;
+        }
+      part->backend_id = backend_hd->backend_id;
+      part->kbx_hd = keybox_new_openpgp (backend_hd->token, 0);
+      if (!part->kbx_hd)
+        {
+          err = gpg_error_from_syserror ();
+          xfree (part);
+          goto leave;
+        }
+      part->next = request->part;
+      request->part = part;
+    }
+
+  if (!desc)
+    err = keybox_search_reset (part->kbx_hd);
+  else
+    err = keybox_search (part->kbx_hd, desc, ndesc, KEYBOX_BLOBTYPE_PGP,
+                         &descindex, &skipped_long_blobs);
+  if (err == -1)
+    err = gpg_error (GPG_ERR_EOF);
+
+  if (desc && !err)
+    {
+      /* Successful search operation.  */
+      void *buffer;
+      size_t buflen;
+      enum pubkey_types pubkey_type;
+
+      err = keybox_get_data (part->kbx_hd, &buffer, &buflen, &pubkey_type);
+      if (err)
+        goto leave;
+      /* Note: be_return_pubkey always takes ownership of BUFFER.  */
+      err = be_return_pubkey (ctrl, buffer, buflen, pubkey_type);
+    }
+
+ leave:
+  return err;
+}

kbx/backend-support.c

@@ -0,0 +1,128 @@
+/* backend-support.c - Supporting functions for the backend.
+ * Copyright (C) 2019 g10 Code GmbH
+ *
+ * This file is part of GnuPG.
+ *
+ * GnuPG is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GnuPG is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ * SPDX-License-Identifier: GPL-3.0+
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+
+#include "keyboxd.h"
+#include "../common/i18n.h"
+#include "../common/asshelp.h"
+#include "backend.h"
+
+
+/* Common definition part of all backend handle.  */
+struct backend_handle_s
+{
+  enum database_types db_type;
+};
+
+
+
+/* Return a string with the name of the database type T.  */
+const char *
+strdbtype (enum database_types t)
+{
+  switch (t)
+    {
+    case DB_TYPE_NONE: return "none";
+    case DB_TYPE_KBX:  return "keybox";
+    }
+  return "?";
+}
+
+
+/* Return a new backend ID.  Backend IDs are used to identify backends
+ * without using the actual object.  The number of backend resources
+ * is limited because they are specified in the config file.  Thus an
+ * overflow check is not required.  */
+unsigned int
+be_new_backend_id (void)
+{
+  static unsigned int last;
+
+  return ++last;
+}
+
+
+/* Release the backend described by HD.  This is a generic function
+ * which dispatches to the the actual backend.  */
+void
+be_generic_release_backend (ctrl_t ctrl, backend_handle_t hd)
+{
+  if (!hd)
+    return;
+  switch (hd->db_type)
+    {
+    case DB_TYPE_NONE:
+      xfree (hd);
+      break;
+    case DB_TYPE_KBX:
+      be_kbx_release_resource (ctrl, hd);
+      break;
+    default:
+      log_error ("%s: faulty backend handle of type %d given\n",
+                 __func__, hd->db_type);
+    }
+}
+
+
+/* Release the request object REQ.  */
+void
+be_release_request (db_request_t req)
+{
+  db_request_part_t part, partn;
+
+  if (!req)
+    return;
+
+  for (part = req->part; part; part = partn)
+    {
+      partn = part->next;
+      be_kbx_release_kbx_hd (part->kbx_hd);
+      xfree (part);
+    }
+}
+
+
+/* Return the public key (BUFFER,BUFLEN) which has the type
+ * PUBVKEY_TYPE to the caller.  Owenership of BUFFER is taken by thgis
+ * function even in the error case.  */
+gpg_error_t
+be_return_pubkey (ctrl_t ctrl, void *buffer, size_t buflen,
+                  enum pubkey_types pubkey_type)
+{
+  gpg_error_t err;
+
+  err = status_printf (ctrl, "PUBKEY_TYPE", "%d", pubkey_type);
+  if (err)
+    goto leave;
+
+  if (ctrl->no_data_return)
+    err = 0;
+  else
+    err = kbxd_write_data_line(ctrl, buffer, buflen);
+
+ leave:
+  xfree (buffer);
+  return err;
+}

kbx/backend.h

@@ -0,0 +1,95 @@
+/* backend.h - Definitions for keyboxd backends
+ * Copyright (C) 2019 g10 Code GmbH
+ *
+ * This file is part of GnuPG.
+ *
+ * GnuPG is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GnuPG is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef KBX_BACKEND_H
+#define KBX_BACKEND_H
+
+#include "keybox-search-desc.h"
+
+/* Forward declaration of the keybox handle type.  */
+struct keybox_handle;
+typedef struct keybox_handle *KEYBOX_HANDLE;
+
+
+/* The types of the backends.  */
+enum database_types
+  {
+   DB_TYPE_NONE,   /* No database at all (unitialized etc.).  */
+   DB_TYPE_KBX     /* Keybox type database (backend-kbx.c).   */
+  };
+
+
+/* Declaration of the backend handle.  Each backend uses its own
+ * hidden handle structure with the only common thing being that the
+ * first field is the database_type to help with debugging.  */
+struct backend_handle_s;
+typedef struct backend_handle_s *backend_handle_t;
+
+
+/* Object to store backend specific databsde information per database
+ * handle.  */
+struct db_request_part_s
+{
+  struct db_request_part_s *next;
+
+  /* Id of the backend instance this object pertains to.  */
+  unsigned int backend_id;
+
+  /* The handle used for a KBX backend or NULL.  */
+  KEYBOX_HANDLE kbx_hd;
+};
+typedef struct db_request_part_s *db_request_part_t;
+
+
+/* A database request handle.  This keeps per session search
+ * information as well as a list of per-backend infos.  */
+struct db_request_s
+{
+  unsigned int any_search:1;  /* Any search has been done.  */
+  unsigned int any_found:1;   /* Any object has been found.  */
+
+  db_request_part_t part;
+
+  /* Counter to track the next to be searched database index.  */
+  unsigned int next_dbidx;
+};
+
+
+
+/*-- backend-support.c --*/
+const char *strdbtype (enum database_types t);
+unsigned int be_new_backend_id (void);
+void be_generic_release_backend (ctrl_t ctrl, backend_handle_t hd);
+void be_release_request (db_request_t req);
+gpg_error_t be_return_pubkey (ctrl_t ctrl, void *buffer, size_t buflen,
+                              enum pubkey_types pubkey_type);
+
+
+/*-- backend-kbx.c --*/
+gpg_error_t be_kbx_add_resource (ctrl_t ctrl, backend_handle_t *r_hd,
+                                 const char *filename, int readonly);
+void be_kbx_release_resource (ctrl_t ctrl, backend_handle_t hd);
+
+void be_kbx_release_kbx_hd (KEYBOX_HANDLE kbx_hd);
+gpg_error_t be_kbx_search (ctrl_t ctrl, backend_handle_t hd,
+                           db_request_t request,
+                           KEYDB_SEARCH_DESC *desc, unsigned int ndesc);
+
+
+#endif /*KBX_BACKEND_H*/

kbx/frontend.c

@@ -0,0 +1,320 @@
+/* frontend.c - Database fronend code for keyboxd
+ * Copyright (C) 2019 g10 Code GmbH
+ *
+ * This file is part of GnuPG.
+ *
+ * GnuPG is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GnuPG is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ * SPDX-License-Identifier: GPL-3.0+
+ */
+
+#include <config.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <stddef.h>
+#include <string.h>
+
+#include "keyboxd.h"
+#include <assuan.h>
+#include "../common/i18n.h"
+#include "../common/userids.h"
+#include "backend.h"
+#include "frontend.h"
+
+
+/* An object to describe a single database.  */
+struct db_desc_s
+{
+  enum database_types db_type;
+  backend_handle_t backend_handle;
+};
+typedef struct db_desc_s *db_desc_t;
+
+
+/* The table of databases and the size of that table.  */
+static db_desc_t databases;
+static unsigned int no_of_databases;
+
+
+
+
+/* Take a lock for reading the databases.  */
+static void
+take_read_lock (ctrl_t ctrl)
+{
+  /* FIXME */
+  (void)ctrl;
+}
+
+
+/* Take a lock for reading and writing the databases.  */
+/* static void */
+/* take_read_write_lock (ctrl_t ctrl) */
+/* { */
+/*   /\* FIXME *\/ */
+/*   (void)ctrl; */
+/* } */
+
+
+/* Release a lock.  It is valid to call this even if no lock has been
+ * taken which which case this is a nop.  */
+static void
+release_lock (ctrl_t ctrl)
+{
+  /* FIXME */
+  (void)ctrl;
+}
+
+
+/* Add a new resource to the database.  Depending on the FILENAME
+ * suffix we decide which one to use.  This function must be called at
+ * daemon startup because it employs no locking.  If FILENAME has no
+ * directory separator, the file is expected or created below
+ * "$GNUPGHOME/public-keys-v1.d/".  In READONLY mode the file must
+ * exists; otherwise it is created.  */
+gpg_error_t
+kbxd_add_resource  (ctrl_t ctrl, const char *filename_arg, int readonly)
+{
+  gpg_error_t err;
+  char *filename;
+  enum database_types db_type;
+  backend_handle_t handle = NULL;
+  unsigned int n, dbidx;
+
+  /* Do tilde expansion etc. */
+  if (strchr (filename_arg, DIRSEP_C)
+#ifdef HAVE_W32_SYSTEM
+      || strchr (filename_arg, '/')  /* Windows also accepts a slash.  */
+#endif
+      )
+    filename = make_filename (filename_arg, NULL);
+  else
+    filename = make_filename (gnupg_homedir (), GNUPG_PUBLIC_KEYS_DIR,
+                              filename_arg, NULL);
+
+  n = strlen (filename);
+  if (n > 4 && !strcmp (filename + n - 4, ".kbx"))
+    db_type = DB_TYPE_KBX;
+  else
+    {
+      log_error (_("can't use file '%s': %s\n"), filename, _("unknown suffix"));
+      err = gpg_error (GPG_ERR_NOT_SUPPORTED);
+      goto leave;
+    }
+
+  err = gpg_error (GPG_ERR_BUG);
+  switch (db_type)
+    {
+    case DB_TYPE_NONE: /* NOTREACHED */
+      break;
+
+    case DB_TYPE_KBX:
+      err = be_kbx_add_resource (ctrl, &handle, filename, readonly);
+      break;
+    }
+  if (err)
+    goto leave;
+
+  /* All good, create an entry in the table. */
+  for (dbidx = 0; dbidx < no_of_databases; dbidx++)
+    if (!databases[dbidx].db_type)
+      break;
+  if (dbidx == no_of_databases)
+    {
+      /* No table yet or table is full.  */
+      if (!databases)
+        {
+          /* Create first set of data bases.  Note that the type for all
+           * entries is DB_TYPE_NONE.  */
+          dbidx = 4;
+          databases = xtrycalloc (dbidx, sizeof *databases);
+          if (!databases)
+            {
+              err = gpg_error_from_syserror ();
+              goto leave;
+            }
+          no_of_databases = dbidx;
+          dbidx = 0; /* Put into first slot.  */
+        }
+      else
+        {
+          db_desc_t newdb;
+
+          dbidx = no_of_databases + (no_of_databases == 4? 12 : 16);
+          newdb = xtrycalloc (dbidx, sizeof *databases);
+          if (!databases)
+            {
+              err = gpg_error_from_syserror ();
+              goto leave;
+            }
+          for (n=0; n < no_of_databases; n++)
+            newdb[n] = databases[n];
+          xfree (databases);
+          databases = newdb;
+          n = no_of_databases;
+          no_of_databases = dbidx;
+          dbidx = n; /* Put into first new slot.  */
+        }
+    }
+
+  databases[dbidx].db_type = db_type;
+  databases[dbidx].backend_handle = handle;
+  handle = NULL;
+
+ leave:
+  if (err)
+    be_generic_release_backend (ctrl, handle);
+  xfree (filename);
+  return err;
+}
+
+
+/* Release all per session objects.  */
+void
+kbxd_release_session_info (ctrl_t ctrl)
+{
+  if (!ctrl)
+    return;
+  be_release_request (ctrl->opgp_req);
+  ctrl->opgp_req = NULL;
+  be_release_request (ctrl->x509_req);
+  ctrl->x509_req = NULL;
+}
+
+
+
+/* Search for the keys described by (DESC,NDESC) and return them to
+ * the caller.  If RESET is set, the search state is first reset. */
+gpg_error_t
+kbxd_search (ctrl_t ctrl, KEYDB_SEARCH_DESC *desc, unsigned int ndesc,
+             int reset)
+{
+  gpg_error_t err;
+  int i;
+  unsigned int dbidx;
+  db_desc_t db;
+  db_request_t request;
+
+  if (DBG_CLOCK)
+    log_clock ("%s: enter", __func__);
+
+  if (DBG_LOOKUP)
+    {
+      log_debug ("%s: %u search descriptions:\n", __func__, ndesc);
+      for (i = 0; i < ndesc; i ++)
+        {
+          /* char *t = keydb_search_desc_dump (&desc[i]); */
+          /* log_debug ("%s   %d: %s\n", __func__, i, t); */
+          /* xfree (t); */
+        }
+    }
+
+  take_read_lock (ctrl);
+
+  /* Allocate a handle object if none exists for this context.  */
+  if (!ctrl->opgp_req)
+    {
+      ctrl->opgp_req = xtrycalloc (1, sizeof *ctrl->opgp_req);
+      if (!ctrl->opgp_req)
+        {
+          err = gpg_error_from_syserror ();
+          goto leave;
+        }
+    }
+  request = ctrl->opgp_req;
+
+  /* If requested do a reset.  Using the reset flag is faster than
+   * letting the caller do a separate call for an intial reset.  */
+  if (!desc || reset)
+    {
+      for (dbidx=0; dbidx < no_of_databases; dbidx++)
+        {
+          db = databases + dbidx;
+          if (!db->db_type)
+            continue;  /* Empty slot.  */
+
+          switch (db->db_type)
+            {
+            case DB_TYPE_NONE: /* NOTREACHED */
+              break;
+
+            case DB_TYPE_KBX:
+              err = be_kbx_search (ctrl, db->backend_handle, request, NULL, 0);
+              break;
+            }
+          if (err)
+            {
+              log_error ("error during the %ssearch reset: %s\n",
+                         reset? "initial ":"", gpg_strerror (err));
+              goto leave;
+            }
+        }
+      request->any_search = 0;
+      request->any_found = 0;
+      request->next_dbidx = 0;
+      if (!desc) /* Reset only mode */
+        {
+          err = 0;
+          goto leave;
+        }
+    }
+
+
+  /* Move to the next non-empty slot.  */
+ next_db:
+  for (dbidx=request->next_dbidx; (dbidx < no_of_databases
+                                   && !databases[dbidx].db_type); dbidx++)
+    ;
+  request->next_dbidx = dbidx;
+  if (!(dbidx < no_of_databases))
+    {
+      /* All databases have been searched.  */
+      err = gpg_error (GPG_ERR_NOT_FOUND);
+      goto leave;
+    }
+  db = databases + dbidx;
+
+  /* Divert to the backend for the actual search.  */
+  switch (db->db_type)
+    {
+    case DB_TYPE_NONE:
+      /* NOTREACHED */
+      err = gpg_error (GPG_ERR_INTERNAL);
+      break;
+
+    case DB_TYPE_KBX:
+      err = be_kbx_search (ctrl, db->backend_handle, request,
+                           desc, ndesc);
+      break;
+    }
+
+  if (DBG_LOOKUP)
+    log_debug ("%s: searched %s (db %u of %u) => %s\n",
+               __func__, strdbtype (db->db_type), dbidx, no_of_databases,
+               gpg_strerror (err));
+  request->any_search = 1;
+  if (!err)
+    request->any_found = 1;
+  else if (gpg_err_code (err) == GPG_ERR_EOF)
+    {
+      request->next_dbidx++;
+      goto next_db;
+    }
+
+
+ leave:
+  release_lock (ctrl);
+  if (DBG_CLOCK)
+    log_clock ("%s: leave (%s)", __func__, err? "not found" : "found");
+  return err;
+}

kbx/frontend.h

@@ -0,0 +1,36 @@
+/* frontend.h - Definitions for the keyboxd frontend
+ * Copyright (C) 2019 g10 Code GmbH
+ *
+ * This file is part of GnuPG.
+ *
+ * GnuPG is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * GnuPG is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ */
+
+#ifndef KBX_FRONTEND_H
+#define KBX_FRONTEND_H
+
+#include "keybox-search-desc.h"
+
+
+gpg_error_t kbxd_add_resource  (ctrl_t ctrl,
+                                const char *filename_arg, int readonly);
+
+void kbxd_release_session_info (ctrl_t ctrl);
+
+gpg_error_t kbxd_search (ctrl_t ctrl,
+                         KEYDB_SEARCH_DESC *desc, unsigned int ndesc,
+                         int reset);
+
+
+#endif /*KBX_FRONTEND_H*/

kbx/kbxserver.c

@@ -1,5 +1,5 @@
 /* kbxserver.c - Handle Assuan commands send to the keyboxd
- * Copyright (C) 2018 g10 Code GmbH
+ * Copyright (C) 2019 g10 Code GmbH
  *
  * This file is part of GnuPG.
  *
@@ -31,10 +31,11 @@
 
 #include "keyboxd.h"
 #include <assuan.h>
-
 #include "../common/i18n.h"
 #include "../common/server-help.h"
-
+#include "../common/userids.h"
+#include "../common/asshelp.h"
+#include "frontend.h"
 
 
 
@@ -65,12 +66,99 @@ struct server_local_s
   unsigned int inhibit_data_logging : 1;
   unsigned int inhibit_data_logging_now : 1;
 
-  /* Dummy option.  */
+  /* This flag is set if the last search command was called with --more.  */
-  int foo;
+  unsigned int search_expecting_more : 1;
+
+  /* This flag is set if the last search command was successful.  */
+  unsigned int search_any_found : 1;
+
+  /* The first is the current search description as parsed by the
+   * cmd_search.  If more than one pattern is required, cmd_search
+   * also allocates and sets multi_search_desc and
+   * multi_search_desc_len.  If a search description has ever been
+   * allocated the allocated size is stored at
+   * multi_search_desc_size.  */
+  KEYBOX_SEARCH_DESC search_desc;
+  KEYBOX_SEARCH_DESC *multi_search_desc;
+  unsigned int multi_search_desc_size;
+  unsigned int multi_search_desc_len;
 };
 
 
 
+
+/* Return the assuan contxt from the local server info in CTRL.  */
+static assuan_context_t
+get_assuan_ctx_from_ctrl (ctrl_t ctrl)
+{
+  if (!ctrl || !ctrl->server_local)
+    return NULL;
+  return ctrl->server_local->assuan_ctx;
+}
+
+
+/* A wrapper around assuan_send_data which makes debugging the output
+ * in verbose mode easier.  It also takes CTRL as argument.  */
+gpg_error_t
+kbxd_write_data_line (ctrl_t ctrl, const void *buffer_arg, size_t size)
+{
+  const char *buffer = buffer_arg;
+  assuan_context_t ctx = get_assuan_ctx_from_ctrl (ctrl);
+  gpg_error_t err;
+
+  if (!ctx) /* Oops - no assuan context.  */
+    return gpg_error (GPG_ERR_NOT_PROCESSED);
+
+  /* If we do not want logging, enable it here.  */
+  if (ctrl && ctrl->server_local && ctrl->server_local->inhibit_data_logging)
+    ctrl->server_local->inhibit_data_logging_now = 1;
+
+  if (opt.verbose && buffer && size)
+    {
+      /* Ease reading of output by limiting the line length.  */
+      size_t n, nbytes;
+
+      nbytes = size;
+      do
+        {
+          n = nbytes > 64? 64 : nbytes;
+          err = assuan_send_data (ctx, buffer, n);
+          if (err)
+            {
+              gpg_err_set_errno (EIO);
+              goto leave;
+            }
+          buffer += n;
+          nbytes -= n;
+          if (nbytes && (err=assuan_send_data (ctx, NULL, 0))) /* Flush line. */
+            {
+              gpg_err_set_errno (EIO);
+              goto leave;
+            }
+        }
+      while (nbytes);
+    }
+  else
+    {
+      err = assuan_send_data (ctx, buffer, size);
+      if (err)
+        {
+          gpg_err_set_errno (EIO);  /* For use by data_line_cookie_write.  */
+          goto leave;
+        }
+    }
+
+ leave:
+  if (ctrl && ctrl->server_local && ctrl->server_local->inhibit_data_logging)
+    {
+      ctrl->server_local->inhibit_data_logging_now = 0;
+      ctrl->server_local->inhibit_data_logging_count += size;
+    }
+
+  return err;
+}
+
+
 
 /* Helper to print a message while leaving a command.  */
 static gpg_error_t
@@ -100,11 +188,7 @@ option_handler (assuan_context_t ctx, const char *key, const char *value)
   ctrl_t ctrl = assuan_get_pointer (ctx);
   gpg_error_t err = 0;
 
-  if (!strcmp (key, "foo"))
+  if (!strcmp (key, "lc-messages"))
-    {
-      ctrl->server_local->foo = 1;
-    }
-  else if (!strcmp (key, "lc-messages"))
     {
       if (ctrl->lc_messages)
         xfree (ctrl->lc_messages);
@@ -120,22 +204,151 @@ option_handler (assuan_context_t ctx, const char *key, const char *value)
 
 
 
-static const char hlp_foo[] =
+static const char hlp_search[] =
-  "FOO <user_id>\n"
+  "SEARCH [--no-data] [[--more] PATTERN]\n"
   "\n"
-  "Bla bla\n"
+  "Search for the keys identified by PATTERN.  With --more more\n"
-  "more bla.";
+  "patterns to be used for the search are expected with the next\n"
+  "command.  With --no-data only the search status is returned but\n"
+  "not the actual data.  See also \"NEXT\".";
 static gpg_error_t
-cmd_foo (assuan_context_t ctx, char *line)
+cmd_search (assuan_context_t ctx, char *line)
 {
   ctrl_t ctrl = assuan_get_pointer (ctx);
+  int opt_more, opt_no_data;
+  gpg_error_t err;
+  unsigned int n, k;
+
+  opt_no_data = has_option (line, "--no-data");
+  opt_more = has_option (line, "--more");
+  line = skip_options (line);
+
+  ctrl->server_local->search_any_found = 0;
+
+  if (!*line && opt_more)
+    {
+      err = set_error (GPG_ERR_INV_ARG, "--more but no pattern");
+      goto leave;
+    }
+  else if (!*line && ctrl->server_local->search_expecting_more)
+    {
+      /* It would be too surprising to first set a pattern but finally
+       * add no pattern to search the entire DB.  */
+      err = set_error (GPG_ERR_INV_ARG, "--more pending but no pattern");
+      goto leave;
+    }
+
+  err = classify_user_id (line, &ctrl->server_local->search_desc, 0);
+  if (err)
+    goto leave;
+  if (opt_more || ctrl->server_local->search_expecting_more)
+    {
+      /* More pattern are expected - store the current one and return
+       * success.  */
+      if (!ctrl->server_local->multi_search_desc_size)
+        {
+          n = 10;
+          ctrl->server_local->multi_search_desc
+            = xtrycalloc (n, sizeof *ctrl->server_local->multi_search_desc);
+          if (!ctrl->server_local->multi_search_desc)
+            {
+              err = gpg_error_from_syserror ();
+              goto leave;
+            }
+          ctrl->server_local->multi_search_desc_size = n;
+        }
+
+      if (ctrl->server_local->multi_search_desc_len
+          == ctrl->server_local->multi_search_desc_size)
+        {
+          KEYBOX_SEARCH_DESC *desc;
+          n = ctrl->server_local->multi_search_desc_size + 10;
+          desc = xtrycalloc (n, sizeof *desc);
+          if (!desc)
+            {
+              err = gpg_error_from_syserror ();
+              goto leave;
+            }
+          for (k=0; k < ctrl->server_local->multi_search_desc_size; k++)
+            desc[k] = ctrl->server_local->multi_search_desc[k];
+          xfree (ctrl->server_local->multi_search_desc);
+          ctrl->server_local->multi_search_desc = desc;
+          ctrl->server_local->multi_search_desc_size = n;
+        }
+      /* Actually store.  */
+      ctrl->server_local->multi_search_desc
+        [ctrl->server_local->multi_search_desc_len++]
+        = ctrl->server_local->search_desc;
+
+      if (opt_more)
+        {
+          /* We need to be called aagain with more pattern.  */
+          ctrl->server_local->search_expecting_more = 1;
+          goto leave;
+        }
+      ctrl->server_local->search_expecting_more = 0;
+      /* Continue with the actual search.  */
+    }
+  else
+    ctrl->server_local->multi_search_desc_len = 0;
+
+  ctrl->no_data_return = opt_no_data;
+  if (ctrl->server_local->multi_search_desc_len)
+    err = kbxd_search (ctrl, ctrl->server_local->multi_search_desc,
+                       ctrl->server_local->multi_search_desc_len, 1);
+  else
+    err = kbxd_search (ctrl, &ctrl->server_local->search_desc, 1, 1);
+  if (err)
+    goto leave;
+
+  /* Set a flag for use by NEXT.  */
+  ctrl->server_local->search_any_found = 1;
+
+ leave:
+  if (err)
+    ctrl->server_local->multi_search_desc_len = 0;
+  ctrl->no_data_return = 0;
+  return leave_cmd (ctx, err);
+}
+
+
+static const char hlp_next[] =
+  "NEXT [--no-data]\n"
+  "\n"
+  "Get the next search result from a previus search.";
+static gpg_error_t
+cmd_next (assuan_context_t ctx, char *line)
+{
+  ctrl_t ctrl = assuan_get_pointer (ctx);
+  int opt_no_data;
   gpg_error_t err;
 
-  (void)ctrl;
+  opt_no_data = has_option (line, "--no-data");
-  (void)line;
+  line = skip_options (line);
 
-  err = gpg_error (GPG_ERR_NOT_IMPLEMENTED);
+  if (*line)
+    {
+      err = set_error (GPG_ERR_INV_ARG, "no args expected");
+      goto leave;
+    }
 
+  if (!ctrl->server_local->search_any_found)
+    {
+      err = set_error (GPG_ERR_NOTHING_FOUND, "no previous SEARCH");
+      goto leave;
+    }
+
+  ctrl->no_data_return = opt_no_data;
+  if (ctrl->server_local->multi_search_desc_len)
+    err = kbxd_search (ctrl, ctrl->server_local->multi_search_desc,
+                       ctrl->server_local->multi_search_desc_len, 0);
+  else
+    err = kbxd_search (ctrl, &ctrl->server_local->search_desc, 1, 0);
+  if (err)
+    goto leave;
+
+ leave:
+  ctrl->no_data_return = 0;
   return leave_cmd (ctx, err);
 }
 
@@ -250,7 +463,8 @@ register_commands (assuan_context_t ctx)
     assuan_handler_t handler;
     const char * const help;
   } table[] = {
-    { "FOO",        cmd_foo,        hlp_foo },
+    { "SEARCH",     cmd_search,     hlp_search },
+    { "NEXT",       cmd_next,       hlp_next   },
     { "GETINFO",    cmd_getinfo,    hlp_getinfo },
     { "KILLKEYBOXD",cmd_killkeyboxd,hlp_killkeyboxd },
     { "RELOADKEYBOXD",cmd_reloadkeyboxd,hlp_reloadkeyboxd },
@@ -306,16 +520,6 @@ kbxd_assuan_log_monitor (assuan_context_t ctx, unsigned int cat,
 }
 
 
-/* Return the assuan contxt from the local server info in CTRL.  */
-static assuan_context_t
-get_assuan_ctx_from_ctrl (ctrl_t ctrl)
-{
-  if (!ctrl || !ctrl->server_local)
-    return NULL;
-  return ctrl->server_local->assuan_ctx;
-}
-
-
 /* Startup the server and run the main command loop.  With FD = -1,
  * use stdin/stdout.  SESSION_ID is either 0 or a unique number
  * identifying a session. */
@@ -441,6 +645,7 @@ kbxd_start_command_handler (ctrl_t ctrl, gnupg_fd_t fd, unsigned int session_id)
                ctrl->refcount);
   else
     {
+      xfree (ctrl->server_local->multi_search_desc);
       xfree (ctrl->server_local);
       ctrl->server_local = NULL;
     }

kbx/keybox-search-desc.h

@@ -47,6 +47,15 @@ typedef enum {
 } KeydbSearchMode;
 
 
+/* Identifiers for the public key types we use in GnuPG.  */
+enum pubkey_types
+  {
+   PUBKEY_TYPE_UNKNOWN = 0,
+   PUBKEY_TYPE_OPGP    = 1,
+   PUBKEY_TYPE_X509    = 2
+  };
+
+
 /* Forward declaration.  See g10/packet.h.  */
 struct gpg_pkt_user_id_s;
 typedef struct gpg_pkt_user_id_s *gpg_pkt_user_id_t;

kbx/keybox-search.c

@@ -1180,11 +1180,70 @@ keybox_search (KEYBOX_HANDLE hd, KEYBOX_SEARCH_DESC *desc, size_t ndesc,
    a successful search operation.
 */
 
+/* Return the raw data from the last found blob.  Caller must release
+ * the value stored at R_BUFFER.  If called with NULL for R_BUFFER
+ * only the needed length for the buffer and the public key type is
+ * returned.  */
+gpg_error_t
+keybox_get_data (KEYBOX_HANDLE hd, void **r_buffer, size_t *r_length,
+                 enum pubkey_types *r_pubkey_type)
+{
+  const unsigned char *buffer;
+  size_t length;
+  size_t image_off, image_len;
+
+  if (r_buffer)
+    *r_buffer = NULL;
+  if (r_length)
+    *r_length = 0;
+  if (r_pubkey_type)
+    *r_pubkey_type = PUBKEY_TYPE_UNKNOWN;
+
+  if (!hd)
+    return gpg_error (GPG_ERR_INV_VALUE);
+  if (!hd->found.blob)
+    return gpg_error (GPG_ERR_NOTHING_FOUND);
+
+  switch (blob_get_type (hd->found.blob))
+    {
+    case KEYBOX_BLOBTYPE_PGP:
+      if (r_pubkey_type)
+        *r_pubkey_type = PUBKEY_TYPE_OPGP;
+      break;
+    case KEYBOX_BLOBTYPE_X509:
+      if (r_pubkey_type)
+        *r_pubkey_type = PUBKEY_TYPE_X509;
+      break;
+    default:
+      return gpg_error (GPG_ERR_WRONG_BLOB_TYPE);
+    }
+
+  buffer = _keybox_get_blob_image (hd->found.blob, &length);
+  if (length < 40)
+    return gpg_error (GPG_ERR_TOO_SHORT);
+  image_off = get32 (buffer+8);
+  image_len = get32 (buffer+12);
+  if ((uint64_t)image_off+(uint64_t)image_len > (uint64_t)length)
+    return gpg_error (GPG_ERR_TOO_SHORT);
+
+  if (r_length)
+    *r_length = image_len;
+  if (r_buffer)
+    {
+      *r_buffer = xtrymalloc (image_len);
+      if (!*r_buffer)
+        return gpg_error_from_syserror ();
+      memcpy (*r_buffer, buffer + image_off, image_len);
+    }
+
+  return 0;
+}
+
 
 /* Return the last found keyblock.  Returns 0 on success and stores a
  * new iobuf at R_IOBUF.  R_UID_NO and R_PK_NO are used to return the
- * number of the key or user id which was matched the search criteria;
+ * index of the key or user id which matched the search criteria; if
- * if not known they are set to 0. */
+ * not known they are set to 0. */
 gpg_error_t
 keybox_get_keyblock (KEYBOX_HANDLE hd, iobuf_t *r_iobuf,
                      int *r_pk_no, int *r_uid_no)

kbx/keybox.h

@@ -85,6 +85,9 @@ gpg_error_t _keybox_write_header_blob (FILE *fp, estream_t stream,
                                        int openpgp_flag);
 
 /*-- keybox-search.c --*/
+gpg_error_t keybox_get_data (KEYBOX_HANDLE hd,
+                             void **r_buffer, size_t *r_length,
+                             enum pubkey_types *r_pubkey_type);
 gpg_error_t keybox_get_keyblock (KEYBOX_HANDLE hd, iobuf_t *r_iobuf,
                                  int *r_uid_no, int *r_pk_no);
 #ifdef KEYBOX_WITH_X509

kbx/keyboxd.c

@@ -56,7 +56,7 @@
 #include "../common/init.h"
 #include "../common/gc-opt-flags.h"
 #include "../common/exechelp.h"
-
+#include "frontend.h"
 
 enum cmd_and_opt_values
   {
@@ -127,6 +127,8 @@ static struct debug_flags_s debug_flags [] =
     { DBG_MEMSTAT_VALUE, "memstat" },
     { DBG_HASHING_VALUE, "hashing" },
     { DBG_IPC_VALUE    , "ipc"     },
+    { DBG_CLOCK_VALUE  , "clock"   },
+    { DBG_LOOKUP_VALUE , "lookup"  },
     { 77, NULL } /* 77 := Do not exit on "help" or "?".  */
   };
 
@@ -727,6 +729,9 @@ main (int argc, char **argv )
           kbxd_exit (1);
         }
       kbxd_init_default_ctrl (ctrl);
+
+      kbxd_add_resource (ctrl, "pubring.kbx", 0);
+
       kbxd_start_command_handler (ctrl, GNUPG_INVALID_FD, 0);
       kbxd_deinit_default_ctrl (ctrl);
       xfree (ctrl);
@@ -848,6 +853,22 @@ main (int argc, char **argv )
           exit (1);
         }
 
+      {
+        ctrl_t ctrl;
+
+        ctrl = xtrycalloc (1, sizeof *ctrl);
+        if (!ctrl)
+          {
+            log_error ("error allocating connection control data: %s\n",
+                       strerror (errno) );
+            kbxd_exit (1);
+          }
+        kbxd_init_default_ctrl (ctrl);
+        kbxd_add_resource (ctrl, "pubring.kbx", 0);
+        kbxd_deinit_default_ctrl (ctrl);
+        xfree (ctrl);
+      }
+
       log_info ("%s %s started\n", strusage(11), strusage(13) );
       handle_connections (fd);
       assuan_sock_close (fd);
@@ -974,6 +995,7 @@ kbxd_deinit_default_ctrl (ctrl_t ctrl)
 {
   if (!ctrl)
     return;
+  kbxd_release_session_info (ctrl);
   ctrl->magic = 0xdeadbeef;
   unregister_progress_cb ();
   xfree (ctrl->lc_messages);

kbx/keyboxd.h

@@ -55,6 +55,8 @@ struct
 #define DBG_MEMSTAT_VALUE 128	/* show memory statistics */
 #define DBG_HASHING_VALUE 512	/* debug hashing operations */
 #define DBG_IPC_VALUE     1024  /* Enable Assuan debugging.  */
+#define DBG_CLOCK_VALUE   4096  /* debug timings (required build option).  */
+#define DBG_LOOKUP_VALUE  8192	/* debug the key lookup */
 
 /* Test macros for the debug option.  */
 #define DBG_CRYPTO  (opt.debug & DBG_CRYPTO_VALUE)
@@ -62,6 +64,14 @@ struct
 #define DBG_CACHE   (opt.debug & DBG_CACHE_VALUE)
 #define DBG_HASHING (opt.debug & DBG_HASHING_VALUE)
 #define DBG_IPC     (opt.debug & DBG_IPC_VALUE)
+#define DBG_CLOCK   (opt.debug & DBG_CLOCK_VALUE)
+#define DBG_LOOKUP  (opt.debug & DBG_LOOKUP_VALUE)
+
+
+/* Declaration of a database request object.  This is used for all
+ * database operation (search, insert, update, delete).  */
+struct db_request_s;
+typedef struct db_request_s *db_request_t;
 
 /* Forward reference for local definitions in command.c.  */
 struct server_local_s;
@@ -95,6 +105,13 @@ struct server_control_s
   unsigned long client_pid;
   int client_uid;
 
+  /* Two database request objects used with a connection.  They are
+   * auto-created as needed.  */
+  db_request_t opgp_req;
+  db_request_t x509_req;
+
+  /* Flags for the current request.  */
+  unsigned int no_data_return : 1;  /* Used by SEARCH and NEXT.  */
 };
 
 
@@ -132,6 +149,8 @@ void kbxd_sighup_action (void);
 
 
 /*-- kbxserver.c --*/
+gpg_error_t kbxd_write_data_line (ctrl_t ctrl,
+                                  const void *buffer_arg, size_t size);
 void kbxd_start_command_handler (ctrl_t, gnupg_fd_t, unsigned int);
 
 #endif /*KEYBOXD_H*/