kbx: First take on a cache for the keyboxd.

* kbx/backend.h (enum database_types): Add DB_TYPE_CACHE.
(struct db_request_part_s): Add seqno fields.
(struct db_request_s): Add infos for the cache backend.
* kbx/backend-support.c (struct backend_handle_s): Add 'backend_id'.
(strdbtype): Support DB_TYPE_CACHE.
(be_generic_release_backend): Ditto.
(be_find_request_part): New.
(be_return_pubkey): New arg UBID and chnage status name.
* kbx/backend-cache.c: New.
* kbx/backend-kbx.c (be_kbx_init_request_part): New.
(be_kbx_search): Factor some code out to a support function.
(be_kbx_seek): New.
* kbx/frontend.c (kbxd_add_resource): Support DB_TYPE_CACHE.
(kbxd_search): Support the NEXR operation with the cache.
* kbx/keybox-search-desc.h (KEYDB_SEARCH_MODE_UBID): New.
(struct keydb_search_desc): Add field u.ubid.
* kbx/keybox-search.c (has_ubid): New.
(keybox_search): Support the UBID search.
--

This adds a caching backend to the keyboxd.  This tries to accommodate
for duplicate use of fingerprints and thus be correct in case a
fingerprint is used in several keys.  It also turned out that we need
to have a unique identifier (UBID) to identify a keyblock or X.509
certificate.  In particular with an OpenPGP keyblob we can't easily
use the primary fingerprint as an identifier because that fingerprint
may also be used as subkey in another key.  Thus using a hash of the
entire keyblock is a better identifier to be used to address a
keyblock for restarting a search or for identifying the keyblock to be
updated.  Note that this new UBID is not a permanent identifier
because it changes with all keyblock update; it should be viewed as a
handle to the keyblock or X509 cert.

kbx/Makefile.am

@@ -77,6 +77,7 @@ keyboxd_SOURCES = \
 	kbxserver.c           \
 	frontend.c frontend.h \
 	backend.h backend-support.c \
+	backend-cache.c \
 	backend-kbx.c \
 	$(common_sources)
 

kbx/backend-kbx.c

@@ -34,7 +34,7 @@
 struct backend_handle_s
 {
   enum database_types db_type; /* Always DB_TYPE_KBX.  */
-  unsigned int backend_id;     /* Id of this backend.  */
+  unsigned int backend_id;     /* Always the id of the backend.  */
 
   void *token;  /* Used to create a new KEYBOX_HANDLE.  */
   char filename[1];
@@ -226,6 +226,17 @@ be_kbx_release_kbx_hd (KEYBOX_HANDLE kbx_hd)
 }
 
 
+/* Helper for be_find_request_part to initialize a kbx request part.  */
+gpg_error_t
+be_kbx_init_request_part (backend_handle_t backend_hd, db_request_part_t part)
+{
+  part->kbx_hd = keybox_new_openpgp (backend_hd->token, 0);
+  if (!part->kbx_hd)
+    return gpg_error_from_syserror ();
+  return 0;
+}
+
+
 /* Search for the keys described by (DESC,NDESC) and return them to
  * the caller.  BACKEND_HD is the handle for this backend and REQUEST
  * is the current database request object.  */
@@ -242,28 +253,9 @@ be_kbx_search (ctrl_t ctrl, backend_handle_t backend_hd, db_request_t request,
   log_assert (request);
 
   /* Find the specific request part or allocate it.  */
-  for (part = request->part; part; part = part->next)
+  err = be_find_request_part (backend_hd, request, &part);
-    if (part->backend_id == backend_hd->backend_id)
+  if (err)
-      break;
+    goto leave;
-  if (!part)
-    {
-      part = xtrycalloc (1, sizeof *part);
-      if (!part)
-        {
-          err = gpg_error_from_syserror ();
-          goto leave;
-        }
-      part->backend_id = backend_hd->backend_id;
-      part->kbx_hd = keybox_new_openpgp (backend_hd->token, 0);
-      if (!part->kbx_hd)
-        {
-          err = gpg_error_from_syserror ();
-          xfree (part);
-          goto leave;
-        }
-      part->next = request->part;
-      request->part = part;
-    }
 
   if (!desc)
     err = keybox_search_reset (part->kbx_hd);
@@ -279,14 +271,56 @@ be_kbx_search (ctrl_t ctrl, backend_handle_t backend_hd, db_request_t request,
       void *buffer;
       size_t buflen;
       enum pubkey_types pubkey_type;
+      unsigned char blobid[20];
 
       err = keybox_get_data (part->kbx_hd, &buffer, &buflen, &pubkey_type);
       if (err)
         goto leave;
-      /* Note: be_return_pubkey always takes ownership of BUFFER.  */
+      gcry_md_hash_buffer (GCRY_MD_SHA1, blobid, buffer, buflen);
-      err = be_return_pubkey (ctrl, buffer, buflen, pubkey_type);
+      err = be_return_pubkey (ctrl, buffer, buflen, pubkey_type, blobid);
+      if (!err)
+        be_cache_pubkey (ctrl, blobid, buffer, buflen, pubkey_type);
+      xfree (buffer);
     }
 
  leave:
   return err;
 }
+
+
+/* Seek in the keybox to the given UBID.  BACKEND_HD is the handle for
+ * this backend and REQUEST is the current database request object.
+ * This does a dummy read so that the next search operation starts
+ * right after that UBID. */
+gpg_error_t
+be_kbx_seek (ctrl_t ctrl, backend_handle_t backend_hd,
+                  db_request_t request, unsigned char *ubid)
+{
+  gpg_error_t err;
+  db_request_part_t part;
+  size_t descindex;
+  unsigned long skipped_long_blobs;
+  KEYDB_SEARCH_DESC desc;
+
+  log_assert (backend_hd && backend_hd->db_type == DB_TYPE_KBX);
+  log_assert (request);
+
+  memset (&desc, 0, sizeof desc);
+  desc.mode = KEYDB_SEARCH_MODE_UBID;
+  memcpy (desc.u.ubid, ubid, 20);
+
+  /* Find the specific request part or allocate it.  */
+  err = be_find_request_part (backend_hd, request, &part);
+  if (err)
+    goto leave;
+
+  err = keybox_search_reset (part->kbx_hd);
+  if (!err)
+    err = keybox_search (part->kbx_hd, &desc, 1, 0,
+                         &descindex, &skipped_long_blobs);
+  if (err == -1)
+    err = gpg_error (GPG_ERR_EOF);
+
+ leave:
+  return err;
+}

kbx/backend-support.c

@@ -28,12 +28,15 @@
 #include "../common/i18n.h"
 #include "../common/asshelp.h"
 #include "backend.h"
+#include "keybox.h"
 
 
-/* Common definition part of all backend handle.  */
+/* Common definition part of all backend handle.  All definitions of
+ * this structure must start with these fields.  */
 struct backend_handle_s
 {
   enum database_types db_type;
+  unsigned int backend_id;
 };
 
 
@@ -45,6 +48,7 @@ strdbtype (enum database_types t)
   switch (t)
     {
     case DB_TYPE_NONE: return "none";
+    case DB_TYPE_CACHE:return "cache";
     case DB_TYPE_KBX:  return "keybox";
     }
   return "?";
@@ -76,6 +80,9 @@ be_generic_release_backend (ctrl_t ctrl, backend_handle_t hd)
     case DB_TYPE_NONE:
       xfree (hd);
       break;
+    case DB_TYPE_CACHE:
+      be_cache_release_resource (ctrl, hd);
+      break;
     case DB_TYPE_KBX:
       be_kbx_release_resource (ctrl, hd);
       break;
@@ -104,16 +111,53 @@ be_release_request (db_request_t req)
 }
 
 
-/* Return the public key (BUFFER,BUFLEN) which has the type
+/* Given the backend handle BACKEND_HD and the REQUEST find or
- * PUBVKEY_TYPE to the caller.  Owenership of BUFFER is taken by thgis
+ * allocate a request part for that backend and store it at R_PART.
- * function even in the error case.  */
+ * On error R_PART is set to NULL and an error returned.  */
 gpg_error_t
-be_return_pubkey (ctrl_t ctrl, void *buffer, size_t buflen,
+be_find_request_part (backend_handle_t backend_hd, db_request_t request,
-                  enum pubkey_types pubkey_type)
+                      db_request_part_t *r_part)
 {
   gpg_error_t err;
+  db_request_part_t part;
 
-  err = status_printf (ctrl, "PUBKEY_TYPE", "%d", pubkey_type);
+  for (part = request->part; part; part = part->next)
+    if (part->backend_id == backend_hd->backend_id)
+      break;
+  if (!part)
+    {
+      part = xtrycalloc (1, sizeof *part);
+      if (!part)
+        return gpg_error_from_syserror ();
+      part->backend_id = backend_hd->backend_id;
+      if (backend_hd->db_type == DB_TYPE_KBX)
+        {
+          err = be_kbx_init_request_part (backend_hd, part);
+          if (err)
+            {
+              xfree (part);
+              return err;
+            }
+        }
+      part->next = request->part;
+      request->part = part;
+    }
+  *r_part = part;
+  return 0;
+}
+
+
+/* Return the public key (BUFFER,BUFLEN) which has the type
+ * PUBKEY_TYPE to the caller.  */
+gpg_error_t
+be_return_pubkey (ctrl_t ctrl, const void *buffer, size_t buflen,
+                  enum pubkey_types pubkey_type, const unsigned char *ubid)
+{
+  gpg_error_t err;
+  char hexubid[41];
+
+  bin2hex (ubid, 20, hexubid);
+  err = status_printf (ctrl, "PUBKEY_INFO", "%d %s", pubkey_type, hexubid);
   if (err)
     goto leave;
 
@@ -123,6 +167,5 @@ be_return_pubkey (ctrl_t ctrl, void *buffer, size_t buflen,
     err = kbxd_write_data_line(ctrl, buffer, buflen);
 
  leave:
-  xfree (buffer);
   return err;
 }

kbx/backend.h

@@ -31,6 +31,7 @@ typedef struct keybox_handle *KEYBOX_HANDLE;
 enum database_types
   {
    DB_TYPE_NONE,   /* No database at all (unitialized etc.).  */
+   DB_TYPE_CACHE,  /* The cache backend (backend-cache.c).    */
    DB_TYPE_KBX     /* Keybox type database (backend-kbx.c).   */
   };
 
@@ -42,7 +43,7 @@ struct backend_handle_s;
 typedef struct backend_handle_s *backend_handle_t;
 
 
-/* Object to store backend specific databsde information per database
+/* Object to store backend specific database information per database
  * handle.  */
 struct db_request_part_s
 {
@@ -53,6 +54,14 @@ struct db_request_part_s
 
   /* The handle used for a KBX backend or NULL.  */
   KEYBOX_HANDLE kbx_hd;
+
+  /* For the CACHE backend the indices into the bloblist for each
+   * index type.  */
+  struct {
+    unsigned int fpr;
+    unsigned int kid;
+    unsigned int grip;
+  } cache_seqno;
 };
 typedef struct db_request_part_s *db_request_part_t;
 
@@ -63,11 +72,24 @@ struct db_request_s
 {
   unsigned int any_search:1;  /* Any search has been done.  */
   unsigned int any_found:1;   /* Any object has been found.  */
+  unsigned int last_cached_valid:1; /* see below */
+  unsigned int last_cached_final:1; /* see below */
+  unsigned int last_cached_fprlen:8;/* see below */
 
   db_request_part_t part;
 
   /* Counter to track the next to be searched database index.  */
   unsigned int next_dbidx;
+
+  /* The last UBID found in the cache and the corresponding keyid and,
+   * if found via fpr, the fingerprint.  For the LAST_CACHE_FPRLEN see
+   * above.  The entry here is only valid if LAST_CACHE_VALID is set;
+   * if LAST_CACHE_FINAL is also set, this indicates that no further
+   * database searches are required.  */
+  unsigned char last_cached_ubid[20];
+  u32 last_cached_kid_h;
+  u32 last_cached_kid_l;
+  unsigned char last_cached_fpr[32];
 };
 
 
@@ -77,8 +99,26 @@ const char *strdbtype (enum database_types t);
 unsigned int be_new_backend_id (void);
 void be_generic_release_backend (ctrl_t ctrl, backend_handle_t hd);
 void be_release_request (db_request_t req);
-gpg_error_t be_return_pubkey (ctrl_t ctrl, void *buffer, size_t buflen,
+gpg_error_t be_find_request_part (backend_handle_t backend_hd,
-                              enum pubkey_types pubkey_type);
+                                  db_request_t request,
+                                  db_request_part_t *r_part);
+gpg_error_t be_return_pubkey (ctrl_t ctrl, const void *buffer, size_t buflen,
+                              enum pubkey_types pubkey_type,
+                              const unsigned char *ubid);
+
+
+/*-- backend-cache.c --*/
+gpg_error_t be_cache_add_resource (ctrl_t ctrl, backend_handle_t *r_hd);
+void be_cache_release_resource (ctrl_t ctrl, backend_handle_t hd);
+gpg_error_t be_cache_search (ctrl_t ctrl, backend_handle_t backend_hd,
+                             db_request_t request,
+                             KEYDB_SEARCH_DESC *desc, unsigned int ndesc);
+void be_cache_mark_final (ctrl_t ctrl, db_request_t request);
+void be_cache_pubkey (ctrl_t ctrl, const unsigned char *ubid,
+                      const void *blob, unsigned int bloblen,
+                      enum pubkey_types pubkey_type);
+void be_cache_not_found (ctrl_t ctrl, enum pubkey_types pubkey_type,
+                         KEYDB_SEARCH_DESC *desc, unsigned int ndesc);
 
 
 /*-- backend-kbx.c --*/
@@ -87,9 +127,13 @@ gpg_error_t be_kbx_add_resource (ctrl_t ctrl, backend_handle_t *r_hd,
 void be_kbx_release_resource (ctrl_t ctrl, backend_handle_t hd);
 
 void be_kbx_release_kbx_hd (KEYBOX_HANDLE kbx_hd);
+gpg_error_t be_kbx_init_request_part (backend_handle_t backend_hd,
+                                      db_request_part_t part);
 gpg_error_t be_kbx_search (ctrl_t ctrl, backend_handle_t hd,
                            db_request_t request,
                            KEYDB_SEARCH_DESC *desc, unsigned int ndesc);
+gpg_error_t be_kbx_seek (ctrl_t ctrl, backend_handle_t backend_hd,
+                         db_request_t request, unsigned char *ubid);
 
 
 #endif /*KBX_BACKEND_H*/

kbx/frontend.c

@@ -67,7 +67,7 @@ take_read_lock (ctrl_t ctrl)
 
 
 /* Release a lock.  It is valid to call this even if no lock has been
- * taken which which case this is a nop.  */
+ * taken in which case this is a nop.  */
 static void
 release_lock (ctrl_t ctrl)
 {
@@ -87,12 +87,17 @@ kbxd_add_resource  (ctrl_t ctrl, const char *filename_arg, int readonly)
 {
   gpg_error_t err;
   char *filename;
-  enum database_types db_type;
+  enum database_types db_type = 0;
   backend_handle_t handle = NULL;
   unsigned int n, dbidx;
 
   /* Do tilde expansion etc. */
-  if (strchr (filename_arg, DIRSEP_C)
+  if (!strcmp (filename_arg, "[cache]"))
+    {
+      filename = xstrdup (filename_arg);
+      db_type = DB_TYPE_CACHE;
+    }
+  else if (strchr (filename_arg, DIRSEP_C)
 #ifdef HAVE_W32_SYSTEM
       || strchr (filename_arg, '/')  /* Windows also accepts a slash.  */
 #endif
@@ -102,8 +107,20 @@ kbxd_add_resource  (ctrl_t ctrl, const char *filename_arg, int readonly)
     filename = make_filename (gnupg_homedir (), GNUPG_PUBLIC_KEYS_DIR,
                               filename_arg, NULL);
 
+  /* If this is the first call to the function and the request is not
+   * for the cache backend, add the cache backend so that it will
+   * always be the first to be queried.  */
+  if (!no_of_databases && !db_type)
+    {
+      err = kbxd_add_resource (ctrl, "[cache]", 0);
+      if (err)
+        goto leave;
+    }
+
   n = strlen (filename);
-  if (n > 4 && !strcmp (filename + n - 4, ".kbx"))
+  if (db_type)
+    ; /* We already know it.  */
+  else if (n > 4 && !strcmp (filename + n - 4, ".kbx"))
     db_type = DB_TYPE_KBX;
   else
     {
@@ -118,6 +135,10 @@ kbxd_add_resource  (ctrl_t ctrl, const char *filename_arg, int readonly)
     case DB_TYPE_NONE: /* NOTREACHED */
       break;
 
+    case DB_TYPE_CACHE:
+      err = be_cache_add_resource (ctrl, &handle);
+      break;
+
     case DB_TYPE_KBX:
       err = be_kbx_add_resource (ctrl, &handle, filename, readonly);
       break;
@@ -134,8 +155,8 @@ kbxd_add_resource  (ctrl_t ctrl, const char *filename_arg, int readonly)
       /* No table yet or table is full.  */
       if (!databases)
         {
-          /* Create first set of data bases.  Note that the type for all
+          /* Create first set of databases.  Note that the initial
-           * entries is DB_TYPE_NONE.  */
+           * type for all entries is DB_TYPE_NONE.  */
           dbidx = 4;
           databases = xtrycalloc (dbidx, sizeof *databases);
           if (!databases)
@@ -204,6 +225,7 @@ kbxd_search (ctrl_t ctrl, KEYDB_SEARCH_DESC *desc, unsigned int ndesc,
   unsigned int dbidx;
   db_desc_t db;
   db_request_t request;
+  int start_at_ubid = 0;
 
   if (DBG_CLOCK)
     log_clock ("%s: enter", __func__);
@@ -248,6 +270,10 @@ kbxd_search (ctrl_t ctrl, KEYDB_SEARCH_DESC *desc, unsigned int ndesc,
             case DB_TYPE_NONE: /* NOTREACHED */
               break;
 
+            case DB_TYPE_CACHE:
+              err = 0; /* Nothing to do.  */
+              break;
+
             case DB_TYPE_KBX:
               err = be_kbx_search (ctrl, db->backend_handle, request, NULL, 0);
               break;
@@ -278,7 +304,10 @@ kbxd_search (ctrl_t ctrl, KEYDB_SEARCH_DESC *desc, unsigned int ndesc,
   request->next_dbidx = dbidx;
   if (!(dbidx < no_of_databases))
     {
-      /* All databases have been searched.  */
+      /* All databases have been searched.  Put the non-found mark
+       * into the cache for all descriptors.
+       * FIXME: We need to see which pubkey type we need to insert.  */
+      be_cache_not_found (ctrl, PUBKEY_TYPE_UNKNOWN, desc, ndesc);
       err = gpg_error (GPG_ERR_NOT_FOUND);
       goto leave;
     }
@@ -292,9 +321,32 @@ kbxd_search (ctrl_t ctrl, KEYDB_SEARCH_DESC *desc, unsigned int ndesc,
       err = gpg_error (GPG_ERR_INTERNAL);
       break;
 
+    case DB_TYPE_CACHE:
+      err = be_cache_search (ctrl, db->backend_handle, request,
+                             desc, ndesc);
+      /* Expected error codes from the cache lookup are:
+       *  0 - found and returned via the cache
+       *  GPG_ERR_NOT_FOUND - marked in the cache as not available
+       *  GPG_ERR_EOF - cache miss. */
+      break;
+
     case DB_TYPE_KBX:
+      if (start_at_ubid)
+        {
+          /* We need to set the startpoint for the search.  */
+          err = be_kbx_seek (ctrl, db->backend_handle, request,
+                             request->last_cached_ubid);
+          if (err)
+            {
+              log_debug ("%s: seeking %s to an UBID failed: %s\n",
+                         __func__, strdbtype (db->db_type), gpg_strerror (err));
+              break;
+            }
+        }
       err = be_kbx_search (ctrl, db->backend_handle, request,
                            desc, ndesc);
+      if (start_at_ubid && gpg_err_code (err) == GPG_ERR_EOF)
+        be_cache_mark_final (ctrl, request);
       break;
     }
 
@@ -303,10 +355,19 @@ kbxd_search (ctrl_t ctrl, KEYDB_SEARCH_DESC *desc, unsigned int ndesc,
                __func__, strdbtype (db->db_type), dbidx, no_of_databases,
                gpg_strerror (err));
   request->any_search = 1;
+  start_at_ubid = 0;
   if (!err)
-    request->any_found = 1;
+    {
+      request->any_found = 1;
+    }
   else if (gpg_err_code (err) == GPG_ERR_EOF)
     {
+      if (db->db_type == DB_TYPE_CACHE && request->last_cached_valid)
+        {
+          if (request->last_cached_final)
+            goto leave;
+          start_at_ubid = 1;
+        }
       request->next_dbidx++;
       goto next_db;
     }

kbx/kbxserver.c

@@ -236,7 +236,7 @@ kbxd_write_data_line (ctrl_t ctrl, const void *buffer_arg, size_t size)
 static gpg_error_t
 leave_cmd (assuan_context_t ctx, gpg_error_t err)
 {
-  if (err)
+  if (err && opt.verbose)
     {
       const char *name = assuan_get_command_name (ctx);
       if (!name)

kbx/keybox-search-desc.h

@@ -42,6 +42,7 @@ typedef enum {
   KEYDB_SEARCH_MODE_SN,
   KEYDB_SEARCH_MODE_SUBJECT,
   KEYDB_SEARCH_MODE_KEYGRIP,
+  KEYDB_SEARCH_MODE_UBID,
   KEYDB_SEARCH_MODE_FIRST,
   KEYDB_SEARCH_MODE_NEXT
 } KeydbSearchMode;
@@ -79,6 +80,7 @@ struct keydb_search_desc
     unsigned char fpr[32];
     u32 kid[2]; /* Note that this is in native endianness.  */
     unsigned char grip[20];
+    unsigned char ubid[20];
   } u;
   byte fprlen;  /* Only used with KEYDB_SEARCH_MODE_FPR.  */
   int exact;    /* Use exactly this key ('!' suffix in gpg).  */

kbx/keybox-search.c

@@ -696,6 +696,26 @@ has_keygrip (KEYBOXBLOB blob, const unsigned char *grip)
   return 0;
 }
 
+static inline int
+has_ubid (KEYBOXBLOB blob, const unsigned char *ubid)
+{
+  size_t length;
+  const unsigned char *buffer;
+  size_t image_off, image_len;
+  unsigned char ubid_blob[20];
+
+  buffer = _keybox_get_blob_image (blob, &length);
+  if (length < 40)
+    return 0; /*GPG_ERR_TOO_SHORT*/
+  image_off = get32 (buffer+8);
+  image_len = get32 (buffer+12);
+  if ((uint64_t)image_off+(uint64_t)image_len > (uint64_t)length)
+    return 0; /*GPG_ERR_TOO_SHORT*/
+
+  gcry_md_hash_buffer (GCRY_MD_SHA1, ubid_blob, buffer + image_off, image_len);
+
+  return !memcmp (ubid, ubid_blob, 20);
+}
 
 static inline int
 has_issuer (KEYBOXBLOB blob, const char *name)
@@ -1119,6 +1139,10 @@ keybox_search (KEYBOX_HANDLE hd, KEYBOX_SEARCH_DESC *desc, size_t ndesc,
               if (has_keygrip (blob, desc[n].u.grip))
                 goto found;
               break;
+            case KEYDB_SEARCH_MODE_UBID:
+              if (has_ubid (blob, desc[n].u.ubid))
+                goto found;
+              break;
             case KEYDB_SEARCH_MODE_FIRST:
               goto found;
               break;