gpg: Add signature cache support to the keybox.

* g10/keydb.c (parse_keyblock_image): Add arg SIGSTATUS.
(keydb_get_keyblock): Handle it.
(build_keyblock_image): Add arg SIGSTATUS.
(keydb_insert_keyblock): Handle it.
* kbx/keybox-blob.c (pgp_create_sig_part): Add arg SIGSTATUS.
(_keybox_create_openpgp_blob): Ditto.
* kbx/kbxutil.c (import_openpgp): Adjust for above change.
* kbx/keybox.h (KEYBOX_FLAG_SIG_INFO): New.
* kbx/keybox-search.c (_keybox_get_flag_location): Handle new flag.
(keybox_get_keyblock): Add arg R_SIGSTATUS.
* kbx/keybox-update.c (keybox_insert_keyblock): Add arg SIGSTATUS.
--

With this change a key listing using the keybox format is now double
as fast as using a keyring.  The memory use dropped as well.  Measured
with about 1500 keys.

g10/keydb.c

@@ -617,13 +617,14 @@ unlock_all (KEYDB_HANDLE hd)
 
 
 static gpg_error_t
-parse_keyblock_image (iobuf_t iobuf, kbnode_t *r_keyblock)
+parse_keyblock_image (iobuf_t iobuf, const u32 *sigstatus, kbnode_t *r_keyblock)
 {
   gpg_error_t err;
   PACKET *pkt;
   kbnode_t keyblock = NULL;
-  kbnode_t node;
+  kbnode_t node, *tail;
   int in_cert, save_mode;
+  u32 n_sigs;
 
   *r_keyblock = NULL;
 
@@ -633,6 +634,8 @@ parse_keyblock_image (iobuf_t iobuf, kbnode_t *r_keyblock)
   init_packet (pkt);
   save_mode = set_packet_list_mode (0);
   in_cert = 0;
+  n_sigs = 0;
+  tail = NULL;
   while ((err = parse_packet (iobuf, pkt)) != -1)
     {
       if (gpg_err_code (err) == GPG_ERR_UNKNOWN_PACKET)
@@ -665,25 +668,57 @@ parse_keyblock_image (iobuf_t iobuf, kbnode_t *r_keyblock)
 
       if (!in_cert && pkt->pkttype != PKT_PUBLIC_KEY)
         {
-          log_error ("error: first packet in a keybox blob is not a "
+          log_error ("parse_keyblock_image: first packet in a keybox blob "
-                     "public key packet\n");
+                     "is not a public key packet\n");
           err = gpg_error (GPG_ERR_INV_KEYRING);
           break;
         }
       if (in_cert && (pkt->pkttype == PKT_PUBLIC_KEY
                       || pkt->pkttype == PKT_SECRET_KEY))
         {
-          log_error ("error: multiple keyblocks in a keybox blob\n");
+          log_error ("parse_keyblock_image: "
+                     "multiple keyblocks in a keybox blob\n");
           err = gpg_error (GPG_ERR_INV_KEYRING);
           break;
         }
       in_cert = 1;
 
+      if (pkt->pkttype == PKT_SIGNATURE && sigstatus)
+        {
+          PKT_signature *sig = pkt->pkt.signature;
+
+          n_sigs++;
+          if (n_sigs > sigstatus[0])
+            {
+              log_error ("parse_keyblock_image: "
+                         "more signatures than found in the meta data\n");
+              err = gpg_error (GPG_ERR_INV_KEYRING);
+              break;
+
+            }
+          if (sigstatus[n_sigs])
+            {
+              sig->flags.checked = 1;
+              if (sigstatus[n_sigs] == 1 )
+                ; /* missing key */
+              else if (sigstatus[n_sigs] == 2 )
+                ; /* bad signature */
+              else if (sigstatus[n_sigs] < 0x10000000)
+                ; /* bad flag */
+              else
+                {
+                  sig->flags.valid = 1;
+                  /* Fixme: Shall we set the expired flag here?  */
+                }
+            }
+        }
+
       node = new_kbnode (pkt);
       if (!keyblock)
         keyblock = node;
       else
-        add_kbnode (keyblock, node);
+        *tail = node;
+      tail = &node->next;
       pkt = xtrymalloc (sizeof *pkt);
       if (!pkt)
         {
@@ -697,6 +732,12 @@ parse_keyblock_image (iobuf_t iobuf, kbnode_t *r_keyblock)
   if (err == -1 && keyblock)
     err = 0; /* Got the entire keyblock.  */
 
+  if (!err && sigstatus && n_sigs != sigstatus[0])
+    {
+      log_error ("parse_keyblock_image: signature count does not match\n");
+      err = gpg_error (GPG_ERR_INV_KEYRING);
+    }
+
   if (err)
     release_kbnode (keyblock);
   else
@@ -737,11 +778,14 @@ keydb_get_keyblock (KEYDB_HANDLE hd, KBNODE *ret_kb)
     case KEYDB_RESOURCE_TYPE_KEYBOX:
       {
         iobuf_t iobuf;
+        u32 *sigstatus;
 
-        err = keybox_get_keyblock (hd->active[hd->found].u.kb, &iobuf);
+        err = keybox_get_keyblock (hd->active[hd->found].u.kb,
+                                   &iobuf, &sigstatus);
         if (!err)
           {
-            err = parse_keyblock_image (iobuf, ret_kb);
+            err = parse_keyblock_image (iobuf, sigstatus, ret_kb);
+            xfree (sigstatus);
             iobuf_close (iobuf);
           }
       }
@@ -753,18 +797,33 @@ keydb_get_keyblock (KEYDB_HANDLE hd, KBNODE *ret_kb)
 
 
 /* Build a keyblock image from KEYBLOCK.  Returns 0 on success and
-   only then stores a new iobuf object at R_IOBUF.  */
+   only then stores a new iobuf object at R_IOBUF and a signature
+   status vecotor at R_SIGSTATUS.  */
 static gpg_error_t
-build_keyblock_image (kbnode_t keyblock, iobuf_t *r_iobuf)
+build_keyblock_image (kbnode_t keyblock, iobuf_t *r_iobuf, u32 **r_sigstatus)
 {
   gpg_error_t err;
   iobuf_t iobuf;
   kbnode_t kbctx, node;
+  u32 n_sigs;
+  u32 *sigstatus;
 
   *r_iobuf = NULL;
+  *r_sigstatus = NULL;
+
+  /* Allocate a vector for the signature cache.  This is an array of
+     u32 values with the first value giving the number of elements to
+     follow and each element descriping the cache status of the
+     signature.  */
+  for (kbctx = NULL, n_sigs = 0; (node = walk_kbnode (keyblock, &kbctx, 0));)
+    if (node->pkt->pkttype == PKT_SIGNATURE)
+      n_sigs++;
+  sigstatus = xtrycalloc (1+n_sigs, sizeof *sigstatus);
+  if (!sigstatus)
+    return gpg_error_from_syserror ();
 
   iobuf = iobuf_temp ();
-  for (kbctx = NULL; (node = walk_kbnode (keyblock, &kbctx, 0)); )
+  for (kbctx = NULL, n_sigs = 0; (node = walk_kbnode (keyblock, &kbctx, 0));)
     {
       /* Make sure to use only packets valid on a keyblock.  */
       switch (node->pkt->pkttype)
@@ -787,9 +846,34 @@ build_keyblock_image (kbnode_t keyblock, iobuf_t *r_iobuf)
           iobuf_close (iobuf);
           return err;
         }
+
+      /* Build signature status vector.  */
+      if (node->pkt->pkttype == PKT_SIGNATURE)
+        {
+          PKT_signature *sig = node->pkt->pkt.signature;
+
+          n_sigs++;
+          /* Fixme: Detect tye "missing key" status.  */
+          if (sig->flags.checked)
+            {
+              if (sig->flags.valid)
+                {
+                  if (!sig->expiredate)
+                    sigstatus[n_sigs] = 0xffffffff;
+                  else if (sig->expiredate < 0x1000000)
+                    sigstatus[n_sigs] = 0x10000000;
+                  else
+                    sigstatus[n_sigs] = sig->expiredate;
+                }
+              else
+                sigstatus[n_sigs] = 0x00000002; /* Bad signature.  */
+            }
+        }
     }
+  sigstatus[0] = n_sigs;
 
   *r_iobuf = iobuf;
+  *r_sigstatus = sigstatus;
   return 0;
 }
 
@@ -876,13 +960,16 @@ keydb_insert_keyblock (KEYDB_HANDLE hd, kbnode_t kb)
            included in the keybox code.  Eventually we can change this
            kludge to have the caller pass the image.  */
         iobuf_t iobuf;
+        u32 *sigstatus;
 
-        err = build_keyblock_image (kb, &iobuf);
+        err = build_keyblock_image (kb, &iobuf, &sigstatus);
         if (!err)
           {
             err = keybox_insert_keyblock (hd->active[idx].u.kb,
                                           iobuf_get_temp_buffer (iobuf),
-                                          iobuf_get_temp_length (iobuf));
+                                          iobuf_get_temp_length (iobuf),
+                                          sigstatus);
+            xfree (sigstatus);
             iobuf_close (iobuf);
           }
       }

kbx/kbxutil.c

@@ -411,7 +411,8 @@ import_openpgp (const char *filename, int dryrun)
             dump_openpgp_key (&info, p);
           else
             {
-              err = _keybox_create_openpgp_blob (&blob, &info, p, nparsed, 0);
+              err = _keybox_create_openpgp_blob (&blob, &info, p, nparsed,
+                                                 NULL, 0);
               if (err)
                 {
                   fflush (stdout);

kbx/keybox-blob.c

@@ -408,13 +408,13 @@ pgp_create_uid_part (KEYBOXBLOB blob, keybox_openpgp_info_t info)
 
 
 static void
-pgp_create_sig_part (KEYBOXBLOB blob)
+pgp_create_sig_part (KEYBOXBLOB blob, u32 *sigstatus)
 {
   int n;
 
   for (n=0; n < blob->nsigs; n++)
     {
-      blob->sigs[n] = 0;  /* FIXME: check the signature here */
+      blob->sigs[n] = sigstatus? sigstatus[n+1] : 0;
     }
 }
 
@@ -658,12 +658,14 @@ create_blob_finish (KEYBOXBLOB blob)
   return 0;
 }
 
+
 
 gpg_error_t
 _keybox_create_openpgp_blob (KEYBOXBLOB *r_blob,
                              keybox_openpgp_info_t info,
                              const unsigned char *image,
                              size_t imagelen,
+                             u32 *sigstatus,
                              int as_ephemeral)
 {
   gpg_error_t err;
@@ -674,6 +676,11 @@ _keybox_create_openpgp_blob (KEYBOXBLOB *r_blob,
   if (!info->nuids || !info->nsigs)
     return gpg_error (GPG_ERR_BAD_PUBKEY);
 
+  /* If we have a signature status vector, check that the number of
+     elements matches the actual number of signatures.  */
+  if (sigstatus && sigstatus[0] != info->nsigs)
+    return gpg_error (GPG_ERR_INTERNAL);
+
   blob = xtrycalloc (1, sizeof *blob);
   if (!blob)
     return gpg_error_from_syserror ();
@@ -704,7 +711,7 @@ _keybox_create_openpgp_blob (KEYBOXBLOB *r_blob,
   if (err)
     goto leave;
   pgp_create_uid_part (blob, info);
-  pgp_create_sig_part (blob);
+  pgp_create_sig_part (blob, sigstatus);
 
   init_membuf (&blob->bufbuf, 1024);
   blob->buf = &blob->bufbuf;

kbx/keybox-defs.h

@@ -160,6 +160,7 @@ gpg_error_t _keybox_create_openpgp_blob (KEYBOXBLOB *r_blob,
                                          keybox_openpgp_info_t info,
                                          const unsigned char *image,
                                          size_t imagelen,
+                                         u32 *sigstatus,
                                          int as_ephemeral);
 #ifdef KEYBOX_WITH_X509
 int _keybox_create_x509_blob (KEYBOXBLOB *r_blob, ksba_cert_t cert,

kbx/keybox-search.c

@@ -102,7 +102,7 @@ _keybox_get_flag_location (const unsigned char *buffer, size_t length,
   size_t nkeys, keyinfolen;
   size_t nuids, uidinfolen;
   size_t nserial;
-  size_t nsigs, siginfolen;
+  size_t nsigs, siginfolen, siginfooff;
 
   switch (what)
     {
@@ -116,6 +116,7 @@ _keybox_get_flag_location (const unsigned char *buffer, size_t length,
     case KEYBOX_FLAG_OWNERTRUST:
     case KEYBOX_FLAG_VALIDITY:
     case KEYBOX_FLAG_CREATED_AT:
+    case KEYBOX_FLAG_SIG_INFO:
       if (length < 20)
         return GPG_ERR_INV_OBJ;
       /* Key info. */
@@ -140,6 +141,7 @@ _keybox_get_flag_location (const unsigned char *buffer, size_t length,
       if (pos+4 > length)
         return GPG_ERR_INV_OBJ ; /* Out of bounds. */
       /* Signature info. */
+      siginfooff = pos;
       nsigs = get16 (buffer + pos); pos += 2;
       siginfolen = get16 (buffer + pos); pos += 2;
       if (siginfolen < 4 )
@@ -158,6 +160,10 @@ _keybox_get_flag_location (const unsigned char *buffer, size_t length,
           *flag_size = 4;
           *flag_off += 1+2+4+4+4;
           break;
+        case KEYBOX_FLAG_SIG_INFO:
+          *flag_size = siginfolen * nsigs;
+          *flag_off = siginfooff;
+          break;
         default:
           break;
         }
@@ -961,15 +967,20 @@ keybox_search (KEYBOX_HANDLE hd, KEYBOX_SEARCH_DESC *desc, size_t ndesc)
 
 
 /* Return the last found keyblock.  Returns 0 on success and stores a
-   new iobuf at R_IOBUF in that case.  */
+   new iobuf at R_IOBUF and a signature status vector at R_SIGSTATUS
+   in that case.  */
 gpg_error_t
-keybox_get_keyblock (KEYBOX_HANDLE hd, iobuf_t *r_iobuf)
+keybox_get_keyblock (KEYBOX_HANDLE hd, iobuf_t *r_iobuf, u32 **r_sigstatus)
 {
-  const unsigned char *buffer;
+  gpg_error_t err;
+  const unsigned char *buffer, *p;
   size_t length;
   size_t image_off, image_len;
+  size_t siginfo_off, siginfo_len;
+  u32 *sigstatus, n, n_sigs, sigilen;
 
   *r_iobuf = NULL;
+  *r_sigstatus = NULL;
 
   if (!hd)
     return gpg_error (GPG_ERR_INV_VALUE);
@@ -987,6 +998,21 @@ keybox_get_keyblock (KEYBOX_HANDLE hd, iobuf_t *r_iobuf)
   if (image_off+image_len > length)
     return gpg_error (GPG_ERR_TOO_SHORT);
 
+  err = _keybox_get_flag_location (buffer, length, KEYBOX_FLAG_SIG_INFO,
+                                   &siginfo_off, &siginfo_len);
+  if (err)
+    return err;
+  n_sigs  = get16 (buffer + siginfo_off);
+  sigilen = get16 (buffer + siginfo_off + 2);
+  p = buffer + siginfo_off + 4;
+  sigstatus = xtrymalloc ((1+n_sigs) * sizeof *sigstatus);
+  if (!sigstatus)
+    return gpg_error_from_syserror ();
+  sigstatus[0] = n_sigs;
+  for (n=1; n <= n_sigs; n++, p += sigilen)
+    sigstatus[n] = get32 (p);
+
+  *r_sigstatus = sigstatus;
   *r_iobuf = iobuf_temp_with_content (buffer+image_off, image_len);
   return 0;
 }

kbx/keybox-update.c

@@ -371,9 +371,12 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob,
 }
 
 
-/* Insert the OpenPGP keyblock {IMAGE,IMAGELEN} into HD.  */
+/* Insert the OpenPGP keyblock {IMAGE,IMAGELEN} into HD.  SIGSTATUS is
+   a vector describing the status of the signatures; its first element
+   gives the number of following elements.  */
 gpg_error_t
-keybox_insert_keyblock (KEYBOX_HANDLE hd, const void *image, size_t imagelen)
+keybox_insert_keyblock (KEYBOX_HANDLE hd, const void *image, size_t imagelen,
+                        u32 *sigstatus)
 {
   gpg_error_t err;
   const char *fname;
@@ -400,7 +403,7 @@ keybox_insert_keyblock (KEYBOX_HANDLE hd, const void *image, size_t imagelen)
     return err;
   assert (nparsed <= imagelen);
   err = _keybox_create_openpgp_blob (&blob, &info, image, imagelen,
-                                     hd->ephemeral);
+                                     sigstatus, hd->ephemeral);
   _keybox_destroy_openpgp_info (&info);
   if (!err)
     {

kbx/keybox.h

@@ -54,7 +54,8 @@ typedef enum
     KEYBOX_FLAG_UID,        /* The user ID flags; requires an uid index. */
     KEYBOX_FLAG_UID_VALIDITY,/* The validity of a specific uid, requires
                                an uid index. */
-    KEYBOX_FLAG_CREATED_AT  /* The date the block was created. */
+    KEYBOX_FLAG_CREATED_AT, /* The date the block was created. */
+    KEYBOX_FLAG_SIG_INFO,   /* The signature info block.  */
   } keybox_flag_t;
 
 /* Flag values used with KEYBOX_FLAG_BLOB.  */
@@ -80,7 +81,8 @@ int keybox_lock (KEYBOX_HANDLE hd, int yes);
 int _keybox_write_header_blob (FILE *fp);
 
 /*-- keybox-search.c --*/
-gpg_error_t keybox_get_keyblock (KEYBOX_HANDLE hd, iobuf_t *r_iobuf);
+gpg_error_t keybox_get_keyblock (KEYBOX_HANDLE hd,
+                                 iobuf_t *r_iobuf, u32 **sigstatus);
 #ifdef KEYBOX_WITH_X509
 int keybox_get_cert (KEYBOX_HANDLE hd, ksba_cert_t *ret_cert);
 #endif /*KEYBOX_WITH_X509*/
@@ -92,7 +94,8 @@ int keybox_search (KEYBOX_HANDLE hd, KEYBOX_SEARCH_DESC *desc, size_t ndesc);
 
 /*-- keybox-update.c --*/
 gpg_error_t keybox_insert_keyblock (KEYBOX_HANDLE hd,
-                                    const void *image, size_t imagelen);
+                                    const void *image, size_t imagelen,
+                                    u32 *sigstatus);
 gpg_error_t keybox_update_keyblock (KEYBOX_HANDLE hd,
                                     const void *image, size_t imagelen);