Add self-signature verification caching

NEWS

@@ -1,4 +1,13 @@
         
+    * The verification status of self-signatures are now cached. To increase
+      the speed of key list operations for existing keys you can do the
+      following in yout GnuPG homedir (~/.gnupg): 
+        $ cp pubring.gpg pubring.gpg.save
+        $ gpg --export-all >x
+        $ rm pubring.gpg
+        $ gpg --import x
+      Note, that only v4 keys (i.e no old RSA keys) benefit from this caching.
+
     * WARNING: The semantics of --verify have changed to address a problem
       with detached signature detection. --verify now ignores signed material
       given on stdin unless this is requested by using a "-" as the name for

TODO

@@ -1,6 +1,10 @@
 
+  * Dlopen does not yet work under W32.
+
+  * key selection is far too slow (due to all the signature checks)
+
   * check whether we can remove all the expire stuff in trustdb because this
-    is now done getkey.  
+    is now done in getkey.  
 
   * ask for alternate filename?
 

g10/ChangeLog

@@ -1,3 +1,19 @@
+2001-03-05  Werner Koch  <wk@gnupg.org>
+
+	* packet.h: Replaced sigsubpkt_t value 101 by PRIV_VERIFY_CACHE.
+	We have never used the old value, so we can do this without any harm.
+	* parse-packet.c (dump_sig_subpkt): Ditto.
+	(parse_one_sig_subpkt): Parse that new sub packet.
+	* build-packet.c (build_sig_subpkt): Removed the old one from the
+	hashed area.
+	(delete_sig_subpkt): New.
+	(build_sig_subpkt): Allow an update of that new subpkt.
+	* sig-check.c (check_key_signature2): Add verification caching
+	(cache_selfsig_result): New.
+	* export.c (do_export_stream): Delete that sig subpkt before exporting.
+	* import.c (remove_bad_stuff): New.
+	(import): Apply that function to all imported data 
+
 2001-03-03  Werner Koch  <wk@gnupg.org>
 
 	* getkey.c: Introduced a new lookup context flag "exact" and used 

g10/build-packet.c

@@ -648,13 +648,83 @@ find_subpkt( byte *buffer, sigsubpkttype_t reqtype,
     return NULL;
 }
 
+/****************
+ * Delete all subpackets of type REQTYPE and return the number of bytes
+ * which are now unused at the end of the buffer.
+ */
+size_t
+delete_sig_subpkt( byte *buffer, sigsubpkttype_t reqtype )
+{
+    int buflen, orig_buflen;
+    sigsubpkttype_t type;
+    byte *bufstart, *orig_buffer;
+    size_t n;
+    size_t unused = 0;
+    int okay = 0;
+
+    if( !buffer )
+	return 0;
+    orig_buffer = buffer;
+    buflen = (*buffer << 8) | buffer[1];
+    buffer += 2;
+    orig_buflen = buflen;
+    for(;;) {
+	if( !buflen ) {
+            okay = 1;
+            break;
+        }
+	bufstart = buffer;
+	n = *buffer++; buflen--;
+	if( n == 255 ) {
+	    if( buflen < 4 )
+		break;
+	    n = (buffer[0] << 24) | (buffer[1] << 16)
+				  | (buffer[2] << 8) | buffer[3];
+	    buffer += 4;
+	    buflen -= 4;
+	}
+	else if( n >= 192 ) {
+	    if( buflen < 2 )
+		break;
+	    n = (( n - 192 ) << 8) + *buffer + 192;
+	    buffer++;
+	    buflen--;
+	}
+	if( buflen < n )
+	    break;
+	type = *buffer & 0x7f;
+	if( type == reqtype ) {
+	    buffer++;
+            buflen--;
+	    n--;
+	    if( n > buflen )
+		break;
+            memmove (bufstart, buffer + n, n + (buffer-bufstart)); /* shift */
+            unused += n + (buffer-bufstart);
+            buffer = bufstart;
+            buflen -= n;
+	}
+        else {
+            buffer += n; buflen -=n;
+        }
+    }
+
+    if (!okay)
+        log_error("delete_subpkt: buffer shorter than subpacket\n");
+    assert (unused <= orig_buflen);
+    orig_buflen -= unused;
+    orig_buffer[0] = (orig_buflen >> 8) & 0xff;
+    orig_buffer[1] = orig_buflen & 0xff;
+    return unused;
+}
+
 
 /****************
  * Create or update a signature subpacket for SIG of TYPE.
  * This functions knows where to put the data (hashed or unhashed).
- * The function may move data from the unhased part to the hashed one.
+ * The function may move data from the unhashed part to the hashed one.
  * Note: All pointers into sig->[un]hashed are not valid after a call
- * to this function.  The data to but into the subpaket should be
+ * to this function.  The data to put into the subpaket should be
  * in buffer with a length of buflen.
  */
 void
@@ -666,6 +736,7 @@ build_sig_subpkt( PKT_signature *sig, sigsubpkttype_t type,
     int found=0;
     int critical, hashed, realloced;
     size_t n, n0;
+    size_t unused = 0;
 
     critical = (type & SIGSUBPKT_FLAG_CRITICAL);
     type &= ~SIGSUBPKT_FLAG_CRITICAL;
@@ -677,6 +748,12 @@ build_sig_subpkt( PKT_signature *sig, sigsubpkttype_t type,
     else if( (data = find_subpkt( sig->unhashed_data, type, &hlen, &dlen )))
 	found = 2;
 
+    if (found==2 && type == SIGSUBPKT_PRIV_VERIFY_CACHE) {
+        unused = delete_sig_subpkt (sig->unhashed_data, type);
+        assert (unused);
+        found = 0;
+    }
+
     if( found )
 	log_bug("build_sig_packet: update nyi\n");
     if( (buflen+1) >= 8384 )
@@ -688,7 +765,6 @@ build_sig_subpkt( PKT_signature *sig, sigsubpkttype_t type,
 
     switch( type ) {
       case SIGSUBPKT_SIG_CREATED:
-      case SIGSUBPKT_PRIV_ADD_SIG:
       case SIGSUBPKT_PREF_SYM:
       case SIGSUBPKT_PREF_HASH:
       case SIGSUBPKT_PREF_COMPR:
@@ -713,10 +789,18 @@ build_sig_subpkt( PKT_signature *sig, sigsubpkttype_t type,
 	n0 = sig->unhashed_data ? ((*sig->unhashed_data << 8)
 				      | sig->unhashed_data[1]) : 0;
 	n = n0 + nlen + 1 + buflen; /* length, type, buffer */
+        if ( sig->unhashed_data && (nlen + 1 + buflen) <= unused ) {
+            /* does fit into the freed area */
+            data = sig->unhashed_data;
+            realloced = 1;
+            log_debug ("updating area of type %d\n", type );
+        }
+        else {
             realloced = !!sig->unhashed_data;
             data = sig->unhashed_data ? m_realloc( sig->unhashed_data, n+2 )
                                       : m_alloc( n+2 );
         }
+    }
 
     if( critical )
 	type |= SIGSUBPKT_FLAG_CRITICAL;

g10/export.c

@@ -197,6 +197,10 @@ do_export_stream( IOBUF out, STRLIST users, int secret, int onlyrfc, int *any )
 				       SIGSUBPKT_EXPORTABLE, NULL );
 		if( p && !*p )
 		    continue; /* not exportable */
+
+                /* delete our verification cache */
+                delete_sig_subpkt (node->pkt->pkt.signature->unhashed_data,
+                                   SIGSUBPKT_PRIV_VERIFY_CACHE);
 	    }
 
 	    if( secret == 2 && node->pkt->pkttype == PKT_SECRET_KEY ) {

g10/g10.c

@@ -108,6 +108,7 @@ enum cmd_and_opt_values { aNull = 0,
     aEnArmor,
     aGenRandom,
     aPipeMode,
+    aRefreshCaches,
 
     oTextmode,
     oFingerprint,

g10/import.c

@@ -57,6 +57,7 @@ static struct {
 static int import( IOBUF inp, int fast, const char* fname, int allow_secret );
 static void print_stats(void);
 static int read_block( IOBUF a, PACKET **pending_pkt, KBNODE *ret_root );
+static void remove_bad_stuff (KBNODE keyblock);
 static int import_one( const char *fname, KBNODE keyblock, int fast );
 static int import_secret_one( const char *fname, KBNODE keyblock, int allow );
 static int import_revoke_cert( const char *fname, KBNODE node );
@@ -171,6 +172,7 @@ import( IOBUF inp, int fast, const char* fname, int allow_secret )
     }
 
     while( !(rc = read_block( inp, &pending_pkt, &keyblock) )) {
+        remove_bad_stuff (keyblock);
 	if( keyblock->pkt->pkttype == PKT_PUBLIC_KEY )
 	    rc = import_one( fname, keyblock, fast );
 	else if( keyblock->pkt->pkttype == PKT_SECRET_KEY ) 
@@ -349,6 +351,21 @@ read_block( IOBUF a, PACKET **pending_pkt, KBNODE *ret_root )
 }
 
 
+static void
+remove_bad_stuff (KBNODE keyblock)
+{
+    KBNODE node;
+
+    for (node=keyblock; node; node = node->next ) {
+        if( node->pkt->pkttype == PKT_SIGNATURE ) {
+            /* delete the subpackets we use for the verification cache */
+            delete_sig_subpkt (node->pkt->pkt.signature->unhashed_data,
+                               SIGSUBPKT_PRIV_VERIFY_CACHE);
+        }
+    }
+}
+
+
 /****************
  * Try to import one keyblock.	Return an error only in serious cases, but
  * never for an invalid keyblock.  It uses log_error to increase the

g10/keylist.c

@@ -515,7 +515,7 @@ list_keyblock( KBNODE keyblock, int secret )
 	    if( opt.with_colons )
 		printf(":%02x:", sig->sig_class );
 	    putchar('\n');
-	    /* FIXME: check or list other sigs here (subpkt PRIV_ADD_SIG)*/
+	    /* fixme: check or list other sigs here */
 	}
     }
     if( !any ) {/* oops, no user id */

g10/packet.h

@@ -272,7 +272,7 @@ typedef enum {
     SIGSUBPKT_KEY_FLAGS    =27, /* key flags */
     SIGSUBPKT_SIGNERS_UID  =28, /* signer's user id */
     SIGSUBPKT_REVOC_REASON =29, /* reason for revocation */
-    SIGSUBPKT_PRIV_ADD_SIG =101,/* signatur is also valid for this uid */
+    SIGSUBPKT_PRIV_VERIFY_CACHE =101, /* cache verification result */
 
     SIGSUBPKT_FLAG_CRITICAL=128
 } sigsubpkttype_t;
@@ -331,6 +331,7 @@ void hash_public_key( MD_HANDLE md, PKT_public_key *pk );
 void build_sig_subpkt( PKT_signature *sig, sigsubpkttype_t type,
 			const byte *buffer, size_t buflen );
 void build_sig_subpkt_from_sig( PKT_signature *sig );
+size_t delete_sig_subpkt( byte *buffer, sigsubpkttype_t type );
 
 /*-- free-packet.c --*/
 void free_symkey_enc( PKT_symkey_enc *enc );

g10/parse-packet.c

@@ -848,8 +848,8 @@ dump_sig_subpkt( int hashed, int type, int critical,
 		printf("%02X", buffer[i] );
 	}
         break;
-      case SIGSUBPKT_PRIV_ADD_SIG:
+      case SIGSUBPKT_PRIV_VERIFY_CACHE:
-	p = "signs additional user ID";
+	p = "verification cache";
 	break;
       default: p = "?"; break;
     }
@@ -900,13 +900,17 @@ parse_one_sig_subpkt( const byte *buffer, size_t n, int type )
           if ( n != 1 )
               break;
           return 0;   
-      case SIGSUBPKT_PRIV_ADD_SIG:
+      case SIGSUBPKT_PRIV_VERIFY_CACHE:
-	/* because we use private data, we check the GNUPG marker */
+        /* "GPG" 0x00 <mode> <stat>
-	if( n < 24 )
+         * where mode == 1: valid data, stat == 0: invalid signature
+         * stat == 1: valid signature 
+	 * (because we use private data, we check our marker) */
+	if( n < 6 )
 	    break;
-	if( buffer[0] != 'G' || buffer[1] != 'P' || buffer[2] != 'G' )
+	if( buffer[0] != 'G' || buffer[1] != 'P'
+            || buffer[2] != 'G' || buffer[3] )
 	    return -2;
-	return 3;
+	return 4;
       default: return -1;
     }
     return -3;

g10/sig-check.c

@@ -437,6 +437,31 @@ hash_uid_node( KBNODE unode, MD_HANDLE md, PKT_signature *sig )
     }
 }
 
+static void
+cache_selfsig_result ( PKT_signature *sig, int result )
+{
+    byte buf[6];
+
+    buf[0] = 'G';
+    buf[1] = 'P';
+    buf[2] = 'G';
+    buf[3] = 0;
+    if ( !result ) {
+        buf[4] = 1; /* mark cache valid */
+        buf[5] = 1; /* mark signature valid */
+    }
+    else if ( result == G10ERR_BAD_SIGN ) {
+        buf[4] = 1; /* mark cache valid */
+        buf[5] = 0; /* mark signature invalid */
+    }
+    else {
+        buf[4] = 0; /* mark cache invalid */
+        buf[5] = 0; 
+    }
+
+    build_sig_subpkt (sig, SIGSUBPKT_PRIV_VERIFY_CACHE, buf, 6 );
+}
+
 /****************
  * check the signature pointed to by NODE. This is a key signature.
  * If the function detects a self-signature, it uses the PK from
@@ -477,13 +502,33 @@ check_key_signature2( KBNODE root, KBNODE node, int *is_selfsig,
 		      sig->flags.valid? "good":"bad" );
   #endif
 
+    /* Check whether we have cached the result of a previous signature check.*/
+    {
+        const byte *p;
+        size_t len;
+
+        p = parse_sig_subpkt( sig->unhashed_data,
+                              SIGSUBPKT_PRIV_VERIFY_CACHE, &len );
+        if ( p && len >= 2 && p[0] == 1 ) { /* cache hit */
+	    if( is_selfsig ) {	
+		u32 keyid[2];	
+
+		keyid_from_pk( pk, keyid );
+		if( keyid[0] == sig->keyid[0] && keyid[1] == sig->keyid[1] )
+		    *is_selfsig = 1;
+	    }
+            return p[1] == 1? 0 : G10ERR_BAD_SIGN;
+        }
+    }
+
     if( (rc=check_digest_algo(algo)) )
 	return rc;
 
-    if( sig->sig_class == 0x20 ) {
+    if( sig->sig_class == 0x20 ) { /* key revocation */
 	md = md_open( algo, 0 );
 	hash_public_key( md, pk );
 	rc = do_check( pk, sig, md, r_expired );
+        cache_selfsig_result ( sig, rc );
 	md_close(md);
     }
     else if( sig->sig_class == 0x28 ) { /* subkey revocation */
@@ -494,6 +539,7 @@ check_key_signature2( KBNODE root, KBNODE node, int *is_selfsig,
 	    hash_public_key( md, pk );
 	    hash_public_key( md, snode->pkt->pkt.public_key );
 	    rc = do_check( pk, sig, md, r_expired );
+            cache_selfsig_result ( sig, rc );
 	    md_close(md);
 	}
 	else {
@@ -516,6 +562,7 @@ check_key_signature2( KBNODE root, KBNODE node, int *is_selfsig,
 	    hash_public_key( md, pk );
 	    hash_public_key( md, snode->pkt->pkt.public_key );
 	    rc = do_check( pk, sig, md, r_expired );
+            cache_selfsig_result ( sig, rc );
 	    md_close(md);
 	}
 	else {
@@ -537,6 +584,7 @@ check_key_signature2( KBNODE root, KBNODE node, int *is_selfsig,
 		if( is_selfsig )
 		    *is_selfsig = 1;
 		rc = do_check( pk, sig, md, r_expired );
+                cache_selfsig_result ( sig, rc );
 	    }
 	    else {
 		rc = do_signature_check( sig, md, r_expiredate, r_expired );