From 6500f338a35f4148606480c79f3a0c1b0d15f13a Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Sat, 20 Jun 2015 15:03:32 +0200 Subject: [PATCH] gpg: Improve speed of --check-sigs and --lish-sigs. * g10/keydb.c (kid_list_t): New. (kid_not_found_table, n_kid_not_found_table): New. (kid_not_found_p, kid_not_found_insert, kid_not_found_flush): New. (keydb_insert_keyblock): Flush the new cache. (keydb_delete_keyblock): Ditto. (keydb_update_keyblock): Ditto. (keydb_search): Use the new cache. (keydb_dump_stats): New. * g10/gpg.c (g10_exit): Dump keydb stats. -- What we do here is to keep track of key searches by long keyids (as stored in all signatures) so that we do not need to scan the keybox again after we already found that this keyid will result in not-found. As soon as we change gpg to run as a co-process we should store this table per session because other instances of gpg may have updated the keybox without us knowing. On a test ring with gpg: 94721 good signatures gpg: 6831 bad signatures gpg: 150703 signatures not checked due to missing keys gpg: 5 signatures not checked due to errors gpg: keydb: kid_not_found_table: total: 14132 this new cache speeds a --check-sigs listing up from 28 minutes to less than 3 minutes. Signed-off-by: Werner Koch --- g10/gpg.c | 2 ++ g10/keydb.c | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++ g10/keydb.h | 2 ++ 3 files changed, 101 insertions(+) diff --git a/g10/gpg.c b/g10/gpg.c index b7b81c9e0..eebb668cd 100644 --- a/g10/gpg.c +++ b/g10/gpg.c @@ -4361,8 +4361,10 @@ g10_exit( int rc ) gcry_control (GCRYCTL_UPDATE_RANDOM_SEED_FILE); if (DBG_CLOCK) log_clock ("stop"); + if ( (opt.debug & DBG_MEMSTAT_VALUE) ) { + keydb_dump_stats (); gcry_control (GCRYCTL_DUMP_MEMORY_STATS); gcry_control (GCRYCTL_DUMP_RANDOM_STATS); } diff --git a/g10/keydb.c b/g10/keydb.c index 6c79903d6..71ea11326 100644 --- a/g10/keydb.c +++ b/g10/keydb.c @@ -75,6 +75,24 @@ struct keydb_handle }; +/* This object is used to keep a list of keyids in a linked list. */ +typedef struct kid_list_s +{ + struct kid_list_s *next; + u32 kid[2]; +} *kid_list_t; + +/* To avoid looking up a key by keyid where we know that it does not + yet exist, we keep a table of keyids where a search resulted in + not-found. This improves the --list-sigs and --check-sigs commands + substantively. To avoid extra complexity we clear the entire table + on any inert or update operation. The array is indexed by the + LSByte of the keyid. N_KID_NOT_FOUND_TABLE is the nu,ber of keys + in the table. */ +static kid_list_t kid_not_found_table[256]; +static unsigned int n_kid_not_found_table; + + /* This is a simple cache used to return the last result of a successful fingerprint search. This works only for keybox resources because (due to lack of a copy_keyblock function) we need to store @@ -100,6 +118,61 @@ static int lock_all (KEYDB_HANDLE hd); static void unlock_all (KEYDB_HANDLE hd); +/* Return true if the keyid KID is in the table of keyids whcih were + not found in a previous searches. */ +static int +kid_not_found_p (u32 *kid) +{ + kid_list_t k; + + for (k = kid_not_found_table[kid[0] % 256]; k; k = k->next) + if (k->kid[0] == kid[0] && k->kid[1] == kid[1]) + return 1; + return 0; +} + + +/* Put the keyid KID into the table of keyids whcih were not found in + previous searches. Note that there is no check whether the keyid + is already in the table, thus kid_not_found_p() should be used prior. */ +static void +kid_not_found_insert (u32 *kid) +{ + kid_list_t k; + + k = xmalloc (sizeof *k); + k->kid[0] = kid[0]; + k->kid[1] = kid[1]; + k->next = kid_not_found_table[kid[0]%256]; + kid_not_found_table[kid[0]%256] = k; + n_kid_not_found_table++; +} + + +/* Flush the entire table of keyids whche were not found in previous + searches. */ +static void +kid_not_found_flush (void) +{ + kid_list_t k, knext; + int i; + + if (!n_kid_not_found_table) + return; + + for (i=0; i < DIM(kid_not_found_table); i++) + { + for (k = kid_not_found_table[i]; k; k = knext) + { + knext = k->next; + xfree (k); + } + kid_not_found_table[i] = NULL; + } + n_kid_not_found_table = 0; +} + + static void keyblock_cache_clear (void) { @@ -529,6 +602,12 @@ keydb_add_resource (const char *url, unsigned int flags) } +void +keydb_dump_stats (void) +{ + if (n_kid_not_found_table) + log_info ("keydb: kid_not_found_table: total: %u\n", n_kid_not_found_table); +} KEYDB_HANDLE @@ -1151,6 +1230,7 @@ keydb_update_keyblock (KEYDB_HANDLE hd, kbnode_t kb) if (!hd) return gpg_error (GPG_ERR_INV_ARG); + kid_not_found_flush (); keyblock_cache_clear (); if (hd->found < 0 || hd->found >= hd->used) @@ -1204,6 +1284,7 @@ keydb_insert_keyblock (KEYDB_HANDLE hd, kbnode_t kb) if (!hd) return gpg_error (GPG_ERR_INV_ARG); + kid_not_found_flush (); keyblock_cache_clear (); if (opt.dry_run) @@ -1266,6 +1347,7 @@ keydb_delete_keyblock (KEYDB_HANDLE hd) if (!hd) return gpg_error (GPG_ERR_INV_ARG); + kid_not_found_flush (); keyblock_cache_clear (); if (hd->found < 0 || hd->found >= hd->used) @@ -1509,6 +1591,15 @@ keydb_search (KEYDB_HANDLE hd, KEYDB_SEARCH_DESC *desc, if (DBG_CACHE) dump_search_desc (hd, "keydb_search", desc, ndesc); + + if (ndesc == 1 && desc[0].mode == KEYDB_SEARCH_MODE_LONG_KID + && kid_not_found_p (desc[0].u.kid)) + { + if (DBG_CLOCK) + log_clock ("keydb_search leave (not found, cached)"); + return gpg_error (GPG_ERR_NOT_FOUND); + } + /* NB: If one of the exact search modes below is used in a loop to walk over all keys (with the same fingerprint) the caching must have been disabled for the handle. */ @@ -1567,6 +1658,12 @@ keydb_search (KEYDB_HANDLE hd, KEYDB_SEARCH_DESC *desc, memcpy (keyblock_cache.fpr, desc[0].u.fpr, 20); } + if (gpg_err_code (rc) == GPG_ERR_NOT_FOUND + && ndesc == 1 && desc[0].mode == KEYDB_SEARCH_MODE_LONG_KID) + { + kid_not_found_insert (desc[0].u.kid); + } + if (DBG_CLOCK) log_clock (rc? "keydb_search leave (not found)" : "keydb_search leave (found)"); diff --git a/g10/keydb.h b/g10/keydb.h index 0e3816f1d..1aa4e0e2f 100644 --- a/g10/keydb.h +++ b/g10/keydb.h @@ -132,6 +132,7 @@ union pref_hint #define KEYDB_RESOURCE_FLAG_READONLY 8 /* Open in read only mode. */ gpg_error_t keydb_add_resource (const char *url, unsigned int flags); +void keydb_dump_stats (void); KEYDB_HANDLE keydb_new (void); void keydb_release (KEYDB_HANDLE hd); @@ -154,6 +155,7 @@ gpg_error_t keydb_search_next (KEYDB_HANDLE hd); gpg_error_t keydb_search_kid (KEYDB_HANDLE hd, u32 *kid); gpg_error_t keydb_search_fpr (KEYDB_HANDLE hd, const byte *fpr); + /*-- pkclist.c --*/ void show_revocation_reason( PKT_public_key *pk, int mode ); int check_signatures_trust( PKT_signature *sig );