From 68613a6a9de4020fe921b661b7403b7eb865518d Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Wed, 3 May 2023 17:39:37 +0200 Subject: [PATCH 1/8] gpgsm: Cache the non-existence of the policy file. * sm/certchain.c (check_cert_policy): Add simple static cache. -- It is quite common that a policy file does not exist. Thus we can avoid the overhead of trying to open it over and over again just to assert that it does not exists. --- sm/certchain.c | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/sm/certchain.c b/sm/certchain.c index 7b782190b..84dbed696 100644 --- a/sm/certchain.c +++ b/sm/certchain.c @@ -307,6 +307,7 @@ allowed_ca (ctrl_t ctrl, static int check_cert_policy (ksba_cert_t cert, int listmode, estream_t fplist) { + static int no_policy_file; gpg_error_t err; char *policies; estream_t fp; @@ -341,12 +342,24 @@ check_cert_policy (ksba_cert_t cert, int listmode, estream_t fplist) return 0; } - fp = es_fopen (opt.policy_file, "r"); + if (no_policy_file) + { + /* Avoid trying to open the policy file if we already know that + * it does not exist. */ + fp = NULL; + gpg_err_set_errno (ENOENT); + } + else + fp = es_fopen (opt.policy_file, "r"); if (!fp) { - if (opt.verbose || errno != ENOENT) + if ((opt.verbose || errno != ENOENT) && !no_policy_file) log_info (_("failed to open '%s': %s\n"), opt.policy_file, strerror (errno)); + + if (errno == ENOENT) + no_policy_file = 1; + xfree (policies); /* With no critical policies this is only a warning */ if (!any_critical) @@ -361,6 +374,8 @@ check_cert_policy (ksba_cert_t cert, int listmode, estream_t fplist) return gpg_error (GPG_ERR_NO_POLICY_MATCH); } + /* FIXME: Cache the policy file content. */ + for (;;) { int c; From a7dbf11954873c6c58a36b57028ced4ecd45bdc6 Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Thu, 4 May 2023 11:55:26 +0200 Subject: [PATCH 2/8] kbx: Add extra flags to fopen for use by Windows. * kbx/keybox-search.c (open_file): Use sysopen and sequential. * kbx/keybox-update.c (create_tmp_file): Ditto. (blob_filecopy): Ditto. (keybox_set_flags): Ditto. (keybox_delete): Ditto. (keybox_compress): Ditto. -- Under Windows "sysopen" requests that direct API calls (CreateFile et al.) are used instead of the libc wrappers. This may or may not improve the performance. Using "sequential" is a hint to Windows to assume that a file is in general access in a sequential manner. This will have an affect only with a future libgpg-error. --- kbx/keybox-search.c | 2 +- kbx/keybox-update.c | 12 ++++++------ sm/keydb.c | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/kbx/keybox-search.c b/kbx/keybox-search.c index 8dea7cb8e..7645fba4f 100644 --- a/kbx/keybox-search.c +++ b/kbx/keybox-search.c @@ -878,7 +878,7 @@ static gpg_error_t open_file (KEYBOX_HANDLE hd) { - hd->fp = es_fopen (hd->kb->fname, "rb"); + hd->fp = es_fopen (hd->kb->fname, "rb,sysopen,sequential"); if (!hd->fp) { hd->error = gpg_error_from_syserror (); diff --git a/kbx/keybox-update.c b/kbx/keybox-update.c index 273265635..eab961a3e 100644 --- a/kbx/keybox-update.c +++ b/kbx/keybox-update.c @@ -78,7 +78,7 @@ create_tmp_file (const char *template, err = keybox_tmp_names (template, 0, r_bakfname, r_tmpfname); if (!err) { - *r_fp = es_fopen (*r_tmpfname, "wb"); + *r_fp = es_fopen (*r_tmpfname, "wb,sysopen,sequential"); if (!*r_fp) { err = gpg_error_from_syserror (); @@ -174,12 +174,12 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, if ((ec = gnupg_access (fname, W_OK))) return gpg_error (ec); - fp = es_fopen (fname, "rb"); + fp = es_fopen (fname, "rb,sysopen,sequential"); if (mode == FILECOPY_INSERT && !fp && errno == ENOENT) { /* Insert mode but file does not exist: Create a new keybox file. */ - newfp = es_fopen (fname, "wb"); + newfp = es_fopen (fname, "wb,sysopen,sequential"); if (!newfp ) return gpg_error_from_syserror (); @@ -536,7 +536,7 @@ keybox_set_flags (KEYBOX_HANDLE hd, int what, int idx, unsigned int value) off += flag_pos; _keybox_close_file (hd); - fp = es_fopen (hd->kb->fname, "r+b"); + fp = es_fopen (hd->kb->fname, "r+b,sysopen,sequential"); if (!fp) return gpg_error_from_syserror (); @@ -601,7 +601,7 @@ keybox_delete (KEYBOX_HANDLE hd) off += 4; _keybox_close_file (hd); - fp = es_fopen (hd->kb->fname, "r+b"); + fp = es_fopen (hd->kb->fname, "r+b,sysopen,sequential"); if (!fp) return gpg_error_from_syserror (); @@ -656,7 +656,7 @@ keybox_compress (KEYBOX_HANDLE hd) if ((ec = gnupg_access (fname, W_OK))) return gpg_error (ec); - fp = es_fopen (fname, "rb"); + fp = es_fopen (fname, "rb,sysopen,sequential"); if (!fp && errno == ENOENT) return 0; /* Ready. File has been deleted right after the access above. */ if (!fp) diff --git a/sm/keydb.c b/sm/keydb.c index a12dba19f..38737c96a 100644 --- a/sm/keydb.c +++ b/sm/keydb.c @@ -693,7 +693,7 @@ keydb_release (KEYDB_HANDLE hd) switch (hd->active[i].type) { case KEYDB_RESOURCE_TYPE_NONE: - break; + break; case KEYDB_RESOURCE_TYPE_KEYBOX: keybox_release (hd->active[i].u.kr); break; From a6c4d6413ae0af3b3ed0d697618699233c8607cc Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Fri, 5 May 2023 11:37:44 +0200 Subject: [PATCH 3/8] kbx: Use wrapper functions for es_fclose and es_fopen. * kbx/keybox-defs.h (KEYBOX_LL_OPEN_READ) (KEYBOX_LL_OPEN_UPDATE, KEYBOX_LL_OPEN_CREATE): New. * kbx/keybox-init.c (_keybox_ll_open): New. Replace all keybox use of es_fopen by this function. (_keybox_ll_close): New. Replace all keybox use of es_fclose by this function. -- Note that this has not been done for the utilities and the backend-kbx of keyboxd. --- kbx/keybox-defs.h | 8 +++ kbx/keybox-init.c | 45 +++++++++++++++- kbx/keybox-search.c | 30 +++-------- kbx/keybox-update.c | 125 ++++++++++++++++++++++---------------------- 4 files changed, 120 insertions(+), 88 deletions(-) diff --git a/kbx/keybox-defs.h b/kbx/keybox-defs.h index 51ba8cd0e..3768beb3d 100644 --- a/kbx/keybox-defs.h +++ b/kbx/keybox-defs.h @@ -136,6 +136,14 @@ typedef struct _keybox_openpgp_info *keybox_openpgp_info_t; /* } keybox_opt; */ /*-- keybox-init.c --*/ + +#define KEYBOX_LL_OPEN_READ 0 +#define KEYBOX_LL_OPEN_UPDATE 1 +#define KEYBOX_LL_OPEN_CREATE 2 +gpg_error_t _keybox_ll_open (estream_t *rfp, const char *fname, + unsigned int mode); +gpg_error_t _keybox_ll_close (estream_t fp); + void _keybox_close_file (KEYBOX_HANDLE hd); diff --git a/kbx/keybox-init.c b/kbx/keybox-init.c index 48af5c7a1..0e37e3d9e 100644 --- a/kbx/keybox-init.c +++ b/kbx/keybox-init.c @@ -180,7 +180,7 @@ keybox_release (KEYBOX_HANDLE hd) _keybox_release_blob (hd->saved_found.blob); if (hd->fp) { - es_fclose (hd->fp); + _keybox_ll_close (hd->fp); hd->fp = NULL; } xfree (hd->word_match.name); @@ -236,6 +236,47 @@ keybox_set_ephemeral (KEYBOX_HANDLE hd, int yes) } +/* Low-level open function to be used for keybox files. This function + * also manages custom buffering. On success 0 is returned and a new + * file pointer stored at RFP; on error an error code is returned and + * NULL is stored at RFP. MODE is one of + * KEYBOX_LL_OPEN_READ(0) := fopen mode is "rb" + * KEYBOX_LL_OPEN_UPDATE := fopen mode is "r+b" + * KEYBOX_LL_OPEN_CREATE := fopen mode is "wb" + */ +gpg_error_t +_keybox_ll_open (estream_t *rfp, const char *fname, unsigned int mode) +{ + estream_t fp; + + *rfp = NULL; + + fp = es_fopen (fname, + mode == KEYBOX_LL_OPEN_CREATE + ? "wb,sysopen,sequential" : + mode == KEYBOX_LL_OPEN_UPDATE + ? "r+b,sysopen,sequential" : + "rb,sysopen,sequential"); + if (!fp) + return gpg_error_from_syserror (); + + *rfp = fp; + return 0; +} + + +/* Wrapper around es_fclose to be used for file opened with + * _keybox_ll_open. */ +gpg_error_t +_keybox_ll_close (estream_t fp) +{ + if (fp && es_fclose (fp)) + return gpg_error_from_syserror (); + return 0; +} + + + /* Close the file of the resource identified by HD. For consistent results this function closes the files of all handles pointing to the resource identified by HD. */ @@ -253,7 +294,7 @@ _keybox_close_file (KEYBOX_HANDLE hd) { if (roverhd->fp) { - es_fclose (roverhd->fp); + _keybox_ll_close (roverhd->fp); roverhd->fp = NULL; } } diff --git a/kbx/keybox-search.c b/kbx/keybox-search.c index 7645fba4f..31ea0ba60 100644 --- a/kbx/keybox-search.c +++ b/kbx/keybox-search.c @@ -873,28 +873,12 @@ release_sn_array (struct sn_array_s *array, size_t size) } -/* Helper to open the file. */ -static gpg_error_t -open_file (KEYBOX_HANDLE hd) -{ - - hd->fp = es_fopen (hd->kb->fname, "rb,sysopen,sequential"); - if (!hd->fp) - { - hd->error = gpg_error_from_syserror (); - return hd->error; - } - - return 0; -} - - /* - - The search API - -*/ + * + * The search API + * + */ gpg_error_t keybox_search_reset (KEYBOX_HANDLE hd) @@ -914,7 +898,7 @@ keybox_search_reset (KEYBOX_HANDLE hd) { /* Ooops. Seek did not work. Close so that the search will * open the file again. */ - es_fclose (hd->fp); + _keybox_ll_close (hd->fp); hd->fp = NULL; } } @@ -992,7 +976,7 @@ keybox_search (KEYBOX_HANDLE hd, KEYBOX_SEARCH_DESC *desc, size_t ndesc, if (!hd->fp) { - rc = open_file (hd); + rc = _keybox_ll_open (&hd->fp, hd->kb->fname, 0); if (rc) { xfree (sn_array); @@ -1480,7 +1464,7 @@ keybox_seek (KEYBOX_HANDLE hd, off_t offset) return 0; } - err = open_file (hd); + err = _keybox_ll_open (&hd->fp, hd->kb->fname, 0); if (err) return err; } diff --git a/kbx/keybox-update.c b/kbx/keybox-update.c index eab961a3e..be49e7b4a 100644 --- a/kbx/keybox-update.c +++ b/kbx/keybox-update.c @@ -78,10 +78,9 @@ create_tmp_file (const char *template, err = keybox_tmp_names (template, 0, r_bakfname, r_tmpfname); if (!err) { - *r_fp = es_fopen (*r_tmpfname, "wb,sysopen,sequential"); - if (!*r_fp) + err = _keybox_ll_open (r_fp, *r_tmpfname, KEYBOX_LL_OPEN_CREATE); + if (err) { - err = gpg_error_from_syserror (); xfree (*r_tmpfname); *r_tmpfname = NULL; xfree (*r_bakfname); @@ -174,31 +173,32 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, if ((ec = gnupg_access (fname, W_OK))) return gpg_error (ec); - fp = es_fopen (fname, "rb,sysopen,sequential"); - if (mode == FILECOPY_INSERT && !fp && errno == ENOENT) + rc = _keybox_ll_open (&fp, fname, 0); + if (mode == FILECOPY_INSERT && gpg_err_code (rc) == GPG_ERR_ENOENT) { /* Insert mode but file does not exist: - Create a new keybox file. */ - newfp = es_fopen (fname, "wb,sysopen,sequential"); - if (!newfp ) - return gpg_error_from_syserror (); + * Create a new keybox file. */ + rc = _keybox_ll_open (&newfp, fname, KEYBOX_LL_OPEN_CREATE); + if (rc) + return rc; rc = _keybox_write_header_blob (newfp, for_openpgp); if (rc) { - es_fclose (newfp); + _keybox_ll_close (newfp); return rc; } rc = _keybox_write_blob (blob, newfp, NULL); if (rc) { - es_fclose (newfp); + _keybox_ll_close (newfp); return rc; } - if ( es_fclose (newfp) ) - return gpg_error_from_syserror (); + rc = _keybox_ll_close (newfp); + if (rc) + return rc; /* if (chmod( fname, S_IRUSR | S_IWUSR )) */ /* { */ @@ -218,7 +218,7 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, rc = create_tmp_file (fname, &bakfname, &tmpfname, &newfp); if (rc) { - es_fclose (fp); + _keybox_ll_close (fp); goto leave; } @@ -242,16 +242,16 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, if (es_fwrite (buffer, nread, 1, newfp) != 1) { rc = gpg_error_from_syserror (); - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } } if (es_ferror (fp)) { rc = gpg_error_from_syserror (); - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } } @@ -275,16 +275,16 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, if (es_fwrite (buffer, nread, 1, newfp) != 1) { rc = gpg_error_from_syserror (); - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } } if (es_ferror (fp)) { rc = gpg_error_from_syserror (); - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } @@ -292,8 +292,8 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, rc = _keybox_read_blob (NULL, fp, NULL); if (rc) { - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } } @@ -304,8 +304,8 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, rc = _keybox_write_blob (blob, newfp, NULL); if (rc) { - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } } @@ -318,32 +318,30 @@ blob_filecopy (int mode, const char *fname, KEYBOXBLOB blob, if (es_fwrite (buffer, nread, 1, newfp) != 1) { rc = gpg_error_from_syserror (); - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } } if (es_ferror (fp)) { rc = gpg_error_from_syserror (); - es_fclose (fp); - es_fclose (newfp); + _keybox_ll_close (fp); + _keybox_ll_close (newfp); goto leave; } } /* Close both files. */ - if (es_fclose(fp)) + rc = _keybox_ll_close (fp); + if (rc) { - rc = gpg_error_from_syserror (); - es_fclose (newfp); - goto leave; - } - if (es_fclose(newfp)) - { - rc = gpg_error_from_syserror (); + _keybox_ll_close (newfp); goto leave; } + rc = _keybox_ll_close (newfp); + if (rc) + goto leave; rc = rename_tmp_file (bakfname, tmpfname, fname, secret); @@ -502,6 +500,7 @@ keybox_update_cert (KEYBOX_HANDLE hd, ksba_cert_t cert, int keybox_set_flags (KEYBOX_HANDLE hd, int what, int idx, unsigned int value) { + gpg_error_t err; off_t off; const char *fname; estream_t fp; @@ -536,9 +535,10 @@ keybox_set_flags (KEYBOX_HANDLE hd, int what, int idx, unsigned int value) off += flag_pos; _keybox_close_file (hd); - fp = es_fopen (hd->kb->fname, "r+b,sysopen,sequential"); - if (!fp) - return gpg_error_from_syserror (); + + err = _keybox_ll_open (&fp, fname, KEYBOX_LL_OPEN_UPDATE); + if (err) + return err; ec = 0; if (es_fseeko (fp, off, SEEK_SET)) @@ -566,10 +566,11 @@ keybox_set_flags (KEYBOX_HANDLE hd, int what, int idx, unsigned int value) } } - if (es_fclose (fp)) + err = _keybox_ll_close (fp); + if (err) { if (!ec) - ec = gpg_err_code_from_syserror (); + ec = gpg_err_code (err); } return gpg_error (ec); @@ -583,7 +584,7 @@ keybox_delete (KEYBOX_HANDLE hd) off_t off; const char *fname; estream_t fp; - int rc; + int rc, rc2; if (!hd) return gpg_error (GPG_ERR_INV_VALUE); @@ -601,9 +602,9 @@ keybox_delete (KEYBOX_HANDLE hd) off += 4; _keybox_close_file (hd); - fp = es_fopen (hd->kb->fname, "r+b,sysopen,sequential"); - if (!fp) - return gpg_error_from_syserror (); + rc = _keybox_ll_open (&fp, hd->kb->fname, KEYBOX_LL_OPEN_UPDATE); + if (rc) + return rc; if (es_fseeko (fp, off, SEEK_SET)) rc = gpg_error_from_syserror (); @@ -612,10 +613,11 @@ keybox_delete (KEYBOX_HANDLE hd) else rc = 0; - if (es_fclose (fp)) + rc2 = _keybox_ll_close (fp); + if (rc2) { if (!rc) - rc = gpg_error_from_syserror (); + rc = rc2; } return rc; @@ -628,7 +630,7 @@ int keybox_compress (KEYBOX_HANDLE hd) { gpg_err_code_t ec; - int read_rc, rc; + int read_rc, rc, rc2; const char *fname; estream_t fp, newfp; char *bakfname = NULL; @@ -656,14 +658,11 @@ keybox_compress (KEYBOX_HANDLE hd) if ((ec = gnupg_access (fname, W_OK))) return gpg_error (ec); - fp = es_fopen (fname, "rb,sysopen,sequential"); - if (!fp && errno == ENOENT) + rc = _keybox_ll_open (&fp, fname, 0); + if (gpg_err_code (rc) == GPG_ERR_ENOENT) return 0; /* Ready. File has been deleted right after the access above. */ - if (!fp) - { - rc = gpg_error_from_syserror (); - return rc; - } + if (rc) + return rc; /* A quick test to see if we need to compress the file at all. We schedule a compress run after 3 hours. */ @@ -679,7 +678,7 @@ keybox_compress (KEYBOX_HANDLE hd) if ( (last_maint + 3*3600) > make_timestamp () ) { - es_fclose (fp); + _keybox_ll_close (fp); _keybox_release_blob (blob); return 0; /* Compress run not yet needed. */ } @@ -693,7 +692,7 @@ keybox_compress (KEYBOX_HANDLE hd) rc = create_tmp_file (fname, &bakfname, &tmpfname, &newfp); if (rc) { - es_fclose (fp); + _keybox_ll_close (fp); return rc;; } @@ -782,10 +781,10 @@ keybox_compress (KEYBOX_HANDLE hd) rc = read_rc; /* Close both files. */ - if (es_fclose(fp) && !rc) - rc = gpg_error_from_syserror (); - if (es_fclose(newfp) && !rc) - rc = gpg_error_from_syserror (); + if ((rc2 = _keybox_ll_close (fp)) && !rc) + rc = rc2; + if ((rc2 = _keybox_ll_close (newfp)) && !rc) + rc = rc2; /* Rename or remove the temporary file. */ if (rc || !any_changes) From 08ff55bd44aea6cd8b25384ee7d127576866ec71 Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Mon, 8 May 2023 09:16:35 +0200 Subject: [PATCH 4/8] kbx: Use custom estream buffering * kbx/keybox-init.c (ll_buffer_size): New var intialized to 128k (stream_buffers): New var. (keybox_set_buffersize): New. (_keybox_ll_open, _keybox_ll_close): Implement buffering. * sm/gpgsm.c (oKbxBufferSize): New. (opts): Add option --kbx-buffer-size. (main): Call keybox_set_buffersize. * g10/gpg.c: Include keybox.h. * (oKbxBufferSize): New. (opts): Add option --kbx-buffer-size. (main): Call keybox_set_buffersize. -- Running a test on Windows using a pubring.kbx with Total number of blobs: 2098 openpgp: 1294 x509: 803 and a size of 42MiB with gpgsm -k --with-validation --disable-dirmngr --kbx-buffer-size N >nul gives these performance figures using procmon | N(k) | file events | time(s) | |------+-------------+---------| | 0 | 4900000 | 86 | | 16 | 2456000 | 58 | | 32 | 1233000 | 43 | | 64 | 622000 | 37 | | 128 | 317000 | 32 | | 256 | 164000 | 31 | | 512 | 88000 | 30 | Using _open instead of CreateFile give the same number of file events but increased the time slight by one second for the measured buffer size of 64k and 128k. Benchmarks for gpg have not been conducted. --- g10/gpg.c | 6 ++++ kbx/keybox-init.c | 81 +++++++++++++++++++++++++++++++++++++++++++++-- kbx/keybox.h | 1 + sm/gpgsm.c | 6 ++++ 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/g10/gpg.c b/g10/gpg.c index b759cc1cf..737753a40 100644 --- a/g10/gpg.c +++ b/g10/gpg.c @@ -68,6 +68,7 @@ #include "../common/shareddefs.h" #include "../common/compliance.h" #include "../common/comopt.h" +#include "../kbx/keybox.h" #if defined(HAVE_DOSISH_SYSTEM) || defined(__CYGWIN__) #define MY_O_BINARY O_BINARY @@ -447,6 +448,7 @@ enum cmd_and_opt_values oCompatibilityFlags, oAddDesigRevoker, oAssertSigner, + oKbxBufferSize, oNoop }; @@ -926,6 +928,7 @@ static gpgrt_opt_t opts[] = { /* Esoteric compatibility options. */ ARGPARSE_s_n (oRFC2440Text, "rfc2440-text", "@"), ARGPARSE_s_n (oNoRFC2440Text, "no-rfc2440-text", "@"), + ARGPARSE_p_u (oKbxBufferSize, "kbx-buffer-size", "@"), ARGPARSE_header (NULL, ""), /* Stop the header group. */ @@ -3743,6 +3746,9 @@ main (int argc, char **argv) add_to_strlist (&opt.assert_signer_list, pargs.r.ret_str); break; + case oKbxBufferSize: + keybox_set_buffersize (pargs.r.ret_ulong, 0); + break; case oNoop: break; diff --git a/kbx/keybox-init.c b/kbx/keybox-init.c index 0e37e3d9e..c83c4712f 100644 --- a/kbx/keybox-init.c +++ b/kbx/keybox-init.c @@ -28,8 +28,20 @@ #include "../common/sysutils.h" #include "../common/mischelp.h" +static unsigned int ll_buffer_size = 128; + static KB_NAME kb_names; +/* This object is used to mahe setvbuf buffers. We use a short arary + * to be able to reuse already allocated buffers. */ +struct stream_buffer_s +{ + int inuse; /* True if used by a stream. */ + size_t bufsize; + char *buf; +}; +static struct stream_buffer_s stream_buffers[5]; + /* Register a filename for plain keybox files. Returns 0 on success, * GPG_ERR_EEXIST if it has already been registered, or another error @@ -85,6 +97,16 @@ keybox_is_writable (void *token) } +/* Change the default buffering to KBYTES KiB; using 0 uses the syste + * buffers. This function must be called early. */ +void +keybox_set_buffersize (unsigned int kbytes, int reserved) +{ + (void)reserved; + /* Round down to 8k multiples. */ + ll_buffer_size = (kbytes + 7)/8 * 8; +} + static KEYBOX_HANDLE do_keybox_new (KB_NAME resource, int secret, int for_openpgp) @@ -248,6 +270,8 @@ gpg_error_t _keybox_ll_open (estream_t *rfp, const char *fname, unsigned int mode) { estream_t fp; + int i; + size_t bufsize; *rfp = NULL; @@ -260,6 +284,37 @@ _keybox_ll_open (estream_t *rfp, const char *fname, unsigned int mode) if (!fp) return gpg_error_from_syserror (); + if (ll_buffer_size) + { + for (i=0; i < DIM (stream_buffers); i++) + if (!stream_buffers[i].inuse) + { + /* There is a free slot - we can use a larger buffer. */ + stream_buffers[i].inuse = 1; + if (!stream_buffers[i].buf) + { + bufsize = ll_buffer_size * 1024; + stream_buffers[i].buf = xtrymalloc (bufsize); + if (stream_buffers[i].buf) + stream_buffers[i].bufsize = bufsize; + else + { + log_info ("can't allocate a large buffer for a kbx file;" + " using default\n"); + stream_buffers[i].inuse = 0; + } + } + + if (stream_buffers[i].buf) + { + es_setvbuf (fp, stream_buffers[i].buf, _IOFBF, + stream_buffers[i].bufsize); + es_opaque_set (fp, stream_buffers + i); + } + break; + } + } + *rfp = fp; return 0; } @@ -270,9 +325,29 @@ _keybox_ll_open (estream_t *rfp, const char *fname, unsigned int mode) gpg_error_t _keybox_ll_close (estream_t fp) { - if (fp && es_fclose (fp)) - return gpg_error_from_syserror (); - return 0; + gpg_error_t err; + struct stream_buffer_s *sbuf; + int i; + + if (!fp) + return 0; + + sbuf = ll_buffer_size? es_opaque_get (fp) : NULL; + if (es_fclose (fp)) + err = gpg_error_from_syserror (); + else + err = 0; + if (sbuf) + { + for (i=0; i < DIM (stream_buffers); i++) + if (stream_buffers + i == sbuf) + break; + log_assert (i < DIM (stream_buffers)); + stream_buffers[i].inuse = 0; + } + + + return err; } diff --git a/kbx/keybox.h b/kbx/keybox.h index 9bff271ea..1146eca31 100644 --- a/kbx/keybox.h +++ b/kbx/keybox.h @@ -66,6 +66,7 @@ typedef enum /*-- keybox-init.c --*/ gpg_error_t keybox_register_file (const char *fname, int secret, void **r_token); +void keybox_set_buffersize (unsigned int kbytes, int reserved); int keybox_is_writable (void *token); KEYBOX_HANDLE keybox_new_openpgp (void *token, int secret); diff --git a/sm/gpgsm.c b/sm/gpgsm.c index aeb6ad7a9..55173f8a2 100644 --- a/sm/gpgsm.c +++ b/sm/gpgsm.c @@ -213,6 +213,7 @@ enum cmd_and_opt_values { oKeyboxdProgram, oRequireCompliance, oCompatibilityFlags, + oKbxBufferSize, oNoAutostart }; @@ -447,6 +448,7 @@ static gpgrt_opt_t opts[] = { ARGPARSE_s_s (oXauthority, "xauthority", "@"), ARGPARSE_s_s (oChUid, "chuid", "@"), ARGPARSE_s_s (oCompatibilityFlags, "compatibility-flags", "@"), + ARGPARSE_p_u (oKbxBufferSize, "kbx-buffer-size", "@"), ARGPARSE_header (NULL, ""), /* Stop the header group. */ @@ -1492,6 +1494,10 @@ main ( int argc, char **argv) case oRequireCompliance: opt.require_compliance = 1; break; + case oKbxBufferSize: + keybox_set_buffersize (pargs.r.ret_ulong, 0); + break; + default: if (configname) pargs.err = ARGPARSE_PRINT_WARNING; From a2c199affb8e92ce1e0206757130e2fb72cc0cfd Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Mon, 8 May 2023 13:09:27 +0200 Subject: [PATCH 5/8] kbx: For non-Windows use 64k buffers by default instead of 128k. * kbx/keybox-init.c (DEFAULT_LL_BUFFER_SIZE): New. -- A simple gpg --check-sigs benchmark showed on Linux a small performance peak at around 64k (5m52 vs. 6m8 for 128k and 6m33 for system size). --- kbx/keybox-init.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/kbx/keybox-init.c b/kbx/keybox-init.c index c83c4712f..439d1c477 100644 --- a/kbx/keybox-init.c +++ b/kbx/keybox-init.c @@ -28,7 +28,13 @@ #include "../common/sysutils.h" #include "../common/mischelp.h" -static unsigned int ll_buffer_size = 128; +#ifdef HAVE_W32_SYSTEM +# define DEFAULT_LL_BUFFER_SIZE 128 +#else +# define DEFAULT_LL_BUFFER_SIZE 64 +#endif + +static unsigned int ll_buffer_size = DEFAULT_LL_BUFFER_SIZE; static KB_NAME kb_names; From ef2c3d50fa8cbc2413645a56d9d060c0ec31e62f Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Mon, 8 May 2023 13:10:47 +0200 Subject: [PATCH 6/8] gpg: Print a warning if no more encryption subkey was left over. * g10/keyedit.c (no_usable_encr_subkeys_warning): New. (keyedit_menu): Call it after running an expire command. * g10/import.c (import_one_real): Call it in the show_key case. -- --- g10/import.c | 1 + g10/keyedit.c | 35 +++++++++++++++++++++++++++++++++++ g10/keyedit.h | 1 + 3 files changed, 37 insertions(+) diff --git a/g10/import.c b/g10/import.c index 5e44942e7..987fef3cd 100644 --- a/g10/import.c +++ b/g10/import.c @@ -2136,6 +2136,7 @@ import_one_real (ctrl_t ctrl, list_keyblock_direct (ctrl, keyblock, from_sk, 0, opt.fingerprint || opt.with_fingerprint, 1); es_fflush (es_stdout); + no_usable_encr_subkeys_warning (keyblock); } /* Write the keyblock to the output and do not actually import. */ diff --git a/g10/keyedit.c b/g10/keyedit.c index a91cc4447..4b767aed6 100644 --- a/g10/keyedit.c +++ b/g10/keyedit.c @@ -1501,6 +1501,7 @@ keyedit_menu (ctrl_t ctrl, const char *username, strlist_t locusr, run_subkey_warnings = 0; if (!count_selected_keys (keyblock)) subkey_expire_warning (keyblock); + no_usable_encr_subkeys_warning (keyblock); } if (delseckey_list_warn) @@ -4258,6 +4259,40 @@ subkey_expire_warning (kbnode_t keyblock) } +/* Print a warning if all encryption (sub|primary)keys are expired. + * The warning is not printed if there is no encryption + * (sub|primary)key at all. This function is called after the expire + * data of the primary key has been changed. */ +void +no_usable_encr_subkeys_warning (kbnode_t keyblock) +{ + kbnode_t node; + PKT_public_key *pk; + int any_encr_key = 0; + + for (node = keyblock; node; node = node->next) + { + if (node->pkt->pkttype == PKT_PUBLIC_KEY + || node->pkt->pkttype == PKT_PUBLIC_SUBKEY) + { + pk = node->pkt->pkt.public_key; + if ((pk->pubkey_usage & PUBKEY_USAGE_ENC)) + { + any_encr_key = 1; + if (pk->flags.valid && !pk->has_expired && !pk->flags.revoked + && !pk->flags.disabled) + { + return; /* Key is usable for encryption */ + } + } + } + } + + if (any_encr_key && !opt.quiet) + log_info (_("WARNING: No valid encryption subkey left over.\n")); +} + + /* * Ask for a new user id, add the self-signature, and update the * keyblock. If UIDSTRING is not NULL the user ID is generated diff --git a/g10/keyedit.h b/g10/keyedit.h index 3ed0d0fea..abf7314af 100644 --- a/g10/keyedit.h +++ b/g10/keyedit.h @@ -63,5 +63,6 @@ int keyedit_print_one_sig (ctrl_t ctrl, estream_t fp, kbnode_t node, int *inv_sigs, int *no_key, int *oth_err, int is_selfsig, int print_without_key, int extended); +void no_usable_encr_subkeys_warning (kbnode_t keyblock); #endif /* GNUPG_G10_KEYEDIT_H */ From c30d5829c9ab076e9fd39c223273b065cdb58a0d Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Tue, 9 May 2023 08:17:30 +0200 Subject: [PATCH 7/8] gpg: New option --debug-ignore-expiration to help with testing. * g10/gpg.c (oDebugIgnoreExpiration): New. (opts): Add option. (main): Set flag. * g10/options.h (opt): Add field ignore_expiration. * g10/pkclist.c (do_we_trust): Handle the option. * g10/getkey.c (skip_unusable): Ditto. (finish_lookup): Ditto. -- GnuPG-bug-id: 2703 --- doc/gpg.texi | 13 +++++++++---- g10/getkey.c | 7 ++++--- g10/gpg.c | 8 +++++++- g10/options.h | 1 + g10/pkclist.c | 6 +++++- 5 files changed, 26 insertions(+), 9 deletions(-) diff --git a/doc/gpg.texi b/doc/gpg.texi index 9fdabc743..6b584a913 100644 --- a/doc/gpg.texi +++ b/doc/gpg.texi @@ -3185,12 +3185,17 @@ and may thus be changed or removed at any time without notice. To facilitate software tests and experiments this option allows to specify a limit of up to 4 EiB (@code{--chunk-size 62}). +@item --debug-ignore-expiration +@opindex debug-ignore-expiration +This option tries to override certain key expiration dates. It is +only useful for certain regression tests. + @item --faked-system-time @var{epoch} @opindex faked-system-time -This option is only useful for testing; it sets the system time back or -forth to @var{epoch} which is the number of seconds elapsed since the year -1970. Alternatively @var{epoch} may be given as a full ISO time string -(e.g. "20070924T154812"). +This option is only useful for testing; it sets the system time back +or forth to @var{epoch} which is the number of seconds elapsed since +the year 1970. Alternatively @var{epoch} may be given as a full ISO +time string (e.g. "20070924T154812"). If you suffix @var{epoch} with an exclamation mark (!), the system time will appear to be frozen at the specified time. diff --git a/g10/getkey.c b/g10/getkey.c index 1b37c597d..15905dc63 100644 --- a/g10/getkey.c +++ b/g10/getkey.c @@ -685,7 +685,7 @@ skip_unusable (void *opaque, u32 * keyid, int uid_no) pk = keyblock->pkt->pkt.public_key; /* Is the key revoked or expired? */ - if (pk->flags.revoked || pk->has_expired) + if (pk->flags.revoked || (pk->has_expired && !opt.ignore_expiration)) unusable = 1; /* Is the user ID in question revoked or expired? */ @@ -704,7 +704,8 @@ skip_unusable (void *opaque, u32 * keyid, int uid_no) if (uids_seen != uid_no) continue; - if (user_id->flags.revoked || user_id->flags.expired) + if (user_id->flags.revoked + || (user_id->flags.expired && !opt.ignore_expiration)) unusable = 1; break; @@ -3736,7 +3737,7 @@ finish_lookup (kbnode_t keyblock, unsigned int req_usage, int want_exact, n_revoked_or_expired++; continue; } - if (pk->has_expired) + if (pk->has_expired && !opt.ignore_expiration) { if (DBG_LOOKUP) log_debug ("\tsubkey has expired\n"); diff --git a/g10/gpg.c b/g10/gpg.c index 737753a40..6e54aa763 100644 --- a/g10/gpg.c +++ b/g10/gpg.c @@ -234,6 +234,7 @@ enum cmd_and_opt_values oDebugIOLBF, oDebugSetIobufSize, oDebugAllowLargeChunks, + oDebugIgnoreExpiration, oStatusFD, oStatusFile, oAttributeFD, @@ -607,7 +608,6 @@ static gpgrt_opt_t opts[] = { N_("|FILE|write server mode logs to FILE")), ARGPARSE_s_s (oLoggerFile, "logger-file", "@"), /* 1.4 compatibility. */ ARGPARSE_s_n (oLogTime, "log-time", "@"), - ARGPARSE_s_n (oQuickRandom, "debug-quick-random", "@"), ARGPARSE_header ("Configuration", @@ -929,6 +929,8 @@ static gpgrt_opt_t opts[] = { ARGPARSE_s_n (oRFC2440Text, "rfc2440-text", "@"), ARGPARSE_s_n (oNoRFC2440Text, "no-rfc2440-text", "@"), ARGPARSE_p_u (oKbxBufferSize, "kbx-buffer-size", "@"), + ARGPARSE_s_n (oQuickRandom, "debug-quick-random", "@"), + ARGPARSE_s_n (oDebugIgnoreExpiration, "debug-ignore-expiration", "@"), ARGPARSE_header (NULL, ""), /* Stop the header group. */ @@ -2851,6 +2853,10 @@ main (int argc, char **argv) allow_large_chunks = 1; break; + case oDebugIgnoreExpiration: + opt.ignore_expiration = 1; + break; + case oCompatibilityFlags: if (parse_compatibility_flags (pargs.r.ret_str, &opt.compat_flags, compatibility_flags)) diff --git a/g10/options.h b/g10/options.h index 9015e321f..914c24849 100644 --- a/g10/options.h +++ b/g10/options.h @@ -208,6 +208,7 @@ struct int ignore_valid_from; int ignore_crc_error; int ignore_mdc_error; + int ignore_expiration; int command_fd; const char *override_session_key; int show_session_key; diff --git a/g10/pkclist.c b/g10/pkclist.c index 2e8932b9c..42e124e9e 100644 --- a/g10/pkclist.c +++ b/g10/pkclist.c @@ -417,7 +417,11 @@ do_we_trust( PKT_public_key *pk, unsigned int trustlevel ) if(trustlevel & TRUST_FLAG_REVOKED || trustlevel & TRUST_FLAG_SUB_REVOKED || (trustlevel & TRUST_MASK) == TRUST_EXPIRED) - BUG(); + { + if (opt.ignore_expiration) + return 0; + BUG (); + } if( opt.trust_model==TM_ALWAYS ) { From 1e41878bf283ca5cb6bcc3ad41d7b7331802a481 Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Wed, 10 May 2023 10:23:59 +0200 Subject: [PATCH 8/8] Prepare new development cycle -- --- NEWS | 2 +- README | 2 +- configure.ac | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NEWS b/NEWS index af2064199..d2f0da2ac 100644 --- a/NEWS +++ b/NEWS @@ -1,4 +1,4 @@ -Noteworthy changes in version 2.4.2 (unreleased) +Noteworthy changes in version 2.5.0 (unreleased) ------------------------------------------------ diff --git a/README b/README index 84a8bacfd..97961369e 100644 --- a/README +++ b/README @@ -1,6 +1,6 @@ The GNU Privacy Guard ======================= - Version 2.4 + Version 2.5 (devel) Copyright 1997-2019 Werner Koch Copyright 1998-2021 Free Software Foundation, Inc. diff --git a/configure.ac b/configure.ac index 24448c157..39cf5cbe0 100644 --- a/configure.ac +++ b/configure.ac @@ -28,8 +28,8 @@ min_automake_version="1.16.3" # another commit and push so that the git magic is able to work. m4_define([mym4_package],[gnupg]) m4_define([mym4_major], [2]) -m4_define([mym4_minor], [4]) -m4_define([mym4_micro], [2]) +m4_define([mym4_minor], [5]) +m4_define([mym4_micro], [0]) # To start a new development series, i.e a new major or minor number # you need to mark an arbitrary commit before the first beta release @@ -53,7 +53,7 @@ AC_INIT([mym4_package],[mym4_version],[https://bugs.gnupg.org]) # When changing the SWDB tag please also adjust the hard coded tags in # build-aux/speedo.mk, build-aux/getswdb.sh, and Makefile.am # As well as the source info for the man pages. -AC_DEFINE_UNQUOTED(GNUPG_SWDB_TAG, "gnupg24", [swdb tag for this branch]) +AC_DEFINE_UNQUOTED(GNUPG_SWDB_TAG, "gnupg26", [swdb tag for this branch]) NEED_GPGRT_VERSION=1.46