From 08ff55bd44aea6cd8b25384ee7d127576866ec71 Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Mon, 8 May 2023 09:16:35 +0200 Subject: [PATCH] kbx: Use custom estream buffering * kbx/keybox-init.c (ll_buffer_size): New var intialized to 128k (stream_buffers): New var. (keybox_set_buffersize): New. (_keybox_ll_open, _keybox_ll_close): Implement buffering. * sm/gpgsm.c (oKbxBufferSize): New. (opts): Add option --kbx-buffer-size. (main): Call keybox_set_buffersize. * g10/gpg.c: Include keybox.h. * (oKbxBufferSize): New. (opts): Add option --kbx-buffer-size. (main): Call keybox_set_buffersize. -- Running a test on Windows using a pubring.kbx with Total number of blobs: 2098 openpgp: 1294 x509: 803 and a size of 42MiB with gpgsm -k --with-validation --disable-dirmngr --kbx-buffer-size N >nul gives these performance figures using procmon | N(k) | file events | time(s) | |------+-------------+---------| | 0 | 4900000 | 86 | | 16 | 2456000 | 58 | | 32 | 1233000 | 43 | | 64 | 622000 | 37 | | 128 | 317000 | 32 | | 256 | 164000 | 31 | | 512 | 88000 | 30 | Using _open instead of CreateFile give the same number of file events but increased the time slight by one second for the measured buffer size of 64k and 128k. Benchmarks for gpg have not been conducted. --- g10/gpg.c | 6 ++++ kbx/keybox-init.c | 81 +++++++++++++++++++++++++++++++++++++++++++++-- kbx/keybox.h | 1 + sm/gpgsm.c | 6 ++++ 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/g10/gpg.c b/g10/gpg.c index b759cc1cf..737753a40 100644 --- a/g10/gpg.c +++ b/g10/gpg.c @@ -68,6 +68,7 @@ #include "../common/shareddefs.h" #include "../common/compliance.h" #include "../common/comopt.h" +#include "../kbx/keybox.h" #if defined(HAVE_DOSISH_SYSTEM) || defined(__CYGWIN__) #define MY_O_BINARY O_BINARY @@ -447,6 +448,7 @@ enum cmd_and_opt_values oCompatibilityFlags, oAddDesigRevoker, oAssertSigner, + oKbxBufferSize, oNoop }; @@ -926,6 +928,7 @@ static gpgrt_opt_t opts[] = { /* Esoteric compatibility options. */ ARGPARSE_s_n (oRFC2440Text, "rfc2440-text", "@"), ARGPARSE_s_n (oNoRFC2440Text, "no-rfc2440-text", "@"), + ARGPARSE_p_u (oKbxBufferSize, "kbx-buffer-size", "@"), ARGPARSE_header (NULL, ""), /* Stop the header group. */ @@ -3743,6 +3746,9 @@ main (int argc, char **argv) add_to_strlist (&opt.assert_signer_list, pargs.r.ret_str); break; + case oKbxBufferSize: + keybox_set_buffersize (pargs.r.ret_ulong, 0); + break; case oNoop: break; diff --git a/kbx/keybox-init.c b/kbx/keybox-init.c index 0e37e3d9e..c83c4712f 100644 --- a/kbx/keybox-init.c +++ b/kbx/keybox-init.c @@ -28,8 +28,20 @@ #include "../common/sysutils.h" #include "../common/mischelp.h" +static unsigned int ll_buffer_size = 128; + static KB_NAME kb_names; +/* This object is used to mahe setvbuf buffers. We use a short arary + * to be able to reuse already allocated buffers. */ +struct stream_buffer_s +{ + int inuse; /* True if used by a stream. */ + size_t bufsize; + char *buf; +}; +static struct stream_buffer_s stream_buffers[5]; + /* Register a filename for plain keybox files. Returns 0 on success, * GPG_ERR_EEXIST if it has already been registered, or another error @@ -85,6 +97,16 @@ keybox_is_writable (void *token) } +/* Change the default buffering to KBYTES KiB; using 0 uses the syste + * buffers. This function must be called early. */ +void +keybox_set_buffersize (unsigned int kbytes, int reserved) +{ + (void)reserved; + /* Round down to 8k multiples. */ + ll_buffer_size = (kbytes + 7)/8 * 8; +} + static KEYBOX_HANDLE do_keybox_new (KB_NAME resource, int secret, int for_openpgp) @@ -248,6 +270,8 @@ gpg_error_t _keybox_ll_open (estream_t *rfp, const char *fname, unsigned int mode) { estream_t fp; + int i; + size_t bufsize; *rfp = NULL; @@ -260,6 +284,37 @@ _keybox_ll_open (estream_t *rfp, const char *fname, unsigned int mode) if (!fp) return gpg_error_from_syserror (); + if (ll_buffer_size) + { + for (i=0; i < DIM (stream_buffers); i++) + if (!stream_buffers[i].inuse) + { + /* There is a free slot - we can use a larger buffer. */ + stream_buffers[i].inuse = 1; + if (!stream_buffers[i].buf) + { + bufsize = ll_buffer_size * 1024; + stream_buffers[i].buf = xtrymalloc (bufsize); + if (stream_buffers[i].buf) + stream_buffers[i].bufsize = bufsize; + else + { + log_info ("can't allocate a large buffer for a kbx file;" + " using default\n"); + stream_buffers[i].inuse = 0; + } + } + + if (stream_buffers[i].buf) + { + es_setvbuf (fp, stream_buffers[i].buf, _IOFBF, + stream_buffers[i].bufsize); + es_opaque_set (fp, stream_buffers + i); + } + break; + } + } + *rfp = fp; return 0; } @@ -270,9 +325,29 @@ _keybox_ll_open (estream_t *rfp, const char *fname, unsigned int mode) gpg_error_t _keybox_ll_close (estream_t fp) { - if (fp && es_fclose (fp)) - return gpg_error_from_syserror (); - return 0; + gpg_error_t err; + struct stream_buffer_s *sbuf; + int i; + + if (!fp) + return 0; + + sbuf = ll_buffer_size? es_opaque_get (fp) : NULL; + if (es_fclose (fp)) + err = gpg_error_from_syserror (); + else + err = 0; + if (sbuf) + { + for (i=0; i < DIM (stream_buffers); i++) + if (stream_buffers + i == sbuf) + break; + log_assert (i < DIM (stream_buffers)); + stream_buffers[i].inuse = 0; + } + + + return err; } diff --git a/kbx/keybox.h b/kbx/keybox.h index 9bff271ea..1146eca31 100644 --- a/kbx/keybox.h +++ b/kbx/keybox.h @@ -66,6 +66,7 @@ typedef enum /*-- keybox-init.c --*/ gpg_error_t keybox_register_file (const char *fname, int secret, void **r_token); +void keybox_set_buffersize (unsigned int kbytes, int reserved); int keybox_is_writable (void *token); KEYBOX_HANDLE keybox_new_openpgp (void *token, int secret); diff --git a/sm/gpgsm.c b/sm/gpgsm.c index aeb6ad7a9..55173f8a2 100644 --- a/sm/gpgsm.c +++ b/sm/gpgsm.c @@ -213,6 +213,7 @@ enum cmd_and_opt_values { oKeyboxdProgram, oRequireCompliance, oCompatibilityFlags, + oKbxBufferSize, oNoAutostart }; @@ -447,6 +448,7 @@ static gpgrt_opt_t opts[] = { ARGPARSE_s_s (oXauthority, "xauthority", "@"), ARGPARSE_s_s (oChUid, "chuid", "@"), ARGPARSE_s_s (oCompatibilityFlags, "compatibility-flags", "@"), + ARGPARSE_p_u (oKbxBufferSize, "kbx-buffer-size", "@"), ARGPARSE_header (NULL, ""), /* Stop the header group. */ @@ -1492,6 +1494,10 @@ main ( int argc, char **argv) case oRequireCompliance: opt.require_compliance = 1; break; + case oKbxBufferSize: + keybox_set_buffersize (pargs.r.ret_ulong, 0); + break; + default: if (configname) pargs.err = ARGPARSE_PRINT_WARNING;