From f84264e8acf742793c73ce78491cab61fac37051 Mon Sep 17 00:00:00 2001 From: Werner Koch Date: Mon, 30 Jan 2023 15:23:38 +0100 Subject: [PATCH] gpgtar: Emit progress status lines in create mode. * tools/gpgtar.h (opt): Add field status_stream. * tools/gpgtar.c (main): Set status_stream. * tools/gpgtar-create.c (global_header_count): Rename to global_total_files. (global_written_files): New. (global_total_data, global_written_data): New. (struct scanctrl_s): Add field file_count. (write_progress): New. (write_file): Add arg skipped_open. Don't bail out immediatly on open error. Write progress lines. (gpgtar_create): Write progress lines. Print info aout skipped files. -- GnuPG-bug-id: 6363 --- doc/DETAILS | 8 ++- tools/gpgtar-create.c | 120 +++++++++++++++++++++++++++++++++++++----- tools/gpgtar.c | 26 +++++++++ tools/gpgtar.h | 1 + 4 files changed, 142 insertions(+), 13 deletions(-) diff --git a/doc/DETAILS b/doc/DETAILS index a3fe802a2..e01b74ac1 100644 --- a/doc/DETAILS +++ b/doc/DETAILS @@ -1151,7 +1151,13 @@ pkd:0:1024:B665B1435F4C2 .... FF26ABB: - learncard :: Send by the agent and gpgsm while learing the data of a smartcard. - card_busy :: A smartcard is still working - - scd_locked :: Waiting for other clients to unlock the scdaemon + - scd_locked :: Waiting for other clients to unlock the + scdaemon + - gpgtar :: Here has a special meaning: 's' + indicates total size and 'c' file count. A + of zero indicates that gpgtar is in the + scanning phase. A positive is used in + the writing phase. When refers to a file path, it may be truncated. diff --git a/tools/gpgtar-create.c b/tools/gpgtar-create.c index f52295b5c..26d37a332 100644 --- a/tools/gpgtar-create.c +++ b/tools/gpgtar-create.c @@ -1,5 +1,5 @@ /* gpgtar-create.c - Create a TAR archive - * Copyright (C) 2016-2017, 2019-2022 g10 Code GmbH + * Copyright (C) 2016-2017, 2019-2023 g10 Code GmbH * Copyright (C) 2010, 2012, 2013 Werner Koch * Copyright (C) 2010 Free Software Foundation, Inc. * @@ -49,10 +49,19 @@ #define lstat(a,b) gnupg_stat ((a), (b)) #endif +/* Number of files to be write. */ +static unsigned long global_total_files; + +/* Count the number of written file and thus headers. Extended + * headers are not counted. */ +static unsigned long global_written_files; + +/* Total data expected to be written. */ +static unsigned long long global_total_data; + +/* Number of data bytes written so far. */ +static unsigned long long global_written_data; -/* Count the number of written headers. Extended headers are not - * counted. */ -static unsigned long global_header_count; /* Object to control the file scanning. */ @@ -62,10 +71,62 @@ struct scanctrl_s { tar_header_t flist; tar_header_t *flist_tail; + unsigned long file_count; int nestlevel; }; +/* See ../g10/progress.c:write_status_progress for some background. */ +static void +write_progress (int countmode, unsigned long long current, + unsigned long long total_arg) +{ + char units[] = "BKMGTPEZY?"; + int unitidx = 0; + uint64_t total = total_arg; + + if (!opt.status_stream) + return; /* Not enabled. */ + + if (countmode) + { + if (total && current > total) + current = total; + } + else if (total) /* Size mode: This may use units. */ + { + if (current > total) + current = total; + + while (total > 1024*1024) + { + total /= 1024; + current /= 1024; + unitidx++; + } + } + else /* Size mode */ + { + while (current > 1024*1024) + { + current /= 1024; + unitidx++; + } + } + + if (unitidx > sizeof units - 1) + unitidx = sizeof units - 1; + + if (countmode) + es_fprintf (opt.status_stream, "[GNUPG:] PROGRESS gpgtar c %zu %zu\n", + (size_t)current, (size_t)total); + else + es_fprintf (opt.status_stream, "[GNUPG:] PROGRESS gpgtar s %zu %zu %c%s\n", + (size_t)current, (size_t)total, + units[unitidx], + unitidx? "iB" : ""); +} + /* On Windows convert name to UTF8 and return it; caller must release * the result. On Unix or if ALREADY_UTF8 is set, this function is a @@ -300,6 +361,12 @@ add_entry (const char *dname, const char *entryname, scanctrl_t scanctrl) gpgtar_print_header (hdr, NULL, log_get_stream ()); *scanctrl->flist_tail = hdr; scanctrl->flist_tail = &hdr->next; + scanctrl->file_count++; + /* Print a progress line during scnanning in increments of 5000 + * and not of 100 as we doing during write: Scanning is of + * course much faster. */ + if (!(scanctrl->file_count % 5000)) + write_progress (1, scanctrl->file_count, 0); } return 0; @@ -709,7 +776,7 @@ build_header (void *record, tar_header_t hdr, strlist_t *r_exthdr) * will do. To ease testing we also put in the PID. The * count is bumped after the header has been written. */ snprintf (raw->name, sizeof raw->name-1, "_@paxheader.%u.%lu", - (unsigned int)getpid(), global_header_count + 1); + (unsigned int)getpid(), global_written_files + 1); } } @@ -881,7 +948,7 @@ write_extended_header (estream_t stream, const void *record, strlist_t exthdr) static gpg_error_t -write_file (estream_t stream, tar_header_t hdr) +write_file (estream_t stream, tar_header_t hdr, unsigned int *skipped_open) { gpg_error_t err; char record[RECORDSIZE]; @@ -895,7 +962,7 @@ write_file (estream_t stream, tar_header_t hdr) { if (gpg_err_code (err) == GPG_ERR_NOT_SUPPORTED) { - log_info ("skipping unsupported file '%s'\n", hdr->name); + log_info ("silently skipping unsupported file '%s'\n", hdr->name); err = 0; } return err; @@ -907,9 +974,12 @@ write_file (estream_t stream, tar_header_t hdr) if (!infp) { err = gpg_error_from_syserror (); - log_error ("can't open '%s': %s - skipped\n", + log_info ("can't open '%s': %s - skipped\n", hdr->name, gpg_strerror (err)); - return err; + ++*skipped_open; + if (!*skipped_open) /* Protect against overflow. */ + --*skipped_open; + return 0; } } else @@ -920,7 +990,9 @@ write_file (estream_t stream, tar_header_t hdr) err = write_record (stream, record); if (err) goto leave; - global_header_count++; + global_written_files++; + if (!(global_written_files % 100)) + write_progress (1, global_written_files, global_total_files); if (hdr->typeflag == TF_REGULAR) { @@ -946,6 +1018,9 @@ write_file (estream_t stream, tar_header_t hdr) err = write_record (stream, record); if (err) goto leave; + global_written_data += nbytes; + if (!((global_written_data/nbytes) % (2048*100))) + write_progress (0, global_written_data, global_total_data); } nread = es_fread (record, 1, 1, infp); if (nread) @@ -995,6 +1070,7 @@ gpgtar_create (char **inpattern, const char *files_from, int null_names, estream_t outstream = NULL; int eof_seen = 0; pid_t pid = (pid_t)(-1); + unsigned int skipped_open = 0; memset (scanctrl, 0, sizeof *scanctrl); scanctrl->flist_tail = &scanctrl->flist; @@ -1137,6 +1213,17 @@ gpgtar_create (char **inpattern, const char *files_from, int null_names, if (files_from_stream && files_from_stream != es_stdin) es_fclose (files_from_stream); + global_total_files = global_total_data = 0; + global_written_files = global_written_data = 0; + for (hdr = scanctrl->flist; hdr; hdr = hdr->next) + { + global_total_files++; + global_total_data += hdr->size; + } + write_progress (1, 0, global_total_files); + write_progress (0, 0, global_total_data); + + if (encrypt || sign) { strlist_t arg; @@ -1228,17 +1315,20 @@ gpgtar_create (char **inpattern, const char *files_from, int null_names, es_set_binary (outstream); } - + skipped_open = 0; for (hdr = scanctrl->flist; hdr; hdr = hdr->next) { - err = write_file (outstream, hdr); + err = write_file (outstream, hdr, &skipped_open); if (err) goto leave; } + err = write_eof_mark (outstream); if (err) goto leave; + write_progress (1, global_written_files, global_total_files); + write_progress (0, global_written_data, global_total_data); if (pid != (pid_t)(-1)) { @@ -1259,6 +1349,12 @@ gpgtar_create (char **inpattern, const char *files_from, int null_names, } } + if (skipped_open) + { + log_info ("number of skipped files: %u\n", skipped_open); + log_error ("exiting with failure status due to previous errors\n"); + } + leave: if (!err) { diff --git a/tools/gpgtar.c b/tools/gpgtar.c index b412f3ffb..5d16b70a7 100644 --- a/tools/gpgtar.c +++ b/tools/gpgtar.c @@ -491,6 +491,32 @@ main (int argc, char **argv) log_info (_("NOTE: '%s' is not considered an option\n"), argv[i]); } + /* Set status stream for our own use of --status-fd. The original + * status fd is passed verbatim to gpg. */ + if (opt.status_fd) + { + int fd = translate_sys2libc_fd_int (opt.status_fd, 1); + + if (!gnupg_fd_valid (fd)) + log_fatal ("status-fd is invalid: %s\n", strerror (errno)); + + if (fd == 1) + opt.status_stream = es_stdout; + else if (fd == 2) + opt.status_stream = es_stderr; + else + { + opt.status_stream = es_fdopen (fd, "w"); + if (opt.status_stream) + es_setvbuf (opt.status_stream, NULL, _IOLBF, 0); + } + if (!opt.status_stream) + { + log_fatal ("can't open fd %d for status output: %s\n", + fd, strerror (errno)); + } + } + if (! opt.gpg_program) opt.gpg_program = gnupg_module_name (GNUPG_MODULE_NAME_GPG); diff --git a/tools/gpgtar.h b/tools/gpgtar.h index 0854064fb..303db0045 100644 --- a/tools/gpgtar.h +++ b/tools/gpgtar.h @@ -45,6 +45,7 @@ struct int answer_yes; int answer_no; int status_fd; + estream_t status_stream; int require_compliance; int with_log; } opt;