gpg: Detect already compressed data also when using a pipe.

* common/iobuf.c (file_filter_ctx_t): Add fields for the peek feature.
(file_filter): Implement peeking.
(iobuf_ioctl): Add new IOBUF_IOCTL_PEEK.
* common/iobuf.h (IOBUF_IOCTL_PEEK, IOBUFCTRL_PEEK): New.
* common/miscellaneous.c (is_file_compressed): Rewrite.  Detect PDF.
* g10/encrypt.c (encrypt_simple): Peek before detecting compression.
(encrypt_crypt): Ditto.
* g10/sign.c (sign_file): Also detect already compressed data.

* g10/options.h (opt): Add explicit_compress_option.
* g10/gpg.c (main): Set opt.explicit_compress_option for -z.

--

Note that this patch also introduces a compression check for signing
which was never done in the past.

GnuPG-bug-id: 6332

g10/encrypt.c

@@ -1,7 +1,7 @@
 /* encrypt.c - Main encryption driver
  * Copyright (C) 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005,
  *               2006, 2009 Free Software Foundation, Inc.
- * Copyright (C) 2016 g10 Code GmbH
+ * Copyright (C) 2016, 2023 g10 Code GmbH
  *
  * This file is part of GnuPG.
  *
@@ -17,6 +17,7 @@
  *
  * You should have received a copy of the GNU General Public License
  * along with this program; if not, see <https://www.gnu.org/licenses/>.
+ * SPDX-License-Identifier: GPL-3.0-or-later
  */
 
 #include <config.h>
@@ -409,6 +410,8 @@ encrypt_simple (const char *filename, int mode, int use_seskey)
   text_filter_context_t tfx;
   progress_filter_context_t *pfx;
   int do_compress = !!default_compress_algo();
+  char peekbuf[32];
+  int  peekbuflen;
 
   if (!gnupg_rng_is_compliant (opt.compliance))
     {
@@ -445,6 +448,14 @@ encrypt_simple (const char *filename, int mode, int use_seskey)
       return rc;
     }
 
+  peekbuflen = iobuf_ioctl (inp, IOBUF_IOCTL_PEEK, sizeof peekbuf, peekbuf);
+  if (peekbuflen < 0)
+    {
+      peekbuflen = 0;
+      if (DBG_FILTER)
+        log_debug ("peeking at input failed\n");
+    }
+
   handle_progress (pfx, inp, filename);
 
   if (opt.textmode)
@@ -509,10 +520,11 @@ encrypt_simple (const char *filename, int mode, int use_seskey)
   if (do_compress
       && cfx.dek
       && (cfx.dek->use_mdc || cfx.dek->use_aead)
-      && is_file_compressed(filename, &rc))
+      && !opt.explicit_compress_option
+      && is_file_compressed (peekbuf, peekbuflen))
     {
       if (opt.verbose)
-        log_info(_("'%s' already compressed\n"), filename);
+        log_info(_("'%s' already compressed\n"), filename? filename: "[stdin]");
       do_compress = 0;
     }
 
@@ -780,6 +792,8 @@ encrypt_crypt (ctrl_t ctrl, int filefd, const char *filename,
   progress_filter_context_t *pfx;
   PK_LIST pk_list;
   int do_compress;
+  char peekbuf[32];
+  int  peekbuflen;
 
   if (filefd != -1 && filename)
     return gpg_error (GPG_ERR_INV_ARG);  /* Both given.  */
@@ -852,6 +866,14 @@ encrypt_crypt (ctrl_t ctrl, int filefd, const char *filename,
   if (opt.verbose)
     log_info (_("reading from '%s'\n"), iobuf_get_fname_nonnull (inp));
 
+  peekbuflen = iobuf_ioctl (inp, IOBUF_IOCTL_PEEK, sizeof peekbuf, peekbuf);
+  if (peekbuflen < 0)
+    {
+      peekbuflen = 0;
+      if (DBG_FILTER)
+        log_debug ("peeking at input failed\n");
+    }
+
   handle_progress (pfx, inp, filename);
 
   if (opt.textmode)
@@ -884,10 +906,11 @@ encrypt_crypt (ctrl_t ctrl, int filefd, const char *filename,
    * ciphertext attacks. */
   if (do_compress
       && (cfx.dek->use_mdc || cfx.dek->use_aead)
-      && is_file_compressed (filename, &rc2))
+      && !opt.explicit_compress_option
+      && is_file_compressed (peekbuf, peekbuflen))
     {
       if (opt.verbose)
-        log_info(_("'%s' already compressed\n"), filename);
+        log_info(_("'%s' already compressed\n"), filename? filename: "[stdin]");
       do_compress = 0;
     }
   if (rc2)

g10/gpg.c

@@ -3203,6 +3203,7 @@ main (int argc, char **argv)
 	  case oCompress:
 	    /* this is the -z command line option */
 	    opt.compress_level = opt.bz2_compress_level = pargs.r.ret_int;
+            opt.explicit_compress_option = 1;
 	    break;
 	  case oCompressLevel: opt.compress_level = pargs.r.ret_int; break;
 	  case oBZ2CompressLevel: opt.bz2_compress_level = pargs.r.ret_int; break;

g10/options.h

@@ -98,6 +98,7 @@ struct
   int def_digest_algo;
   int cert_digest_algo;
   int compress_algo;
+  int explicit_compress_option; /* A compress option was explicitly given. */
   int compress_level;
   int bz2_compress_level;
   int bz2_decompress_lowmem;

g10/sign.c

@@ -1037,6 +1037,9 @@ sign_file (ctrl_t ctrl, strlist_t filenames, int detached, strlist_t locusr,
   int multifile = 0;
   u32 duration=0;
   pt_extra_hash_data_t extrahash = NULL;
+  char peekbuf[32];
+  int  peekbuflen = 0;
+
 
   pfx = new_progress_context ();
   afx = new_armor_context ();
@@ -1095,6 +1098,14 @@ sign_file (ctrl_t ctrl, strlist_t filenames, int detached, strlist_t locusr,
           goto leave;
 	}
 
+      peekbuflen = iobuf_ioctl (inp, IOBUF_IOCTL_PEEK, sizeof peekbuf, peekbuf);
+      if (peekbuflen < 0)
+        {
+          peekbuflen = 0;
+          if (DBG_FILTER)
+            log_debug ("peeking at input failed\n");
+        }
+
       handle_progress (pfx, inp, fname);
     }
 
@@ -1251,8 +1262,14 @@ sign_file (ctrl_t ctrl, strlist_t filenames, int detached, strlist_t locusr,
     {
       int compr_algo = opt.compress_algo;
 
-      /* If not forced by user */
-      if (compr_algo==-1)
+      if (!opt.explicit_compress_option
+          && is_file_compressed (peekbuf, peekbuflen))
+        {
+          if (opt.verbose)
+            log_info(_("'%s' already compressed\n"), fname? fname: "[stdin]");
+          compr_algo = 0;
+        }
+      else if (compr_algo==-1)
         {
           /* If we're not encrypting, then select_algo_from_prefs
            * will fail and we'll end up with the default.  If we are