diff --git a/agent/findkey.c b/agent/findkey.c
index d3a3b335c..060cb786d 100644
--- a/agent/findkey.c
+++ b/agent/findkey.c
@@ -1186,6 +1186,15 @@ agent_key_from_file (ctrl_t ctrl, const char *cache_nonce,
     return gpg_error (GPG_ERR_NO_SECKEY);
 
   err = read_key_file (grip? grip : ctrl->keygrip, &s_skey, &keymeta);
+  if (err)
+    {
+      if (gpg_err_code (err) == GPG_ERR_ENOENT)
+        err = gpg_error (GPG_ERR_NO_SECKEY);
+      else
+        log_error ("findkey: error reading key file: %s\n",
+                   gpg_strerror (err));
+      return err;
+    }
 
   /* For use with the protection functions we also need the key as an
      canonical encoded S-expression in a buffer.  Create this buffer
diff --git a/agent/pkdecrypt.c b/agent/pkdecrypt.c
index 82818f863..c26f21d35 100644
--- a/agent/pkdecrypt.c
+++ b/agent/pkdecrypt.c
@@ -74,8 +74,7 @@ agent_pkdecrypt (ctrl_t ctrl, const char *desc_text,
     no_shadow_info = 1;
   else if (err)
     {
-      if (gpg_err_code (err) != GPG_ERR_NO_SECKEY)
-        log_error ("failed to read the secret key\n");
+      log_error ("failed to read the secret key\n");
       goto leave;
     }
 
@@ -88,7 +87,7 @@ agent_pkdecrypt (ctrl_t ctrl, const char *desc_text,
           goto leave;
         }
 
-      if (agent_is_tpm2_key (s_skey))
+      if (s_skey && agent_is_tpm2_key (s_skey))
 	err = divert_tpm2_pkdecrypt (ctrl, ciphertext, shadow_info,
                                      &buf, &len, r_padding);
       else
@@ -96,7 +95,15 @@ agent_pkdecrypt (ctrl_t ctrl, const char *desc_text,
                                 &buf, &len, r_padding);
       if (err)
         {
-          log_error ("smartcard decryption failed: %s\n", gpg_strerror (err));
+          /* We restore the original error (ie. no seckey) is no card
+           * has been found and we have no shadow key.  This avoids a
+           * surprising "card removed" error code.  */
+          if ((gpg_err_code (err) == GPG_ERR_CARD_REMOVED
+               || gpg_err_code (err) == GPG_ERR_CARD_NOT_PRESENT)
+              && no_shadow_info)
+            err = gpg_error (GPG_ERR_NO_SECKEY);
+          else
+            log_error ("smartcard decryption failed: %s\n", gpg_strerror (err));
           goto leave;
         }