diff --git a/agent/findkey.c b/agent/findkey.c
index c5e7ae74e..3cf8d0cc1 100644
--- a/agent/findkey.c
+++ b/agent/findkey.c
@@ -1311,7 +1311,7 @@ agent_delete_key (ctrl_t ctrl, const char *desc_text,
       break;
 
     case PRIVATE_KEY_SHADOWED:
-      err = gpg_error (GPG_ERR_KEY_ON_CARD);
+      err = remove_key_file (grip);
       break;
 
     default: