diff --git a/dirmngr/server.c b/dirmngr/server.c
index 651f67cf8..87a0d7784 100644
--- a/dirmngr/server.c
+++ b/dirmngr/server.c
@@ -3135,8 +3135,10 @@ start_command_handler (assuan_fd_t fd, unsigned int session_id)
                ctrl->refcount);
   else
     {
+#if USE_LDAP
       ks_ldap_free_state (ctrl->ks_get_state);
       ctrl->ks_get_state = NULL;
+#endif
       release_ctrl_ocsp_certs (ctrl);
       xfree (ctrl->server_local);
       dirmngr_deinit_default_ctrl (ctrl);