diff --git a/src/buffer_manager.cc b/src/buffer_manager.cc
index 9568c33e..bb9f7902 100644
--- a/src/buffer_manager.cc
+++ b/src/buffer_manager.cc
@@ -18,7 +18,8 @@ struct name_not_unique : runtime_error
 
 BufferManager::~BufferManager()
 {
-    kak_assert(not ClientManager::has_instance());
+    // Make sure not clients exists
+    ClientManager::instance().clear();
 
     // delete remaining buffers
     while (not m_buffers.empty())
diff --git a/src/client_manager.cc b/src/client_manager.cc
index 675d2e66..b625fdc5 100644
--- a/src/client_manager.cc
+++ b/src/client_manager.cc
@@ -14,6 +14,12 @@ namespace Kakoune
 ClientManager::ClientManager() = default;
 ClientManager::~ClientManager()
 {
+    clear();
+}
+
+void ClientManager::clear()
+{
+    m_free_windows.clear();
     // So that clients destructor find the client manager empty
     // so that local UI does not fork.
     ClientList clients = std::move(m_clients);
diff --git a/src/client_manager.hh b/src/client_manager.hh
index cd7f12bf..c6ba1c35 100644
--- a/src/client_manager.hh
+++ b/src/client_manager.hh
@@ -33,6 +33,8 @@ public:
     bool   empty() const { return m_clients.empty(); }
     size_t count() const { return m_clients.size(); }
 
+    void clear();
+
     void   ensure_no_client_uses_buffer(Buffer& buffer);
 
     WindowAndSelections get_free_window(Buffer& buffer);
diff --git a/src/main.cc b/src/main.cc
index 1b6f38cb..b1c5a7df 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -463,8 +463,8 @@ int run_server(StringView session, StringView init_command,
     HighlighterRegistry highlighter_registry;
     DefinedHighlighters defined_highlighters;
     FaceRegistry        face_registry;
-    BufferManager       buffer_manager;
     ClientManager       client_manager;
+    BufferManager       buffer_manager;
 
     register_options();
     register_env_vars();
@@ -590,8 +590,8 @@ int run_filter(StringView keystr, StringView commands, ConstArrayView<StringView
     ShellManager    shell_manager;
     CommandManager  command_manager;
     RegisterManager register_manager;
-    BufferManager   buffer_manager;
     ClientManager   client_manager;
+    BufferManager   buffer_manager;
 
     register_options();
     register_env_vars();