diff --git a/src/remote.cc b/src/remote.cc
index 850fa59b..ba8ff05c 100644
--- a/src/remote.cc
+++ b/src/remote.cc
@@ -669,7 +669,8 @@ void Server::close_session()
 
 Server::~Server()
 {
-    close_session();
+    if (m_listener)
+        close_session();
 }
 
 void Server::remove_accepter(Accepter* accepter)