diff --git a/src/buffer.cc b/src/buffer.cc
index fb4c61ad..824006df 100644
--- a/src/buffer.cc
+++ b/src/buffer.cc
@@ -41,6 +41,7 @@ Buffer::Buffer(const std::string& name, Type type,
 Buffer::~Buffer()
 {
     m_windows.clear();
+    BufferManager::instance().unregister_buffer(this);
     assert(m_modification_listeners.empty());
 }
 
diff --git a/src/buffer_manager.cc b/src/buffer_manager.cc
index c3211f7c..d8f71861 100644
--- a/src/buffer_manager.cc
+++ b/src/buffer_manager.cc
@@ -2,7 +2,6 @@
 
 #include "assert.hh"
 #include "buffer.hh"
-#include "window.hh"
 #include "exception.hh"
 
 namespace Kakoune
@@ -17,15 +16,18 @@ void BufferManager::register_buffer(Buffer* buffer)
     if (m_buffers.find(name) != m_buffers.end())
         throw name_not_unique();
 
-    m_buffers[name] = std::unique_ptr<Buffer>(buffer);
+    m_buffers[name] = buffer;
 }
 
-void BufferManager::delete_buffer(Buffer* buffer)
+void BufferManager::unregister_buffer(Buffer* buffer)
 {
     assert(buffer);
     auto buffer_it = m_buffers.find(buffer->name());
-    assert(buffer_it->second.get() == buffer);
-    m_buffers.erase(buffer_it);
+    if (buffer_it != m_buffers.end())
+    {
+        assert(buffer_it->second == buffer);
+        m_buffers.erase(buffer_it);
+    }
 }
 
 Buffer* BufferManager::get_buffer(const std::string& name)
@@ -33,7 +35,7 @@ Buffer* BufferManager::get_buffer(const std::string& name)
     if (m_buffers.find(name) == m_buffers.end())
         return nullptr;
 
-    return m_buffers[name].get();
+    return m_buffers[name];
 }
 
 CandidateList BufferManager::complete_buffername(const std::string& prefix,
diff --git a/src/buffer_manager.hh b/src/buffer_manager.hh
index ee3473cd..7109dc11 100644
--- a/src/buffer_manager.hh
+++ b/src/buffer_manager.hh
@@ -1,20 +1,20 @@
 #ifndef buffer_manager_hh_INCLUDED
 #define buffer_manager_hh_INCLUDED
 
-#include "buffer.hh"
 #include "completion.hh"
 #include "utils.hh"
 
 #include <unordered_map>
-#include <memory>
 
 namespace Kakoune
 {
 
+class Buffer;
+
 class BufferManager : public Singleton<BufferManager>
 {
 public:
-    typedef std::unordered_map<std::string, std::unique_ptr<Buffer>> BufferMap;
+    typedef std::unordered_map<std::string, Buffer*> BufferMap;
 
     struct iterator : public BufferMap::const_iterator
     {
@@ -23,11 +23,11 @@ public:
         iterator() {}
         iterator(const parent_type& other) : parent_type(other) {}
         Buffer& operator*()  const { return *(parent_type::operator*().second); }
-        Buffer* operator->() const { return parent_type::operator*().second.get(); }
+        Buffer* operator->() const { return parent_type::operator*().second; }
     };
 
     void register_buffer(Buffer* buffer);
-    void delete_buffer(Buffer* buffer);
+    void unregister_buffer(Buffer* buffer);
 
     iterator begin() const { return iterator(m_buffers.begin()); }
     iterator end() const { return iterator(m_buffers.end()); }
diff --git a/src/file.cc b/src/file.cc
index ca074e2f..08ce8ae6 100644
--- a/src/file.cc
+++ b/src/file.cc
@@ -77,7 +77,7 @@ Buffer* create_buffer_from_file(const std::string& filename)
     std::string content = read_file(filename);
 
     if (Buffer* buffer = BufferManager::instance().get_buffer(filename))
-        BufferManager::instance().delete_buffer(buffer);
+        delete buffer;
 
     return new Buffer(filename, Buffer::Type::File, content);
 }