diff --git a/src/event_manager.cc b/src/event_manager.cc
index 2396a04e..d5d0fa2f 100644
--- a/src/event_manager.cc
+++ b/src/event_manager.cc
@@ -12,38 +12,38 @@ void EventManager::watch(int fd, EventHandler handler)
     if (event != m_events.end())
         throw runtime_error("fd already watched");
 
-    m_events.push_back(pollfd{ fd, POLLIN | POLLPRI, 0 });
-    m_handlers.emplace(fd, std::move(handler));
+    m_events.emplace_back(pollfd{ fd, POLLIN | POLLPRI, 0 });
+    m_handlers.emplace_back(new EventHandler(std::move(handler)));
 }
 
 void EventManager::unwatch(int fd)
 {
-    // do not unwatch now, do that at the end of handle_next_events,
-    // so that if unwatch(fd) is called from fd event handler,
-    // it is not deleted now.
-    m_unwatched.push_back(fd);
+    auto event = std::find_if(m_events.begin(), m_events.end(),
+                              [&](const pollfd& pfd) { return pfd.fd == fd; });
+    assert(event != m_events.end());
+    auto handler = m_handlers.begin() + (event - m_events.begin());
+
+    // keep handler in m_handlers_trash so that it does not die now,
+    // but at the end of handle_next_events. We do this as handler might
+    // be our caller.
+    m_handlers_trash.emplace_back(std::move(*handler));
+    m_handlers.erase(handler);
+    m_events.erase(event);
 }
 
 void EventManager::handle_next_events()
 {
     const int timeout_ms = 100;
     poll(m_events.data(), m_events.size(), timeout_ms);
-    for (auto& event : m_events)
+    for (size_t i = 0; i < m_events.size(); ++i)
     {
+        auto& event = m_events[i];
         const int fd = event.fd;
-        if ((event.revents or contains(m_forced, fd)) and not contains(m_unwatched, fd))
-            m_handlers[fd](fd);
+        if (event.revents or contains(m_forced, fd))
+            (*m_handlers[i])(fd);
     }
 
-    // remove unwatched.
-    for (auto fd : m_unwatched)
-    {
-        auto it = std::find_if(m_events.begin(), m_events.end(),
-                               [fd](pollfd& p) { return p.fd == fd; });
-        m_events.erase(it);
-        m_handlers.erase(fd);
-    }
-    m_unwatched.clear();
+    m_handlers_trash.clear();
     m_forced.clear();
 }
 
diff --git a/src/event_manager.hh b/src/event_manager.hh
index 02f86986..c3b2a92e 100644
--- a/src/event_manager.hh
+++ b/src/event_manager.hh
@@ -35,10 +35,10 @@ public:
     void force_signal(int fd);
 
 private:
-    std::vector<pollfd>                   m_events;
-    std::unordered_map<int, EventHandler> m_handlers;
-    std::vector<int>                      m_forced;
-    std::vector<int>                      m_unwatched;
+    std::vector<pollfd>                        m_events;
+    std::vector<std::unique_ptr<EventHandler>> m_handlers;
+    std::vector<std::unique_ptr<EventHandler>> m_handlers_trash;
+    std::vector<int>                           m_forced;
 };
 
 }