Make word insert completion work better with unicode char

src/insert_completer.cc

@@ -10,6 +10,7 @@
 #include "user_interface.hh"
 #include "window.hh"
 #include "word_db.hh"
+#include "utf8_iterator.hh"
 
 #include <numeric>
 
@@ -75,23 +76,24 @@ WordDB& get_word_db(const Buffer& buffer)
 template<bool other_buffers>
 InsertCompletion complete_word(const Buffer& buffer, ByteCoord cursor_pos)
 {
-   auto pos = buffer.iterator_at(cursor_pos);
-   if (pos == buffer.begin() or not is_word(*utf8::previous(pos, buffer.begin())))
-       return {};
+    using Utf8It = utf8::iterator<BufferIterator>;
+    Utf8It pos{buffer.iterator_at(cursor_pos), buffer};
+    if (pos == buffer.begin() or not is_word(*(pos-1)))
+        return {};
 
-    auto end = buffer.iterator_at(cursor_pos);
+    auto end = Utf8It{buffer.iterator_at(cursor_pos), buffer};
     auto begin = end-1;
     while (begin != buffer.begin() and is_word(*begin))
         --begin;
     if (not is_word(*begin))
         ++begin;
 
-    String prefix{begin, end};
+    String prefix{begin.base(), end.base()};
 
     while (end != buffer.end() and is_word(*end))
         ++end;
 
-    String current_word{begin, end};
+    String current_word{begin.base(), end.base()};
 
     struct RankedMatchAndBuffer : RankedMatch
     {
@@ -153,7 +155,7 @@ InsertCompletion complete_word(const Buffer& buffer, ByteCoord cursor_pos)
         candidates.push_back({m.candidate().str(), "", std::move(menu_entry)});
     }
 
-    return { begin.coord(), cursor_pos, std::move(candidates), buffer.timestamp() };
+    return { begin.base().coord(), cursor_pos, std::move(candidates), buffer.timestamp() };
 }
 
 template<bool require_slash>

src/ranked_match.cc

@@ -1,55 +1,59 @@
 #include "ranked_match.hh"
 
+#include "utf8_iterator.hh"
 #include "unit_tests.hh"
 
 namespace Kakoune
 {
 
+using Utf8It = utf8::iterator<const char*>;
+
 static int count_word_boundaries_match(StringView candidate, StringView query)
 {
     int count = 0;
-    auto it = query.begin();
-    char prev = 0;
-    for (auto c : candidate)
+    Utf8It qit{query.begin(), query};
+    Codepoint prev = 0;
+    for (Utf8It it{candidate.begin(), candidate}; it != candidate.end(); ++it)
     {
+        const Codepoint c = *it;
         const bool is_word_boundary = prev == 0 or
-                                      (ispunct(prev) and is_word(c)) or
+                                      (!iswalnum(prev) and iswalnum(c)) or
                                       (islower(prev) and isupper(c));
         prev = c;
 
         if (not is_word_boundary)
             continue;
 
-        const char lc = tolower(c);
-        for (; it != query.end(); ++it)
+        const Codepoint lc = tolower(c);
+        for (; qit != query.end(); ++qit)
         {
-            const char qc = *it;
+            const Codepoint qc = *qit;
             if (qc == (islower(qc) ? lc  : c))
             {
                 ++count;
-                ++it;
+                ++qit;
                 break;
             }
         }
-        if (it == query.end())
+        if (qit == query.end())
             break;
     }
     return count;
 }
 
-static bool smartcase_eq(char query, char candidate)
+static bool smartcase_eq(Codepoint query, Codepoint candidate)
 {
     return query == (islower(query) ? tolower(candidate) : candidate);
 }
 
 static bool subsequence_match_smart_case(StringView str, StringView subseq)
 {
-    auto it = str.begin();
-    for (auto& c : subseq)
+    Utf8It it{str.begin(), str};
+    for (Utf8It subseq_it{subseq.begin(), subseq}; subseq_it != subseq.end(); ++subseq_it)
     {
         if (it == str.end())
             return false;
-        while (not smartcase_eq(c, *it))
+        while (not smartcase_eq(*subseq_it, *it))
         {
             if (++it == str.end())
                 return false;
@@ -98,9 +102,9 @@ bool RankedMatch::operator<(const RankedMatch& other) const
         return m_first_char_match;
 
     return std::lexicographical_compare(
-        m_candidate.begin(), m_candidate.end(),
-        other.m_candidate.begin(), other.m_candidate.end(),
-        [](char a, char b) {
+        Utf8It{m_candidate.begin(), m_candidate}, Utf8It{m_candidate.end(), m_candidate},
+        Utf8It{other.m_candidate.begin(), other.m_candidate}, Utf8It{other.m_candidate.end(), other.m_candidate},
+        [](Codepoint a, Codepoint b) {
             const bool low_a = islower(a), low_b = islower(b);
             return low_a == low_b ? a < b : low_a;
         });

src/utf8_iterator.hh

@@ -25,7 +25,7 @@ public:
 
     template<typename Container>
     iterator(Iterator it, const Container& c)
-        : m_it{std::move(it)}, m_begin{begin(c)}, m_end{end(c)}
+        : m_it{std::move(it)}, m_begin{std::begin(c)}, m_end{std::end(c)}
     {}
 
     iterator& operator++()