Add an expand_unprintable highlighter which replaces unprintable char with U+XXXX

src/highlighters.cc

@@ -7,6 +7,10 @@
 #include "context.hh"
 #include "string.hh"
 #include "utf8.hh"
+#include "utf8_iterator.hh"
+
+#include <sstream>
+#include <locale>
 
 namespace Kakoune
 {
@@ -279,6 +283,35 @@ void highlight_selections(Window& window, DisplayBuffer& display_buffer)
                     [](DisplayAtom& atom) { atom.attribute |= Attributes::Bold; });
 }
 
+void expand_unprintable(DisplayBuffer& display_buffer)
+{
+    for (auto& line : display_buffer.lines())
+    {
+        for (auto& atom : line)
+        {
+            if (atom.content.type() == AtomContent::BufferRange)
+            {
+                using Utf8It = utf8::utf8_iterator<BufferIterator>;
+                for (Utf8It it = atom.content.begin(), end = atom.content.end(); it != end; ++it)
+                {
+                    Codepoint cp = *it;
+                    if (cp != '\n' and not std::isprint((wchar_t)cp, std::locale()))
+                    {
+                        std::ostringstream oss;
+                        oss << "U+" << std::hex << cp;
+                        String str = oss.str();
+                        highlight_range(display_buffer,
+                                        it.underlying_iterator(), (it+1).underlying_iterator(),
+                                        true, [&str](DisplayAtom& atom){ atom.content.replace(str);
+                                                                         atom.bg_color = Color::Red;
+                                                                         atom.fg_color = Color::Black; });
+                    }
+                }
+            }
+        }
+    }
+}
+
 template<void (*highlighter_func)(DisplayBuffer&)>
 class SimpleHighlighterFactory
 {
@@ -324,6 +357,7 @@ void register_highlighters()
 
     registry.register_func("highlight_selections", WindowHighlighterFactory<highlight_selections>("highlight_selections"));
     registry.register_func("expand_tabs", WindowHighlighterFactory<expand_tabulations>("expand_tabs"));
+    registry.register_func("expand_unprintable", SimpleHighlighterFactory<expand_unprintable>("expand_unprintable"));
     registry.register_func("number_lines", WindowHighlighterFactory<show_line_numbers>("number_lines"));
     registry.register_func("regex", colorize_regex_factory);
     registry.register_func("search", highlight_search_factory);

src/main.cc

@@ -34,6 +34,8 @@
 #include <unistd.h>
 #include <fcntl.h>
 
+#include <locale>
+
 using namespace Kakoune;
 using namespace std::placeholders;
 
@@ -808,6 +810,7 @@ int main(int argc, char* argv[])
 {
     try
     {
+        std::locale::global(std::locale("en_US.UTF-8"));
         signal(SIGSEGV, sigsegv_handler);
         std::vector<String> params;
         for (size_t i = 1; i < argc; ++i)

src/window.cc

@@ -23,6 +23,7 @@ Window::Window(Buffer& buffer)
     m_options.register_watcher(*this);
 
     m_highlighters.append(registry["expand_tabs"](*this, {}));
+    m_highlighters.append(registry["expand_unprintable"](*this, {}));
     m_highlighters.append(registry["highlight_selections"](*this, {}));
 
     for (auto& option : m_options.flatten_options())