Refactor highlighting logic

Always start with full buffer lines and trim the display buffer at
the very end, treat non-range display atoms as non-trimable in that
case and keep track of how many columns are occupied by "widgets"
such as line numbers or flags.

Fixes #4659

src/display_buffer.cc

@@ -192,10 +192,16 @@ ColumnCount DisplayLine::length() const
     return len;
 }
 
-bool DisplayLine::trim(ColumnCount first_col, ColumnCount col_count)
+bool DisplayLine::trim(ColumnCount first_col, ColumnCount col_count, bool only_buffer)
 {
     for (auto it = begin(); first_col > 0 and it != end(); )
     {
+        if (only_buffer and !it->has_buffer_range())
+        {
+            ++it;
+            continue;
+        }
+
         auto len = it->length();
         if (len <= first_col)
         {

src/display_buffer.hh

@@ -27,6 +27,9 @@ public:
     DisplayAtom(const Buffer& buffer, BufferCoord begin, BufferCoord end)
         : m_type(Range), m_buffer(&buffer), m_range{begin, end} {}
 
+    DisplayAtom(const Buffer& buffer, BufferCoord begin, BufferCoord end, String str)
+        : m_type(ReplacedRange), m_buffer(&buffer), m_range{begin, end}, m_text{std::move(str)} {}
+
     DisplayAtom(String str, Face face)
         : face(face), m_type(Text), m_text(std::move(str)) {}
 
@@ -143,7 +146,7 @@ public:
 
     // remove first_col from the begining of the line, and make sure
     // the line is less that col_count character
-    bool trim(ColumnCount first_col, ColumnCount col_count);
+    bool trim(ColumnCount first_col, ColumnCount col_count, bool only_buffer = false);
 
     // Merge together consecutive atoms sharing the same display attributes
     void optimize();

src/highlighter.hh

@@ -38,16 +38,14 @@ struct Highlighter;
 
 struct DisplaySetup
 {
-    // Window position relative to the buffer origin
+    LineCount first_line;
-    DisplayCoord window_pos;
+    LineCount line_count;
-    // Range of lines and columns from the buffer that will get displayed
+    ColumnCount first_column;
-    DisplayCoord window_range;
+    ColumnCount widget_columns;
     // Position of the cursor in the window
     DisplayCoord cursor_pos;
     // Offset of line and columns that must remain visible around cursor
     DisplayCoord scroll_offset;
-    // Put full lines in the initial display buffer
-    bool full_lines;
 };
 
 using HighlighterIdList = ConstArrayView<StringView>;

src/highlighters.cc

@@ -638,14 +638,13 @@ std::unique_ptr<Highlighter> create_column_highlighter(HighlighterParameters par
             return;
 
         const auto face = context.context.faces()[facespec];
-        const auto win_column = context.setup.window_pos.column;
+        if (column < context.setup.first_column or column >= context.setup.first_column + context.context.window().dimensions().column)
-        const auto target_col = column - win_column;
-        if (target_col < 0 or target_col >= context.setup.window_range.column)
             return;
 
+        const Buffer& buffer = context.context.buffer();
         for (auto& line : display_buffer.lines())
         {
-            auto remaining_col = target_col;
+            auto remaining_col = column;
             bool found = false;
             auto first_buf = find_if(line, [](auto& atom) { return atom.has_buffer_range(); });
             for (auto atom_it = first_buf; atom_it != line.end(); ++atom_it)
@@ -667,7 +666,7 @@ std::unique_ptr<Highlighter> create_column_highlighter(HighlighterParameters par
                 continue;
 
             if (remaining_col > 0)
-                line.push_back({String{' ', remaining_col}, {}});
+                line.push_back({buffer, buffer.end_coord(), buffer.end_coord(), String{' ', remaining_col}});
             line.push_back({" ", face});
         }
     };
@@ -702,7 +701,7 @@ struct WrapHighlighter : Highlighter
         if (contains(context.disabled_ids, ms_id))
             return;
 
-        const ColumnCount wrap_column = std::min(m_max_width, context.setup.window_range.column);
+        const ColumnCount wrap_column = std::min(m_max_width, context.context.window().dimensions().column - context.setup.widget_columns);
         if (wrap_column <= 0)
             return;
 
@@ -778,7 +777,7 @@ struct WrapHighlighter : Highlighter
         if (contains(context.disabled_ids, ms_id))
             return;
 
-        const ColumnCount wrap_column = std::min(setup.window_range.column, m_max_width);
+        const ColumnCount wrap_column = std::min(m_max_width, context.context.window().dimensions().column - setup.widget_columns);
         if (wrap_column <= 0)
             return;
 
@@ -804,16 +803,15 @@ struct WrapHighlighter : Highlighter
         const auto win_height = context.context.window().dimensions().line;
 
         // Disable horizontal scrolling when using a WrapHighlighter
-        setup.window_pos.column = 0;
+        setup.first_column = 0;
-        setup.window_range.line = 0;
+        setup.line_count = 0;
         setup.scroll_offset.column = 0;
-        setup.full_lines = true;
 
         const ColumnCount marker_len = zero_if_greater(m_marker.column_length(), wrap_column);
 
-        for (auto buf_line = setup.window_pos.line, win_line = 0_line;
+        for (auto buf_line = setup.first_line, win_line = 0_line;
              win_line < win_height or buf_line <= cursor.line;
-             ++buf_line, ++setup.window_range.line)
+             ++buf_line, ++setup.line_count)
         {
             if (buf_line >= buffer.line_count())
                 break;
@@ -840,18 +838,17 @@ struct WrapHighlighter : Highlighter
                     }
                     pos = next_pos;
                 }
-                kak_assert(setup.cursor_pos.column >= 0 and setup.cursor_pos.column < setup.window_range.column);
             }
             const auto wrap_count = line_wrap_count(buf_line, prefix_len);
             win_line += wrap_count + 1;
 
             // scroll window to keep cursor visible, and update range as lines gets removed
-            while (buf_line >= cursor.line and setup.window_pos.line < cursor.line and
+            while (buf_line >= cursor.line and setup.first_line < cursor.line and
                    setup.cursor_pos.line + setup.scroll_offset.line >= win_height)
             {
-                auto remove_count = 1 + line_wrap_count(setup.window_pos.line, indent);
+                auto remove_count = 1 + line_wrap_count(setup.first_line, indent);
-                ++setup.window_pos.line;
+                ++setup.first_line;
-                --setup.window_range.line;
+                --setup.line_count;
                 setup.cursor_pos.line -= std::min(win_height, remove_count);
                 win_line -= remove_count;
                 kak_assert(setup.cursor_pos.line >= 0);
@@ -977,7 +974,6 @@ struct TabulationHighlighter : Highlighter
     {
         const ColumnCount tabstop = context.context.options()["tabstop"].get<int>();
         const auto& buffer = context.context.buffer();
-        auto win_column = context.setup.window_pos.column;
         for (auto& line : display_buffer.lines())
         {
             for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
@@ -997,8 +993,7 @@ struct TabulationHighlighter : Highlighter
                             atom_it = line.split(atom_it, (it+1).coord());
 
                         const ColumnCount column = get_column(buffer, tabstop, it.coord());
-                        const ColumnCount count = tabstop - (column % tabstop) -
+                        const ColumnCount count = tabstop - (column % tabstop);
-                                                  std::max(win_column - column, 0_col);
                         atom_it->replace(String{' ', count});
                         break;
                     }
@@ -1018,10 +1013,10 @@ struct TabulationHighlighter : Highlighter
         const ColumnCount tabstop = context.context.options()["tabstop"].get<int>();
         const ColumnCount column = get_column(buffer, tabstop, cursor);
         const ColumnCount width = tabstop - (column % tabstop);
-        const ColumnCount win_end = setup.window_pos.column + setup.window_range.column;
+        const ColumnCount win_end = setup.first_column + context.context.window().dimensions().column - setup.widget_columns;
         const ColumnCount offset = std::max(column + width - win_end, 0_col);
 
-        setup.window_pos.column += offset;
+        setup.first_column += offset;
         setup.cursor_pos.column -= offset;
     }
 };
@@ -1067,7 +1062,7 @@ private:
         const int tabstop = context.context.options()["tabstop"].get<int>();
         auto whitespaceface = context.context.faces()["Whitespace"];
         const auto& buffer = context.context.buffer();
-        auto win_column = context.setup.window_pos.column;
+        auto win_column = context.setup.first_column;
         for (auto& line : display_buffer.lines())
         {
             for (auto atom_it = line.begin(); atom_it != line.end(); ++atom_it)
@@ -1201,7 +1196,7 @@ private:
             return;
 
         ColumnCount width = compute_digit_count(context.context) + m_separator.column_length();
-        setup.window_range.column -= std::min(width, setup.window_range.column);
+        setup.widget_columns += width;
     }
 
     void fill_unique_ids(Vector<StringView>& unique_ids) const override
@@ -1499,7 +1494,7 @@ private:
             return;
         }
 
-        setup.window_range.column -= std::min(width, setup.window_range.column);
+        setup.widget_columns += width;
     }
 
     String m_option_name;
@@ -1698,6 +1693,7 @@ private:
         auto& buffer = context.context.buffer();
         auto& sels = context.context.selections();
         auto& range_and_faces = get_option(context);
+        const int tabstop = context.context.options()["tabstop"].get<int>();
         update_ranges(buffer, range_and_faces.prefix, range_and_faces.list);
 
         for (auto& [range, spec] : range_and_faces.list)
@@ -1705,15 +1701,27 @@ private:
             if (!is_valid(buffer, range.first) or (!is_empty(range) and !is_valid(buffer, range.last)) or !is_fully_selected(sels, range))
                 continue;
 
-            if (range.first.line < setup.window_pos.line and range.last.line >= setup.window_pos.line)
+            auto last = is_empty(range) ? range.first : range.last;
-                setup.window_pos.line = range.first.line;
+            if (range.first.line < setup.first_line and last.line >= setup.first_line)
+                setup.first_line = range.first.line;
 
-            if (range.last.line >= setup.window_pos.line and
+            const auto& cursor = context.context.selections().main().cursor();
-                range.first.line <= setup.window_pos.line + setup.window_range.line and
+            if (cursor.line == last.line and cursor.column >= last.column)
-                range.first.line != range.last.line)
             {
-                auto removed_count = range.last.line - range.first.line;
+                auto first_column = get_column(buffer, tabstop, range.first);
-                setup.window_range.line += removed_count;
+                auto last_column = get_column(buffer, tabstop, last);
+                auto replacement = parse_display_line(spec, context.context.faces());
+                auto cursor_move = replacement.length() - ((range.first.line == last.line) ? last_column - first_column : last_column);
+                setup.cursor_pos.line -= last.line - range.first.line;
+                setup.cursor_pos.column += cursor_move;
+            }
+
+            if (last.line >= setup.first_line and
+                range.first.line <= setup.first_line + setup.line_count and
+                range.first.line != last.line)
+            {
+                auto removed_count = last.line - range.first.line;
+                setup.line_count += removed_count;
             }
         }
     }

src/main.cc

@@ -366,8 +366,8 @@ static const EnvVarDesc builtin_env_vars[] = { {
         [](StringView name, const Context& context) -> Vector<String>
         {
             auto setup = context.window().compute_display_setup(context);
-            return {format("{} {} {} {}", setup.window_pos.line, setup.window_pos.column,
+            return {format("{} {} {} {}", setup.first_line, setup.first_column,
-                                          setup.window_range.line, setup.window_range.column)};
+                                          setup.line_count, 0)};
         }
     }, {
         "history", false,

src/window.cc

@@ -145,20 +145,12 @@ const DisplayBuffer& Window::update_display_buffer(const Context& context)
     kak_assert(&buffer() == &context.buffer());
     const DisplaySetup setup = compute_display_setup(context);
 
-    const int tabstop = context.options()["tabstop"].get<int>();
+    for (LineCount line = 0; line < setup.line_count; ++line)
-    for (LineCount line = 0; line < setup.window_range.line; ++line)
     {
-        LineCount buffer_line = setup.window_pos.line + line;
+        LineCount buffer_line = setup.first_line + line;
         if (buffer_line >= buffer().line_count())
             break;
-        auto beg_byte = get_byte_to_column(buffer(), tabstop, {buffer_line, setup.window_pos.column});
+        lines.emplace_back(AtomList{{buffer(), buffer_line, {buffer_line, buffer()[buffer_line].length()}}});
-        auto end_byte = setup.full_lines ?
-            buffer()[buffer_line].length()
-          : get_byte_to_column(buffer(), tabstop, {buffer_line, setup.window_pos.column + setup.window_range.column});
-
-        // The display buffer always has at least one buffer atom, which might be empty if
-        // beg_byte == end_byte
-        lines.emplace_back(AtomList{ {buffer(), {buffer_line, beg_byte}, {buffer_line, end_byte}} });
     }
 
     m_display_buffer.compute_range();
@@ -166,9 +158,12 @@ const DisplayBuffer& Window::update_display_buffer(const Context& context)
     for (auto pass : { HighlightPass::Wrap, HighlightPass::Move, HighlightPass::Colorize })
         m_builtin_highlighters.highlight({context, setup, pass, {}}, m_display_buffer, range);
 
+    for (auto& line : m_display_buffer.lines())
+        line.trim(setup.first_column, m_dimensions.column, true);
+
     m_display_buffer.optimize();
 
-    set_position(setup.window_pos);
+    set_position({setup.first_line, setup.first_column});
     m_last_setup = build_setup(context);
 
     if (profile and not (buffer().flags() & Buffer::Flags::Debug))
@@ -209,10 +204,9 @@ void Window::run_resize_hook_ifn()
 
 static void check_display_setup(const DisplaySetup& setup, const Window& window)
 {
-    kak_assert(setup.window_pos.line >= 0 and setup.window_pos.line < window.buffer().line_count());
+    kak_assert(setup.first_line >= 0 and setup.first_line < window.buffer().line_count());
-    kak_assert(setup.window_pos.column >= 0);
+    kak_assert(setup.first_column >= 0);
-    kak_assert(setup.window_range.column >= 0);
+    kak_assert(setup.line_count >= 0);
-    kak_assert(setup.window_range.line >= 0);
 }
 
 DisplaySetup Window::compute_display_setup(const Context& context) const
@@ -233,12 +227,13 @@ DisplaySetup Window::compute_display_setup(const Context& context) const
         win_pos.line = std::min(buffer().line_count()-1, cursor.line + offset.line - m_dimensions.line + 1);
 
     DisplaySetup setup{
-        win_pos,
+        win_pos.line,
-        m_dimensions,
+        m_dimensions.line,
+        win_pos.column,
+        0_col,
         {cursor.line - win_pos.line,
          get_column(buffer(), tabstop, cursor) - win_pos.column},
-        offset,
+        offset
-        false
     };
     for (auto pass : { HighlightPass::Move, HighlightPass::Wrap })
         m_builtin_highlighters.compute_display_setup({context, setup, pass, {}}, setup);
@@ -246,17 +241,17 @@ DisplaySetup Window::compute_display_setup(const Context& context) const
 
     // now ensure the cursor column is visible
     {
-        auto underflow = std::max(-setup.window_pos.column,
+        auto underflow = std::max(-setup.first_column,
                                   setup.cursor_pos.column - setup.scroll_offset.column);
         if (underflow < 0)
         {
-            setup.window_pos.column += underflow;
+            setup.first_column += underflow;
             setup.cursor_pos.column -= underflow;
         }
-        auto overflow = setup.cursor_pos.column + setup.scroll_offset.column - setup.window_range.column + 1;
+        auto overflow = setup.cursor_pos.column + setup.scroll_offset.column - (m_dimensions.column - setup.widget_columns)  + 1;
         if (overflow > 0)
         {
-            setup.window_pos.column += overflow;
+            setup.first_column += overflow;
             setup.cursor_pos.column -= overflow;
         }
         check_display_setup(setup, *this);

test/highlight/number-lines/cmd

@@ -0,0 +1 @@
+gl

test/highlight/number-lines/in

@@ -0,0 +1 @@
+xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

test/highlight/number-lines/rc

@@ -0,0 +1 @@
+add-highlighter window/ number-lines

test/highlight/number-lines/script

@@ -0,0 +1,2 @@
+ui_out -ignore 1
+ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " 1│" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "x" }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'

test/regression/1382-column-highlighter-broken-on-horizontal-scroll/script

@@ -1,5 +1,5 @@
 ui_out '{ "jsonrpc": "2.0", "method": "set_ui_options", "params": [{}] }'
-ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxxxxxxxxx" }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": "x" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "x" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxx\u000a" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "      " }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": " " }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "          " }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": " " }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "          " }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": " " }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'
+ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxxxxxxxxx" }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": "x" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "x" }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxx\u000a" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "      " }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": " " }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "          " }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": " " }], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "          " }, { "face": { "fg": "default", "bg": "blue", "underline": "default", "attributes": [] }, "contents": " " }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'
 ui_out '{ "jsonrpc": "2.0", "method": "menu_hide", "params": [] }'
 ui_out '{ "jsonrpc": "2.0", "method": "info_hide", "params": [] }'
 ui_out '{ "jsonrpc": "2.0", "method": "draw_status", "params": [[], [{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "out 1:89 " }, { "face": { "fg": "black", "bg": "yellow", "underline": "default", "attributes": [] }, "contents": "" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " " }, { "face": { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }, "contents": "1 sel" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " - client0@[kak-tests]" }], { "fg": "cyan", "bg": "default", "underline": "default", "attributes": [] }] }'

test/regression/4659-scroll-issue-with-replace-ranges/cmd

@@ -0,0 +1 @@
+6l

test/regression/4659-scroll-issue-with-replace-ranges/in

@@ -0,0 +1 @@
+hello world

test/regression/4659-scroll-issue-with-replace-ranges/rc

@@ -0,0 +1,3 @@
+decl range-specs range
+addhl window/ replace-ranges range
+set buffer range %val{timestamp} 1.6+0|xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx

test/regression/4659-scroll-issue-with-replace-ranges/script

@@ -0,0 +1,2 @@
+ui_out -ignore 1
+ui_out '{ "jsonrpc": "2.0", "method": "draw", "params": [[[{ "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" }, { "face": { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, "contents": " " }, { "face": { "fg": "black", "bg": "white", "underline": "default", "attributes": [] }, "contents": "w" }]], { "fg": "default", "bg": "default", "underline": "default", "attributes": [] }, { "fg": "blue", "bg": "default", "underline": "default", "attributes": [] }] }'