diff --git a/src/buffer.cc b/src/buffer.cc index c6ad506a..1fb9ab7e 100644 --- a/src/buffer.cc +++ b/src/buffer.cc @@ -59,10 +59,10 @@ BufferCoord Buffer::line_and_column_at(const BufferIterator& iterator) const return iterator.m_coord; } -CharCount Buffer::line_length(LineCount line) const +ByteCount Buffer::line_length(LineCount line) const { assert(line < line_count()); - CharCount end = (line < line_count() - 1) ? + ByteCount end = (line < line_count() - 1) ? m_lines[line + 1].start : character_count(); return end - m_lines[line].start; } @@ -75,8 +75,8 @@ BufferCoord Buffer::clamp(const BufferCoord& line_and_column, BufferCoord result(line_and_column.line, line_and_column.column); result.line = Kakoune::clamp(result.line, 0_line, line_count() - 1); - CharCount max_col = std::max(0_char, line_length(result.line) - (avoid_eol ? 2 : 1)); - result.column = Kakoune::clamp(result.column, 0_char, max_col); + ByteCount max_col = std::max(0_byte, line_length(result.line) - (avoid_eol ? 2 : 1)); + result.column = Kakoune::clamp(result.column, 0_byte, max_col); return result; } @@ -116,7 +116,7 @@ BufferIterator Buffer::end() const return BufferIterator(*this, { line_count()-1, m_lines.back().length() }); } -CharCount Buffer::character_count() const +ByteCount Buffer::character_count() const { if (m_lines.empty()) return 0; @@ -133,10 +133,10 @@ String Buffer::string(const BufferIterator& begin, const BufferIterator& end) co String res; for (LineCount line = begin.line(); line <= end.line(); ++line) { - CharCount start = 0; + ByteCount start = 0; if (line == begin.line()) start = begin.column(); - CharCount count = -1; + ByteCount count = -1; if (line == end.line()) count = end.column() - start; res += m_lines[line].content.substr(start, count); @@ -224,7 +224,7 @@ void Buffer::reset_undo_data() void Buffer::check_invariant() const { - CharCount start = 0; + ByteCount start = 0; assert(not m_lines.empty()); for (auto& line : m_lines) { @@ -239,7 +239,7 @@ void Buffer::do_insert(const BufferIterator& pos, const String& content) { assert(pos.is_end() or utf8::is_character_start(pos)); ++m_timestamp; - CharCount offset = pos.offset(); + ByteCount offset = pos.offset(); // all following lines advanced by length for (LineCount i = pos.line()+1; i < line_count(); ++i) @@ -251,8 +251,8 @@ void Buffer::do_insert(const BufferIterator& pos, const String& content) // line without inserting a '\n' if (pos == end() and (pos == begin() or *(pos-1) == '\n')) { - CharCount start = 0; - for (CharCount i = 0; i < content.length(); ++i) + ByteCount start = 0; + for (ByteCount i = 0; i < content.length(); ++i) { if (content[i] == '\n') { @@ -274,8 +274,8 @@ void Buffer::do_insert(const BufferIterator& pos, const String& content) auto line_it = m_lines.begin() + (int)pos.line(); line_it = m_lines.erase(line_it); - CharCount start = 0; - for (CharCount i = 0; i < content.length(); ++i) + ByteCount start = 0; + for (ByteCount i = 0; i < content.length(); ++i) { if (content[i] == '\n') { @@ -317,7 +317,7 @@ void Buffer::do_erase(const BufferIterator& begin, const BufferIterator& end) assert(utf8::is_character_start(begin) and (end.is_end() or utf8::is_character_start(end))); ++m_timestamp; - const CharCount length = end - begin; + const ByteCount length = end - begin; String prefix = m_lines[begin.line()].content.substr(0, begin.column()); String suffix = m_lines[end.line()].content.substr(end.column()); Line new_line = { m_lines[begin.line()].start, prefix + suffix }; @@ -351,7 +351,7 @@ void Buffer::apply_modification(const Modification& modification) } case Modification::Erase: { - CharCount count = modification.content.length(); + ByteCount count = modification.content.length(); BufferIterator end = modification.position + count; assert(string(modification.position, end) == modification.content); do_erase(modification.position, end); diff --git a/src/buffer.hh b/src/buffer.hh index 1a4e6694..d955bb14 100644 --- a/src/buffer.hh +++ b/src/buffer.hh @@ -17,14 +17,10 @@ namespace Kakoune class Buffer; class Window; -struct BufferCoord : LineAndColumn +struct BufferCoord : LineAndColumn { - constexpr BufferCoord(LineCount line = 0, CharCount column = 0) + constexpr BufferCoord(LineCount line = 0, ByteCount column = 0) : LineAndColumn(line, column) {} - - template - explicit constexpr BufferCoord(const LineAndColumn& other) - : LineAndColumn(other.line, other.column) {} }; // A BufferIterator permits to iterate over the characters of a buffer @@ -50,11 +46,11 @@ public: char operator* () const; size_t operator- (const BufferIterator& iterator) const; - BufferIterator operator+ (CharCount size) const; - BufferIterator operator- (CharCount size) const; + BufferIterator operator+ (ByteCount size) const; + BufferIterator operator- (ByteCount size) const; - BufferIterator& operator+= (CharCount size); - BufferIterator& operator-= (CharCount size); + BufferIterator& operator+= (ByteCount size); + BufferIterator& operator-= (ByteCount size); BufferIterator& operator++ (); BufferIterator& operator-- (); @@ -74,10 +70,10 @@ public: const Buffer& buffer() const; const BufferCoord& coord() const { return m_coord; } LineCount line() const { return m_coord.line; } - CharCount column() const { return m_coord.column; } + ByteCount column() const { return m_coord.column; } private: - CharCount offset() const; + ByteCount offset() const; const Buffer* m_buffer; BufferCoord m_coord; @@ -130,9 +126,9 @@ public: BufferIterator begin() const; BufferIterator end() const; - CharCount character_count() const; + ByteCount character_count() const; LineCount line_count() const; - CharCount line_length(LineCount line) const; + ByteCount line_length(LineCount line) const; // returns an iterator at given coordinates. line_and_column is // clamped according to avoid_eol. @@ -188,10 +184,10 @@ private: struct Line { - CharCount start; + ByteCount start; String content; - CharCount length() const { return content.length(); } + ByteCount length() const { return content.length(); } }; struct LineList : std::vector { diff --git a/src/buffer_iterator.inl.hh b/src/buffer_iterator.inl.hh index 0738263a..950b7dfb 100644 --- a/src/buffer_iterator.inl.hh +++ b/src/buffer_iterator.inl.hh @@ -114,7 +114,7 @@ inline char BufferIterator::operator*() const return m_buffer->m_lines[line()].content[column()]; } -inline CharCount BufferIterator::offset() const +inline ByteCount BufferIterator::offset() const { assert(m_buffer); return line() == 0 ? column() @@ -127,12 +127,12 @@ inline size_t BufferIterator::operator-(const BufferIterator& iterator) const return (size_t)(int)(offset() - iterator.offset()); } -inline BufferIterator BufferIterator::operator+(CharCount size) const +inline BufferIterator BufferIterator::operator+(ByteCount size) const { assert(m_buffer); if (size >= 0) { - CharCount o = std::min(m_buffer->character_count(), offset() + size); + ByteCount o = std::min(m_buffer->character_count(), offset() + size); for (LineCount i = line() + 1; i < m_buffer->line_count(); ++i) { if (m_buffer->m_lines[i].start > o) @@ -144,12 +144,12 @@ inline BufferIterator BufferIterator::operator+(CharCount size) const return operator-(-size); } -inline BufferIterator BufferIterator::operator-(CharCount size) const +inline BufferIterator BufferIterator::operator-(ByteCount size) const { assert(m_buffer); if (size >= 0) { - CharCount o = std::max(0_char, offset() - size); + ByteCount o = std::max(0_byte, offset() - size); for (LineCount i = line(); i >= 0; --i) { if (m_buffer->m_lines[i].start <= o) @@ -160,12 +160,12 @@ inline BufferIterator BufferIterator::operator-(CharCount size) const return operator+(-size); } -inline BufferIterator& BufferIterator::operator+=(CharCount size) +inline BufferIterator& BufferIterator::operator+=(ByteCount size) { return *this = (*this + size); } -inline BufferIterator& BufferIterator::operator-=(CharCount size) +inline BufferIterator& BufferIterator::operator-=(ByteCount size) { return *this = (*this - size); } diff --git a/src/buffer_manager.cc b/src/buffer_manager.cc index d2f7857f..764e1271 100644 --- a/src/buffer_manager.cc +++ b/src/buffer_manager.cc @@ -63,7 +63,7 @@ void BufferManager::set_last_used_buffer(Buffer& buffer) } CandidateList BufferManager::complete_buffername(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { String real_prefix = prefix.substr(0, cursor_pos); CandidateList result; diff --git a/src/buffer_manager.hh b/src/buffer_manager.hh index ca84c9e8..d8065578 100644 --- a/src/buffer_manager.hh +++ b/src/buffer_manager.hh @@ -30,7 +30,7 @@ public: void set_last_used_buffer(Buffer& buffer); CandidateList complete_buffername(const String& prefix, - CharCount cursor_pos = -1); + ByteCount cursor_pos = -1); private: BufferList m_buffers; diff --git a/src/client.cc b/src/client.cc index 11996f82..9802f623 100644 --- a/src/client.cc +++ b/src/client.cc @@ -134,7 +134,7 @@ public: m_completer(completer), m_callback(callback) { m_history_it = ms_history[m_prompt].end(); - context.ui().print_status(m_prompt, m_prompt.length()); + context.ui().print_status(m_prompt, m_prompt.char_length()); } void on_key(const Key& key, Context& context) override @@ -174,7 +174,7 @@ public: m_saved_result = m_result; auto it = m_history_it; // search for the previous history entry matching typed prefix - CharCount prefix_length = m_saved_result.length(); + ByteCount prefix_length = m_saved_result.length(); do { --it; @@ -182,7 +182,7 @@ public: { m_history_it = it; m_result = *it; - m_cursor_pos = m_result.length(); + m_cursor_pos = m_result.char_length(); break; } } while (it != history.begin()); @@ -193,7 +193,7 @@ public: { if (m_history_it != history.end()) { - CharCount prefix_length = m_saved_result.length(); + ByteCount prefix_length = m_saved_result.length(); // search for the next history entry matching typed prefix ++m_history_it; while (m_history_it != history.end() and @@ -204,7 +204,7 @@ public: m_result = *m_history_it; else m_result = m_saved_result; - m_cursor_pos = m_result.length(); + m_cursor_pos = m_result.char_length(); } } else if (key == Key::Left or @@ -216,7 +216,7 @@ public: else if (key == Key::Right or key == Key{Key::Modifiers::Control, 'f'}) { - if (m_cursor_pos < m_result.length()) + if (m_cursor_pos < m_result.char_length()) ++m_cursor_pos; } else if (key == Key::Backspace) @@ -240,7 +240,7 @@ public: m_current_completion = -1; m_result = m_result.substr(0, m_cursor_pos) + reg + m_result.substr(m_cursor_pos, String::npos); - m_cursor_pos += reg.length(); + m_cursor_pos += reg.char_length(); } else if (key == Key(Key::Modifiers::Control, 'i') or // tab completion key == Key::BackTab) @@ -250,7 +250,8 @@ public: // first try, we need to ask our completer for completions if (m_current_completion == -1) { - m_completions = m_completer(context, m_result, m_cursor_pos); + m_completions = m_completer(context, m_result, + m_result.byte_count_to(m_cursor_pos)); if (candidates.empty()) return; @@ -272,16 +273,20 @@ public: context.ui().menu_select(m_current_completion); m_result = m_result.substr(0, m_completions.start) + completion + m_result.substr(m_cursor_pos); - m_cursor_pos = m_completions.start + completion.length(); + m_cursor_pos = m_result.char_count_to(m_completions.start) + + completion.char_length(); } else { context.ui().menu_hide(); m_current_completion = -1; - m_result = m_result.substr(0, m_cursor_pos) + key.key + m_result.substr(m_cursor_pos, String::npos); + std::string keystr; + auto inserter = back_inserter(keystr); + utf8::dump(inserter, key.key); + m_result = m_result.substr(0, m_cursor_pos) + keystr + m_result.substr(m_cursor_pos, String::npos); ++m_cursor_pos; } - context.ui().print_status(m_prompt + m_result, m_prompt.length() + m_cursor_pos); + context.ui().print_status(m_prompt + m_result, m_prompt.char_length() + m_cursor_pos); } private: diff --git a/src/command_manager.cc b/src/command_manager.cc index b21921da..3998693d 100644 --- a/src/command_manager.cc +++ b/src/command_manager.cc @@ -61,7 +61,7 @@ private: using TokenList = std::vector; -using TokenPosList = std::vector>; +using TokenPosList = std::vector>; bool is_command_separator(char c) { @@ -78,8 +78,8 @@ TokenList parse(const String& line, { TokenList result; - CharCount length = line.length(); - CharCount pos = 0; + ByteCount length = line.length(); + ByteCount pos = 0; while (pos < length) { while (pos != length) @@ -97,7 +97,7 @@ TokenList parse(const String& line, break; } - CharCount token_start = pos; + ByteCount token_start = pos; Token::Type type = Token::Type::Raw; if (line[pos] == '"' or line[pos] == '\'') @@ -112,7 +112,7 @@ TokenList parse(const String& line, } else if (line[pos] == '%') { - CharCount type_start = ++pos; + ByteCount type_start = ++pos; while (isalpha(line[pos])) ++pos; String type_name = line.substr(type_start, pos - type_start); @@ -260,7 +260,7 @@ void CommandManager::execute(const String& command_line, } Completions CommandManager::complete(const Context& context, - const String& command_line, CharCount cursor_pos) + const String& command_line, ByteCount cursor_pos) { TokenPosList pos_info; TokenList tokens = parse(command_line, &pos_info); @@ -277,7 +277,7 @@ Completions CommandManager::complete(const Context& context, if (token_to_complete == 0 or tokens.empty()) // command name completion { - CharCount cmd_start = tokens.empty() ? 0 : pos_info[0].first; + ByteCount cmd_start = tokens.empty() ? 0 : pos_info[0].first; Completions result(cmd_start, cursor_pos); String prefix = command_line.substr(cmd_start, cursor_pos - cmd_start); @@ -302,10 +302,10 @@ Completions CommandManager::complete(const Context& context, if (command_it == m_commands.end() or not command_it->second.completer) return Completions(); - CharCount start = token_to_complete < tokens.size() ? + ByteCount start = token_to_complete < tokens.size() ? pos_info[token_to_complete].first : cursor_pos; Completions result(start , cursor_pos); - CharCount cursor_pos_in_token = cursor_pos - start; + ByteCount cursor_pos_in_token = cursor_pos - start; std::vector params; for (auto token_it = tokens.begin()+1; token_it != tokens.end(); ++token_it) @@ -319,7 +319,7 @@ Completions CommandManager::complete(const Context& context, CandidateList PerArgumentCommandCompleter::operator()(const Context& context, const CommandParameters& params, size_t token_to_complete, - CharCount pos_in_token) const + ByteCount pos_in_token) const { if (token_to_complete >= m_completers.size()) return CandidateList(); diff --git a/src/command_manager.hh b/src/command_manager.hh index 84f764ee..454ff95e 100644 --- a/src/command_manager.hh +++ b/src/command_manager.hh @@ -28,13 +28,13 @@ typedef std::function CommandCompleter; + size_t, ByteCount)> CommandCompleter; class PerArgumentCommandCompleter { public: typedef std::function ArgumentCompleter; + const String&, ByteCount)> ArgumentCompleter; typedef memoryview ArgumentCompleterList; PerArgumentCommandCompleter(const ArgumentCompleterList& completers) @@ -43,7 +43,7 @@ public: CandidateList operator()(const Context& context, const CommandParameters& params, size_t token_to_complete, - CharCount pos_in_token) const; + ByteCount pos_in_token) const; private: std::vector m_completers; @@ -57,7 +57,7 @@ public: const EnvVarMap& env_vars = {}); Completions complete(const Context& context, - const String& command_line, CharCount cursor_pos); + const String& command_line, ByteCount cursor_pos); bool command_defined(const String& command_name) const; diff --git a/src/commands.cc b/src/commands.cc index ddba6d61..97db7e5f 100644 --- a/src/commands.cc +++ b/src/commands.cc @@ -62,7 +62,7 @@ struct ParametersParser { if (params[i][0] == '-') { - auto it = options.find(params[i].substr(1)); + auto it = options.find(params[i].substr(1_byte)); if (it == options.end()) throw unknown_option(params[i]); @@ -88,7 +88,7 @@ struct ParametersParser assert(m_options.find(name) != m_options.end()); for (auto& param : m_params) { - if (param[0] == '-' and param.substr(1) == name) + if (param[0] == '-' and param.substr(1_byte) == name) return true; if (param == "--") @@ -107,7 +107,7 @@ struct ParametersParser for (size_t i = 0; i < m_params.size(); ++i) { - if (m_params[i][0] == '-' and m_params[i].substr(1) == name) + if (m_params[i][0] == '-' and m_params[i].substr(1_byte) == name) return m_params[i+1]; if (m_params[i] == "--") @@ -578,7 +578,7 @@ void define_command(const CommandParameters& params, Context& context) if (parser.has_option("file-completion")) { completer = [](const Context& context, const CommandParameters& params, - size_t token_to_complete, CharCount pos_in_token) + size_t token_to_complete, ByteCount pos_in_token) { const String& prefix = token_to_complete < params.size() ? params[token_to_complete] : String(); @@ -589,7 +589,7 @@ void define_command(const CommandParameters& params, Context& context) { String shell_cmd = parser.option_value("shell-completion"); completer = [=](const Context& context, const CommandParameters& params, - size_t token_to_complete, CharCount pos_in_token) + size_t token_to_complete, ByteCount pos_in_token) { EnvVarMap vars = { {"token_to_complete", int_to_str(token_to_complete) }, @@ -778,7 +778,7 @@ void register_commands() cm.register_command("wq!", write_and_quit); PerArgumentCommandCompleter buffer_completer({ - [](const Context& context, const String& prefix, CharCount cursor_pos) + [](const Context& context, const String& prefix, ByteCount cursor_pos) { return BufferManager::instance().complete_buffername(prefix, cursor_pos); } }); cm.register_commands({ "b", "buffer" }, show_buffer, buffer_completer); @@ -786,7 +786,7 @@ void register_commands() cm.register_commands({ "ah", "addhl" }, add_highlighter, [](const Context& context, const CommandParameters& params, - size_t token_to_complete, CharCount pos_in_token) + size_t token_to_complete, ByteCount pos_in_token) { Window& w = context.window(); const String& arg = token_to_complete < params.size() ? @@ -800,7 +800,7 @@ void register_commands() }); cm.register_commands({ "rh", "rmhl" }, rm_highlighter, [](const Context& context, const CommandParameters& params, - size_t token_to_complete, CharCount pos_in_token) + size_t token_to_complete, ByteCount pos_in_token) { Window& w = context.window(); const String& arg = token_to_complete < params.size() ? @@ -814,7 +814,7 @@ void register_commands() }); cm.register_commands({ "af", "addfilter" }, add_filter, [](const Context& context, const CommandParameters& params, - size_t token_to_complete, CharCount pos_in_token) + size_t token_to_complete, ByteCount pos_in_token) { Window& w = context.window(); const String& arg = token_to_complete < params.size() ? @@ -828,7 +828,7 @@ void register_commands() }); cm.register_commands({ "rf", "rmfilter" }, rm_filter, [](const Context& context, const CommandParameters& params, - size_t token_to_complete, CharCount pos_in_token) + size_t token_to_complete, ByteCount pos_in_token) { Window& w = context.window(); const String& arg = token_to_complete < params.size() ? @@ -856,21 +856,21 @@ void register_commands() [](const CommandParameters& params, Context& context) { set_option(GlobalOptionManager::instance(), params, context); }, PerArgumentCommandCompleter({ - [](const Context& context, const String& prefix, CharCount cursor_pos) + [](const Context& context, const String& prefix, ByteCount cursor_pos) { return GlobalOptionManager::instance().complete_option_name(prefix, cursor_pos); } })); cm.register_commands({ "setb", "setbuffer" }, [](const CommandParameters& params, Context& context) { set_option(context.buffer().option_manager(), params, context); }, PerArgumentCommandCompleter({ - [](const Context& context, const String& prefix, CharCount cursor_pos) + [](const Context& context, const String& prefix, ByteCount cursor_pos) { return context.buffer().option_manager().complete_option_name(prefix, cursor_pos); } })); cm.register_commands({ "setw", "setwindow" }, [](const CommandParameters& params, Context& context) { set_option(context.window().option_manager(), params, context); }, PerArgumentCommandCompleter({ - [](const Context& context, const String& prefix, CharCount cursor_pos) + [](const Context& context, const String& prefix, ByteCount cursor_pos) { return context.window().option_manager().complete_option_name(prefix, cursor_pos); } })); diff --git a/src/completion.cc b/src/completion.cc index 5db7ffb7..da5e1387 100644 --- a/src/completion.cc +++ b/src/completion.cc @@ -12,15 +12,15 @@ namespace Kakoune CandidateList complete_filename(const Context& context, const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { String real_prefix = parse_filename(prefix.substr(0, cursor_pos)); String dirname = "./"; String dirprefix; String fileprefix = real_prefix; - CharCount dir_end = -1; - for (CharCount i = 0; i < real_prefix.length(); ++i) + ByteCount dir_end = -1; + for (ByteCount i = 0; i < real_prefix.length(); ++i) { if (real_prefix[i] == '/') dir_end = i; diff --git a/src/completion.hh b/src/completion.hh index 60462975..cf626ec5 100644 --- a/src/completion.hh +++ b/src/completion.hh @@ -16,25 +16,25 @@ typedef std::vector CandidateList; struct Completions { CandidateList candidates; - CharCount start; - CharCount end; + ByteCount start; + ByteCount end; Completions() : start(0), end(0) {} - Completions(CharCount start, CharCount end) + Completions(ByteCount start, ByteCount end) : start(start), end(end) {} }; CandidateList complete_filename(const Context& context, const String& prefix, - CharCount cursor_pos = -1); + ByteCount cursor_pos = -1); typedef std::function Completer; + const String&, ByteCount)> Completer; inline Completions complete_nothing(const Context& context, - const String&, CharCount cursor_pos) + const String&, ByteCount cursor_pos) { return Completions(cursor_pos, cursor_pos); } diff --git a/src/display_buffer.hh b/src/display_buffer.hh index e1f1442f..aa54f1f6 100644 --- a/src/display_buffer.hh +++ b/src/display_buffer.hh @@ -7,18 +7,15 @@ #include "color.hh" #include "line_and_column.hh" #include "buffer.hh" +#include "utf8.hh" namespace Kakoune { -struct DisplayCoord : LineAndColumn +struct DisplayCoord : LineAndColumn { constexpr DisplayCoord(LineCount line = 0, CharCount column = 0) : LineAndColumn(line, column) {} - - template - explicit constexpr DisplayCoord(const LineAndColumn& other) - : LineAndColumn(other.line, other.column) {} }; typedef int Attribute; @@ -64,10 +61,10 @@ public: switch (m_type) { case BufferRange: - return m_end - m_begin; + return utf8::distance(m_begin, m_end); case Text: case ReplacedBufferRange: - return m_text.length(); + return m_text.char_length(); } } diff --git a/src/editor.cc b/src/editor.cc index ff77e357..1a4cfdb7 100644 --- a/src/editor.cc +++ b/src/editor.cc @@ -129,9 +129,9 @@ void Editor::move_selections(CharCount offset, SelectMode mode) for (auto& sel : m_selections) { auto last = sel.last(); - last = clamp(utf8::advance(last, offset), - buffer().iterator_at_line_begin(last), - utf8::previous(buffer().iterator_at_line_end(last))); + auto limit = offset < 0 ? buffer().iterator_at_line_begin(last) + : utf8::previous(buffer().iterator_at_line_end(last)); + last = utf8::advance(last, limit, offset); sel.selection = Selection(mode == SelectMode::Extend ? sel.first() : last, last); } merge_overlapping(m_selections); diff --git a/src/file.cc b/src/file.cc index 2723893e..b05b5782 100644 --- a/src/file.cc +++ b/src/file.cc @@ -4,6 +4,8 @@ #include "buffer_manager.hh" #include "assert.hh" +#include "unicode.hh" + #include #include #include @@ -14,25 +16,20 @@ namespace Kakoune { -bool isidentifier(char c) -{ - return std::isalnum(c) or c == '_'; -} - String parse_filename(const String& filename) { if (filename.length() >= 2 and filename[0] == '~' and filename[1] == '/') - return parse_filename("$HOME/" + filename.substr(2)); + return parse_filename("$HOME/" + filename.substr(2_byte)); - CharCount pos = 0; + ByteCount pos = 0; String result; - for (CharCount i = 0; i < filename.length(); ++i) + for (ByteCount i = 0; i < filename.length(); ++i) { if (filename[i] == '$' and (i == 0 or filename[i-1] != '\\')) { result += filename.substr(pos, i - pos); - CharCount end = i+1; - while (end != filename.length() and isidentifier(filename[end])) + ByteCount end = i+1; + while (end != filename.length() and is_word(filename[end])) ++end; String var_name = filename.substr(i+1, end - i - 1); const char* var_value = getenv(var_name.c_str()); diff --git a/src/filter_group.cc b/src/filter_group.cc index ea44bffa..ac10faee 100644 --- a/src/filter_group.cc +++ b/src/filter_group.cc @@ -40,13 +40,13 @@ FilterGroup& FilterGroup::get_group(const String& id) CandidateList FilterGroup::complete_id(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { return m_filters.complete_id(prefix, cursor_pos); } CandidateList FilterGroup::complete_group_id(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { return m_filters.complete_id_if( prefix, cursor_pos, diff --git a/src/filter_group.hh b/src/filter_group.hh index 8c93936b..df3cbb9f 100644 --- a/src/filter_group.hh +++ b/src/filter_group.hh @@ -19,8 +19,8 @@ public: FilterGroup& get_group(const String& id); - CandidateList complete_id(const String& prefix, CharCount cursor_pos); - CandidateList complete_group_id(const String& prefix, CharCount cursor_pos); + CandidateList complete_id(const String& prefix, ByteCount cursor_pos); + CandidateList complete_group_id(const String& prefix, ByteCount cursor_pos); private: idvaluemap m_filters; diff --git a/src/filter_registry.cc b/src/filter_registry.cc index 65070bdf..1a2a1754 100644 --- a/src/filter_registry.cc +++ b/src/filter_registry.cc @@ -31,7 +31,7 @@ void FilterRegistry::add_filter_to_group(FilterGroup& group, } CandidateList FilterRegistry::complete_filter(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { return m_factories.complete_id(prefix, cursor_pos); } diff --git a/src/filter_registry.hh b/src/filter_registry.hh index 77763782..0756091c 100644 --- a/src/filter_registry.hh +++ b/src/filter_registry.hh @@ -30,7 +30,7 @@ public: const FilterParameters& parameters); CandidateList complete_filter(const String& prefix, - CharCount cursor_pos); + ByteCount cursor_pos); private: idvaluemap m_factories; diff --git a/src/highlighter_group.cc b/src/highlighter_group.cc index 57b21a5c..1dac5fe9 100644 --- a/src/highlighter_group.cc +++ b/src/highlighter_group.cc @@ -39,13 +39,13 @@ HighlighterGroup& HighlighterGroup::get_group(const String& id) CandidateList HighlighterGroup::complete_id(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { return m_highlighters.complete_id(prefix, cursor_pos); } CandidateList HighlighterGroup::complete_group_id(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { return m_highlighters.complete_id_if( prefix, cursor_pos, diff --git a/src/highlighter_group.hh b/src/highlighter_group.hh index 9b4f1bf0..50c883e7 100644 --- a/src/highlighter_group.hh +++ b/src/highlighter_group.hh @@ -22,8 +22,8 @@ public: HighlighterGroup& get_group(const String& id); - CandidateList complete_id(const String& prefix, CharCount cursor_pos); - CandidateList complete_group_id(const String& prefix, CharCount cursor_pos); + CandidateList complete_id(const String& prefix, ByteCount cursor_pos); + CandidateList complete_group_id(const String& prefix, ByteCount cursor_pos); private: idvaluemap m_highlighters; diff --git a/src/highlighter_registry.cc b/src/highlighter_registry.cc index 46038aa3..90cbe36f 100644 --- a/src/highlighter_registry.cc +++ b/src/highlighter_registry.cc @@ -33,7 +33,7 @@ void HighlighterRegistry::add_highlighter_to_group(Window& window, } CandidateList HighlighterRegistry::complete_highlighter(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { return m_factories.complete_id(prefix, cursor_pos); } diff --git a/src/highlighter_registry.hh b/src/highlighter_registry.hh index 2a3f740d..9690318d 100644 --- a/src/highlighter_registry.hh +++ b/src/highlighter_registry.hh @@ -30,7 +30,7 @@ public: const HighlighterParameters& parameters); CandidateList complete_highlighter(const String& prefix, - CharCount cursor_pos); + ByteCount cursor_pos); private: idvaluemap m_factories; diff --git a/src/idvaluemap.hh b/src/idvaluemap.hh index c65f35fc..e11c327a 100644 --- a/src/idvaluemap.hh +++ b/src/idvaluemap.hh @@ -65,7 +65,7 @@ public: template CandidateList complete_id_if(const String& prefix, - CharCount cursor_pos, + ByteCount cursor_pos, _Condition condition) { String real_prefix = prefix.substr(0, cursor_pos); @@ -83,7 +83,7 @@ public: } CandidateList complete_id(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { return complete_id_if( prefix, cursor_pos, [](const value_type&) { return true; }); diff --git a/src/keys.cc b/src/keys.cc index 18b31d77..6b2479ff 100644 --- a/src/keys.cc +++ b/src/keys.cc @@ -25,11 +25,11 @@ static std::unordered_map keynamemap = { KeyList parse_keys(const String& str) { KeyList result; - for (CharCount pos = 0; pos < str.length(); ++pos) + for (ByteCount pos = 0; pos < str.length(); ++pos) { if (str[pos] == '<') { - CharCount end_pos = pos; + ByteCount end_pos = pos; while (end_pos < str.length() and str[end_pos] != '>') ++end_pos; @@ -43,12 +43,12 @@ KeyList parse_keys(const String& str) if (tolower(keyname[0]) == 'c' and keyname[1] == '-') { modifier = Key::Modifiers::Control; - keyname = keyname.substr(2); + keyname = keyname.substr(2_byte); } if (tolower(keyname[0]) == 'a' and keyname[1] == '-') { modifier = Key::Modifiers::Alt; - keyname = keyname.substr(2); + keyname = keyname.substr(2_byte); } } if (keyname.length() == 1) diff --git a/src/line_and_column.hh b/src/line_and_column.hh index 5a278a4f..aaffb222 100644 --- a/src/line_and_column.hh +++ b/src/line_and_column.hh @@ -6,13 +6,13 @@ namespace Kakoune { -template +template struct LineAndColumn { - LineCount line; - CharCount column; + LineType line; + ColumnType column; - constexpr LineAndColumn(LineCount line = 0, CharCount column = 0) + constexpr LineAndColumn(LineType line = 0, ColumnType column = 0) : line(line), column(column) {} constexpr EffectiveType operator+(const EffectiveType& other) const diff --git a/src/main.cc b/src/main.cc index acd32819..4761d6ff 100644 --- a/src/main.cc +++ b/src/main.cc @@ -263,20 +263,22 @@ void do_scroll(Context& context) "do_scrool only implements PageUp and PageDown"); Window& window = context.window(); Buffer& buffer = context.buffer(); - BufferCoord position = window.position(); - BufferIterator cursor_pos; + DisplayCoord position = window.position(); + LineCount cursor_line = 0; if (key == Key::PageUp) { position.line -= (window.dimensions().line - 2); - cursor_pos = buffer.iterator_at(position); + cursor_line = position.line; } else if (key == Key::PageDown) { position.line += (window.dimensions().line - 2); - cursor_pos = buffer.iterator_at(position + BufferCoord{window.dimensions().line - 1, 0}); + cursor_line = position.line + window.dimensions().line - 1; } - + auto cursor_pos = utf8::advance(buffer.iterator_at_line_begin(position.line), + buffer.iterator_at_line_end(position.line), + position.column); window.select(cursor_pos); window.set_position(position); } @@ -473,7 +475,7 @@ int main(int argc, char* argv[]) { return runtime_directory(); }); shell_manager.register_env_var("opt_.+", [](const String& name, const Context& context) - { return context.option_manager()[name.substr(4)].as_string(); }); + { return context.option_manager()[name.substr(4_byte)].as_string(); }); shell_manager.register_env_var("reg_.+", [](const String& name, const Context& context) { return RegisterManager::instance()[name[4]].values(context)[0]; }); diff --git a/src/ncurses.cc b/src/ncurses.cc index 68001876..2f9ab649 100644 --- a/src/ncurses.cc +++ b/src/ncurses.cc @@ -227,21 +227,17 @@ void NCursesUI::print_status(const String& status, CharCount cursor_pos) move(y-1, 0); clrtoeol(); if (cursor_pos == -1) - addstr(status.c_str()); - else if (cursor_pos < status.length()) - { - addstr(status.substr(0, cursor_pos).c_str()); - set_attribute(A_REVERSE, 1); - addch(status[cursor_pos]); - set_attribute(A_REVERSE, 0); - addstr(status.substr(cursor_pos+1, -1).c_str()); - } + addutf8str(status.begin(), status.end()); else { - addstr(status.c_str()); - set_attribute(A_REVERSE, 1); - addch(' '); - set_attribute(A_REVERSE, 0); + auto cursor_it = utf8::advance(status.begin(), status.end(), (int)cursor_pos); + auto end = status.end(); + addutf8str(status.begin(), cursor_it); + set_attribute(A_REVERSE, 1); + addch((cursor_it == end) ? ' ' : utf8::codepoint(cursor_it)); + set_attribute(A_REVERSE, 0); + if (cursor_it != end) + addutf8str(utf8::next(cursor_it), end); } redraw(m_menu_win); } @@ -256,7 +252,7 @@ void NCursesUI::menu_show(const memoryview& choices, for (int i = 0; i < m_choices.size(); ++i) { m_items.push_back(new_item(m_choices[i].c_str(), "")); - longest = std::max(longest, m_choices[i].length()); + longest = std::max(longest, m_choices[i].char_length()); } m_items.push_back(nullptr); longest += 1; diff --git a/src/option_manager.cc b/src/option_manager.cc index 1a09361d..b2123cfe 100644 --- a/src/option_manager.cc +++ b/src/option_manager.cc @@ -57,7 +57,7 @@ const Option& OptionManager::operator[](const String& name) const } CandidateList OptionManager::complete_option_name(const String& prefix, - CharCount cursor_pos) + ByteCount cursor_pos) { String real_prefix = prefix.substr(0, cursor_pos); CandidateList result; diff --git a/src/option_manager.hh b/src/option_manager.hh index 221239fd..2fa2f296 100644 --- a/src/option_manager.hh +++ b/src/option_manager.hh @@ -57,7 +57,7 @@ public: void set_option(const String& name, const Option& value); CandidateList complete_option_name(const String& prefix, - CharCount cursor_pos); + ByteCount cursor_pos); typedef std::unordered_map OptionMap; OptionMap flatten_options() const; diff --git a/src/string.hh b/src/string.hh index dfcd9cbb..0b24a15f 100644 --- a/src/string.hh +++ b/src/string.hh @@ -3,10 +3,12 @@ #include #include +#include #include #include "memoryview.hh" #include "units.hh" +#include "utf8.hh" namespace Kakoune { @@ -25,8 +27,11 @@ public: template String(Iterator begin, Iterator end) : m_content(begin, end) {} - char operator[](CharCount pos) const { return m_content[(int)pos]; } - CharCount length() const { return m_content.length(); } + char operator[](ByteCount pos) const { return m_content[(int)pos]; } + ByteCount length() const { return m_content.length(); } + CharCount char_length() const { return utf8::distance(begin(), end()); } + ByteCount byte_count_to(CharCount count) const { return utf8::advance(begin(), end(), (int)count) - begin(); } + CharCount char_count_to(ByteCount count) const { return utf8::distance(begin(), begin() + (int)count); } bool empty() const { return m_content.empty(); } bool operator== (const String& other) const { return m_content == other.m_content; } @@ -45,7 +50,13 @@ public: memoryview data() const { return memoryview(m_content.data(), m_content.size()); } const char* c_str() const { return m_content.c_str(); } - String substr(CharCount pos, CharCount length = -1) const { return String(m_content.substr((int)pos, (int)length)); } + String substr(ByteCount pos, ByteCount length = -1) const { return String(m_content.substr((int)pos, (int)length)); } + String substr(CharCount pos, CharCount length = INT_MAX) const + { + auto b = utf8::advance(begin(), end(), (int)pos); + auto e = utf8::advance(b, end(), (int)length); + return String(b,e); + } String replace(const String& expression, const String& replacement) const; using iterator = std::string::const_iterator; diff --git a/src/utf8.hh b/src/utf8.hh index 057f1d59..0f136c68 100644 --- a/src/utf8.hh +++ b/src/utf8.hh @@ -43,17 +43,17 @@ Iterator previous(Iterator it) // dth character after (or before if d < 0) the character // pointed by it template -Iterator advance(Iterator it, Distance d) +Iterator advance(Iterator it, Iterator end, Distance d) { if (d < 0) { - while (d++) - it = previous(it); + while (it != end and d++) + it = utf8::previous(it); } else { - while (d--) - it = next(it); + while (it != end and d--) + it = utf8::next(it); } return it; } diff --git a/src/window.cc b/src/window.cc index 7e62264a..e74098c0 100644 --- a/src/window.cc +++ b/src/window.cc @@ -51,18 +51,14 @@ void Window::update_display_buffer() LineCount buffer_line = m_position.line + line; if (buffer_line >= buffer().line_count()) break; - BufferIterator pos = buffer().iterator_at({ buffer_line, m_position.column }); - BufferIterator line_begin = buffer().iterator_at_line_begin(pos); - BufferIterator line_end = buffer().iterator_at_line_end(pos); + BufferIterator line_begin = buffer().iterator_at_line_begin(buffer_line); + BufferIterator line_end = buffer().iterator_at_line_end(buffer_line); - BufferIterator end; - if (CharCount(line_end - pos) > m_dimensions.column) - end = pos + m_dimensions.column; - else - end = line_end; + BufferIterator begin = utf8::advance(line_begin, line_end, (int)m_position.column); + BufferIterator end = utf8::advance(begin, line_end, (int)m_dimensions.column); lines.push_back(DisplayLine(buffer_line)); - lines.back().push_back(DisplayAtom(AtomContent(pos,end))); + lines.back().push_back(DisplayAtom(AtomContent(begin, end))); } m_display_buffer.compute_range(); @@ -107,20 +103,20 @@ void Window::scroll_to_keep_cursor_visible_ifn() atom.content.begin() <= cursor and atom.content.end() > cursor) { if (atom.content.type() == AtomContent::BufferRange) - column += cursor - atom.content.begin(); + column += utf8::distance(atom.content.begin(), cursor); else - column += atom.content.content().length(); + column += atom.content.content().char_length(); // we could early out on this, but having scrolling left // faster than not scrolling at all is not really useful. - if (cursor.column() < m_position.column) - m_position.column = cursor.column(); + if (column < m_position.column) + m_position.column = column; else if (column >= m_position.column + m_dimensions.column) m_position.column = column - (m_dimensions.column - 1); return; } - column += atom.content.content().length(); + column += atom.content.content().char_length(); } if (cursor != buffer().end()) { diff --git a/src/window.hh b/src/window.hh index 6e2d59d6..79310ddb 100644 --- a/src/window.hh +++ b/src/window.hh @@ -24,8 +24,8 @@ class Window : public Editor, public OptionManagerWatcher public: ~Window(); - const BufferCoord& position() const { return m_position; } - void set_position(const BufferCoord& position) { m_position = buffer().clamp(position); } + const DisplayCoord& position() const { return m_position; } + void set_position(const DisplayCoord& position) { m_position = position; } const DisplayCoord& dimensions() const { return m_dimensions; } void set_dimensions(const DisplayCoord& dimensions); @@ -58,7 +58,7 @@ private: void scroll_to_keep_cursor_visible_ifn(); - BufferCoord m_position; + DisplayCoord m_position; DisplayCoord m_dimensions; DisplayBuffer m_display_buffer;