xenia/kakoune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Regex: Refactor ThreadedRegexVM state handling

Browse Source 
Remove ExecState to store threads inside the ThreadedRegexVM so that
memory buffers can be reused between executions. Extract an ExecConfig
struct with all the data thats execution specific to avoid storing
it needlessly inside the ThreadedRegexVM.
This commit is contained in:
Maxime Coste 2018-04-21 12:44:54 +10:00
parent eca53db2bf
commit fa17c46653
1 changed files with 101 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

194
src/regex_impl.hh
View File

@ -173,35 +173,13 @@ public:
constexpr bool forward = direction == MatchDirection::Forward; constexpr bool forward = direction == MatchDirection::Forward;
m_begin = EffectiveIt{Utf8It{forward ? begin : end, subject_begin, subject_end}};
m_end = EffectiveIt{Utf8It{forward ? end : begin, subject_begin, subject_end}};
m_subject_begin = EffectiveIt{Utf8It{forward ? subject_begin : subject_end, subject_begin, subject_end}}; if (not forward) // Flip line begin/end flags as we flipped the instructions on compilation.
m_subject_end = EffectiveIt{Utf8It{forward ? subject_end : subject_begin, subject_begin, subject_end}}; flags = (RegexExecFlags)(flags & ~(RegexExecFlags::NotEndOfLine | RegexExecFlags::NotBeginOfLine)) |
if (forward)
m_flags = flags;
else // Flip line begin/end flags as we flipped the instructions on compilation.
m_flags = (RegexExecFlags)(flags & ~(RegexExecFlags::NotEndOfLine | RegexExecFlags::NotBeginOfLine)) |
((flags & RegexExecFlags::NotEndOfLine) ? RegexExecFlags::NotBeginOfLine : RegexExecFlags::None) | ((flags & RegexExecFlags::NotEndOfLine) ? RegexExecFlags::NotBeginOfLine : RegexExecFlags::None) |
((flags & RegexExecFlags::NotBeginOfLine) ? RegexExecFlags::NotEndOfLine : RegexExecFlags::None); ((flags & RegexExecFlags::NotBeginOfLine) ? RegexExecFlags::NotEndOfLine : RegexExecFlags::None);
const bool search = (flags & RegexExecFlags::Search); const bool search = (flags & RegexExecFlags::Search);
EffectiveIt start{m_begin};
const auto& start_desc = direction == MatchDirection::Forward ? m_program.forward_start_desc
: m_program.backward_start_desc;
if (start_desc)
{
if (search)
{
to_next_start(start, m_end, *start_desc);
if (start == m_end) // If start_desc is not null, it means we consume at least one char
return false;
}
else if (start != m_end and
not start_desc->map[std::min(*start, CompiledRegex::StartDesc::other)])
return false;
}
ConstArrayView<CompiledRegex::Instruction> instructions{m_program.instructions}; ConstArrayView<CompiledRegex::Instruction> instructions{m_program.instructions};
if (direction == MatchDirection::Forward) if (direction == MatchDirection::Forward)
@ -211,7 +189,33 @@ public:
if (not search) if (not search)
instructions = instructions.subrange(CompiledRegex::search_prefix_size); instructions = instructions.subrange(CompiledRegex::search_prefix_size);
return exec_program(start, instructions);
const ExecConfig config{
EffectiveIt{Utf8It{forward ? begin : end, subject_begin, subject_end}},
EffectiveIt{Utf8It{forward ? end : begin, subject_begin, subject_end}},
EffectiveIt{Utf8It{forward ? subject_begin : subject_end, subject_begin, subject_end}},
EffectiveIt{Utf8It{forward ? subject_end : subject_begin, subject_begin, subject_end}},
flags,
instructions
};
EffectiveIt start{config.begin};
const auto& start_desc = direction == MatchDirection::Forward ? m_program.forward_start_desc
: m_program.backward_start_desc;
if (start_desc)
{
if (search)
{
to_next_start(start, config.end, *start_desc);
if (start == config.end) // If start_desc is not null, it means we consume at least one char
return false;
}
else if (start != config.end and
not start_desc->map[std::min(*start, CompiledRegex::StartDesc::other)])
return false;
}
return exec_program(std::move(start), config);
} }
ArrayView<const Iterator> captures() const ArrayView<const Iterator> captures() const
@ -277,37 +281,40 @@ private:
using EffectiveIt = std::conditional_t<direction == MatchDirection::Forward, using EffectiveIt = std::conditional_t<direction == MatchDirection::Forward,
Utf8It, std::reverse_iterator<Utf8It>>; Utf8It, std::reverse_iterator<Utf8It>>;
struct ExecState struct ExecConfig
{ {
Vector<Thread, MemoryDomain::Regex> current_threads; const EffectiveIt begin;
Vector<Thread, MemoryDomain::Regex> next_threads; const EffectiveIt end;
uint16_t step = -1; const EffectiveIt subject_begin;
const EffectiveIt subject_end;
const RegexExecFlags flags;
ConstArrayView<CompiledRegex::Instruction> instructions;
}; };
enum class StepResult { Consumed, Matched, Failed, FindNextStart }; enum class StepResult { Consumed, Matched, Failed, FindNextStart };
// Steps a thread until it consumes the current character, matches or fail // Steps a thread until it consumes the current character, matches or fail
StepResult step(EffectiveIt& pos, Thread& thread, ExecState& state) StepResult step(EffectiveIt& pos, uint16_t current_step, Thread& thread, const ExecConfig& config)
{ {
const bool no_saves = (m_flags & RegexExecFlags::NoSaves); const bool no_saves = (config.flags & RegexExecFlags::NoSaves);
auto* instructions = m_program.instructions.data(); auto* instructions = m_program.instructions.data();
while (true) while (true)
{ {
auto& inst = *thread.inst++; auto& inst = *thread.inst++;
// if this instruction was already executed for this step in another thread, // if this instruction was already executed for this step in another thread,
// then this thread is redundant and can be dropped // then this thread is redundant and can be dropped
if (inst.last_step == state.step) if (inst.last_step == current_step)
return StepResult::Failed; return StepResult::Failed;
inst.last_step = state.step; inst.last_step = current_step;
switch (inst.op) switch (inst.op)
{ {
case CompiledRegex::Literal: case CompiledRegex::Literal:
if (pos != m_end and inst.param == *pos) if (pos != config.end and inst.param == *pos)
return StepResult::Consumed; return StepResult::Consumed;
return StepResult::Failed; return StepResult::Failed;
case CompiledRegex::Literal_IgnoreCase: case CompiledRegex::Literal_IgnoreCase:
if (pos != m_end and inst.param == to_lower(*pos)) if (pos != config.end and inst.param == to_lower(*pos))
return StepResult::Consumed; return StepResult::Consumed;
return StepResult::Failed; return StepResult::Failed;
case CompiledRegex::AnyChar: case CompiledRegex::AnyChar:
@ -319,14 +326,14 @@ private:
{ {
if (thread.saves) if (thread.saves)
++thread.saves->refcount; ++thread.saves->refcount;
state.current_threads.push_back({instructions + inst.param, thread.saves}); m_current_threads.push_back({instructions + inst.param, thread.saves});
break; break;
} }
case CompiledRegex::Split_PrioritizeChild: case CompiledRegex::Split_PrioritizeChild:
{ {
if (thread.saves) if (thread.saves)
++thread.saves->refcount; ++thread.saves->refcount;
state.current_threads.push_back({thread.inst, thread.saves}); m_current_threads.push_back({thread.inst, thread.saves});
thread.inst = instructions + inst.param; thread.inst = instructions + inst.param;
break; break;
} }
@ -345,66 +352,66 @@ private:
break; break;
} }
case CompiledRegex::Class: case CompiledRegex::Class:
if (pos == m_end) if (pos == config.end)
return StepResult::Failed; return StepResult::Failed;
return is_character_class(m_program.character_classes[inst.param], *pos) ? return is_character_class(m_program.character_classes[inst.param], *pos) ?
StepResult::Consumed : StepResult::Failed; StepResult::Consumed : StepResult::Failed;
case CompiledRegex::CharacterType: case CompiledRegex::CharacterType:
if (pos == m_end) if (pos == config.end)
return StepResult::Failed; return StepResult::Failed;
return is_ctype((CharacterType)inst.param, *pos) ? return is_ctype((CharacterType)inst.param, *pos) ?
StepResult::Consumed : StepResult::Failed;; StepResult::Consumed : StepResult::Failed;;
case CompiledRegex::LineStart: case CompiledRegex::LineStart:
if (not is_line_start(pos)) if (not is_line_start(pos, config))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::LineEnd: case CompiledRegex::LineEnd:
if (not is_line_end(pos)) if (not is_line_end(pos, config))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::WordBoundary: case CompiledRegex::WordBoundary:
if (not is_word_boundary(pos)) if (not is_word_boundary(pos, config))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::NotWordBoundary: case CompiledRegex::NotWordBoundary:
if (is_word_boundary(pos)) if (is_word_boundary(pos, config))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::SubjectBegin: case CompiledRegex::SubjectBegin:
if (pos != m_subject_begin) if (pos != config.subject_begin)
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::SubjectEnd: case CompiledRegex::SubjectEnd:
if (pos != m_subject_end) if (pos != config.subject_end)
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::LookAhead: case CompiledRegex::LookAhead:
case CompiledRegex::NegativeLookAhead: case CompiledRegex::NegativeLookAhead:
if (lookaround<MatchDirection::Forward, false>(inst.param, pos) != if (lookaround<MatchDirection::Forward, false>(inst.param, pos, config) !=
(inst.op == CompiledRegex::LookAhead)) (inst.op == CompiledRegex::LookAhead))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::LookAhead_IgnoreCase: case CompiledRegex::LookAhead_IgnoreCase:
case CompiledRegex::NegativeLookAhead_IgnoreCase: case CompiledRegex::NegativeLookAhead_IgnoreCase:
if (lookaround<MatchDirection::Forward, true>(inst.param, pos) != if (lookaround<MatchDirection::Forward, true>(inst.param, pos, config) !=
(inst.op == CompiledRegex::LookAhead_IgnoreCase)) (inst.op == CompiledRegex::LookAhead_IgnoreCase))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::LookBehind: case CompiledRegex::LookBehind:
case CompiledRegex::NegativeLookBehind: case CompiledRegex::NegativeLookBehind:
if (lookaround<MatchDirection::Backward, false>(inst.param, pos) != if (lookaround<MatchDirection::Backward, false>(inst.param, pos, config) !=
(inst.op == CompiledRegex::LookBehind)) (inst.op == CompiledRegex::LookBehind))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::LookBehind_IgnoreCase: case CompiledRegex::LookBehind_IgnoreCase:
case CompiledRegex::NegativeLookBehind_IgnoreCase: case CompiledRegex::NegativeLookBehind_IgnoreCase:
if (lookaround<MatchDirection::Backward, true>(inst.param, pos) != if (lookaround<MatchDirection::Backward, true>(inst.param, pos, config) !=
(inst.op == CompiledRegex::LookBehind_IgnoreCase)) (inst.op == CompiledRegex::LookBehind_IgnoreCase))
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::FindNextStart: case CompiledRegex::FindNextStart:
kak_assert(state.current_threads.empty()); // search thread should by construction be the lower priority one kak_assert(m_current_threads.empty()); // search thread should by construction be the lower priority one
if (state.next_threads.empty()) if (m_next_threads.empty())
return StepResult::FindNextStart; return StepResult::FindNextStart;
return StepResult::Consumed; return StepResult::Consumed;
case CompiledRegex::Match: case CompiledRegex::Match:
@ -414,35 +421,38 @@ private:
return StepResult::Failed; return StepResult::Failed;
} }
bool exec_program(EffectiveIt pos, ConstArrayView<CompiledRegex::Instruction> instructions) bool exec_program(EffectiveIt pos, const ExecConfig& config)
{ {
ExecState state; kak_assert(m_current_threads.empty() and m_next_threads.empty());
state.current_threads.push_back({instructions.begin(), nullptr}); release_saves(m_captures);
m_captures = nullptr;
m_current_threads.push_back({config.instructions.begin(), nullptr});
const auto& start_desc = direction == MatchDirection::Forward ? m_program.forward_start_desc const auto& start_desc = direction == MatchDirection::Forward ? m_program.forward_start_desc
: m_program.backward_start_desc; : m_program.backward_start_desc;
uint16_t current_step = -1;
bool found_match = false; bool found_match = false;
while (true) // Iterate on all codepoints and once at the end while (true) // Iterate on all codepoints and once at the end
{ {
if (++state.step == 0) if (++current_step == 0)
{ {
// We wrapped, avoid potential collision on inst.last_step by resetting them // We wrapped, avoid potential collision on inst.last_step by resetting them
for (auto& inst : instructions) for (auto& inst : config.instructions)
inst.last_step = 0; inst.last_step = 0;
state.step = 1; // step 0 is never valid current_step = 1; // step 0 is never valid
} }
bool find_next_start = false; bool find_next_start = false;
while (not state.current_threads.empty()) while (not m_current_threads.empty())
{ {
auto thread = state.current_threads.back(); auto thread = m_current_threads.back();
state.current_threads.pop_back(); m_current_threads.pop_back();
switch (step(pos, thread, state)) switch (step(pos, current_step, thread, config))
{ {
case StepResult::Matched: case StepResult::Matched:
if ((pos != m_end and not (m_flags & RegexExecFlags::Search)) or if ((pos != config.end and not (config.flags & RegexExecFlags::Search)) or
(m_flags & RegexExecFlags::NotInitialNull and pos == m_begin)) (config.flags & RegexExecFlags::NotInitialNull and pos == config.begin))
{ {
release_saves(thread.saves); release_saves(thread.saves);
continue; continue;
@ -453,9 +463,9 @@ private:
found_match = true; found_match = true;
// remove this and lower priority threads // remove this and lower priority threads
for (auto& t : state.current_threads) for (auto& t : m_current_threads)
release_saves(t.saves); release_saves(t.saves);
state.current_threads.clear(); m_current_threads.clear();
break; break;
case StepResult::Failed: case StepResult::Failed:
release_saves(thread.saves); release_saves(thread.saves);
@ -467,31 +477,33 @@ private:
continue; continue;
} }
thread.inst->scheduled = true; thread.inst->scheduled = true;
state.next_threads.push_back(thread); m_next_threads.push_back(thread);
break; break;
case StepResult::FindNextStart: case StepResult::FindNextStart:
state.next_threads.push_back(thread); m_next_threads.push_back(thread);
find_next_start = true; find_next_start = true;
break; break;
} }
} }
for (auto& thread : state.next_threads) kak_assert(m_current_threads.empty());
for (auto& thread : m_next_threads)
thread.inst->scheduled = false; thread.inst->scheduled = false;
if (pos == m_end or state.next_threads.empty() or if (pos == config.end or m_next_threads.empty() or
(found_match and (m_flags & RegexExecFlags::AnyMatch))) (found_match and (config.flags & RegexExecFlags::AnyMatch)))
{ {
for (auto& t : state.next_threads) for (auto& t : m_next_threads)
release_saves(t.saves); release_saves(t.saves);
m_next_threads.clear();
return found_match; return found_match;
} }
std::swap(state.current_threads, state.next_threads); std::swap(m_current_threads, m_next_threads);
std::reverse(state.current_threads.begin(), state.current_threads.end()); std::reverse(m_current_threads.begin(), m_current_threads.end());
++pos; ++pos;
if (find_next_start and start_desc) if (find_next_start and start_desc)
to_next_start(pos, m_end, *start_desc); to_next_start(pos, config.end, *start_desc);
} }
} }
@ -504,9 +516,9 @@ private:
} }
template<MatchDirection look_direction, bool ignore_case> template<MatchDirection look_direction, bool ignore_case>
bool lookaround(uint32_t index, EffectiveIt pos) const bool lookaround(uint32_t index, EffectiveIt pos, const ExecConfig& config) const
{ {
const auto end = (look_direction == MatchDirection::Forward ? m_subject_end : m_subject_begin); const auto end = (look_direction == MatchDirection::Forward ? config.subject_end : config.subject_begin);
for (auto it = m_program.lookarounds.begin() + index; *it != -1; ++it) for (auto it = m_program.lookarounds.begin() + index; *it != -1; ++it)
{ {
if (pos == end) if (pos == end)
@ -536,26 +548,26 @@ private:
return true; return true;
} }
bool is_line_start(const EffectiveIt& pos) const static bool is_line_start(const EffectiveIt& pos, const ExecConfig& config)
{ {
if (pos == m_subject_begin) if (pos == config.subject_begin)
return not (m_flags & RegexExecFlags::NotBeginOfLine); return not (config.flags & RegexExecFlags::NotBeginOfLine);
return *(pos-1) == '\n'; return *(pos-1) == '\n';
} }
bool is_line_end(const EffectiveIt& pos) const static bool is_line_end(const EffectiveIt& pos, const ExecConfig& config)
{ {
if (pos == m_subject_end) if (pos == config.subject_end)
return not (m_flags & RegexExecFlags::NotEndOfLine); return not (config.flags & RegexExecFlags::NotEndOfLine);
return *pos == '\n'; return *pos == '\n';
} }
bool is_word_boundary(const EffectiveIt& pos) const static bool is_word_boundary(const EffectiveIt& pos, const ExecConfig& config)
{ {
if (pos == m_subject_begin) if (pos == config.subject_begin)
return not (m_flags & RegexExecFlags::NotBeginOfWord); return not (config.flags & RegexExecFlags::NotBeginOfWord);
if (pos == m_subject_end) if (pos == config.subject_end)
return not (m_flags & RegexExecFlags::NotEndOfWord); return not (config.flags & RegexExecFlags::NotEndOfWord);
return is_word(*(pos-1)) != is_word(*pos); return is_word(*(pos-1)) != is_word(*pos);
} }
@ -564,15 +576,11 @@ private:
const CompiledRegex& m_program; const CompiledRegex& m_program;
EffectiveIt m_begin; Vector<Thread, MemoryDomain::Regex> m_current_threads;
EffectiveIt m_end; Vector<Thread, MemoryDomain::Regex> m_next_threads;
EffectiveIt m_subject_begin;
EffectiveIt m_subject_end;
RegexExecFlags m_flags;
Vector<Saves*, MemoryDomain::Regex> m_saves; Vector<Saves*, MemoryDomain::Regex> m_saves;
Saves* m_first_free = nullptr; Saves* m_first_free = nullptr;
Saves* m_captures = nullptr; Saves* m_captures = nullptr;
}; };