xenia/home
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Regex: cleanup and reorganize regex code and improve capture support

Browse Source 
Introduce the CompiledRegex class, rename ThreadedExecutor to
ThreadedRegexVM, remove the RegexProgram namespace.
This commit is contained in:
Maxime Coste 2017-09-19 17:44:58 +09:00
parent 023511deff
commit f7b8c1c79d
1 changed files with 161 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

285
src/regex_impl.cc
View File

@ -9,7 +9,7 @@
namespace Kakoune namespace Kakoune
{ {
namespace RegexProgram struct CompiledRegex
{ {
enum Op : char enum Op : char
{ {
@ -28,7 +28,10 @@ enum Op : char
}; };
using Offset = size_t; using Offset = size_t;
}
Vector<char> bytecode;
size_t save_count;
};
namespace RegexCompiler namespace RegexCompiler
{ {
@ -100,6 +103,8 @@ struct Parser
return disjunction(pos, end, 0); return disjunction(pos, end, 0);
} }
size_t capture_count() const { return m_next_capture; }
private: private:
AstNodePtr disjunction(Iterator& pos, Iterator end, char capture = -1) AstNodePtr disjunction(Iterator& pos, Iterator end, char capture = -1)
{ {
@ -224,40 +229,40 @@ private:
char m_next_capture = 1; char m_next_capture = 1;
}; };
RegexProgram::Offset compile_node(Vector<char>& program, const AstNodePtr& node); CompiledRegex::Offset compile_node(CompiledRegex& program, const AstNodePtr& node);
RegexProgram::Offset alloc_offset(Vector<char>& instructions) CompiledRegex::Offset alloc_offset(CompiledRegex& program)
{ {
auto pos = instructions.size(); auto pos = program.bytecode.size();
instructions.resize(instructions.size() + sizeof(RegexProgram::Offset)); program.bytecode.resize(pos + sizeof(CompiledRegex::Offset));
return pos; return pos;
} }
RegexProgram::Offset& get_offset(Vector<char>& instructions, RegexProgram::Offset base) CompiledRegex::Offset& get_offset(CompiledRegex& program, CompiledRegex::Offset pos)
{ {
return *reinterpret_cast<RegexProgram::Offset*>(&instructions[base]); return *reinterpret_cast<CompiledRegex::Offset*>(&program.bytecode[pos]);
} }
RegexProgram::Offset compile_node_inner(Vector<char>& program, const AstNodePtr& node) CompiledRegex::Offset compile_node_inner(CompiledRegex& program, const AstNodePtr& node)
{ {
const auto start_pos = program.size(); const auto start_pos = program.bytecode.size();
const char capture = (node->op == Op::Alternation or node->op == Op::Sequence) ? node->value : -1; const char capture = (node->op == Op::Alternation or node->op == Op::Sequence) ? node->value : -1;
if (capture >= 0) if (capture >= 0)
{ {
program.push_back(RegexProgram::Save); program.bytecode.push_back(CompiledRegex::Save);
program.push_back(capture * 2); program.bytecode.push_back(capture * 2);
} }
Vector<RegexProgram::Offset> goto_inner_end_offsets; Vector<CompiledRegex::Offset> goto_inner_end_offsets;
switch (node->op) switch (node->op)
{ {
case Op::Literal: case Op::Literal:
program.push_back(RegexProgram::Literal); program.bytecode.push_back(CompiledRegex::Literal);
program.push_back(node->value); program.bytecode.push_back(node->value);
break; break;
case Op::AnyChar: case Op::AnyChar:
program.push_back(RegexProgram::AnyChar); program.bytecode.push_back(CompiledRegex::AnyChar);
break; break;
case Op::Sequence: case Op::Sequence:
for (auto& child : node->children) for (auto& child : node->children)
@ -268,11 +273,11 @@ RegexProgram::Offset compile_node_inner(Vector<char>& program, const AstNodePtr&
auto& children = node->children; auto& children = node->children;
kak_assert(children.size() == 2); kak_assert(children.size() == 2);
program.push_back(RegexProgram::Split); program.bytecode.push_back(CompiledRegex::Split);
auto offset = alloc_offset(program); auto offset = alloc_offset(program);
compile_node(program, children[0]); compile_node(program, children[0]);
program.push_back(RegexProgram::Jump); program.bytecode.push_back(CompiledRegex::Jump);
goto_inner_end_offsets.push_back(alloc_offset(program)); goto_inner_end_offsets.push_back(alloc_offset(program));
auto right_pos = compile_node(program, children[1]); auto right_pos = compile_node(program, children[1]);
@ -281,45 +286,45 @@ RegexProgram::Offset compile_node_inner(Vector<char>& program, const AstNodePtr&
break; break;
} }
case Op::LineStart: case Op::LineStart:
program.push_back(RegexProgram::LineStart); program.bytecode.push_back(CompiledRegex::LineStart);
break; break;
case Op::LineEnd: case Op::LineEnd:
program.push_back(RegexProgram::LineEnd); program.bytecode.push_back(CompiledRegex::LineEnd);
break; break;
case Op::WordBoundary: case Op::WordBoundary:
program.push_back(RegexProgram::WordBoundary); program.bytecode.push_back(CompiledRegex::WordBoundary);
break; break;
case Op::NotWordBoundary: case Op::NotWordBoundary:
program.push_back(RegexProgram::NotWordBoundary); program.bytecode.push_back(CompiledRegex::NotWordBoundary);
break; break;
case Op::SubjectBegin: case Op::SubjectBegin:
program.push_back(RegexProgram::SubjectBegin); program.bytecode.push_back(CompiledRegex::SubjectBegin);
break; break;
case Op::SubjectEnd: case Op::SubjectEnd:
program.push_back(RegexProgram::SubjectEnd); program.bytecode.push_back(CompiledRegex::SubjectEnd);
break; break;
} }
for (auto& offset : goto_inner_end_offsets) for (auto& offset : goto_inner_end_offsets)
get_offset(program, offset) = program.size(); get_offset(program, offset) = program.bytecode.size();
if (capture >= 0) if (capture >= 0)
{ {
program.push_back(RegexProgram::Save); program.bytecode.push_back(CompiledRegex::Save);
program.push_back(capture * 2 + 1); program.bytecode.push_back(capture * 2 + 1);
} }
return start_pos; return start_pos;
} }
RegexProgram::Offset compile_node(Vector<char>& program, const AstNodePtr& node) CompiledRegex::Offset compile_node(CompiledRegex& program, const AstNodePtr& node)
{ {
RegexProgram::Offset pos = program.size(); CompiledRegex::Offset pos = program.bytecode.size();
Vector<RegexProgram::Offset> goto_end_offsets; Vector<CompiledRegex::Offset> goto_end_offsets;
if (node->quantifier.allows_none()) if (node->quantifier.allows_none())
{ {
program.push_back(RegexProgram::Split); program.bytecode.push_back(CompiledRegex::Split);
goto_end_offsets.push_back(alloc_offset(program)); goto_end_offsets.push_back(alloc_offset(program));
} }
@ -330,94 +335,96 @@ RegexProgram::Offset compile_node(Vector<char>& program, const AstNodePtr& node)
if (node->quantifier.allows_infinite_repeat()) if (node->quantifier.allows_infinite_repeat())
{ {
program.push_back(RegexProgram::Split); program.bytecode.push_back(CompiledRegex::Split);
get_offset(program, alloc_offset(program)) = inner_pos; get_offset(program, alloc_offset(program)) = inner_pos;
} }
// Write the node as an optional match for the min -> max counts // Write the node as an optional match for the min -> max counts
else for (int i = std::max(1, node->quantifier.min); // STILL UGLY ! else for (int i = std::max(1, node->quantifier.min); // STILL UGLY !
i < node->quantifier.max; ++i) i < node->quantifier.max; ++i)
{ {
program.push_back(RegexProgram::Split); program.bytecode.push_back(CompiledRegex::Split);
goto_end_offsets.push_back(alloc_offset(program)); goto_end_offsets.push_back(alloc_offset(program));
compile_node_inner(program, node); compile_node_inner(program, node);
} }
for (auto offset : goto_end_offsets) for (auto offset : goto_end_offsets)
get_offset(program, offset) = program.size(); get_offset(program, offset) = program.bytecode.size();
return pos; return pos;
} }
Vector<char> compile(const AstNodePtr& node) CompiledRegex compile(const AstNodePtr& node, size_t capture_count)
{ {
Vector<char> res; CompiledRegex res;
compile_node(res, node); compile_node(res, node);
res.push_back(RegexProgram::Match); res.bytecode.push_back(CompiledRegex::Match);
res.save_count = capture_count * 2;
return res; return res;
} }
template<typename Iterator> template<typename Iterator>
Vector<char> compile(Iterator begin, Iterator end) CompiledRegex compile(Iterator begin, Iterator end)
{ {
return compile(Parser<Iterator>{}.parse(begin, end)); Parser<Iterator> parser;
} auto node = parser.parse(begin, end);
return compile(node, parser.capture_count());
} }
namespace RegexProgram }
void dump(const CompiledRegex& program)
{ {
void dump(ConstArrayView<char> program) for (auto pos = program.bytecode.begin(); pos < program.bytecode.end(); )
{ {
for (size_t pos = 0; pos < program.size(); ) printf("%4zd ", pos - program.bytecode.begin());
switch ((CompiledRegex::Op)*pos++)
{ {
printf("%4zd ", pos); case CompiledRegex::Literal:
switch ((RegexProgram::Op)program[pos++]) printf("literal %c\n", *pos++);
{
case RegexProgram::Literal:
printf("literal %c\n", program[pos++]);
break; break;
case RegexProgram::AnyChar: case CompiledRegex::AnyChar:
printf("any char\n"); printf("any char\n");
break; break;
case RegexProgram::Jump: case CompiledRegex::Jump:
printf("jump %zd\n", *reinterpret_cast<const RegexProgram::Offset*>(&program[pos])); printf("jump %zd\n", *reinterpret_cast<const CompiledRegex::Offset*>(&*pos));
pos += sizeof(RegexProgram::Offset); pos += sizeof(CompiledRegex::Offset);
break; break;
case RegexProgram::Split: case CompiledRegex::Split:
{ {
printf("split %zd\n", *reinterpret_cast<const RegexProgram::Offset*>(&program[pos])); printf("split %zd\n", *reinterpret_cast<const CompiledRegex::Offset*>(&*pos));
pos += sizeof(RegexProgram::Offset); pos += sizeof(CompiledRegex::Offset);
break; break;
} }
case RegexProgram::Save: case CompiledRegex::Save:
printf("save %d\n", program[pos++]); printf("save %d\n", *pos++);
break; break;
case RegexProgram::LineStart: case CompiledRegex::LineStart:
printf("line start\n"); printf("line start\n");
break; break;
case RegexProgram::LineEnd: case CompiledRegex::LineEnd:
printf("line end\n"); printf("line end\n");
break; break;
case RegexProgram::WordBoundary: case CompiledRegex::WordBoundary:
printf("word boundary\n"); printf("word boundary\n");
break; break;
case RegexProgram::NotWordBoundary: case CompiledRegex::NotWordBoundary:
printf("not word boundary\n"); printf("not word boundary\n");
break; break;
case RegexProgram::SubjectBegin: case CompiledRegex::SubjectBegin:
printf("subject begin\n"); printf("subject begin\n");
break; break;
case RegexProgram::SubjectEnd: case CompiledRegex::SubjectEnd:
printf("subject end\n"); printf("subject end\n");
break; break;
case RegexProgram::Match: case CompiledRegex::Match:
printf("match\n"); printf("match\n");
} }
} }
} }
struct ThreadedExecutor struct ThreadedRegexVM
{ {
ThreadedExecutor(ConstArrayView<char> program) : m_program{program} {} ThreadedRegexVM(const CompiledRegex& program) : m_program{program} {}
struct Thread struct Thread
{ {
@ -432,18 +439,18 @@ struct ThreadedExecutor
{ {
auto& thread = m_threads[thread_index]; auto& thread = m_threads[thread_index];
char c = m_pos == m_subject.end() ? 0 : *m_pos; char c = m_pos == m_subject.end() ? 0 : *m_pos;
const RegexProgram::Op op = (RegexProgram::Op)*thread.inst++; const CompiledRegex::Op op = (CompiledRegex::Op)*thread.inst++;
switch (op) switch (op)
{ {
case RegexProgram::Literal: case CompiledRegex::Literal:
if (*thread.inst++ == c) if (*thread.inst++ == c)
return StepResult::Consumed; return StepResult::Consumed;
return StepResult::Failed; return StepResult::Failed;
case RegexProgram::AnyChar: case CompiledRegex::AnyChar:
return StepResult::Consumed; return StepResult::Consumed;
case RegexProgram::Jump: case CompiledRegex::Jump:
{ {
auto inst = m_program.begin() + *reinterpret_cast<const RegexProgram::Offset*>(thread.inst); auto inst = m_program.bytecode.data() + *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst);
// if instruction is already going to be executed, drop this thread // if instruction is already going to be executed, drop this thread
if (std::find_if(m_threads.begin(), m_threads.end(), if (std::find_if(m_threads.begin(), m_threads.end(),
[inst](const Thread& t) { return t.inst == inst; }) != m_threads.end()) [inst](const Thread& t) { return t.inst == inst; }) != m_threads.end())
@ -451,54 +458,54 @@ struct ThreadedExecutor
thread.inst = inst; thread.inst = inst;
break; break;
} }
case RegexProgram::Split: case CompiledRegex::Split:
{ {
add_thread(*reinterpret_cast<const RegexProgram::Offset*>(thread.inst), thread.saves); add_thread(*reinterpret_cast<const CompiledRegex::Offset*>(thread.inst), thread.saves);
// thread is invalidated now, as we mutated the m_thread vector // thread is invalidated now, as we mutated the m_thread vector
m_threads[thread_index].inst += sizeof(RegexProgram::Offset); m_threads[thread_index].inst += sizeof(CompiledRegex::Offset);
break; break;
} }
case RegexProgram::Save: case CompiledRegex::Save:
{ {
const char index = *thread.inst++; const char index = *thread.inst++;
thread.saves[index] = m_pos; thread.saves[index] = m_pos;
break; break;
} }
case RegexProgram::LineStart: case CompiledRegex::LineStart:
if (not is_line_start()) if (not is_line_start())
return StepResult::Failed; return StepResult::Failed;
break; break;
case RegexProgram::LineEnd: case CompiledRegex::LineEnd:
if (not is_line_end()) if (not is_line_end())
return StepResult::Failed; return StepResult::Failed;
break; break;
case RegexProgram::WordBoundary: case CompiledRegex::WordBoundary:
if (not is_word_boundary()) if (not is_word_boundary())
return StepResult::Failed; return StepResult::Failed;
break; break;
case RegexProgram::NotWordBoundary: case CompiledRegex::NotWordBoundary:
if (is_word_boundary()) if (is_word_boundary())
return StepResult::Failed; return StepResult::Failed;
break; break;
case RegexProgram::SubjectBegin: case CompiledRegex::SubjectBegin:
if (m_pos != m_subject.begin()) if (m_pos != m_subject.begin())
return StepResult::Failed; return StepResult::Failed;
break; break;
case RegexProgram::SubjectEnd: case CompiledRegex::SubjectEnd:
if (m_pos != m_subject.end()) if (m_pos != m_subject.end())
return StepResult::Failed; return StepResult::Failed;
break; break;
case RegexProgram::Match: case CompiledRegex::Match:
return StepResult::Matched; return StepResult::Matched;
} }
} }
return StepResult::Failed; return StepResult::Failed;
} }
bool match(ConstArrayView<char> program, StringView data) bool match(StringView data)
{ {
m_threads.clear(); m_threads.clear();
add_thread(0, Vector<const char*>(10, nullptr)); add_thread(0, Vector<const char*>(m_program.save_count, nullptr));
m_subject = data; m_subject = data;
m_pos = data.begin(); m_pos = data.begin();
@ -534,9 +541,9 @@ struct ThreadedExecutor
return false; return false;
} }
void add_thread(RegexProgram::Offset pos, Vector<const char*> saves) void add_thread(CompiledRegex::Offset pos, Vector<const char*> saves)
{ {
const char* inst = m_program.begin() + pos; const char* inst = m_program.bytecode.data() + pos;
if (std::find_if(m_threads.begin(), m_threads.end(), if (std::find_if(m_threads.begin(), m_threads.end(),
[inst](const Thread& t) { return t.inst == inst; }) == m_threads.end()) [inst](const Thread& t) { return t.inst == inst; }) == m_threads.end())
m_threads.push_back({inst, std::move(saves)}); m_threads.push_back({inst, std::move(saves)});
@ -559,104 +566,102 @@ struct ThreadedExecutor
is_word(*(m_pos-1)) != is_word(*m_pos); is_word(*(m_pos-1)) != is_word(*m_pos);
} }
ConstArrayView<char> m_program; const CompiledRegex& m_program;
Vector<Thread> m_threads; Vector<Thread> m_threads;
Vector<const char*> m_captures; Vector<const char*> m_captures;
StringView m_subject; StringView m_subject;
const char* m_pos; const char* m_pos;
}; };
}
auto test_regex = UnitTest{[]{ auto test_regex = UnitTest{[]{
using Exec = RegexProgram::ThreadedExecutor;
{ {
StringView re = R"(a*b)"; StringView re = R"(a*b)";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(exec.match(program, "b")); kak_assert(vm.match("b"));
kak_assert(exec.match(program, "ab")); kak_assert(vm.match("ab"));
kak_assert(exec.match(program, "aaab")); kak_assert(vm.match("aaab"));
kak_assert(not exec.match(program, "acb")); kak_assert(not vm.match("acb"));
kak_assert(not exec.match(program, "")); kak_assert(not vm.match(""));
} }
{ {
StringView re = R"(^a.*b$)"; StringView re = R"(^a.*b$)";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(exec.match(program, "afoob")); kak_assert(vm.match("afoob"));
kak_assert(exec.match(program, "ab")); kak_assert(vm.match("ab"));
kak_assert(not exec.match(program, "bab")); kak_assert(not vm.match("bab"));
kak_assert(not exec.match(program, "")); kak_assert(not vm.match(""));
} }
{ {
StringView re = R"(^(foo|qux|baz)+(bar)?baz$)"; StringView re = R"(^(foo|qux|baz)+(bar)?baz$)";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(exec.match(program, "fooquxbarbaz")); kak_assert(vm.match("fooquxbarbaz"));
kak_assert(StringView{exec.m_captures[2], exec.m_captures[3]} == "qux"); kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "qux");
kak_assert(not exec.match(program, "fooquxbarbaze")); kak_assert(not vm.match("fooquxbarbaze"));
kak_assert(not exec.match(program, "quxbar")); kak_assert(not vm.match("quxbar"));
kak_assert(not exec.match(program, "blahblah")); kak_assert(not vm.match("blahblah"));
kak_assert(exec.match(program, "bazbaz")); kak_assert(vm.match("bazbaz"));
kak_assert(exec.match(program, "quxbaz")); kak_assert(vm.match("quxbaz"));
} }
{ {
StringView re = R"(.*\b(foo|bar)\b.*)"; StringView re = R"(.*\b(foo|bar)\b.*)";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(exec.match(program, "qux foo baz")); kak_assert(vm.match("qux foo baz"));
kak_assert(StringView{exec.m_captures[2], exec.m_captures[3]} == "foo"); kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "foo");
kak_assert(not exec.match(program, "quxfoobaz")); kak_assert(not vm.match("quxfoobaz"));
kak_assert(exec.match(program, "bar")); kak_assert(vm.match("bar"));
kak_assert(not exec.match(program, "foobar")); kak_assert(not vm.match("foobar"));
} }
{ {
StringView re = R"(\`(foo|bar)\')"; StringView re = R"(\`(foo|bar)\')";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(exec.match(program, "foo")); kak_assert(vm.match("foo"));
kak_assert(exec.match(program, "bar")); kak_assert(vm.match("bar"));
kak_assert(not exec.match(program, "foobar")); kak_assert(not vm.match("foobar"));
} }
{ {
StringView re = R"(\`a{3,5}b\')"; StringView re = R"(\`a{3,5}b\')";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(not exec.match(program, "aab")); kak_assert(not vm.match("aab"));
kak_assert(exec.match(program, "aaab")); kak_assert(vm.match("aaab"));
kak_assert(not exec.match(program, "aaaaaab")); kak_assert(not vm.match("aaaaaab"));
kak_assert(exec.match(program, "aaaaab")); kak_assert(vm.match("aaaaab"));
} }
{ {
StringView re = R"(\`a{3,}b\')"; StringView re = R"(\`a{3,}b\')";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(not exec.match(program, "aab")); kak_assert(not vm.match("aab"));
kak_assert(exec.match(program, "aaab")); kak_assert(vm.match("aaab"));
kak_assert(exec.match(program, "aaaaab")); kak_assert(vm.match("aaaaab"));
} }
{ {
StringView re = R"(\`a{,3}b\')"; StringView re = R"(\`a{,3}b\')";
auto program = RegexCompiler::compile(re.begin(), re.end()); auto program = RegexCompiler::compile(re.begin(), re.end());
RegexProgram::dump(program); dump(program);
Exec exec{program}; ThreadedRegexVM vm{program};
kak_assert(exec.match(program, "b")); kak_assert(vm.match("b"));
kak_assert(exec.match(program, "ab")); kak_assert(vm.match("ab"));
kak_assert(exec.match(program, "aaab")); kak_assert(vm.match("aaab"));
kak_assert(not exec.match(program, "aaaab")); kak_assert(not vm.match("aaaab"));
} }
}}; }};