xenia/kakoune
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Regex: work on unicode codepoints instead of raw bytes

Browse Source
This commit is contained in:
Maxime Coste 2017-09-25 22:24:24 +09:00
parent 75608ea223
commit 002aba562f
1 changed files with 118 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

217
src/regex_impl.cc
View File

@ -3,6 +3,8 @@
#include "unit_tests.hh" #include "unit_tests.hh"
#include "string.hh" #include "string.hh"
#include "unicode.hh" #include "unicode.hh"
#include "utf8.hh"
#include "utf8_iterator.hh"
#include "exception.hh" #include "exception.hh"
#include "array_view.hh" #include "array_view.hh"
@ -86,14 +88,14 @@ enum class Op
struct AstNode struct AstNode
{ {
Op op; Op op;
char value; Codepoint value;
Quantifier quantifier; Quantifier quantifier;
Vector<std::unique_ptr<AstNode>> children; Vector<std::unique_ptr<AstNode>> children;
}; };
using AstNodePtr = std::unique_ptr<AstNode>; using AstNodePtr = std::unique_ptr<AstNode>;
struct CharRange { char min, max; }; struct CharRange { Codepoint min, max; };
struct ParsedRegex struct ParsedRegex
{ {
@ -102,7 +104,7 @@ struct ParsedRegex
Vector<Vector<CharRange>> ranges; Vector<Vector<CharRange>> ranges;
}; };
AstNodePtr make_ast_node(Op op, char value = -1, AstNodePtr make_ast_node(Op op, Codepoint value = -1,
Quantifier quantifier = {Quantifier::One}) Quantifier quantifier = {Quantifier::One})
{ {
return AstNodePtr{new AstNode{op, value, quantifier, {}}}; return AstNodePtr{new AstNode{op, value, quantifier, {}}};
@ -110,19 +112,26 @@ AstNodePtr make_ast_node(Op op, char value = -1,
// Recursive descent parser based on naming used in the ECMAScript // Recursive descent parser based on naming used in the ECMAScript
// standard, although the syntax is not fully compatible. // standard, although the syntax is not fully compatible.
template<typename Iterator>
struct Parser struct Parser
{ {
static ParsedRegex parse(Iterator pos, Iterator end) struct InvalidPolicy
{
Codepoint operator()(Codepoint cp) { throw runtime_error{"Invalid utf8 in regex"}; }
};
using Iterator = utf8::iterator<const char*, Codepoint, int, InvalidPolicy>;
static ParsedRegex parse(StringView re)
{ {
ParsedRegex res; ParsedRegex res;
res.capture_count = 1; res.capture_count = 1;
Iterator pos{re.begin(), re}, end{re.end(), re};
res.ast = disjunction(res, pos, end, 0); res.ast = disjunction(res, pos, end, 0);
return res; return res;
} }
private: private:
static AstNodePtr disjunction(ParsedRegex& parsed_regex, Iterator& pos, Iterator end, char capture = -1) static AstNodePtr disjunction(ParsedRegex& parsed_regex, Iterator& pos, Iterator end, unsigned capture = -1)
{ {
AstNodePtr node = alternative(parsed_regex, pos, end); AstNodePtr node = alternative(parsed_regex, pos, end);
if (pos == end or *pos != '|') if (pos == end or *pos != '|')
@ -143,6 +152,8 @@ private:
AstNodePtr res = make_ast_node(Op::Sequence); AstNodePtr res = make_ast_node(Op::Sequence);
while (auto node = term(parsed_regex, pos, end)) while (auto node = term(parsed_regex, pos, end))
res->children.push_back(std::move(node)); res->children.push_back(std::move(node));
if (res->children.empty())
throw runtime_error{"Parse error in alternative"};
return res; return res;
} }
@ -160,6 +171,9 @@ private:
static AstNodePtr assertion(ParsedRegex& parsed_regex, Iterator& pos, Iterator end) static AstNodePtr assertion(ParsedRegex& parsed_regex, Iterator& pos, Iterator end)
{ {
if (pos == end)
return nullptr;
switch (*pos) switch (*pos)
{ {
case '^': ++pos; return make_ast_node(Op::LineStart); case '^': ++pos; return make_ast_node(Op::LineStart);
@ -182,8 +196,11 @@ private:
static AstNodePtr atom(ParsedRegex& parsed_regex, Iterator& pos, Iterator end) static AstNodePtr atom(ParsedRegex& parsed_regex, Iterator& pos, Iterator end)
{ {
const auto c = *pos; if (pos == end)
switch (c) return nullptr;
const Codepoint cp = *pos;
switch (cp)
{ {
case '.': ++pos; return make_ast_node(Op::AnyChar); case '.': ++pos; return make_ast_node(Op::AnyChar);
case '(': case '(':
@ -203,30 +220,30 @@ private:
++pos; ++pos;
return character_class(parsed_regex, pos, end); return character_class(parsed_regex, pos, end);
default: default:
if (contains("^$.*+?()[]{}|", c)) if (contains("^$.*+?()[]{}|", cp))
return nullptr; return nullptr;
++pos; ++pos;
return make_ast_node(Op::Literal, c); return make_ast_node(Op::Literal, cp);
} }
} }
static AstNodePtr atom_escape(ParsedRegex& parsed_regex, Iterator& pos, Iterator end) static AstNodePtr atom_escape(ParsedRegex& parsed_regex, Iterator& pos, Iterator end)
{ {
const auto c = *pos; const Codepoint cp = *pos;
struct { char name; char value; } control_escapes[] = { struct { Codepoint name; Codepoint value; } control_escapes[] = {
{ 'f', '\f' }, { 'n', '\n' }, { 'r', '\r' }, { 't', '\t' }, { 'v', '\v' } { 'f', '\f' }, { 'n', '\n' }, { 'r', '\r' }, { 't', '\t' }, { 'v', '\v' }
}; };
for (auto& control : control_escapes) for (auto& control : control_escapes)
{ {
if (control.name == c) if (control.name == cp)
return make_ast_node(Op::Literal, control.value); return make_ast_node(Op::Literal, control.value);
} }
// TOOD: \c..., \0..., '\0x...', \u... // TOOD: \c..., \0..., '\0x...', \u...
if (contains("^$\\.*+?()[]{}|", c)) // SyntaxCharacter if (contains("^$\\.*+?()[]{}|", cp)) // SyntaxCharacter
return make_ast_node(Op::Literal, c); return make_ast_node(Op::Literal, cp);
throw runtime_error{"Unknown atom escape"}; throw runtime_error{"Unknown atom escape"};
} }
@ -239,8 +256,8 @@ private:
Vector<CharRange> ranges; Vector<CharRange> ranges;
while (pos != end and *pos != ']') while (pos != end and *pos != ']')
{ {
const auto c = *pos++; const auto cp = *pos++;
if (c == '-') if (cp == '-')
{ {
ranges.push_back({ '-', 0 }); ranges.push_back({ '-', 0 });
continue; continue;
@ -249,7 +266,7 @@ private:
if (pos == end) if (pos == end)
break; break;
CharRange range = { c, 0 }; CharRange range = { cp, 0 };
if (*pos == '-') if (*pos == '-')
{ {
if (++pos == end) if (++pos == end)
@ -272,14 +289,17 @@ private:
static Quantifier quantifier(ParsedRegex& parsed_regex, Iterator& pos, Iterator end) static Quantifier quantifier(ParsedRegex& parsed_regex, Iterator& pos, Iterator end)
{ {
if (pos == end)
return {Quantifier::One};
auto read_int = [](Iterator& pos, Iterator begin, Iterator end) { auto read_int = [](Iterator& pos, Iterator begin, Iterator end) {
int res = 0; int res = 0;
for (; pos != end; ++pos) for (; pos != end; ++pos)
{ {
const auto c = *pos; const auto cp = *pos;
if (c < '0' or c > '9') if (cp < '0' or cp > '9')
return pos == begin ? -1 : res; return pos == begin ? -1 : res;
res = res * 10 + c - '0'; res = res * 10 + cp - '0';
} }
return res; return res;
}; };
@ -321,14 +341,19 @@ CompiledRegex::Offset& get_offset(CompiledRegex& program, CompiledRegex::Offset
return *reinterpret_cast<CompiledRegex::Offset*>(&program.bytecode[pos]); return *reinterpret_cast<CompiledRegex::Offset*>(&program.bytecode[pos]);
} }
void push_codepoint(CompiledRegex& program, Codepoint cp)
{
utf8::dump(std::back_inserter(program.bytecode), cp);
}
CompiledRegex::Offset compile_node(CompiledRegex& program, const ParsedRegex& parsed_regex, const AstNodePtr& node); CompiledRegex::Offset compile_node(CompiledRegex& program, const ParsedRegex& parsed_regex, const AstNodePtr& node);
CompiledRegex::Offset compile_node_inner(CompiledRegex& program, const ParsedRegex& parsed_regex, const AstNodePtr& node) CompiledRegex::Offset compile_node_inner(CompiledRegex& program, const ParsedRegex& parsed_regex, const AstNodePtr& node)
{ {
const auto start_pos = program.bytecode.size(); const auto start_pos = program.bytecode.size();
const char capture = (node->op == Op::Alternation or node->op == Op::Sequence) ? node->value : -1; const Codepoint capture = (node->op == Op::Alternation or node->op == Op::Sequence) ? node->value : -1;
if (capture >= 0) if (capture != -1)
{ {
program.bytecode.push_back(CompiledRegex::Save); program.bytecode.push_back(CompiledRegex::Save);
program.bytecode.push_back(capture * 2); program.bytecode.push_back(capture * 2);
@ -339,7 +364,7 @@ CompiledRegex::Offset compile_node_inner(CompiledRegex& program, const ParsedReg
{ {
case Op::Literal: case Op::Literal:
program.bytecode.push_back(CompiledRegex::Literal); program.bytecode.push_back(CompiledRegex::Literal);
program.bytecode.push_back(node->value); push_codepoint(program, node->value);
break; break;
case Op::AnyChar: case Op::AnyChar:
program.bytecode.push_back(CompiledRegex::AnyChar); program.bytecode.push_back(CompiledRegex::AnyChar);
@ -358,14 +383,14 @@ CompiledRegex::Offset compile_node_inner(CompiledRegex& program, const ParsedReg
for (auto& r : ranges) for (auto& r : ranges)
{ {
if (r.max == 0) if (r.max == 0)
program.bytecode.push_back(r.min); push_codepoint(program, r.min);
} }
for (auto& r : ranges) for (auto& r : ranges)
{ {
if (r.max != 0) if (r.max != 0)
{ {
program.bytecode.push_back(r.min); push_codepoint(program, r.min);
program.bytecode.push_back(r.max); push_codepoint(program, r.max);
} }
} }
break; break;
@ -414,7 +439,7 @@ CompiledRegex::Offset compile_node_inner(CompiledRegex& program, const ParsedReg
for (auto& offset : goto_inner_end_offsets) for (auto& offset : goto_inner_end_offsets)
get_offset(program, offset) = program.bytecode.size(); get_offset(program, offset) = program.bytecode.size();
if (capture >= 0) if (capture != -1)
{ {
program.bytecode.push_back(CompiledRegex::Save); program.bytecode.push_back(CompiledRegex::Save);
program.bytecode.push_back(capture * 2 + 1); program.bytecode.push_back(capture * 2 + 1);
@ -482,24 +507,24 @@ CompiledRegex compile(const ParsedRegex& parsed_regex)
return res; return res;
} }
template<typename Iterator> CompiledRegex compile(StringView re)
CompiledRegex compile(Iterator begin, Iterator end)
{ {
return compile(Parser<Iterator>::parse(begin, end)); return compile(Parser::parse(re));
} }
} }
void dump(const CompiledRegex& program) void dump(const CompiledRegex& program)
{ {
for (auto pos = program.bytecode.begin(); pos < program.bytecode.end(); ) for (auto pos = program.bytecode.data(), end = program.bytecode.data() + program.bytecode.size();
pos < end; )
{ {
printf("%4zd ", pos - program.bytecode.begin()); printf("%4zd ", pos - program.bytecode.data());
const auto op = (CompiledRegex::Op)*pos++; const auto op = (CompiledRegex::Op)*pos++;
switch (op) switch (op)
{ {
case CompiledRegex::Literal: case CompiledRegex::Literal:
printf("literal %c\n", *pos++); printf("literal %lc\n", utf8::read_codepoint(pos, (const char*)nullptr));
break; break;
case CompiledRegex::AnyChar: case CompiledRegex::AnyChar:
printf("any char\n"); printf("any char\n");
@ -526,14 +551,14 @@ void dump(const CompiledRegex& program)
auto single_count = *pos++; auto single_count = *pos++;
auto range_count = *pos++; auto range_count = *pos++;
for (int i = 0; i < single_count; ++i) for (int i = 0; i < single_count; ++i)
printf("%c", *pos++); printf("%lc", utf8::read_codepoint(pos, (const char*)nullptr));
printf("]"); printf("]");
for (int i = 0; i < range_count; ++i) for (int i = 0; i < range_count; ++i)
{ {
auto min = *pos++; Codepoint min = utf8::read_codepoint(pos, (const char*)nullptr);
auto max = *pos++; Codepoint max = utf8::read_codepoint(pos, (const char*)nullptr);
printf(" [%c-%c]", min, max); printf(" [%lc-%lc]", min, max);
} }
printf("\n"); printf("\n");
break; break;
@ -562,9 +587,11 @@ void dump(const CompiledRegex& program)
} }
} }
template<typename Iterator>
struct ThreadedRegexVM struct ThreadedRegexVM
{ {
ThreadedRegexVM(const CompiledRegex& program) : m_program{program} {} ThreadedRegexVM(const CompiledRegex& program)
: m_program{program} {}
struct Thread struct Thread
{ {
@ -575,22 +602,24 @@ struct ThreadedRegexVM
enum class StepResult { Consumed, Matched, Failed }; enum class StepResult { Consumed, Matched, Failed };
StepResult step(size_t thread_index) StepResult step(size_t thread_index)
{ {
const auto prog_start = m_program.bytecode.data();
const auto prog_end = prog_start + m_program.bytecode.size();
while (true) while (true)
{ {
auto& thread = m_threads[thread_index]; auto& thread = m_threads[thread_index];
char c = m_pos == m_subject.end() ? 0 : *m_pos; const Codepoint cp = m_pos == m_end ? 0 : *m_pos;
const CompiledRegex::Op op = (CompiledRegex::Op)*thread.inst++; const CompiledRegex::Op op = (CompiledRegex::Op)*thread.inst++;
switch (op) switch (op)
{ {
case CompiledRegex::Literal: case CompiledRegex::Literal:
if (*thread.inst++ == c) if (utf8::read_codepoint(thread.inst, prog_end) == cp)
return StepResult::Consumed; return StepResult::Consumed;
return StepResult::Failed; return StepResult::Failed;
case CompiledRegex::AnyChar: case CompiledRegex::AnyChar:
return StepResult::Consumed; return StepResult::Consumed;
case CompiledRegex::Jump: case CompiledRegex::Jump:
{ {
auto inst = m_program.bytecode.data() + *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst); auto inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst);
// if instruction is already going to be executed by another thread, drop this thread // if instruction is already going to be executed by another thread, drop this thread
if (std::find_if(m_threads.begin(), m_threads.end(), if (std::find_if(m_threads.begin(), m_threads.end(),
[inst](const Thread& t) { return t.inst == inst; }) != m_threads.end()) [inst](const Thread& t) { return t.inst == inst; }) != m_threads.end())
@ -607,7 +636,6 @@ struct ThreadedRegexVM
} }
case CompiledRegex::Split_PrioritizeChild: case CompiledRegex::Split_PrioritizeChild:
{ {
auto prog_start = m_program.bytecode.data();
add_thread(thread_index+1, thread.inst + sizeof(CompiledRegex::Offset) - prog_start, thread.saves); add_thread(thread_index+1, thread.inst + sizeof(CompiledRegex::Offset) - prog_start, thread.saves);
// thread is invalidated now, as we mutated the m_thread vector // thread is invalidated now, as we mutated the m_thread vector
m_threads[thread_index].inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(m_threads[thread_index].inst); m_threads[thread_index].inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(m_threads[thread_index].inst);
@ -616,34 +644,32 @@ struct ThreadedRegexVM
case CompiledRegex::Save: case CompiledRegex::Save:
{ {
const char index = *thread.inst++; const char index = *thread.inst++;
thread.saves[index] = m_pos; thread.saves[index] = m_pos.base();
break; break;
} }
case CompiledRegex::CharRange: case CompiledRegex::NegativeCharRange: case CompiledRegex::CharRange: case CompiledRegex::NegativeCharRange:
{ {
auto single_count = *thread.inst++; const int single_count = *thread.inst++;
auto range_count = *thread.inst++; const int range_count = *thread.inst++;
const char* end = thread.inst + single_count + 2 * range_count;
for (int i = 0; i < single_count; ++i) for (int i = 0; i < single_count; ++i)
{ {
auto candidate = *thread.inst++; auto candidate = utf8::read_codepoint(thread.inst, prog_end);
if (c == candidate) if (cp == candidate)
{ {
thread.inst = end; thread.inst = utf8::advance(thread.inst, prog_end, CharCount{single_count - (i + 1) + range_count * 2});
return op == CompiledRegex::CharRange ? StepResult::Consumed : StepResult::Failed; return op == CompiledRegex::CharRange ? StepResult::Consumed : StepResult::Failed;
} }
} }
for (int i = 0; i < range_count; ++i) for (int i = 0; i < range_count; ++i)
{ {
auto min = *thread.inst++; auto min = utf8::read_codepoint(thread.inst, prog_end);
auto max = *thread.inst++; auto max = utf8::read_codepoint(thread.inst, prog_end);
if (min <= c and c <= max) if (min <= cp and cp <= max)
{ {
thread.inst = end; thread.inst = utf8::advance(thread.inst, prog_end, CharCount{(range_count - (i + 1)) * 2});
return op == CompiledRegex::CharRange ? StepResult::Consumed : StepResult::Failed; return op == CompiledRegex::CharRange ? StepResult::Consumed : StepResult::Failed;
} }
} }
kak_assert(thread.inst == end);
return op == CompiledRegex::CharRange ? StepResult::Failed : StepResult::Consumed; return op == CompiledRegex::CharRange ? StepResult::Failed : StepResult::Consumed;
} }
case CompiledRegex::LineStart: case CompiledRegex::LineStart:
@ -663,11 +689,11 @@ struct ThreadedRegexVM
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::SubjectBegin: case CompiledRegex::SubjectBegin:
if (m_pos != m_subject.begin()) if (m_pos != m_begin)
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::SubjectEnd: case CompiledRegex::SubjectEnd:
if (m_pos != m_subject.end()) if (m_pos != m_end)
return StepResult::Failed; return StepResult::Failed;
break; break;
case CompiledRegex::Match: case CompiledRegex::Match:
@ -685,10 +711,10 @@ struct ThreadedRegexVM
add_thread(0, match ? RegexCompiler::prefix_size : 0, add_thread(0, match ? RegexCompiler::prefix_size : 0,
Vector<const char*>(m_program.save_count, nullptr)); Vector<const char*>(m_program.save_count, nullptr));
m_subject = data; m_begin = data.begin();
m_pos = data.begin(); m_end = data.end();
for (m_pos = m_subject.begin(); m_pos != m_subject.end(); ++m_pos) for (m_pos = Utf8It{m_begin, m_begin, m_end}; m_pos != m_end; ++m_pos)
{ {
for (int i = 0; i < m_threads.size(); ++i) for (int i = 0; i < m_threads.size(); ++i)
{ {
@ -738,35 +764,37 @@ struct ThreadedRegexVM
bool is_line_start() const bool is_line_start() const
{ {
return m_pos == m_subject.begin() or *(m_pos-1) == '\n'; return m_pos == m_begin or *(m_pos-1) == '\n';
} }
bool is_line_end() const bool is_line_end() const
{ {
return m_pos == m_subject.end() or *m_pos == '\n'; return m_pos == m_end or *m_pos == '\n';
} }
bool is_word_boundary() const bool is_word_boundary() const
{ {
return m_pos == m_subject.begin() or return m_pos == m_begin or m_pos == m_end or
m_pos == m_subject.end() or
is_word(*(m_pos-1)) != is_word(*m_pos); is_word(*(m_pos-1)) != is_word(*m_pos);
} }
const CompiledRegex& m_program; const CompiledRegex& m_program;
Vector<Thread> m_threads; Vector<Thread> m_threads;
StringView m_subject;
const char* m_pos; using Utf8It = utf8::iterator<Iterator>;
Iterator m_begin;
Iterator m_end;
Utf8It m_pos;
Vector<const char*> m_captures; Vector<const char*> m_captures;
}; };
auto test_regex = UnitTest{[]{ auto test_regex = UnitTest{[]{
{ {
StringView re = R"(a*b)"; auto program = RegexCompiler::compile(R"(a*b)");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("b")); kak_assert(vm.exec("b"));
kak_assert(vm.exec("ab")); kak_assert(vm.exec("ab"));
kak_assert(vm.exec("aaab")); kak_assert(vm.exec("aaab"));
@ -776,10 +804,9 @@ auto test_regex = UnitTest{[]{
} }
{ {
StringView re = R"(^a.*b$)"; auto program = RegexCompiler::compile(R"(^a.*b$)");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("afoob")); kak_assert(vm.exec("afoob"));
kak_assert(vm.exec("ab")); kak_assert(vm.exec("ab"));
kak_assert(not vm.exec("bab")); kak_assert(not vm.exec("bab"));
@ -787,10 +814,9 @@ auto test_regex = UnitTest{[]{
} }
{ {
StringView re = R"(^(foo|qux|baz)+(bar)?baz$)"; auto program = RegexCompiler::compile(R"(^(foo|qux|baz)+(bar)?baz$)");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("fooquxbarbaz")); kak_assert(vm.exec("fooquxbarbaz"));
kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "qux"); kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "qux");
kak_assert(not vm.exec("fooquxbarbaze")); kak_assert(not vm.exec("fooquxbarbaze"));
@ -801,10 +827,9 @@ auto test_regex = UnitTest{[]{
} }
{ {
StringView re = R"(.*\b(foo|bar)\b.*)"; auto program = RegexCompiler::compile(R"(.*\b(foo|bar)\b.*)");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("qux foo baz")); kak_assert(vm.exec("qux foo baz"));
kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "foo"); kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "foo");
kak_assert(not vm.exec("quxfoobaz")); kak_assert(not vm.exec("quxfoobaz"));
@ -812,20 +837,18 @@ auto test_regex = UnitTest{[]{
kak_assert(not vm.exec("foobar")); kak_assert(not vm.exec("foobar"));
} }
{ {
StringView re = R"(\`(foo|bar)\')"; auto program = RegexCompiler::compile(R"(\`(foo|bar)\')");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("foo")); kak_assert(vm.exec("foo"));
kak_assert(vm.exec("bar")); kak_assert(vm.exec("bar"));
kak_assert(not vm.exec("foobar")); kak_assert(not vm.exec("foobar"));
} }
{ {
StringView re = R"(\`a{3,5}b\')"; auto program = RegexCompiler::compile(R"(\`a{3,5}b\')");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(not vm.exec("aab")); kak_assert(not vm.exec("aab"));
kak_assert(vm.exec("aaab")); kak_assert(vm.exec("aaab"));
kak_assert(not vm.exec("aaaaaab")); kak_assert(not vm.exec("aaaaaab"));
@ -833,20 +856,18 @@ auto test_regex = UnitTest{[]{
} }
{ {
StringView re = R"(\`a{3,}b\')"; auto program = RegexCompiler::compile(R"(\`a{3,}b\')");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(not vm.exec("aab")); kak_assert(not vm.exec("aab"));
kak_assert(vm.exec("aaab")); kak_assert(vm.exec("aaab"));
kak_assert(vm.exec("aaaaab")); kak_assert(vm.exec("aaaaab"));
} }
{ {
StringView re = R"(\`a{,3}b\')"; auto program = RegexCompiler::compile(R"(\`a{,3}b\')");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("b")); kak_assert(vm.exec("b"));
kak_assert(vm.exec("ab")); kak_assert(vm.exec("ab"));
kak_assert(vm.exec("aaab")); kak_assert(vm.exec("aaab"));
@ -854,10 +875,9 @@ auto test_regex = UnitTest{[]{
} }
{ {
StringView re = R"(f.*a(.*o))"; auto program = RegexCompiler::compile(R"(f.*a(.*o))");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("blahfoobarfoobaz", false, true)); kak_assert(vm.exec("blahfoobarfoobaz", false, true));
kak_assert(StringView{vm.m_captures[0], vm.m_captures[1]} == "foobarfoo"); kak_assert(StringView{vm.m_captures[0], vm.m_captures[1]} == "foobarfoo");
kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "rfoo"); kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "rfoo");
@ -867,13 +887,12 @@ auto test_regex = UnitTest{[]{
} }
{ {
StringView re = R"([ab-dX-Z]{3,5})"; auto program = RegexCompiler::compile(R"([àb-dX-Z]{3,5})");
auto program = RegexCompiler::compile(re.begin(), re.end());
dump(program); dump(program);
ThreadedRegexVM vm{program}; ThreadedRegexVM<const char*> vm{program};
kak_assert(vm.exec("acY")); kak_assert(vm.exec("càY"));
kak_assert(not vm.exec("aeY")); kak_assert(not vm.exec("àeY"));
kak_assert(vm.exec("abcdX")); kak_assert(vm.exec("dcbàX"));
kak_assert(not vm.exec("efg")); kak_assert(not vm.exec("efg"));
} }
}}; }};