2017-09-17 10:50:53 +02:00
|
|
|
#include "regex_impl.hh"
|
|
|
|
#include "vector.hh"
|
|
|
|
#include "unit_tests.hh"
|
|
|
|
#include "string.hh"
|
2017-09-18 04:56:14 +02:00
|
|
|
#include "unicode.hh"
|
2017-09-25 15:24:24 +02:00
|
|
|
#include "utf8.hh"
|
|
|
|
#include "utf8_iterator.hh"
|
2017-09-17 10:50:53 +02:00
|
|
|
#include "exception.hh"
|
2017-09-17 12:15:43 +02:00
|
|
|
#include "array_view.hh"
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 09:44:30 +02:00
|
|
|
#include "buffer_utils.hh"
|
|
|
|
|
2017-09-17 10:50:53 +02:00
|
|
|
namespace Kakoune
|
|
|
|
{
|
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
struct ParsedRegex
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
enum Op
|
2017-09-19 10:44:58 +02:00
|
|
|
{
|
|
|
|
Literal,
|
|
|
|
AnyChar,
|
2017-09-26 11:03:12 +02:00
|
|
|
Matcher,
|
2017-09-26 16:04:47 +02:00
|
|
|
Sequence,
|
|
|
|
Alternation,
|
2017-09-19 10:44:58 +02:00
|
|
|
LineStart,
|
|
|
|
LineEnd,
|
|
|
|
WordBoundary,
|
|
|
|
NotWordBoundary,
|
|
|
|
SubjectBegin,
|
|
|
|
SubjectEnd,
|
|
|
|
};
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
struct Quantifier
|
|
|
|
{
|
|
|
|
enum Type
|
|
|
|
{
|
|
|
|
One,
|
|
|
|
Optional,
|
|
|
|
RepeatZeroOrMore,
|
|
|
|
RepeatOneOrMore,
|
|
|
|
RepeatMinMax,
|
|
|
|
};
|
|
|
|
Type type = One;
|
|
|
|
int min = -1, max = -1;
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
bool allows_none() const
|
|
|
|
{
|
|
|
|
return type == Quantifier::Optional or
|
|
|
|
type == Quantifier::RepeatZeroOrMore or
|
|
|
|
(type == Quantifier::RepeatMinMax and min <= 0);
|
|
|
|
}
|
2017-09-19 11:26:24 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
bool allows_infinite_repeat() const
|
|
|
|
{
|
|
|
|
return type == Quantifier::RepeatZeroOrMore or
|
|
|
|
type == Quantifier::RepeatOneOrMore or
|
|
|
|
(type == Quantifier::RepeatMinMax and max == -1);
|
|
|
|
};
|
2017-09-18 11:22:11 +02:00
|
|
|
};
|
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
struct AstNode
|
2017-09-18 11:22:11 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
Op op;
|
|
|
|
Codepoint value;
|
|
|
|
Quantifier quantifier;
|
|
|
|
Vector<std::unique_ptr<AstNode>> children;
|
2017-09-18 11:22:11 +02:00
|
|
|
};
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
using AstNodePtr = std::unique_ptr<AstNode>;
|
2017-09-23 09:18:21 +02:00
|
|
|
|
|
|
|
AstNodePtr ast;
|
|
|
|
size_t capture_count;
|
2017-09-26 11:03:12 +02:00
|
|
|
Vector<std::function<bool (Codepoint)>> matchers;
|
2017-09-23 09:18:21 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
// Recursive descent parser based on naming used in the ECMAScript
|
2017-09-19 04:57:02 +02:00
|
|
|
// standard, although the syntax is not fully compatible.
|
2017-09-26 15:38:56 +02:00
|
|
|
struct RegexParser
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
RegexParser(StringView re)
|
|
|
|
: m_regex{re}, m_pos{re.begin(), re}
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
m_parsed_regex.capture_count = 1;
|
|
|
|
m_parsed_regex.ast = disjunction(0);
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
ParsedRegex get_parsed_regex() { return std::move(m_parsed_regex); }
|
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
static ParsedRegex parse(StringView re) { return RegexParser{re}.get_parsed_regex(); }
|
|
|
|
|
2017-09-17 12:15:43 +02:00
|
|
|
private:
|
2017-09-26 09:44:30 +02:00
|
|
|
struct InvalidPolicy
|
|
|
|
{
|
|
|
|
Codepoint operator()(Codepoint cp) { throw runtime_error{"Invalid utf8 in regex"}; }
|
|
|
|
};
|
|
|
|
|
|
|
|
using Iterator = utf8::iterator<const char*, Codepoint, int, InvalidPolicy>;
|
2017-09-26 16:04:47 +02:00
|
|
|
using AstNodePtr = ParsedRegex::AstNodePtr;
|
2017-09-26 09:44:30 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr disjunction(unsigned capture = -1)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr node = alternative();
|
|
|
|
if (at_end() or *m_pos != '|')
|
2017-09-19 09:56:21 +02:00
|
|
|
{
|
|
|
|
node->value = capture;
|
2017-09-17 12:15:43 +02:00
|
|
|
return node;
|
2017-09-19 09:56:21 +02:00
|
|
|
}
|
2017-09-17 12:15:43 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
++m_pos;
|
2017-09-26 16:04:47 +02:00
|
|
|
AstNodePtr res = new_node(ParsedRegex::Alternation);
|
2017-09-17 12:15:43 +02:00
|
|
|
res->children.push_back(std::move(node));
|
2017-09-26 15:38:56 +02:00
|
|
|
res->children.push_back(disjunction());
|
2017-09-19 09:56:21 +02:00
|
|
|
res->value = capture;
|
2017-09-17 12:15:43 +02:00
|
|
|
return res;
|
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr alternative()
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
AstNodePtr res = new_node(ParsedRegex::Sequence);
|
2017-09-26 15:38:56 +02:00
|
|
|
while (auto node = term())
|
2017-09-17 10:50:53 +02:00
|
|
|
res->children.push_back(std::move(node));
|
2017-09-25 15:24:24 +02:00
|
|
|
if (res->children.empty())
|
2017-09-26 15:38:56 +02:00
|
|
|
parse_error("empty alternative");
|
2017-09-17 12:15:43 +02:00
|
|
|
return res;
|
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr term()
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
if (auto node = assertion())
|
2017-09-17 12:15:43 +02:00
|
|
|
return node;
|
2017-09-26 15:38:56 +02:00
|
|
|
if (auto node = atom())
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
node->quantifier = quantifier();
|
2017-09-17 12:15:43 +02:00
|
|
|
return node;
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
2017-09-17 12:15:43 +02:00
|
|
|
return nullptr;
|
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr assertion()
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
if (at_end())
|
2017-09-25 15:24:24 +02:00
|
|
|
return nullptr;
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
switch (*m_pos)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
case '^': ++m_pos; return new_node(ParsedRegex::LineStart);
|
|
|
|
case '$': ++m_pos; return new_node(ParsedRegex::LineEnd);
|
2017-09-18 04:56:14 +02:00
|
|
|
case '\\':
|
2017-09-26 15:38:56 +02:00
|
|
|
if (m_pos+1 == m_regex.end())
|
2017-09-18 04:56:14 +02:00
|
|
|
return nullptr;
|
2017-09-26 15:38:56 +02:00
|
|
|
switch (*(m_pos+1))
|
2017-09-18 04:56:14 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
case 'b': m_pos += 2; return new_node(ParsedRegex::WordBoundary);
|
|
|
|
case 'B': m_pos += 2; return new_node(ParsedRegex::NotWordBoundary);
|
|
|
|
case '`': m_pos += 2; return new_node(ParsedRegex::SubjectBegin);
|
|
|
|
case '\'': m_pos += 2; return new_node(ParsedRegex::SubjectEnd);
|
2017-09-18 04:56:14 +02:00
|
|
|
}
|
|
|
|
break;
|
2017-09-19 04:57:02 +02:00
|
|
|
/* TODO: look ahead, look behind */
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
2017-09-17 12:15:43 +02:00
|
|
|
return nullptr;
|
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr atom()
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
if (at_end())
|
2017-09-25 15:24:24 +02:00
|
|
|
return nullptr;
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
const Codepoint cp = *m_pos;
|
2017-09-25 15:24:24 +02:00
|
|
|
switch (cp)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
case '.': ++m_pos; return new_node(ParsedRegex::AnyChar);
|
2017-09-17 12:15:43 +02:00
|
|
|
case '(':
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
++m_pos;
|
|
|
|
auto content = disjunction(m_parsed_regex.capture_count++);
|
2017-09-19 09:56:21 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
if (at_end() or *m_pos != ')')
|
|
|
|
parse_error("unclosed parenthesis");
|
|
|
|
++m_pos;
|
2017-09-17 12:15:43 +02:00
|
|
|
return content;
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
2017-09-23 09:18:21 +02:00
|
|
|
case '\\':
|
2017-09-26 15:38:56 +02:00
|
|
|
++m_pos;
|
|
|
|
return atom_escape();
|
2017-09-23 09:18:21 +02:00
|
|
|
case '[':
|
2017-09-26 15:38:56 +02:00
|
|
|
++m_pos;
|
|
|
|
return character_class();
|
2017-09-26 16:42:54 +02:00
|
|
|
case '|': case ')':
|
|
|
|
return nullptr;
|
2017-09-17 12:15:43 +02:00
|
|
|
default:
|
2017-09-26 16:42:54 +02:00
|
|
|
if (contains("^$.*+?[]{}", cp))
|
|
|
|
parse_error(format("unexpected '{}'", cp));
|
2017-09-26 15:38:56 +02:00
|
|
|
++m_pos;
|
2017-09-26 16:04:47 +02:00
|
|
|
return new_node(ParsedRegex::Literal, cp);
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
2017-09-17 12:15:43 +02:00
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr atom_escape()
|
2017-09-23 09:18:21 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
const Codepoint cp = *m_pos++;
|
2017-09-23 09:18:21 +02:00
|
|
|
|
2017-09-26 15:04:41 +02:00
|
|
|
// CharacterClassEscape
|
|
|
|
for (auto& character_class : character_class_escapes)
|
|
|
|
{
|
|
|
|
if (character_class.cp == cp)
|
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
auto matcher_id = m_parsed_regex.matchers.size();
|
|
|
|
m_parsed_regex.matchers.push_back(
|
2017-09-26 15:04:41 +02:00
|
|
|
[ctype = wctype(character_class.ctype),
|
|
|
|
chars = character_class.additional_chars] (Codepoint cp) {
|
|
|
|
return iswctype(cp, ctype) or contains(chars, cp);
|
|
|
|
});
|
2017-09-26 16:04:47 +02:00
|
|
|
return new_node(ParsedRegex::Matcher, matcher_id);
|
2017-09-26 15:04:41 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// CharacterEscape
|
2017-09-25 15:24:24 +02:00
|
|
|
struct { Codepoint name; Codepoint value; } control_escapes[] = {
|
2017-09-23 09:18:21 +02:00
|
|
|
{ 'f', '\f' }, { 'n', '\n' }, { 'r', '\r' }, { 't', '\t' }, { 'v', '\v' }
|
|
|
|
};
|
|
|
|
for (auto& control : control_escapes)
|
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
if (control.name == cp)
|
2017-09-26 16:04:47 +02:00
|
|
|
return new_node(ParsedRegex::Literal, control.value);
|
2017-09-23 09:18:21 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// TOOD: \c..., \0..., '\0x...', \u...
|
|
|
|
|
2017-09-25 15:24:24 +02:00
|
|
|
if (contains("^$\\.*+?()[]{}|", cp)) // SyntaxCharacter
|
2017-09-26 16:04:47 +02:00
|
|
|
return new_node(ParsedRegex::Literal, cp);
|
2017-09-26 15:38:56 +02:00
|
|
|
parse_error("unknown atom escape");
|
2017-09-23 09:18:21 +02:00
|
|
|
}
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
AstNodePtr character_class()
|
2017-09-23 09:18:21 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
const bool negative = m_pos != m_regex.end() and *m_pos == '^';
|
2017-09-23 09:18:21 +02:00
|
|
|
if (negative)
|
2017-09-26 15:38:56 +02:00
|
|
|
++m_pos;
|
2017-09-23 09:18:21 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
struct CharRange { Codepoint min, max; };
|
2017-09-23 09:18:21 +02:00
|
|
|
Vector<CharRange> ranges;
|
2017-09-26 15:04:41 +02:00
|
|
|
Vector<std::pair<wctype_t, bool>> ctypes;
|
2017-09-26 15:38:56 +02:00
|
|
|
while (m_pos != m_regex.end() and *m_pos != ']')
|
2017-09-23 09:18:21 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
const auto cp = *m_pos++;
|
2017-09-25 15:24:24 +02:00
|
|
|
if (cp == '-')
|
2017-09-23 09:18:21 +02:00
|
|
|
{
|
2017-09-26 11:03:12 +02:00
|
|
|
ranges.push_back({ '-', '-' });
|
2017-09-23 09:18:21 +02:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
if (at_end())
|
2017-09-23 09:18:21 +02:00
|
|
|
break;
|
|
|
|
|
2017-09-26 15:04:41 +02:00
|
|
|
if (cp == '\\')
|
|
|
|
{
|
|
|
|
auto it = find_if(character_class_escapes,
|
2017-09-26 15:38:56 +02:00
|
|
|
[cp = *m_pos](auto& t) { return t.cp == cp; });
|
2017-09-26 15:04:41 +02:00
|
|
|
if (it != std::end(character_class_escapes))
|
|
|
|
{
|
|
|
|
ctypes.push_back({wctype(it->ctype), not it->neg});
|
|
|
|
for (auto& c : it->additional_chars)
|
|
|
|
ranges.push_back({(Codepoint)c, (Codepoint)c});
|
2017-09-26 15:38:56 +02:00
|
|
|
++m_pos;
|
2017-09-26 15:04:41 +02:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-26 11:03:12 +02:00
|
|
|
CharRange range = { cp, cp };
|
2017-09-26 15:38:56 +02:00
|
|
|
if (*m_pos == '-')
|
2017-09-23 09:18:21 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
if (++m_pos == m_regex.end())
|
2017-09-23 09:18:21 +02:00
|
|
|
break;
|
2017-09-26 16:37:21 +02:00
|
|
|
if (*m_pos != ']')
|
|
|
|
{
|
|
|
|
range.max = *m_pos++;
|
|
|
|
if (range.min > range.max)
|
|
|
|
parse_error("invalid range specified");
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
ranges.push_back(range);
|
|
|
|
range = { '-', '-' };
|
|
|
|
}
|
2017-09-23 09:18:21 +02:00
|
|
|
}
|
|
|
|
ranges.push_back(range);
|
|
|
|
}
|
2017-09-26 15:38:56 +02:00
|
|
|
if (at_end())
|
|
|
|
parse_error("unclosed character class");
|
|
|
|
++m_pos;
|
2017-09-23 09:18:21 +02:00
|
|
|
|
2017-09-26 15:04:41 +02:00
|
|
|
auto matcher = [ranges = std::move(ranges),
|
|
|
|
ctypes = std::move(ctypes), negative] (Codepoint cp) {
|
2017-09-26 11:03:12 +02:00
|
|
|
auto found = contains_that(ranges, [cp](auto& r) {
|
|
|
|
return r.min <= cp and cp <= r.max;
|
2017-09-26 15:04:41 +02:00
|
|
|
}) or contains_that(ctypes, [cp](auto& c) {
|
|
|
|
return (bool)iswctype(cp, c.first) == c.second;
|
2017-09-26 11:03:12 +02:00
|
|
|
});
|
|
|
|
return negative ? not found : found;
|
|
|
|
};
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
auto matcher_id = m_parsed_regex.matchers.size();
|
|
|
|
m_parsed_regex.matchers.push_back(std::move(matcher));
|
2017-09-23 09:18:21 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
return new_node(ParsedRegex::Matcher, matcher_id);
|
2017-09-23 09:18:21 +02:00
|
|
|
}
|
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
ParsedRegex::Quantifier quantifier()
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
if (at_end())
|
2017-09-26 16:04:47 +02:00
|
|
|
return {ParsedRegex::Quantifier::One};
|
2017-09-25 15:24:24 +02:00
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
auto read_int = [](auto& pos, auto begin, auto end) {
|
2017-09-18 11:22:11 +02:00
|
|
|
int res = 0;
|
|
|
|
for (; pos != end; ++pos)
|
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
const auto cp = *pos;
|
|
|
|
if (cp < '0' or cp > '9')
|
2017-09-18 11:22:11 +02:00
|
|
|
return pos == begin ? -1 : res;
|
2017-09-25 15:24:24 +02:00
|
|
|
res = res * 10 + cp - '0';
|
2017-09-18 11:22:11 +02:00
|
|
|
}
|
|
|
|
return res;
|
|
|
|
};
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
switch (*m_pos)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
case '*': ++m_pos; return {ParsedRegex::Quantifier::RepeatZeroOrMore};
|
|
|
|
case '+': ++m_pos; return {ParsedRegex::Quantifier::RepeatOneOrMore};
|
|
|
|
case '?': ++m_pos; return {ParsedRegex::Quantifier::Optional};
|
2017-09-18 11:22:11 +02:00
|
|
|
case '{':
|
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
auto it = m_pos+1;
|
|
|
|
const int min = read_int(it, it, m_regex.end());
|
2017-09-26 13:44:01 +02:00
|
|
|
int max = min;
|
2017-09-18 11:22:11 +02:00
|
|
|
if (*it == ',')
|
|
|
|
{
|
|
|
|
++it;
|
2017-09-26 15:38:56 +02:00
|
|
|
max = read_int(it, it, m_regex.end());
|
2017-09-18 11:22:11 +02:00
|
|
|
}
|
|
|
|
if (*it++ != '}')
|
2017-09-26 15:38:56 +02:00
|
|
|
parse_error("expected closing bracket");
|
|
|
|
m_pos = it;
|
2017-09-26 16:04:47 +02:00
|
|
|
return {ParsedRegex::Quantifier::RepeatMinMax, min, max};
|
2017-09-18 11:22:11 +02:00
|
|
|
}
|
2017-09-26 16:04:47 +02:00
|
|
|
default: return {ParsedRegex::Quantifier::One};
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
2017-09-17 12:15:43 +02:00
|
|
|
}
|
2017-09-26 15:04:41 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
static AstNodePtr new_node(ParsedRegex::Op op, Codepoint value = -1,
|
|
|
|
ParsedRegex::Quantifier quantifier = {ParsedRegex::Quantifier::One})
|
|
|
|
{
|
|
|
|
return AstNodePtr{new ParsedRegex::AstNode{op, value, quantifier, {}}};
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-09-26 15:38:56 +02:00
|
|
|
bool at_end() const { return m_pos == m_regex.end(); }
|
|
|
|
|
|
|
|
[[gnu::noreturn]]
|
|
|
|
void parse_error(StringView error)
|
|
|
|
{
|
|
|
|
throw runtime_error(format("regex parse error: {} at '{}<<<HERE>>>{}'", error,
|
|
|
|
StringView{m_regex.begin(), m_pos.base()},
|
|
|
|
StringView{m_pos.base(), m_regex.end()}));
|
|
|
|
}
|
|
|
|
|
|
|
|
ParsedRegex m_parsed_regex;
|
|
|
|
StringView m_regex;
|
|
|
|
Iterator m_pos;
|
|
|
|
|
2017-09-26 15:04:41 +02:00
|
|
|
struct CharacterClassEscape {
|
|
|
|
Codepoint cp;
|
|
|
|
const char* ctype;
|
|
|
|
StringView additional_chars;
|
|
|
|
bool neg;
|
|
|
|
};
|
|
|
|
|
|
|
|
static const CharacterClassEscape character_class_escapes[6];
|
|
|
|
};
|
|
|
|
|
|
|
|
// For some reason Gcc fails to link if this is constexpr
|
2017-09-26 15:38:56 +02:00
|
|
|
const RegexParser::CharacterClassEscape RegexParser::character_class_escapes[6] = {
|
2017-09-26 15:04:41 +02:00
|
|
|
{ 'd', "digit", "", false },
|
|
|
|
{ 'D', "digit", "", true },
|
|
|
|
{ 'w', "alnum", "_", false },
|
|
|
|
{ 'W', "alnum", "_", true },
|
|
|
|
{ 's', "space", "", false },
|
|
|
|
{ 's', "space", "", true },
|
2017-09-17 12:15:43 +02:00
|
|
|
};
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
struct CompiledRegex
|
2017-09-18 11:22:11 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
enum Op : char
|
|
|
|
{
|
|
|
|
Match,
|
|
|
|
Literal,
|
|
|
|
AnyChar,
|
|
|
|
Matcher,
|
|
|
|
Jump,
|
|
|
|
Split_PrioritizeParent,
|
|
|
|
Split_PrioritizeChild,
|
|
|
|
Save,
|
|
|
|
LineStart,
|
|
|
|
LineEnd,
|
|
|
|
WordBoundary,
|
|
|
|
NotWordBoundary,
|
|
|
|
SubjectBegin,
|
|
|
|
SubjectEnd,
|
|
|
|
};
|
2017-09-17 12:15:43 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
using Offset = unsigned;
|
2017-09-17 12:15:43 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
Vector<char> bytecode;
|
|
|
|
Vector<std::function<bool (Codepoint)>> matchers;
|
|
|
|
size_t save_count;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct RegexCompiler
|
2017-09-25 15:24:24 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
RegexCompiler(const ParsedRegex& parsed_regex)
|
|
|
|
: m_parsed_regex{parsed_regex}
|
|
|
|
{
|
|
|
|
write_search_prefix();
|
|
|
|
compile_node(m_parsed_regex.ast);
|
|
|
|
push_op(CompiledRegex::Match);
|
|
|
|
m_program.matchers = m_parsed_regex.matchers;
|
|
|
|
m_program.save_count = m_parsed_regex.capture_count * 2;
|
|
|
|
}
|
2017-09-25 15:24:24 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
CompiledRegex get_compiled_regex() { return std::move(m_program); }
|
2017-09-23 09:18:21 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
using Offset = CompiledRegex::Offset;
|
|
|
|
static constexpr Offset search_prefix_size = 3 + 2 * sizeof(Offset);
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
static CompiledRegex compile(StringView re)
|
2017-09-19 09:56:21 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
return RegexCompiler{RegexParser::parse(re)}.get_compiled_regex();
|
2017-09-19 09:56:21 +02:00
|
|
|
}
|
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
private:
|
|
|
|
Offset compile_node_inner(const ParsedRegex::AstNodePtr& node)
|
|
|
|
{
|
|
|
|
const auto start_pos = m_program.bytecode.size();
|
|
|
|
|
|
|
|
const Codepoint capture = (node->op == ParsedRegex::Alternation or node->op == ParsedRegex::Sequence) ? node->value : -1;
|
|
|
|
if (capture != -1)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
push_op(CompiledRegex::Save);
|
|
|
|
push_byte(capture * 2);
|
|
|
|
}
|
2017-09-17 12:15:43 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
Vector<Offset> goto_inner_end_offsets;
|
|
|
|
switch (node->op)
|
|
|
|
{
|
|
|
|
case ParsedRegex::Literal:
|
|
|
|
push_op(CompiledRegex::Literal);
|
|
|
|
push_codepoint(node->value);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::AnyChar:
|
|
|
|
push_op(CompiledRegex::AnyChar);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::Matcher:
|
|
|
|
push_op(CompiledRegex::Matcher);
|
|
|
|
push_byte(node->value);
|
|
|
|
case ParsedRegex::Sequence:
|
|
|
|
for (auto& child : node->children)
|
|
|
|
compile_node(child);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::Alternation:
|
|
|
|
{
|
|
|
|
auto& children = node->children;
|
|
|
|
kak_assert(children.size() == 2);
|
|
|
|
|
|
|
|
push_op(CompiledRegex::Split_PrioritizeParent);
|
|
|
|
auto offset = alloc_offset();
|
2017-09-17 13:08:10 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
compile_node(children[0]);
|
|
|
|
push_op(CompiledRegex::Jump);
|
|
|
|
goto_inner_end_offsets.push_back(alloc_offset());
|
2017-09-17 13:08:10 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
auto right_pos = compile_node(children[1]);
|
|
|
|
get_offset(offset) = right_pos;
|
2017-09-17 13:08:10 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
case ParsedRegex::LineStart:
|
|
|
|
push_op(CompiledRegex::LineStart);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::LineEnd:
|
|
|
|
push_op(CompiledRegex::LineEnd);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::WordBoundary:
|
|
|
|
push_op(CompiledRegex::WordBoundary);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::NotWordBoundary:
|
|
|
|
push_op(CompiledRegex::NotWordBoundary);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::SubjectBegin:
|
|
|
|
push_op(CompiledRegex::SubjectBegin);
|
|
|
|
break;
|
|
|
|
case ParsedRegex::SubjectEnd:
|
|
|
|
push_op(CompiledRegex::SubjectEnd);
|
|
|
|
break;
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
for (auto& offset : goto_inner_end_offsets)
|
|
|
|
get_offset(offset) = m_program.bytecode.size();
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
if (capture != -1)
|
|
|
|
{
|
|
|
|
push_op(CompiledRegex::Save);
|
|
|
|
push_byte(capture * 2 + 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
return start_pos;
|
2017-09-19 09:56:21 +02:00
|
|
|
}
|
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
Offset compile_node(const ParsedRegex::AstNodePtr& node)
|
|
|
|
{
|
|
|
|
Offset pos = m_program.bytecode.size();
|
|
|
|
Vector<Offset> goto_end_offsets;
|
2017-09-18 11:22:11 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
if (node->quantifier.allows_none())
|
|
|
|
{
|
|
|
|
push_op(CompiledRegex::Split_PrioritizeParent);
|
|
|
|
goto_end_offsets.push_back(alloc_offset());
|
|
|
|
}
|
2017-09-18 11:22:11 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
auto inner_pos = compile_node_inner(node);
|
|
|
|
// Write the node multiple times when we have a min count quantifier
|
|
|
|
for (int i = 1; i < node->quantifier.min; ++i)
|
|
|
|
inner_pos = compile_node_inner(node);
|
2017-09-18 11:22:11 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
if (node->quantifier.allows_infinite_repeat())
|
|
|
|
{
|
|
|
|
push_op(CompiledRegex::Split_PrioritizeChild);
|
|
|
|
get_offset(alloc_offset()) = inner_pos;
|
|
|
|
}
|
|
|
|
// Write the node as an optional match for the min -> max counts
|
|
|
|
else for (int i = std::max(1, node->quantifier.min); // STILL UGLY !
|
|
|
|
i < node->quantifier.max; ++i)
|
|
|
|
{
|
|
|
|
push_op(CompiledRegex::Split_PrioritizeParent);
|
|
|
|
goto_end_offsets.push_back(alloc_offset());
|
|
|
|
compile_node_inner(node);
|
|
|
|
}
|
2017-09-18 11:22:11 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
for (auto offset : goto_end_offsets)
|
|
|
|
get_offset(offset) = m_program.bytecode.size();
|
|
|
|
|
|
|
|
return pos;
|
2017-09-18 11:22:11 +02:00
|
|
|
}
|
2017-09-26 16:04:47 +02:00
|
|
|
|
|
|
|
// Add a '.*' as the first instructions for the search use case
|
|
|
|
void write_search_prefix()
|
2017-09-18 11:22:11 +02:00
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
kak_assert(m_program.bytecode.empty());
|
|
|
|
push_op(CompiledRegex::Split_PrioritizeChild);
|
|
|
|
get_offset(alloc_offset()) = search_prefix_size;
|
|
|
|
push_op(CompiledRegex::AnyChar);
|
|
|
|
push_op(CompiledRegex::Split_PrioritizeParent);
|
|
|
|
get_offset(alloc_offset()) = 1 + sizeof(Offset);
|
2017-09-17 12:15:43 +02:00
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
Offset alloc_offset()
|
|
|
|
{
|
|
|
|
auto pos = m_program.bytecode.size();
|
|
|
|
m_program.bytecode.resize(pos + sizeof(Offset));
|
|
|
|
return pos;
|
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
Offset& get_offset(Offset pos)
|
|
|
|
{
|
|
|
|
return *reinterpret_cast<Offset*>(&m_program.bytecode[pos]);
|
|
|
|
}
|
2017-09-19 11:26:24 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
void push_op(CompiledRegex::Op op)
|
|
|
|
{
|
|
|
|
m_program.bytecode.push_back(op);
|
|
|
|
}
|
2017-09-19 11:26:24 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
void push_byte(char byte)
|
|
|
|
{
|
|
|
|
m_program.bytecode.push_back(byte);
|
|
|
|
}
|
2017-09-17 12:15:43 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
void push_codepoint(Codepoint cp)
|
|
|
|
{
|
|
|
|
utf8::dump(std::back_inserter(m_program.bytecode), cp);
|
|
|
|
}
|
2017-09-19 10:44:58 +02:00
|
|
|
|
2017-09-26 16:04:47 +02:00
|
|
|
CompiledRegex m_program;
|
|
|
|
const ParsedRegex& m_parsed_regex;
|
|
|
|
};
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 16:31:18 +02:00
|
|
|
void dump_regex(const CompiledRegex& program)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
for (auto pos = program.bytecode.data(), end = program.bytecode.data() + program.bytecode.size();
|
|
|
|
pos < end; )
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
printf("%4zd ", pos - program.bytecode.data());
|
2017-09-23 09:18:21 +02:00
|
|
|
const auto op = (CompiledRegex::Op)*pos++;
|
|
|
|
switch (op)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Literal:
|
2017-09-25 15:24:24 +02:00
|
|
|
printf("literal %lc\n", utf8::read_codepoint(pos, (const char*)nullptr));
|
2017-09-17 10:50:53 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::AnyChar:
|
2017-09-17 10:50:53 +02:00
|
|
|
printf("any char\n");
|
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Jump:
|
2017-09-19 11:26:24 +02:00
|
|
|
printf("jump %u\n", *reinterpret_cast<const CompiledRegex::Offset*>(&*pos));
|
2017-09-19 10:44:58 +02:00
|
|
|
pos += sizeof(CompiledRegex::Offset);
|
2017-09-17 10:50:53 +02:00
|
|
|
break;
|
2017-09-24 16:14:35 +02:00
|
|
|
case CompiledRegex::Split_PrioritizeParent:
|
|
|
|
case CompiledRegex::Split_PrioritizeChild:
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-24 16:14:35 +02:00
|
|
|
printf("split (prioritize %s) %u\n",
|
|
|
|
op == CompiledRegex::Split_PrioritizeParent ? "parent" : "child",
|
|
|
|
*reinterpret_cast<const CompiledRegex::Offset*>(&*pos));
|
2017-09-19 10:44:58 +02:00
|
|
|
pos += sizeof(CompiledRegex::Offset);
|
2017-09-17 10:50:53 +02:00
|
|
|
break;
|
|
|
|
}
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Save:
|
|
|
|
printf("save %d\n", *pos++);
|
2017-09-19 09:56:21 +02:00
|
|
|
break;
|
2017-09-26 11:03:12 +02:00
|
|
|
case CompiledRegex::Matcher:
|
|
|
|
printf("matcher %d\n", *pos++);
|
2017-09-23 09:18:21 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::LineStart:
|
2017-09-17 10:50:53 +02:00
|
|
|
printf("line start\n");
|
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::LineEnd:
|
2017-09-17 10:50:53 +02:00
|
|
|
printf("line end\n");
|
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::WordBoundary:
|
2017-09-18 04:56:14 +02:00
|
|
|
printf("word boundary\n");
|
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::NotWordBoundary:
|
2017-09-18 04:56:14 +02:00
|
|
|
printf("not word boundary\n");
|
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::SubjectBegin:
|
2017-09-18 05:47:10 +02:00
|
|
|
printf("subject begin\n");
|
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::SubjectEnd:
|
2017-09-18 05:47:10 +02:00
|
|
|
printf("subject end\n");
|
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Match:
|
2017-09-17 10:50:53 +02:00
|
|
|
printf("match\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-25 15:24:24 +02:00
|
|
|
template<typename Iterator>
|
2017-09-19 10:44:58 +02:00
|
|
|
struct ThreadedRegexVM
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
ThreadedRegexVM(const CompiledRegex& program)
|
|
|
|
: m_program{program} {}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-19 09:56:21 +02:00
|
|
|
struct Thread
|
2017-09-17 12:48:23 +02:00
|
|
|
{
|
2017-09-19 09:56:21 +02:00
|
|
|
const char* inst;
|
|
|
|
Vector<const char*> saves = {};
|
2017-09-17 12:48:23 +02:00
|
|
|
};
|
|
|
|
|
2017-09-19 09:56:21 +02:00
|
|
|
enum class StepResult { Consumed, Matched, Failed };
|
|
|
|
StepResult step(size_t thread_index)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
const auto prog_start = m_program.bytecode.data();
|
|
|
|
const auto prog_end = prog_start + m_program.bytecode.size();
|
2017-09-17 12:48:23 +02:00
|
|
|
while (true)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-19 09:56:21 +02:00
|
|
|
auto& thread = m_threads[thread_index];
|
2017-09-25 15:24:24 +02:00
|
|
|
const Codepoint cp = m_pos == m_end ? 0 : *m_pos;
|
2017-09-19 10:44:58 +02:00
|
|
|
const CompiledRegex::Op op = (CompiledRegex::Op)*thread.inst++;
|
2017-09-17 12:48:23 +02:00
|
|
|
switch (op)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Literal:
|
2017-09-25 15:24:24 +02:00
|
|
|
if (utf8::read_codepoint(thread.inst, prog_end) == cp)
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Consumed;
|
|
|
|
return StepResult::Failed;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::AnyChar:
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Consumed;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Jump:
|
2017-09-19 09:56:21 +02:00
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
auto inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst);
|
2017-09-19 11:26:24 +02:00
|
|
|
// if instruction is already going to be executed by another thread, drop this thread
|
2017-09-19 09:56:21 +02:00
|
|
|
if (std::find_if(m_threads.begin(), m_threads.end(),
|
|
|
|
[inst](const Thread& t) { return t.inst == inst; }) != m_threads.end())
|
|
|
|
return StepResult::Failed;
|
|
|
|
thread.inst = inst;
|
2017-09-17 12:48:23 +02:00
|
|
|
break;
|
2017-09-19 09:56:21 +02:00
|
|
|
}
|
2017-09-24 16:14:35 +02:00
|
|
|
case CompiledRegex::Split_PrioritizeParent:
|
2017-09-17 12:48:23 +02:00
|
|
|
{
|
2017-09-24 16:14:35 +02:00
|
|
|
add_thread(thread_index+1, *reinterpret_cast<const CompiledRegex::Offset*>(thread.inst), thread.saves);
|
2017-09-19 09:56:21 +02:00
|
|
|
// thread is invalidated now, as we mutated the m_thread vector
|
2017-09-19 10:44:58 +02:00
|
|
|
m_threads[thread_index].inst += sizeof(CompiledRegex::Offset);
|
2017-09-19 09:56:21 +02:00
|
|
|
break;
|
|
|
|
}
|
2017-09-24 16:14:35 +02:00
|
|
|
case CompiledRegex::Split_PrioritizeChild:
|
|
|
|
{
|
|
|
|
add_thread(thread_index+1, thread.inst + sizeof(CompiledRegex::Offset) - prog_start, thread.saves);
|
|
|
|
// thread is invalidated now, as we mutated the m_thread vector
|
|
|
|
m_threads[thread_index].inst = prog_start + *reinterpret_cast<const CompiledRegex::Offset*>(m_threads[thread_index].inst);
|
|
|
|
break;
|
|
|
|
}
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Save:
|
2017-09-19 09:56:21 +02:00
|
|
|
{
|
|
|
|
const char index = *thread.inst++;
|
2017-09-25 15:24:24 +02:00
|
|
|
thread.saves[index] = m_pos.base();
|
2017-09-17 12:48:23 +02:00
|
|
|
break;
|
|
|
|
}
|
2017-09-26 11:03:12 +02:00
|
|
|
case CompiledRegex::Matcher:
|
2017-09-23 09:18:21 +02:00
|
|
|
{
|
2017-09-26 11:03:12 +02:00
|
|
|
const int matcher_id = *thread.inst++;
|
|
|
|
return m_program.matchers[matcher_id](*m_pos) ?
|
|
|
|
StepResult::Consumed : StepResult::Failed;
|
2017-09-23 09:18:21 +02:00
|
|
|
}
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::LineStart:
|
2017-09-17 12:48:23 +02:00
|
|
|
if (not is_line_start())
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Failed;
|
2017-09-17 12:48:23 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::LineEnd:
|
2017-09-17 12:48:23 +02:00
|
|
|
if (not is_line_end())
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Failed;
|
2017-09-17 12:48:23 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::WordBoundary:
|
2017-09-18 04:56:14 +02:00
|
|
|
if (not is_word_boundary())
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Failed;
|
2017-09-18 04:56:14 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::NotWordBoundary:
|
2017-09-18 04:56:14 +02:00
|
|
|
if (is_word_boundary())
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Failed;
|
2017-09-18 04:56:14 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::SubjectBegin:
|
2017-09-25 15:24:24 +02:00
|
|
|
if (m_pos != m_begin)
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Failed;
|
2017-09-18 05:47:10 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::SubjectEnd:
|
2017-09-25 15:24:24 +02:00
|
|
|
if (m_pos != m_end)
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Failed;
|
2017-09-18 05:47:10 +02:00
|
|
|
break;
|
2017-09-19 10:44:58 +02:00
|
|
|
case CompiledRegex::Match:
|
2017-09-24 16:14:35 +02:00
|
|
|
thread.inst = nullptr;
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Matched;
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
|
|
|
}
|
2017-09-19 09:56:21 +02:00
|
|
|
return StepResult::Failed;
|
2017-09-17 12:15:43 +02:00
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-24 16:14:35 +02:00
|
|
|
bool exec(StringView data, bool match = true, bool longest = false)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-24 16:14:35 +02:00
|
|
|
bool found_match = false;
|
2017-09-19 09:56:21 +02:00
|
|
|
m_threads.clear();
|
2017-09-26 16:04:47 +02:00
|
|
|
add_thread(0, match ? RegexCompiler::search_prefix_size : 0,
|
2017-09-19 11:26:24 +02:00
|
|
|
Vector<const char*>(m_program.save_count, nullptr));
|
2017-09-19 09:56:21 +02:00
|
|
|
|
2017-09-25 15:24:24 +02:00
|
|
|
m_begin = data.begin();
|
|
|
|
m_end = data.end();
|
2017-09-17 12:48:23 +02:00
|
|
|
|
2017-09-25 15:24:24 +02:00
|
|
|
for (m_pos = Utf8It{m_begin, m_begin, m_end}; m_pos != m_end; ++m_pos)
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-17 12:48:23 +02:00
|
|
|
for (int i = 0; i < m_threads.size(); ++i)
|
|
|
|
{
|
2017-09-19 09:56:21 +02:00
|
|
|
const auto res = step(i);
|
|
|
|
if (res == StepResult::Matched)
|
|
|
|
{
|
2017-09-24 16:23:02 +02:00
|
|
|
if (match)
|
|
|
|
continue; // We are not at end, this is not a full match
|
|
|
|
|
2017-09-19 09:56:21 +02:00
|
|
|
m_captures = std::move(m_threads[i].saves);
|
2017-09-24 16:14:35 +02:00
|
|
|
found_match = true;
|
|
|
|
m_threads.resize(i); // remove this and lower priority threads
|
|
|
|
if (not longest)
|
|
|
|
return true;
|
2017-09-19 09:56:21 +02:00
|
|
|
}
|
|
|
|
else if (res == StepResult::Failed)
|
|
|
|
m_threads[i].inst = nullptr;
|
2017-09-17 12:48:23 +02:00
|
|
|
}
|
2017-09-19 09:56:21 +02:00
|
|
|
m_threads.erase(std::remove_if(m_threads.begin(), m_threads.end(),
|
|
|
|
[](const Thread& t) { return t.inst == nullptr; }), m_threads.end());
|
2017-09-17 12:48:23 +02:00
|
|
|
if (m_threads.empty())
|
2017-09-19 09:56:21 +02:00
|
|
|
return false;
|
2017-09-17 12:48:23 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
// Step remaining threads to see if they match without consuming anything else
|
|
|
|
for (int i = 0; i < m_threads.size(); ++i)
|
|
|
|
{
|
2017-09-19 09:56:21 +02:00
|
|
|
if (step(i) == StepResult::Matched)
|
|
|
|
{
|
|
|
|
m_captures = std::move(m_threads[i].saves);
|
2017-09-24 16:14:35 +02:00
|
|
|
found_match = true;
|
|
|
|
m_threads.resize(i); // remove this and lower priority threads
|
|
|
|
if (not longest)
|
|
|
|
return true;
|
2017-09-19 09:56:21 +02:00
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
2017-09-24 16:14:35 +02:00
|
|
|
return found_match;
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
|
|
|
|
2017-09-24 16:14:35 +02:00
|
|
|
void add_thread(int index, CompiledRegex::Offset pos, Vector<const char*> saves)
|
2017-09-18 04:30:01 +02:00
|
|
|
{
|
2017-09-19 10:44:58 +02:00
|
|
|
const char* inst = m_program.bytecode.data() + pos;
|
2017-09-19 09:56:21 +02:00
|
|
|
if (std::find_if(m_threads.begin(), m_threads.end(),
|
|
|
|
[inst](const Thread& t) { return t.inst == inst; }) == m_threads.end())
|
2017-09-24 16:14:35 +02:00
|
|
|
m_threads.insert(m_threads.begin() + index, {inst, std::move(saves)});
|
2017-09-18 04:30:01 +02:00
|
|
|
}
|
|
|
|
|
2017-09-17 12:48:23 +02:00
|
|
|
bool is_line_start() const
|
2017-09-17 10:50:53 +02:00
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
return m_pos == m_begin or *(m_pos-1) == '\n';
|
2017-09-17 10:50:53 +02:00
|
|
|
}
|
2017-09-17 12:48:23 +02:00
|
|
|
|
|
|
|
bool is_line_end() const
|
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
return m_pos == m_end or *m_pos == '\n';
|
2017-09-17 12:48:23 +02:00
|
|
|
}
|
|
|
|
|
2017-09-18 04:56:14 +02:00
|
|
|
bool is_word_boundary() const
|
|
|
|
{
|
2017-09-25 15:24:24 +02:00
|
|
|
return m_pos == m_begin or m_pos == m_end or
|
2017-09-18 04:56:14 +02:00
|
|
|
is_word(*(m_pos-1)) != is_word(*m_pos);
|
|
|
|
}
|
|
|
|
|
2017-09-19 10:44:58 +02:00
|
|
|
const CompiledRegex& m_program;
|
2017-09-19 09:56:21 +02:00
|
|
|
Vector<Thread> m_threads;
|
2017-09-25 15:24:24 +02:00
|
|
|
|
|
|
|
using Utf8It = utf8::iterator<Iterator>;
|
|
|
|
|
|
|
|
Iterator m_begin;
|
|
|
|
Iterator m_end;
|
|
|
|
Utf8It m_pos;
|
2017-09-24 16:14:35 +02:00
|
|
|
|
|
|
|
Vector<const char*> m_captures;
|
2017-09-17 12:48:23 +02:00
|
|
|
};
|
2017-09-17 10:50:53 +02:00
|
|
|
|
2017-09-26 09:44:30 +02:00
|
|
|
void validate_regex(StringView re)
|
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
2017-09-26 16:04:47 +02:00
|
|
|
RegexParser{re};
|
2017-09-26 09:44:30 +02:00
|
|
|
}
|
|
|
|
catch (runtime_error& err)
|
|
|
|
{
|
2017-09-26 15:38:56 +02:00
|
|
|
write_to_debug_buffer(err.what());
|
2017-09-26 09:44:30 +02:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-09-17 10:50:53 +02:00
|
|
|
auto test_regex = UnitTest{[]{
|
2017-09-26 16:16:35 +02:00
|
|
|
struct TestVM : ThreadedRegexVM<const char*>
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 16:31:18 +02:00
|
|
|
TestVM(StringView re, bool dump = false)
|
2017-09-26 16:16:35 +02:00
|
|
|
: ThreadedRegexVM{m_program},
|
|
|
|
m_program{RegexCompiler::compile(re)}
|
2017-09-26 16:31:18 +02:00
|
|
|
{ if (dump) dump_regex(m_program); }
|
2017-09-26 16:16:35 +02:00
|
|
|
|
|
|
|
CompiledRegex m_program;
|
|
|
|
};
|
|
|
|
|
|
|
|
{
|
|
|
|
TestVM vm{R"(a*b)"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(vm.exec("b"));
|
|
|
|
kak_assert(vm.exec("ab"));
|
|
|
|
kak_assert(vm.exec("aaab"));
|
|
|
|
kak_assert(not vm.exec("acb"));
|
2017-09-24 16:23:02 +02:00
|
|
|
kak_assert(not vm.exec("abc"));
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(not vm.exec(""));
|
2017-09-17 12:15:43 +02:00
|
|
|
}
|
2017-09-17 13:12:46 +02:00
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(^a.*b$)"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(vm.exec("afoob"));
|
|
|
|
kak_assert(vm.exec("ab"));
|
|
|
|
kak_assert(not vm.exec("bab"));
|
|
|
|
kak_assert(not vm.exec(""));
|
2017-09-17 13:12:46 +02:00
|
|
|
}
|
|
|
|
|
2017-09-17 12:15:43 +02:00
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(^(foo|qux|baz)+(bar)?baz$)"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(vm.exec("fooquxbarbaz"));
|
2017-09-19 10:44:58 +02:00
|
|
|
kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "qux");
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(not vm.exec("fooquxbarbaze"));
|
|
|
|
kak_assert(not vm.exec("quxbar"));
|
|
|
|
kak_assert(not vm.exec("blahblah"));
|
|
|
|
kak_assert(vm.exec("bazbaz"));
|
|
|
|
kak_assert(vm.exec("quxbaz"));
|
2017-09-17 12:15:43 +02:00
|
|
|
}
|
2017-09-18 04:56:14 +02:00
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(.*\b(foo|bar)\b.*)"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(vm.exec("qux foo baz"));
|
2017-09-19 10:44:58 +02:00
|
|
|
kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "foo");
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(not vm.exec("quxfoobaz"));
|
|
|
|
kak_assert(vm.exec("bar"));
|
|
|
|
kak_assert(not vm.exec("foobar"));
|
2017-09-18 04:56:14 +02:00
|
|
|
}
|
2017-09-26 16:16:35 +02:00
|
|
|
|
2017-09-18 05:47:10 +02:00
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"((foo|bar))"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(vm.exec("foo"));
|
|
|
|
kak_assert(vm.exec("bar"));
|
|
|
|
kak_assert(not vm.exec("foobar"));
|
2017-09-18 05:47:10 +02:00
|
|
|
}
|
2017-09-18 11:22:11 +02:00
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(a{3,5}b)"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(not vm.exec("aab"));
|
|
|
|
kak_assert(vm.exec("aaab"));
|
|
|
|
kak_assert(not vm.exec("aaaaaab"));
|
|
|
|
kak_assert(vm.exec("aaaaab"));
|
2017-09-18 11:22:11 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(a{3}b)"};
|
2017-09-26 13:44:01 +02:00
|
|
|
kak_assert(not vm.exec("aab"));
|
|
|
|
kak_assert(vm.exec("aaab"));
|
|
|
|
kak_assert(not vm.exec("aaaab"));
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(a{3,}b)"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(not vm.exec("aab"));
|
|
|
|
kak_assert(vm.exec("aaab"));
|
|
|
|
kak_assert(vm.exec("aaaaab"));
|
2017-09-18 11:22:11 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(a{,3}b)"};
|
2017-09-19 11:26:24 +02:00
|
|
|
kak_assert(vm.exec("b"));
|
|
|
|
kak_assert(vm.exec("ab"));
|
|
|
|
kak_assert(vm.exec("aaab"));
|
|
|
|
kak_assert(not vm.exec("aaaab"));
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(f.*a(.*o))"};
|
2017-09-24 16:14:35 +02:00
|
|
|
kak_assert(vm.exec("blahfoobarfoobaz", false, true));
|
|
|
|
kak_assert(StringView{vm.m_captures[0], vm.m_captures[1]} == "foobarfoo");
|
|
|
|
kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == "rfoo");
|
|
|
|
kak_assert(vm.exec("mais que fais la police", false, true));
|
|
|
|
kak_assert(StringView{vm.m_captures[0], vm.m_captures[1]} == "fais la po");
|
|
|
|
kak_assert(StringView{vm.m_captures[2], vm.m_captures[3]} == " po");
|
2017-09-18 11:22:11 +02:00
|
|
|
}
|
2017-09-23 09:18:21 +02:00
|
|
|
|
|
|
|
{
|
2017-09-26 16:37:21 +02:00
|
|
|
TestVM vm{R"([àb-dX-Z-]{3,5})"};
|
|
|
|
kak_assert(vm.exec("cà-Y"));
|
2017-09-25 15:24:24 +02:00
|
|
|
kak_assert(not vm.exec("àeY"));
|
|
|
|
kak_assert(vm.exec("dcbàX"));
|
2017-09-23 09:18:21 +02:00
|
|
|
kak_assert(not vm.exec("efg"));
|
|
|
|
}
|
2017-09-26 15:04:41 +02:00
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"(\d{3})"};
|
2017-09-26 15:04:41 +02:00
|
|
|
kak_assert(vm.exec("123"));
|
|
|
|
kak_assert(not vm.exec("1x3"));
|
|
|
|
}
|
|
|
|
|
|
|
|
{
|
2017-09-26 16:16:35 +02:00
|
|
|
TestVM vm{R"([-\d]+)"};
|
2017-09-26 15:04:41 +02:00
|
|
|
kak_assert(vm.exec("123-456"));
|
|
|
|
kak_assert(not vm.exec("123_456"));
|
|
|
|
}
|
2017-09-17 10:50:53 +02:00
|
|
|
}};
|
|
|
|
|
|
|
|
}
|