From 7cdbe1d3d24c1cc13bd7cbc3fe252f1e88747ffb Mon Sep 17 00:00:00 2001 From: Frank LENORMAND Date: Wed, 13 Nov 2019 09:54:17 +0100 Subject: [PATCH 1/2] src: Move JSON parsing code to its own file The `json_ui.cc` file contained both data-parsing and UI-related code. This commit moves the JSON parsing code to its own `json.cc` file, to separate concerns, make compilation faster when changes are made to either UI or parsing code, and make the parsing code more accessible to fuzzers. The signature of the following function: ``` auto parse_json(StringView json); ``` was changed to: ``` JsonResult parse_json(StringView json); ``` to avoid `auto` deduction issues at compile-time. --- src/json.cc | 175 +++++++++++++++++++++++++++++++++++++++++++++ src/json.hh | 41 +++++++++++ src/json_ui.cc | 187 +------------------------------------------------ 3 files changed, 217 insertions(+), 186 deletions(-) create mode 100644 src/json.cc create mode 100644 src/json.hh diff --git a/src/json.cc b/src/json.cc new file mode 100644 index 00000000..dcb4005a --- /dev/null +++ b/src/json.cc @@ -0,0 +1,175 @@ +#include "json.hh" + +#include "exception.hh" +#include "string_utils.hh" +#include "unit_tests.hh" +#include "utils.hh" + +#include + +namespace Kakoune +{ + +String to_json(int i) { return to_string(i); } +String to_json(bool b) { return b ? "true" : "false"; } +String to_json(StringView str) +{ + String res; + res.reserve(str.length() + 4); + res += '"'; + for (auto it = str.begin(), end = str.end(); it != end; ) + { + auto next = std::find_if(it, end, [](char c) { + return c == '\\' or c == '"' or (c >= 0 and c <= 0x1F); + }); + + res += StringView{it, next}; + if (next == end) + break; + + char buf[7] = {'\\', *next, 0}; + if (*next >= 0 and *next <= 0x1F) + sprintf(buf, "\\u%04x", *next); + + res += buf; + it = next+1; + } + res += '"'; + return res; +} + +static bool is_digit(char c) { return c >= '0' and c <= '9'; } + +JsonResult parse_json(const char* pos, const char* end) +{ + if (not skip_while(pos, end, is_blank)) + return {}; + + if (is_digit(*pos) or *pos == '-') + { + auto digit_end = pos + 1; + skip_while(digit_end, end, is_digit); + return { Value{str_to_int({pos, digit_end})}, digit_end }; + } + if (end - pos > 4 and StringView{pos, pos+4} == "true") + return { Value{true}, pos+4 }; + if (end - pos > 5 and StringView{pos, pos+5} == "false") + return { Value{false}, pos+5 }; + if (*pos == '"') + { + String value; + bool escaped = false; + ++pos; + for (auto string_end = pos; string_end != end; ++string_end) + { + if (escaped) + { + escaped = false; + value += StringView{pos, string_end}; + value.back() = *string_end; + pos = string_end+1; + continue; + } + if (*string_end == '\\') + escaped = true; + if (*string_end == '"') + { + value += StringView{pos, string_end}; + return {std::move(value), string_end+1}; + } + } + return {}; + } + if (*pos == '[') + { + JsonArray array; + if (++pos == end) + throw runtime_error("unable to parse array"); + if (*pos == ']') + return {std::move(array), pos+1}; + + while (true) + { + auto [element, new_pos] = parse_json(pos, end); + if (not element) + return {}; + pos = new_pos; + array.push_back(std::move(element)); + if (not skip_while(pos, end, is_blank)) + return {}; + + if (*pos == ',') + ++pos; + else if (*pos == ']') + return {std::move(array), pos+1}; + else + throw runtime_error("unable to parse array, expected ',' or ']'"); + } + } + if (*pos == '{') + { + if (++pos == end) + throw runtime_error("unable to parse object"); + JsonObject object; + if (*pos == '}') + return {std::move(object), pos+1}; + + while (true) + { + auto [name_value, name_end] = parse_json(pos, end); + if (not name_value) + return {}; + pos = name_end; + String& name = name_value.as(); + if (not skip_while(pos, end, is_blank)) + return {}; + if (*pos++ != ':') + throw runtime_error("expected :"); + + auto [element, element_end] = parse_json(pos, end); + if (not element) + return {}; + pos = element_end; + object.insert({ std::move(name), std::move(element) }); + if (not skip_while(pos, end, is_blank)) + return {}; + + if (*pos == ',') + ++pos; + else if (*pos == '}') + return {std::move(object), pos+1}; + else + throw runtime_error("unable to parse object, expected ',' or '}'"); + } + } + throw runtime_error("unable to parse json"); +} + +JsonResult parse_json(StringView json) { return parse_json(json.begin(), json.end()); } + +UnitTest test_json_parser{[]() +{ + { + auto value = parse_json(R"({ "jsonrpc": "2.0", "method": "keys", "params": [ "b", "l", "a", "h" ] })").value; + kak_assert(value); + } + + { + auto value = parse_json("[10,20]").value; + kak_assert(value and value.is_a()); + kak_assert(value.as().at(1).as() == 20); + } + + { + auto value = parse_json("-1").value; + kak_assert(value.as() == -1); + } + + { + auto value = parse_json("{}").value; + kak_assert(value and value.is_a()); + kak_assert(value.as().empty()); + } +}}; + +} diff --git a/src/json.hh b/src/json.hh new file mode 100644 index 00000000..1f76739c --- /dev/null +++ b/src/json.hh @@ -0,0 +1,41 @@ +#ifndef json_hh_INCLUDED +#define json_hh_INCLUDED + +#include "hash_map.hh" +#include "string.hh" +#include "value.hh" + +namespace Kakoune +{ + +using JsonArray = Vector; +using JsonObject = HashMap; + +template +String to_json(ArrayView array) +{ + return "[" + join(array | transform([](auto&& elem) { return to_json(elem); }), ", ") + "]"; +} + +template +String to_json(const Vector& vec) { return to_json(ArrayView{vec}); } + +template +String to_json(const HashMap& map) +{ + return "{" + join(map | transform([](auto&& i) { return format("{}: {}", to_json(i.key), to_json(i.value)); }), + ',', false) + "}"; +} + +String to_json(int i); +String to_json(bool b); +String to_json(StringView str); + +struct JsonResult { Value value; const char* new_pos; }; + +JsonResult parse_json(const char* pos, const char* end); +JsonResult parse_json(StringView json); + +} + +#endif // json_hh_INCLUDED diff --git a/src/json_ui.cc b/src/json_ui.cc index 51fde617..0f1d6f3b 100644 --- a/src/json_ui.cc +++ b/src/json_ui.cc @@ -4,13 +4,11 @@ #include "event_manager.hh" #include "exception.hh" #include "file.hh" +#include "json.hh" #include "keys.hh" #include "ranges.hh" #include "string_utils.hh" -#include "unit_tests.hh" -#include "value.hh" -#include #include #include @@ -23,50 +21,6 @@ struct invalid_rpc_request : runtime_error { : runtime_error(format("invalid json rpc request ({})", message)) {} }; -template -String to_json(ArrayView array) -{ - return "[" + join(array | transform([](auto&& elem) { return to_json(elem); }), ", ") + "]"; -} - -template -String to_json(const Vector& vec) { return to_json(ArrayView{vec}); } - -template -String to_json(const HashMap& map) -{ - return "{" + join(map | transform([](auto&& i) { return format("{}: {}", to_json(i.key), to_json(i.value)); }), - ',', false) + "}"; -} - -String to_json(int i) { return to_string(i); } -String to_json(bool b) { return b ? "true" : "false"; } -String to_json(StringView str) -{ - String res; - res.reserve(str.length() + 4); - res += '"'; - for (auto it = str.begin(), end = str.end(); it != end; ) - { - auto next = std::find_if(it, end, [](char c) { - return c == '\\' or c == '"' or (c >= 0 and c <= 0x1F); - }); - - res += StringView{it, next}; - if (next == end) - break; - - char buf[7] = {'\\', *next, 0}; - if (*next >= 0 and *next <= 0x1F) - sprintf(buf, "\\u%04x", *next); - - res += buf; - it = next+1; - } - res += '"'; - return res; -} - String to_json(Color color) { if (color.color == Kakoune::Color::RGB) @@ -254,120 +208,6 @@ void JsonUI::set_on_key(OnKeyCallback callback) m_on_key = std::move(callback); } -using JsonArray = Vector; -using JsonObject = HashMap; - -static bool is_digit(char c) { return c >= '0' and c <= '9'; } - -struct JsonResult { Value value; const char* new_pos; }; - -JsonResult parse_json(const char* pos, const char* end) -{ - if (not skip_while(pos, end, is_blank)) - return {}; - - if (is_digit(*pos) or *pos == '-') - { - auto digit_end = pos + 1; - skip_while(digit_end, end, is_digit); - return { Value{str_to_int({pos, digit_end})}, digit_end }; - } - if (end - pos > 4 and StringView{pos, pos+4} == "true") - return { Value{true}, pos+4 }; - if (end - pos > 5 and StringView{pos, pos+5} == "false") - return { Value{false}, pos+5 }; - if (*pos == '"') - { - String value; - bool escaped = false; - ++pos; - for (auto string_end = pos; string_end != end; ++string_end) - { - if (escaped) - { - escaped = false; - value += StringView{pos, string_end}; - value.back() = *string_end; - pos = string_end+1; - continue; - } - if (*string_end == '\\') - escaped = true; - if (*string_end == '"') - { - value += StringView{pos, string_end}; - return {std::move(value), string_end+1}; - } - } - return {}; - } - if (*pos == '[') - { - JsonArray array; - if (++pos == end) - throw runtime_error("unable to parse array"); - if (*pos == ']') - return {std::move(array), pos+1}; - - while (true) - { - auto [element, new_pos] = parse_json(pos, end); - if (not element) - return {}; - pos = new_pos; - array.push_back(std::move(element)); - if (not skip_while(pos, end, is_blank)) - return {}; - - if (*pos == ',') - ++pos; - else if (*pos == ']') - return {std::move(array), pos+1}; - else - throw runtime_error("unable to parse array, expected ',' or ']'"); - } - } - if (*pos == '{') - { - if (++pos == end) - throw runtime_error("unable to parse object"); - JsonObject object; - if (*pos == '}') - return {std::move(object), pos+1}; - - while (true) - { - auto [name_value, name_end] = parse_json(pos, end); - if (not name_value) - return {}; - pos = name_end; - String& name = name_value.as(); - if (not skip_while(pos, end, is_blank)) - return {}; - if (*pos++ != ':') - throw runtime_error("expected :"); - - auto [element, element_end] = parse_json(pos, end); - if (not element) - return {}; - pos = element_end; - object.insert({ std::move(name), std::move(element) }); - if (not skip_while(pos, end, is_blank)) - return {}; - - if (*pos == ',') - ++pos; - else if (*pos == '}') - return {std::move(object), pos+1}; - else - throw runtime_error("unable to parse object, expected ',' or '}'"); - } - } - throw runtime_error("unable to parse json"); -} - -auto parse_json(StringView json) { return parse_json(json.begin(), json.end()); } - void JsonUI::eval_json(const Value& json) { if (not json.is_a()) @@ -510,29 +350,4 @@ void JsonUI::parse_requests(EventMode mode) } } -UnitTest test_json_parser{[]() -{ - { - auto value = parse_json(R"({ "jsonrpc": "2.0", "method": "keys", "params": [ "b", "l", "a", "h" ] })").value; - kak_assert(value); - } - - { - auto value = parse_json("[10,20]").value; - kak_assert(value and value.is_a()); - kak_assert(value.as().at(1).as() == 20); - } - - { - auto value = parse_json("-1").value; - kak_assert(value.as() == -1); - } - - { - auto value = parse_json("{}").value; - kak_assert(value and value.is_a()); - kak_assert(value.as().empty()); - } -}}; - } From 19f1754a2a9db3f6a84b2e611b8956fd51ad748e Mon Sep 17 00:00:00 2001 From: Frank LENORMAND Date: Wed, 13 Nov 2019 10:30:25 +0100 Subject: [PATCH 2/2] src json: Limit the recursion depth to 100 --- src/json.cc | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/json.cc b/src/json.cc index dcb4005a..82db4576 100644 --- a/src/json.cc +++ b/src/json.cc @@ -40,11 +40,16 @@ String to_json(StringView str) static bool is_digit(char c) { return c >= '0' and c <= '9'; } -JsonResult parse_json(const char* pos, const char* end) +static constexpr size_t max_parsing_depth = 100; + +JsonResult parse_json_impl(const char* pos, const char* end, size_t depth) { if (not skip_while(pos, end, is_blank)) return {}; + if (depth >= max_parsing_depth) + throw runtime_error("maximum parsing depth reached"); + if (is_digit(*pos) or *pos == '-') { auto digit_end = pos + 1; @@ -90,7 +95,7 @@ JsonResult parse_json(const char* pos, const char* end) while (true) { - auto [element, new_pos] = parse_json(pos, end); + auto [element, new_pos] = parse_json_impl(pos, end, depth+1); if (not element) return {}; pos = new_pos; @@ -116,7 +121,7 @@ JsonResult parse_json(const char* pos, const char* end) while (true) { - auto [name_value, name_end] = parse_json(pos, end); + auto [name_value, name_end] = parse_json_impl(pos, end, depth+1); if (not name_value) return {}; pos = name_end; @@ -126,7 +131,7 @@ JsonResult parse_json(const char* pos, const char* end) if (*pos++ != ':') throw runtime_error("expected :"); - auto [element, element_end] = parse_json(pos, end); + auto [element, element_end] = parse_json_impl(pos, end, depth+1); if (not element) return {}; pos = element_end; @@ -145,7 +150,8 @@ JsonResult parse_json(const char* pos, const char* end) throw runtime_error("unable to parse json"); } -JsonResult parse_json(StringView json) { return parse_json(json.begin(), json.end()); } +JsonResult parse_json(const char* pos, const char* end) { return parse_json_impl(pos, end, 0); } +JsonResult parse_json(StringView json) { return parse_json_impl(json.begin(), json.end(), 0); } UnitTest test_json_parser{[]() { @@ -170,6 +176,16 @@ UnitTest test_json_parser{[]() kak_assert(value and value.is_a()); kak_assert(value.as().empty()); } + + { + String big_nested_array = {"", max_parsing_depth*2+2}; + for (size_t i = 0; i < max_parsing_depth+1; i++) + { + big_nested_array[i] = '['; + big_nested_array[i+max_parsing_depth+1] = ']'; + } + kak_expect_throw(runtime_error, parse_json(big_nested_array)); + } }}; }