diff --git a/include/nlohmann/detail/input/lexer.hpp b/include/nlohmann/detail/input/lexer.hpp index eab64f406d..d5e243e632 100644 --- a/include/nlohmann/detail/input/lexer.hpp +++ b/include/nlohmann/detail/input/lexer.hpp @@ -837,6 +837,7 @@ class lexer : public lexer_base { switch (get()) { + // single-line comments skip input until a newline or EOF is read case '/': { while (true) @@ -845,6 +846,8 @@ class lexer : public lexer_base { case '\n': case '\r': + case std::char_traits::eof(): + case '\0': return true; default: @@ -853,6 +856,7 @@ class lexer : public lexer_base } } + // multi-line comments skip input until */ is read case '*': { while (true) @@ -877,10 +881,14 @@ class lexer : public lexer_base } } } + + default: + break; } } } + // unexpected character after reading '/' default: return false; } diff --git a/include/nlohmann/detail/input/parser.hpp b/include/nlohmann/detail/input/parser.hpp index 0546b88cb8..c79b492aab 100644 --- a/include/nlohmann/detail/input/parser.hpp +++ b/include/nlohmann/detail/input/parser.hpp @@ -63,8 +63,11 @@ class parser /// a parser reading from an input adapter explicit parser(InputAdapterType&& adapter, const parser_callback_t cb = nullptr, - const bool allow_exceptions_ = true) - : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_) + const bool allow_exceptions_ = true, + const bool skip_comments = false) + : callback(cb) + , m_lexer(std::move(adapter), skip_comments) + , allow_exceptions(allow_exceptions_) { // read first token get_token(); diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 409a6e7993..8698a9bb8a 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -196,10 +196,12 @@ class basic_json static ::nlohmann::detail::parser parser( InputAdapterType adapter, detail::parser_callback_tcb = nullptr, - bool allow_exceptions = true + const bool allow_exceptions = true, + const bool ignore_comments = false ) { - return ::nlohmann::detail::parser(std::move(adapter), std::move(cb), allow_exceptions); + return ::nlohmann::detail::parser(std::move(adapter), + std::move(cb), allow_exceptions, ignore_comments); } using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; @@ -6563,6 +6565,8 @@ class basic_json (optional) @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default) @return deserialized JSON value; in case of a parse error and @a allow_exceptions set to `false`, the return value will be @@ -6591,16 +6595,18 @@ class basic_json @liveexample{The example below demonstrates the `parse()` function reading from a contiguous container.,parse__contiguouscontainer__parser_callback_t} - @since version 2.0.3 (contiguous containers) + @since version 2.0.3 (contiguous containers); version 3.9.0 allowed to + ignore comments. */ template JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json parse(InputType&& i, const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const bool ignore_comments = false) { basic_json result; - parser(detail::input_adapter(std::forward(i)), cb, allow_exceptions).parse(true, result); + parser(detail::input_adapter(std::forward(i)), cb, allow_exceptions, ignore_comments).parse(true, result); return result; } @@ -6617,6 +6623,8 @@ class basic_json (optional) @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default) @return deserialized JSON value; in case of a parse error and @a allow_exceptions set to `false`, the return value will be @@ -6632,10 +6640,11 @@ class basic_json static basic_json parse(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const bool ignore_comments = false) { basic_json result; - parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions).parse(true, result); + parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions, ignore_comments).parse(true, result); return result; } @@ -6643,10 +6652,11 @@ class basic_json JSON_HEDLEY_DEPRECATED_FOR(3.8.0, parse(ptr, ptr + len)) static basic_json parse(detail::span_input_adapter&& i, const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const bool ignore_comments = false) { basic_json result; - parser(i.get(), cb, allow_exceptions).parse(true, result); + parser(i.get(), cb, allow_exceptions, ignore_comments).parse(true, result); return result; } @@ -6666,6 +6676,8 @@ class basic_json iterators. @param[in] i input to read from + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default) @return Whether the input read from @a i is valid JSON. @@ -6678,22 +6690,25 @@ class basic_json from a string.,accept__string} */ template - static bool accept(InputType&& i) + static bool accept(InputType&& i, + const bool ignore_comments = false) { - return parser(detail::input_adapter(std::forward(i))).accept(true); + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments).accept(true); } template - static bool accept(IteratorType first, IteratorType last) + static bool accept(IteratorType first, IteratorType last, + const bool ignore_comments = false) { - return parser(detail::input_adapter(std::move(first), std::move(last))).accept(true); + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments).accept(true); } JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DEPRECATED_FOR(3.8.0, accept(ptr, ptr + len)) - static bool accept(detail::span_input_adapter&& i) + static bool accept(detail::span_input_adapter&& i, + const bool ignore_comments = false) { - return parser(i.get()).accept(true); + return parser(i.get(), nullptr, false, ignore_comments).accept(true); } /*! @@ -6713,6 +6728,9 @@ class basic_json @param[in,out] sax SAX event listener @param[in] format the format to parse (JSON, CBOR, MessagePack, or UBJSON) @param[in] strict whether the input has to be consumed completely + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default); only applieds to + the JSON file format. @return return value of the last processed SAX event @@ -6737,11 +6755,12 @@ class basic_json JSON_HEDLEY_NON_NULL(2) static bool sax_parse(InputType&& i, SAX* sax, input_format_t format = input_format_t::json, - const bool strict = true) + const bool strict = true, + const bool ignore_comments = false) { auto ia = detail::input_adapter(std::forward(i)); return format == input_format_t::json - ? parser(std::move(ia)).sax_parse(sax, strict) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); } @@ -6749,11 +6768,12 @@ class basic_json JSON_HEDLEY_NON_NULL(3) static bool sax_parse(IteratorType first, IteratorType last, SAX* sax, input_format_t format = input_format_t::json, - const bool strict = true) + const bool strict = true, + const bool ignore_comments = false) { auto ia = detail::input_adapter(std::move(first), std::move(last)); return format == input_format_t::json - ? parser(std::move(ia)).sax_parse(sax, strict) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); } @@ -6762,11 +6782,12 @@ class basic_json JSON_HEDLEY_NON_NULL(2) static bool sax_parse(detail::span_input_adapter&& i, SAX* sax, input_format_t format = input_format_t::json, - const bool strict = true) + const bool strict = true, + const bool ignore_comments = false) { auto ia = i.get(); return format == input_format_t::json - ? parser(std::move(ia)).sax_parse(sax, strict) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); } diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index bdd97a145a..099fdd8eaf 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -8904,6 +8904,7 @@ class lexer : public lexer_base { switch (get()) { + // single-line comments skip input until a newline or EOF is read case '/': { while (true) @@ -8912,6 +8913,8 @@ class lexer : public lexer_base { case '\n': case '\r': + case std::char_traits::eof(): + case '\0': return true; default: @@ -8920,6 +8923,7 @@ class lexer : public lexer_base } } + // multi-line comments skip input until */ is read case '*': { while (true) @@ -8944,10 +8948,14 @@ class lexer : public lexer_base } } } + + default: + break; } } } + // unexpected character after reading '/' default: return false; } @@ -9742,8 +9750,11 @@ class parser /// a parser reading from an input adapter explicit parser(InputAdapterType&& adapter, const parser_callback_t cb = nullptr, - const bool allow_exceptions_ = true) - : callback(cb), m_lexer(std::move(adapter)), allow_exceptions(allow_exceptions_) + const bool allow_exceptions_ = true, + const bool skip_comments = false) + : callback(cb) + , m_lexer(std::move(adapter), skip_comments) + , allow_exceptions(allow_exceptions_) { // read first token get_token(); @@ -16051,10 +16062,12 @@ class basic_json static ::nlohmann::detail::parser parser( InputAdapterType adapter, detail::parser_callback_tcb = nullptr, - bool allow_exceptions = true + const bool allow_exceptions = true, + const bool ignore_comments = false ) { - return ::nlohmann::detail::parser(std::move(adapter), std::move(cb), allow_exceptions); + return ::nlohmann::detail::parser(std::move(adapter), + std::move(cb), allow_exceptions, ignore_comments); } using primitive_iterator_t = ::nlohmann::detail::primitive_iterator_t; @@ -22418,6 +22431,8 @@ class basic_json (optional) @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default) @return deserialized JSON value; in case of a parse error and @a allow_exceptions set to `false`, the return value will be @@ -22446,16 +22461,18 @@ class basic_json @liveexample{The example below demonstrates the `parse()` function reading from a contiguous container.,parse__contiguouscontainer__parser_callback_t} - @since version 2.0.3 (contiguous containers) + @since version 2.0.3 (contiguous containers); version 3.9.0 allowed to + ignore comments. */ template JSON_HEDLEY_WARN_UNUSED_RESULT static basic_json parse(InputType&& i, const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const bool ignore_comments = false) { basic_json result; - parser(detail::input_adapter(std::forward(i)), cb, allow_exceptions).parse(true, result); + parser(detail::input_adapter(std::forward(i)), cb, allow_exceptions, ignore_comments).parse(true, result); return result; } @@ -22472,6 +22489,8 @@ class basic_json (optional) @param[in] allow_exceptions whether to throw exceptions in case of a parse error (optional, true by default) + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default) @return deserialized JSON value; in case of a parse error and @a allow_exceptions set to `false`, the return value will be @@ -22487,10 +22506,11 @@ class basic_json static basic_json parse(IteratorType first, IteratorType last, const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const bool ignore_comments = false) { basic_json result; - parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions).parse(true, result); + parser(detail::input_adapter(std::move(first), std::move(last)), cb, allow_exceptions, ignore_comments).parse(true, result); return result; } @@ -22498,10 +22518,11 @@ class basic_json JSON_HEDLEY_DEPRECATED_FOR(3.8.0, parse(ptr, ptr + len)) static basic_json parse(detail::span_input_adapter&& i, const parser_callback_t cb = nullptr, - const bool allow_exceptions = true) + const bool allow_exceptions = true, + const bool ignore_comments = false) { basic_json result; - parser(i.get(), cb, allow_exceptions).parse(true, result); + parser(i.get(), cb, allow_exceptions, ignore_comments).parse(true, result); return result; } @@ -22521,6 +22542,8 @@ class basic_json iterators. @param[in] i input to read from + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default) @return Whether the input read from @a i is valid JSON. @@ -22533,22 +22556,25 @@ class basic_json from a string.,accept__string} */ template - static bool accept(InputType&& i) + static bool accept(InputType&& i, + const bool ignore_comments = false) { - return parser(detail::input_adapter(std::forward(i))).accept(true); + return parser(detail::input_adapter(std::forward(i)), nullptr, false, ignore_comments).accept(true); } template - static bool accept(IteratorType first, IteratorType last) + static bool accept(IteratorType first, IteratorType last, + const bool ignore_comments = false) { - return parser(detail::input_adapter(std::move(first), std::move(last))).accept(true); + return parser(detail::input_adapter(std::move(first), std::move(last)), nullptr, false, ignore_comments).accept(true); } JSON_HEDLEY_WARN_UNUSED_RESULT JSON_HEDLEY_DEPRECATED_FOR(3.8.0, accept(ptr, ptr + len)) - static bool accept(detail::span_input_adapter&& i) + static bool accept(detail::span_input_adapter&& i, + const bool ignore_comments = false) { - return parser(i.get()).accept(true); + return parser(i.get(), nullptr, false, ignore_comments).accept(true); } /*! @@ -22568,6 +22594,9 @@ class basic_json @param[in,out] sax SAX event listener @param[in] format the format to parse (JSON, CBOR, MessagePack, or UBJSON) @param[in] strict whether the input has to be consumed completely + @param[in] ignore_comments whether comments should be ignored (true) or + yield a parse error (true); (optional, false by default); only applieds to + the JSON file format. @return return value of the last processed SAX event @@ -22592,11 +22621,12 @@ class basic_json JSON_HEDLEY_NON_NULL(2) static bool sax_parse(InputType&& i, SAX* sax, input_format_t format = input_format_t::json, - const bool strict = true) + const bool strict = true, + const bool ignore_comments = false) { auto ia = detail::input_adapter(std::forward(i)); return format == input_format_t::json - ? parser(std::move(ia)).sax_parse(sax, strict) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); } @@ -22604,11 +22634,12 @@ class basic_json JSON_HEDLEY_NON_NULL(3) static bool sax_parse(IteratorType first, IteratorType last, SAX* sax, input_format_t format = input_format_t::json, - const bool strict = true) + const bool strict = true, + const bool ignore_comments = false) { auto ia = detail::input_adapter(std::move(first), std::move(last)); return format == input_format_t::json - ? parser(std::move(ia)).sax_parse(sax, strict) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); } @@ -22617,11 +22648,12 @@ class basic_json JSON_HEDLEY_NON_NULL(2) static bool sax_parse(detail::span_input_adapter&& i, SAX* sax, input_format_t format = input_format_t::json, - const bool strict = true) + const bool strict = true, + const bool ignore_comments = false) { auto ia = i.get(); return format == input_format_t::json - ? parser(std::move(ia)).sax_parse(sax, strict) + ? parser(std::move(ia), nullptr, true, ignore_comments).sax_parse(sax, strict) : detail::binary_reader(std::move(ia)).sax_parse(format, sax, strict); } diff --git a/test/src/unit-class_lexer.cpp b/test/src/unit-class_lexer.cpp index c4423e6023..aee4703fcd 100644 --- a/test/src/unit-class_lexer.cpp +++ b/test/src/unit-class_lexer.cpp @@ -37,11 +37,11 @@ using nlohmann::json; namespace { // shortcut to scan a string literal -json::lexer::token_type scan_string(const char* s); -json::lexer::token_type scan_string(const char* s) +json::lexer::token_type scan_string(const char* s, const bool ignore_comments = false); +json::lexer::token_type scan_string(const char* s, const bool ignore_comments) { auto ia = nlohmann::detail::input_adapter(s); - return nlohmann::detail::lexer(std::move(ia)).scan(); + return nlohmann::detail::lexer(std::move(ia), ignore_comments).scan(); } } @@ -163,9 +163,6 @@ TEST_CASE("lexer class") break; } - // case ('/'): - // break; - // anything else is not expected default: { @@ -185,18 +182,39 @@ TEST_CASE("lexer class") CHECK((scan_string(s.c_str()) == json::lexer::token_type::value_string)); } - // SECTION("ignore comments") - // { - // CHECK((scan_string("/") == json::lexer::token_type::parse_error)); - // - // CHECK((scan_string("/!") == json::lexer::token_type::parse_error)); - // CHECK((scan_string("/*") == json::lexer::token_type::parse_error)); - // CHECK((scan_string("/**") == json::lexer::token_type::parse_error)); - // - // CHECK((scan_string("//") == json::lexer::token_type::end_of_input)); - // CHECK((scan_string("/**/") == json::lexer::token_type::end_of_input)); - // CHECK((scan_string("/** /") == json::lexer::token_type::parse_error)); - // - // CHECK((scan_string("/***/") == json::lexer::token_type::end_of_input)); - // } + SECTION("fail on comments") + { + CHECK((scan_string("/", false) == json::lexer::token_type::parse_error)); + + CHECK((scan_string("/!", false) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/*", false) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/**", false) == json::lexer::token_type::parse_error)); + + CHECK((scan_string("//", false) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/**/", false) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/** /", false) == json::lexer::token_type::parse_error)); + + CHECK((scan_string("/***/", false) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/* true */", false) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/*/**/", false) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/*/* */", false) == json::lexer::token_type::parse_error)); + } + + SECTION("ignore comments") + { + CHECK((scan_string("/", true) == json::lexer::token_type::parse_error)); + + CHECK((scan_string("/!", true) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/*", true) == json::lexer::token_type::parse_error)); + CHECK((scan_string("/**", true) == json::lexer::token_type::parse_error)); + + CHECK((scan_string("//", true) == json::lexer::token_type::end_of_input)); + CHECK((scan_string("/**/", true) == json::lexer::token_type::end_of_input)); + CHECK((scan_string("/** /", true) == json::lexer::token_type::parse_error)); + + CHECK((scan_string("/***/", true) == json::lexer::token_type::end_of_input)); + CHECK((scan_string("/* true */", true) == json::lexer::token_type::end_of_input)); + CHECK((scan_string("/*/**/", true) == json::lexer::token_type::end_of_input)); + CHECK((scan_string("/*/* */", true) == json::lexer::token_type::end_of_input)); + } }