Skip to content

Commit b1ab8c4

Browse files
danielaparkerdanielaparker
committed
json_parser
1 parent 862ac78 commit b1ab8c4

File tree

5 files changed

+141
-77
lines changed

5 files changed

+141
-77
lines changed

‎include/jsoncons/json_parser.hpp‎

Lines changed: 10 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -215,68 +215,9 @@ class basic_json_parser
215215

216216
void parse_some(basic_json_visitor<char_type>& visitor, std::error_code& ec)
217217
{
218-
auto r = tokenizer_.try_next_token();
219-
if (JSONCONS_UNLIKELY(!r))
220-
{
221-
if (r.ec != json_errc::unexpected_eof)
222-
{
223-
ec = r.ec;
224-
}
225-
return;
226-
}
227-
bool test = false;
228218
while (!tokenizer_.done() && !tokenizer_.source_exhausted())
229219
{
230-
test = true;
231-
switch (tokenizer_.token_kind())
232-
{
233-
case generic_token_kind::string_value:
234-
if (tokenizer_.is_key())
235-
{
236-
visitor.key(tokenizer_.get_string_view(),
237-
tokenizer_.get_context());
238-
}
239-
else
240-
{
241-
visitor.string_value(tokenizer_.get_string_view(),
242-
tokenizer_.tag(), tokenizer_.get_context());
243-
}
244-
break;
245-
case generic_token_kind::null_value:
246-
visitor.null_value(tokenizer_.tag(), tokenizer_.get_context());
247-
break;
248-
case generic_token_kind::bool_value:
249-
visitor.bool_value(tokenizer_.get_bool(),
250-
tokenizer_.tag(), tokenizer_.get_context());
251-
break;
252-
case generic_token_kind::int64_value:
253-
visitor.int64_value(tokenizer_.get_int64(),
254-
tokenizer_.tag(), tokenizer_.get_context());
255-
break;
256-
case generic_token_kind::uint64_value:
257-
visitor.uint64_value(tokenizer_.get_uint64(),
258-
tokenizer_.tag(), tokenizer_.get_context());
259-
break;
260-
case generic_token_kind::double_value:
261-
visitor.double_value(tokenizer_.get_double(),
262-
tokenizer_.tag(), tokenizer_.get_context());
263-
break;
264-
case generic_token_kind::begin_map:
265-
visitor.begin_object(tokenizer_.tag(), tokenizer_.get_context());
266-
break;
267-
case generic_token_kind::end_map:
268-
visitor.end_object(tokenizer_.get_context());
269-
break;
270-
case generic_token_kind::begin_array:
271-
visitor.begin_array(tokenizer_.tag(), tokenizer_.get_context());
272-
break;
273-
case generic_token_kind::end_array:
274-
visitor.end_array(tokenizer_.get_context());
275-
break;
276-
default:
277-
break;
278-
}
279-
r = tokenizer_.try_next_token();
220+
auto r = tokenizer_.try_next_token();
280221
if (JSONCONS_UNLIKELY(!r))
281222
{
282223
if (r.ec != json_errc::unexpected_eof)
@@ -285,9 +226,6 @@ class basic_json_parser
285226
}
286227
return;
287228
}
288-
}
289-
if (!test)
290-
{
291229
switch (tokenizer_.token_kind())
292230
{
293231
case generic_token_kind::string_value:
@@ -363,6 +301,15 @@ class basic_json_parser
363301
}
364302
while (!tokenizer_.done())
365303
{
304+
auto r = tokenizer_.try_next_token();
305+
if (JSONCONS_UNLIKELY(!r))
306+
{
307+
if (!r)
308+
{
309+
ec = r.ec;
310+
}
311+
return;
312+
}
366313
switch (tokenizer_.token_kind())
367314
{
368315
case generic_token_kind::string_value:
@@ -411,15 +358,6 @@ class basic_json_parser
411358
default:
412359
break;
413360
}
414-
auto r = tokenizer_.try_next_token();
415-
if (JSONCONS_UNLIKELY(!r))
416-
{
417-
if (!r)
418-
{
419-
ec = r.ec;
420-
}
421-
return;
422-
}
423361
}
424362
}
425363
};

‎include/jsoncons/json_tokenizer.hpp‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -302,6 +302,7 @@ class basic_json_tokenizer : private ser_context
302302
}
303303
if (state_ == parse_state::accept)
304304
{
305+
token_kind_ = generic_token_kind{};
305306
state_ = parse_state::done;
306307
more_ = false;
307308
return from_json_result{};
@@ -336,6 +337,7 @@ class basic_json_tokenizer : private ser_context
336337
}
337338
break;
338339
case parse_state::accept:
340+
token_kind_ = generic_token_kind{};
339341
state_ = parse_state::done;
340342
more_ = false;
341343
break;
@@ -628,6 +630,7 @@ class basic_json_tokenizer : private ser_context
628630
token_kind_ = generic_token_kind{};
629631
if (state_ == parse_state::accept)
630632
{
633+
token_kind_ = generic_token_kind{};
631634
state_ = parse_state::done;
632635
more_ = false;
633636
return;
@@ -639,6 +642,7 @@ class basic_json_tokenizer : private ser_context
639642
switch (state_)
640643
{
641644
case parse_state::accept:
645+
token_kind_ = generic_token_kind{};
642646
state_ = parse_state::done;
643647
more_ = false;
644648
break;

‎test/corelib/src/json_parser_tests.cpp‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,3 +479,4 @@ TEST_CASE("json_parser skip space tests")
479479
}
480480
}
481481

482+

‎test/corelib/src/json_tokenizer_tests.cpp‎

Lines changed: 118 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
using namespace jsoncons;
1313

14+
#if 0
1415
TEST_CASE("json_tokenizer constructor test")
1516
{
1617
SECTION("default constructor")
@@ -340,10 +341,11 @@ TEST_CASE("json_tokenizer update test")
340341
}
341342
}
342343
}
344+
#endif
343345

344346
TEST_CASE("json_tokenizer incremental update tests")
345347
{
346-
SECTION("test 1")
348+
/*SECTION("test 1")
347349
{
348350
std::string data{"123456"};
349351
std::string more_data{"78"};
@@ -653,6 +655,121 @@ TEST_CASE("json_tokenizer incremental update tests")
653655
std::cout << "ec: " << rc.ec << "\n";
654656
} while (!tokenizer.done());
655657
658+
}*/
659+
660+
SECTION("test 8")
661+
{
662+
std::string data = R"([
663+
{
664+
"given":
665+
[{"name": "JS"}]
666+
}
667+
]
668+
)";
669+
670+
json_tokenizer tokenizer{};
671+
672+
std::istringstream is(data);
673+
stream_source<char> source(is, 8);
674+
675+
auto chunk = source.read_buffer();
676+
tokenizer.update(chunk.data(), chunk.size());
677+
678+
auto r = tokenizer.try_next_token();
679+
REQUIRE(r);
680+
CHECK_FALSE(tokenizer.done());
681+
CHECK(tokenizer.started());
682+
CHECK_FALSE(tokenizer.done());
683+
CHECK_FALSE(tokenizer.source_exhausted());
684+
CHECK(generic_token_kind::begin_array == tokenizer.token_kind());
685+
686+
r = tokenizer.try_next_token();
687+
REQUIRE(r);
688+
CHECK_FALSE(tokenizer.done());
689+
CHECK(generic_token_kind::begin_map == tokenizer.token_kind());
690+
CHECK_FALSE(tokenizer.source_exhausted());
691+
692+
r = tokenizer.try_next_token();
693+
REQUIRE(r);
694+
CHECK_FALSE(tokenizer.done());
695+
CHECK(generic_token_kind{} == tokenizer.token_kind());
696+
CHECK(tokenizer.source_exhausted());
697+
chunk = source.read_buffer();
698+
tokenizer.update(chunk.data(), chunk.size());
699+
700+
r = tokenizer.try_next_token();
701+
REQUIRE(r);
702+
CHECK_FALSE(tokenizer.done());
703+
CHECK(generic_token_kind{} == tokenizer.token_kind());
704+
CHECK(tokenizer.source_exhausted());
705+
chunk = source.read_buffer();
706+
tokenizer.update(chunk.data(), chunk.size());
707+
708+
r = tokenizer.try_next_token();
709+
REQUIRE(r);
710+
CHECK_FALSE(tokenizer.done());
711+
CHECK_FALSE(tokenizer.source_exhausted());
712+
CHECK(generic_token_kind::string_value == tokenizer.token_kind());
713+
CHECK(tokenizer.is_key());
714+
715+
r = tokenizer.try_next_token();
716+
REQUIRE(r);
717+
CHECK_FALSE(tokenizer.done());
718+
CHECK(generic_token_kind{} == tokenizer.token_kind());
719+
CHECK(tokenizer.source_exhausted());
720+
chunk = source.read_buffer();
721+
tokenizer.update(chunk.data(), chunk.size());
722+
723+
r = tokenizer.try_next_token();
724+
REQUIRE(r);
725+
CHECK_FALSE(tokenizer.done());
726+
CHECK(generic_token_kind{} == tokenizer.token_kind());
727+
CHECK(tokenizer.source_exhausted());
728+
chunk = source.read_buffer();
729+
tokenizer.update(chunk.data(), chunk.size());
730+
731+
r = tokenizer.try_next_token();
732+
REQUIRE(r);
733+
CHECK_FALSE(tokenizer.done());
734+
CHECK_FALSE(tokenizer.source_exhausted());
735+
CHECK(generic_token_kind::begin_array == tokenizer.token_kind());
736+
CHECK_FALSE(tokenizer.is_key());
737+
738+
r = tokenizer.try_next_token();
739+
REQUIRE(r);
740+
CHECK_FALSE(tokenizer.done());
741+
CHECK(generic_token_kind::begin_map == tokenizer.token_kind());
742+
CHECK_FALSE(tokenizer.source_exhausted());
743+
744+
r = tokenizer.try_next_token();
745+
REQUIRE(r);
746+
CHECK_FALSE(tokenizer.done());
747+
CHECK(generic_token_kind{} == tokenizer.token_kind());
748+
CHECK(tokenizer.source_exhausted());
749+
chunk = source.read_buffer();
750+
tokenizer.update(chunk.data(), chunk.size());
751+
752+
r = tokenizer.try_next_token();
753+
REQUIRE(r);
754+
CHECK_FALSE(tokenizer.done());
755+
CHECK_FALSE(tokenizer.source_exhausted());
756+
CHECK(generic_token_kind::string_value == tokenizer.token_kind());
757+
CHECK(tokenizer.is_key());
758+
759+
r = tokenizer.try_next_token();
760+
REQUIRE(r);
761+
CHECK_FALSE(tokenizer.done());
762+
CHECK_FALSE(tokenizer.source_exhausted());
763+
CHECK(generic_token_kind::string_value == tokenizer.token_kind());
764+
CHECK_FALSE(tokenizer.is_key());
765+
766+
r = tokenizer.try_next_token();
767+
REQUIRE(r);
768+
CHECK_FALSE(tokenizer.done());
769+
CHECK(generic_token_kind::end_map == tokenizer.token_kind()); // missing event
770+
CHECK(tokenizer.source_exhausted());
771+
chunk = source.read_buffer();
772+
tokenizer.update(chunk.data(), chunk.size());
656773
}
657774
}
658775

‎test/jsonpath/src/jsonpath_test_suite.cpp‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ using namespace jsoncons;
2323

2424
void jsonpath_tests(const std::string& fpath)
2525
{
26-
std::cout << "Test " << fpath << '\n';
26+
std::cout << "Test: " << fpath << '\n';
2727

2828
std::fstream is(fpath);
2929
if (!is)
@@ -54,6 +54,8 @@ void jsonpath_tests(const std::string& fpath)
5454
auto expression = jsoncons::jsonpath::make_expression<json>(expr);
5555
if (test_case.contains("result"))
5656
{
57+
std::cout << "Input:\n" << pretty_print(instance) << "\n\n";
58+
std::cout << "Expression: " << expr << "\n\n";
5759
jsonpath::result_options rflags = options;
5860
json actual = expression.evaluate(instance, rflags);
5961
const json& expected = test_case["result"];
@@ -130,6 +132,7 @@ TEST_CASE("jsonpath-tests")
130132
{
131133
SECTION("compliance")
132134
{
135+
/*
133136
#if defined(JSONCONS_HAS_STD_REGEX)
134137
jsonpath_tests("./jsonpath/input/test_data/regex.json");
135138
#endif
@@ -138,15 +141,16 @@ TEST_CASE("jsonpath-tests")
138141
jsonpath_tests("./jsonpath/input/test_data/indices.json");
139142
jsonpath_tests("./jsonpath/input/test_data/wildcard.json");
140143
jsonpath_tests("./jsonpath/input/test_data/recursive-descent.json");
141-
jsonpath_tests("./jsonpath/input/test_data/union.json");
144+
jsonpath_tests("./jsonpath/input/test_data/union.json");
145+
*/
142146
jsonpath_tests("./jsonpath/input/test_data/filters.json");
143-
jsonpath_tests("./jsonpath/input/test_data/functions.json");
147+
/*jsonpath_tests("./jsonpath/input/test_data/functions.json");
144148
jsonpath_tests("./jsonpath/input/test_data/expressions.json");
145149
jsonpath_tests("./jsonpath/input/test_data/syntax.json");
146150
jsonpath_tests("./jsonpath/input/test_data/functions.json");
147151
jsonpath_tests("./jsonpath/input/test_data/slice.json");
148152
jsonpath_tests("./jsonpath/input/test_data/parent-operator.json");
149-
jsonpath_tests("./jsonpath/input/test.json");
153+
jsonpath_tests("./jsonpath/input/test.json");*/
150154
}
151155
}
152156

0 commit comments

Comments
 (0)