Felipe97/llama-cpp-compiled
01.1k
1#include "peg-parser.h"2#include "tests.h"3 4void test_basic(testing & t) {5 t.test("chars", [](testing & t) {6 // Test common escape sequences - newline7 t.test("escape_sequence_newline", [](testing &t) {8 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });9 10 common_peg_parse_context ctx;11 common_peg_parse_result result;12 13 ctx = common_peg_parse_context("\n");14 result = common_chat_combinator_parser.parse(ctx);15 t.assert_equal("escape_sequence_newline", true, result.success());16 });17 18 // Test common escape sequences - tab19 t.test("escape_sequence_tab", [](testing &t) {20 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });21 22 common_peg_parse_context ctx;23 common_peg_parse_result result;24 25 ctx = common_peg_parse_context("\t");26 result = common_chat_combinator_parser.parse(ctx);27 t.assert_equal("escape_sequence_tab", true, result.success());28 });29 30 // Test common escape sequences - backslash31 t.test("escape_sequence_backslash", [](testing &t) {32 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });33 34 common_peg_parse_context ctx;35 common_peg_parse_result result;36 37 ctx = common_peg_parse_context("\\");38 result = common_chat_combinator_parser.parse(ctx);39 t.assert_equal("escape_sequence_backslash", true, result.success());40 });41 42 // Test common escape sequences - space (should ())43 t.test("escape_sequence_space_fail", [](testing &t) {44 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[\\n\\t\\\\]"); });45 46 common_peg_parse_context ctx;47 common_peg_parse_result result;48 49 ctx = common_peg_parse_context(" ");50 result = common_chat_combinator_parser.parse(ctx);51 t.assert_equal("escape_sequence_space_fail", true, result.fail());52 });53 54 // Test escaped dash - 'a' should succeed55 t.test("escaped_dash_a", [](testing &t) {56 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });57 58 common_peg_parse_context ctx;59 common_peg_parse_result result;60 61 ctx = common_peg_parse_context("a");62 result = common_chat_combinator_parser.parse(ctx);63 t.assert_equal("escaped_dash_a", true, result.success());64 });65 66 // Test escaped dash - '-' should succeed (literal dash)67 t.test("escaped_dash_literal", [](testing &t) {68 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });69 70 common_peg_parse_context ctx;71 common_peg_parse_result result;72 73 ctx = common_peg_parse_context("-");74 result = common_chat_combinator_parser.parse(ctx);75 t.assert_equal("escaped_dash_literal", true, result.success());76 });77 78 // Test escaped dash - 'z' should succeed79 t.test("escaped_dash_z", [](testing &t) {80 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });81 82 common_peg_parse_context ctx;83 common_peg_parse_result result;84 85 ctx = common_peg_parse_context("z");86 result = common_chat_combinator_parser.parse(ctx);87 t.assert_equal("escaped_dash_z", true, result.success());88 });89 90 // Test escaped dash - 'b' should NOT match (since \- is literal dash, not range)91 t.test("escaped_dash_b_fail", [](testing &t) {92 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("[a\\-z]"); });93 94 common_peg_parse_context ctx;95 common_peg_parse_result result;96 97 ctx = common_peg_parse_context("b");98 result = common_chat_combinator_parser.parse(ctx);99 t.assert_equal("escaped_dash_b_fail", true, result.fail());100 });101 });102 103 104 t.test("optional", [](testing & t) {105 // Full match with optional part present106 t.test("optional_present", [](testing &t) {107 auto parser = build_peg_parser([](common_peg_parser_builder & p) {108 return p.literal("hello") + p.optional(p.literal(" world"));109 });110 111 auto ctx = common_peg_parse_context("hello world");112 auto result = parser.parse(ctx);113 t.assert_equal("optional_present", true, result.success());114 t.assert_equal("optional_present_end", 11u, result.end);115 });116 117 // Full match with optional part absent118 t.test("optional_absent", [](testing &t) {119 auto parser = build_peg_parser([](common_peg_parser_builder & p) {120 return p.literal("hello") + p.optional(p.literal(" world"));121 });122 123 auto ctx = common_peg_parse_context("hello");124 auto result = parser.parse(ctx);125 t.assert_equal("optional_absent", true, result.success());126 t.assert_equal("optional_absent_end", 5u, result.end);127 });128 129 // Partial match - waiting for more input to determine if optional matches130 t.test("partial_match_need_more", [](testing &t) {131 auto parser = build_peg_parser([](common_peg_parser_builder & p) {132 return p.literal("hello") + p.optional(p.literal(" world"));133 });134 135 auto ctx = common_peg_parse_context("hello ", COMMON_PEG_PARSE_FLAG_LENIENT);136 auto result = parser.parse(ctx);137 t.assert_equal("partial_match_need_more", true, result.need_more_input());138 });139 });140 141 t.test("partial parsing", [](testing & t) {142 // Literals - Basic Success143 t.test("literal_success", [&](testing & t) {144 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello"); });145 146 common_peg_parse_context ctx;147 common_peg_parse_result result;148 149 ctx = common_peg_parse_context("hello");150 result = parser.parse(ctx);151 t.assert_equal("literal_success", true, result.success());152 });153 154 // Char Classes - Basic Lowercase Success155 t.test("char_class_lowercase_success", [&](testing & t) {156 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });157 158 common_peg_parse_context ctx;159 common_peg_parse_result result;160 161 ctx = common_peg_parse_context("a");162 result = parser.parse(ctx);163 t.assert_equal("char_class_lowercase_success", true, result.success());164 });165 166 // Char Classes - Uppercase Fail167 t.test("char_class_uppercase_fail", [&](testing & t) {168 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z"); });169 170 common_peg_parse_context ctx;171 common_peg_parse_result result;172 173 ctx = common_peg_parse_context("A");174 result = parser.parse(ctx);175 t.assert_equal("char_class_uppercase_fail", true, result.fail());176 });177 178 // Char Classes with Dash - Lowercase Success179 t.test("char_class_with_dash_lowercase", [&](testing & t) {180 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });181 182 common_peg_parse_context ctx;183 common_peg_parse_result result;184 185 ctx = common_peg_parse_context("f");186 result = parser.parse(ctx);187 t.assert_equal("char_class_with_dash_lowercase", true, result.success());188 });189 190 // Char Classes with Dash - Literal Dash Success191 t.test("char_class_with_dash_literal_dash", [&](testing & t) {192 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });193 194 common_peg_parse_context ctx;195 common_peg_parse_result result;196 197 ctx = common_peg_parse_context("-");198 result = parser.parse(ctx);199 t.assert_equal("char_class_with_dash_literal_dash", true, result.success());200 });201 202 // Char Classes with Dash - Uppercase Fail203 t.test("char_class_with_dash_uppercase_fail", [&](testing & t) {204 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.chars("a-z-"); });205 206 common_peg_parse_context ctx;207 common_peg_parse_result result;208 209 ctx = common_peg_parse_context("A");210 result = parser.parse(ctx);211 t.assert_equal("char_class_with_dash_uppercase_fail", true, result.fail());212 });213 214 // Sequences - Partial Match 1215 t.test("sequence_partial_match_1", [&](testing & t) {216 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });217 218 auto ctx = common_peg_parse_context("<thi", COMMON_PEG_PARSE_FLAG_LENIENT);219 auto result = parser.parse(ctx);220 t.assert_equal("sequence_partial_match_1", true, result.need_more_input());221 });222 223 // Sequences - Partial Match 2224 t.test("sequence_partial_match_2", [&](testing & t) {225 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("begin") + p.literal("end"); });226 227 auto ctx = common_peg_parse_context("begin", COMMON_PEG_PARSE_FLAG_LENIENT);228 auto result = parser.parse(ctx);229 t.assert_equal("sequence_partial_match_2", true, result.need_more_input());230 });231 232 // Sequences - Partial Match 3233 t.test("sequence_partial_match_3", [&](testing & t) {234 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });235 236 auto ctx = common_peg_parse_context("<think></", COMMON_PEG_PARSE_FLAG_LENIENT);237 auto result = parser.parse(ctx);238 t.assert_equal("sequence_partial_match_3", true, result.need_more_input());239 });240 241 // Sequences - Full Match242 t.test("sequence_full_match", [&](testing & t) {243 auto common_chat_combinator_parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("hello") + p.literal("world"); });244 245 auto ctx = common_peg_parse_context("helloworld");246 auto result = common_chat_combinator_parser.parse(ctx);247 t.assert_equal("sequence_full_match", true, result.success());248 });249 250 // Sequences - No Match251 t.test("sequence_no_match", [&](testing & t) {252 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("<think>") + p.literal("</think>"); });253 254 auto ctx = common_peg_parse_context("<think>I am common_chat_combinator_parser", COMMON_PEG_PARSE_FLAG_LENIENT);255 auto result = parser.parse(ctx);256 t.assert_equal("sequence_no_match", true, result.fail());257 });258 259 // Choices - Partial Match 1260 t.test("choices_partial_match_1", [&](testing & t) {261 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("option1") | p.literal("option2"); });262 263 auto ctx = common_peg_parse_context("opt", COMMON_PEG_PARSE_FLAG_LENIENT);264 auto result = parser.parse(ctx);265 t.assert_equal("choices_partial_match_1", true, result.need_more_input());266 });267 268 // Choices - Partial Match 2269 t.test("choices_partial_match_2", [&](testing & t) {270 auto parser =271 build_peg_parser([](common_peg_parser_builder & p) { return p.literal("choice_a") | p.literal("choice_b"); });272 273 auto ctx = common_peg_parse_context("choice", COMMON_PEG_PARSE_FLAG_LENIENT);274 auto result = parser.parse(ctx);275 t.assert_equal("choices_partial_match_2", true, result.need_more_input());276 });277 278 // Choices - Full Match 1279 t.test("choices_full_match_1", [&](testing & t) {280 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("first") | p.literal("second"); });281 282 auto ctx = common_peg_parse_context("first");283 auto result = parser.parse(ctx);284 t.assert_equal("choices_full_match_1", true, result.success());285 });286 287 // Choices - Full Match 2288 t.test("choices_full_match_2", [&](testing & t) {289 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("alpha") | p.literal("beta"); });290 291 auto ctx = common_peg_parse_context("beta");292 auto result = parser.parse(ctx);293 t.assert_equal("choices_full_match_2", true, result.success());294 });295 296 // Choices - No Match297 t.test("choices_no_match", [&](testing & t) {298 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.literal("good") | p.literal("better"); });299 300 auto ctx = common_peg_parse_context("best");301 auto result = parser.parse(ctx);302 t.assert_equal("choices_no_match", true, result.fail());303 });304 305 // Zero or More - Partial Match 1306 t.test("zero_or_more_partial_match_1", [&](testing & t) {307 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("ab")); });308 309 auto ctx = common_peg_parse_context("a", COMMON_PEG_PARSE_FLAG_LENIENT);310 auto result = parser.parse(ctx);311 t.assert_equal("zero_or_more_partial_match_1", true, result.need_more_input());312 });313 314 // Zero or More - Partial Match 2315 t.test("zero_or_more_partial_match_2", [&](testing & t) {316 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("xy")); });317 318 auto ctx = common_peg_parse_context("xyx", COMMON_PEG_PARSE_FLAG_LENIENT);319 auto result = parser.parse(ctx);320 t.assert_equal("zero_or_more_partial_match_2", true, result.need_more_input());321 });322 323 // Zero or More - Full Match324 t.test("zero_or_more_full_match", [&](testing & t) {325 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.zero_or_more(p.literal("test")); });326 327 auto ctx = common_peg_parse_context("test");328 auto result = parser.parse(ctx);329 t.assert_equal("zero_or_more_full_match", true, result.success());330 });331 332 // One or More - Partial Match 1333 t.test("one_or_more_partial_match_1", [&](testing & t) {334 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("repeat")); });335 336 auto ctx = common_peg_parse_context("rep", COMMON_PEG_PARSE_FLAG_LENIENT);337 auto result = parser.parse(ctx);338 t.assert_equal("one_or_more_partial_match_1", true, result.need_more_input());339 });340 341 // One or More - Partial Match 2342 t.test("one_or_more_partial_match_2", [&](testing & t) {343 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("ab")); });344 345 auto ctx = common_peg_parse_context("aba", COMMON_PEG_PARSE_FLAG_LENIENT);346 auto result = parser.parse(ctx);347 t.assert_equal("one_or_more_partial_match_2", true, result.need_more_input());348 });349 350 // One or More - Full Match351 t.test("one_or_more_full_match", [&](testing & t) {352 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("single")); });353 354 auto ctx = common_peg_parse_context("single");355 auto result = parser.parse(ctx);356 t.assert_equal("one_or_more_full_match", true, result.success());357 });358 359 // One or More - No Match360 t.test("one_or_more_no_match", [&](testing & t) {361 auto parser = build_peg_parser([](common_peg_parser_builder & p) { return p.one_or_more(p.literal("()")); });362 363 auto ctx = common_peg_parse_context("success");364 auto result = parser.parse(ctx);365 t.assert_equal("one_or_more_no_match", true, result.fail());366 });367 });368 369 370 t.test("recursive rules", [](testing &t) {371 // Test simple number372 t.test("simple_number", [](testing &t) {373 auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {374 p.rule("number", p.chars("0-9"));375 p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));376 return p.rule("value", p.ref("number") | p.ref("list"));377 });378 379 common_peg_parse_context ctx("1");380 auto result = value_parser.parse(ctx);381 382 t.assert_equal("result_is_success", true, result.success());383 });384 385 // Test simple list386 t.test("simple_list", [](testing &t) {387 auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {388 p.rule("number", p.chars("0-9"));389 p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));390 return p.rule("value", p.ref("number") | p.ref("list"));391 });392 393 common_peg_parse_context ctx("[1]");394 auto result = value_parser.parse(ctx);395 396 t.assert_equal("result_is_success", true, result.success());397 });398 399 // Test nested list400 t.test("nested_list", [](testing &t) {401 auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {402 p.rule("number", p.chars("0-9"));403 p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));404 return p.rule("value", p.ref("number") | p.ref("list"));405 });406 407 common_peg_parse_context ctx("[[2]]");408 auto result = value_parser.parse(ctx);409 410 t.assert_equal("result_is_success", true, result.success());411 });412 413 // Test deeply nested list414 t.test("deeply_nested_list", [](testing &t) {415 auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {416 p.rule("number", p.chars("0-9"));417 p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));418 return p.rule("value", p.ref("number") | p.ref("list"));419 });420 421 common_peg_parse_context ctx("[[[3]]]");422 auto result = value_parser.parse(ctx);423 424 t.assert_equal("result_is_success", true, result.success());425 });426 427 // Test need_more_input match428 t.test("need_more_input_match", [](testing &t) {429 auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {430 p.rule("number", p.chars("0-9"));431 p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));432 return p.rule("value", p.ref("number") | p.ref("list"));433 });434 435 common_peg_parse_context ctx("[[", COMMON_PEG_PARSE_FLAG_LENIENT);436 auto result = value_parser.parse(ctx);437 438 t.assert_equal("result_is_need_more_input", true, result.need_more_input());439 });440 441 // Test no match442 t.test("no_match", [](testing &t) {443 auto value_parser = build_peg_parser([](common_peg_parser_builder & p) {444 p.rule("number", p.chars("0-9"));445 p.rule("list", p.literal("[") + p.ref("value") + p.literal("]"));446 return p.rule("value", p.ref("number") | p.ref("list"));447 });448 449 common_peg_parse_context ctx("[a]");450 auto result = value_parser.parse(ctx);451 452 t.assert_equal("result_is_fail", true, result.fail());453 });454 455 // Test markers456 t.test("marker", [](testing &t) {457 auto bracket_parser = build_peg_parser([](common_peg_parser_builder & p) {458 return p.marker();459 });460 461 common_peg_parse_context ctx_square("[marker]");462 common_peg_parse_context ctx_sharp("<marker>");463 464 auto result_square = bracket_parser.parse(ctx_square);465 auto result_sharp = bracket_parser.parse(ctx_sharp);466 467 t.assert_true("result_square_is_success", result_square.success());468 t.assert_true("result_sharp_is_success", result_sharp.success());469 });470 });471}472