Codeprocastinator/optimized-tinyllama-covalent
0119
1#ifdef NDEBUG2# undef NDEBUG3#endif4 5#include "unicode.h"6#include "sampling.h"7 8#include <cassert>9#include <string>10#include <vector>11 12static const llama_vocab * vocab;13 14static bool match_string(const std::string & input, llama_sampler * grammar) {15 llama_sampler_reset(grammar);16 auto tokens = common_tokenize(vocab, input, false, false);17 18 auto n_vocab = llama_vocab_n_tokens(vocab);19 20 std::vector<llama_token_data> cur;21 cur.reserve(n_vocab);22 for (llama_token token_id = 0; token_id < (llama_token) n_vocab; token_id++) {23 cur.emplace_back(llama_token_data{ token_id, 0.0f, 0.0f });24 }25 auto tok_arr = llama_token_data_array{ cur.data(), cur.size(), -1, false };26 27 for (const auto token : tokens) {28 for (llama_token token_id = 0; token_id < (llama_token) n_vocab; token_id++) {29 cur[token_id].logit = 0.0f;30 }31 llama_sampler_apply(grammar, &tok_arr);32 if (cur[token].logit < 0.0f) {33 return false;34 }35 llama_sampler_accept(grammar, token);36 }37 38 // do we allow EOS at the end? if so the grammar is accepting39 40 auto tok_eos = llama_vocab_eot(vocab);41 if (tok_eos == LLAMA_TOKEN_NULL) {42 tok_eos = llama_vocab_eos(vocab);43 }44 45 cur[tok_eos].logit = 0.0f;46 llama_sampler_apply(grammar, &tok_arr);47 48 return cur[tok_eos].logit >= 0.0f;49}50 51static void test(const std::string & test_desc, const std::string & grammar_str,52 const std::vector<std::string> & passing_strings, const std::vector<std::string> & failing_strings) {53 fprintf(stderr, "⚫ Testing %s\n%s\n", test_desc.c_str(), grammar_str.c_str());54 fflush(stderr);55 56 auto * grammar = llama_sampler_init_llg(vocab, "lark", grammar_str.c_str());57 58 fprintf(stderr, " 🔵 Valid strings:\n");59 60 // Passing strings61 for (const auto & test_string : passing_strings) {62 fprintf(stderr, " \"%s\" ", test_string.c_str());63 fflush(stderr);64 65 bool matched = match_string(test_string, grammar);66 67 if (!matched) {68 fprintf(stderr, "❌ (failed to match)\n");69 70 // DEBUG: Write strings to files so that we can analyze more easily with gbnf-validator program to see exactly where things failed.71 // DEBUG: Write the grammar_str to test-grammar-integration.grammar.gbnf72 FILE * grammar_file = fopen("test-grammar-integration.grammar.gbnf", "w");73 if (grammar_file) {74 fprintf(grammar_file, "%s", grammar_str.c_str());75 fclose(grammar_file);76 }77 78 // DEBUG: Write the test string to test-grammar-integration.string.txt79 FILE * string_file = fopen("test-grammar-integration.string.txt", "w");80 if (string_file) {81 fprintf(string_file, "%s", test_string.c_str());82 fclose(string_file);83 }84 85 fprintf(stderr,86 "\n NOTE: Debug grammar file generated. To analyze this failure in detail, run the following "87 "command: ./llama-gbnf-validator test-grammar-integration.grammar.gbnf "88 "test-grammar-integration.string.txt\n\n");89 } else {90 fprintf(stdout, "✅︎\n");91 }92 93 assert(matched);94 }95 96 fprintf(stderr, " 🟠 Invalid strings:\n");97 98 // Failing strings99 for (const auto & test_string : failing_strings) {100 fprintf(stderr, " \"%s\" ", test_string.c_str());101 fflush(stderr);102 103 bool matched = match_string(test_string, grammar);104 105 if (matched) {106 fprintf(stderr, "❌ (incorrectly matched)\n");107 } else {108 fprintf(stdout, "✅︎\n");109 }110 assert(!matched);111 }112 113 llama_sampler_free(grammar);114}115 116static void test_grammar(const std::string & test_desc, const std::string & grammar_str,117 const std::vector<std::string> & passing_strings,118 const std::vector<std::string> & failing_strings) {119 test(test_desc + ". Grammar: " + grammar_str, grammar_str, passing_strings, failing_strings);120}121 122static void test_schema(const std::string & test_desc, const std::string & schema_str,123 const std::vector<std::string> & passing_strings,124 const std::vector<std::string> & failing_strings) {125 test(test_desc + ". Schema: " + schema_str, "%llguidance {}\nstart: %json " + schema_str, passing_strings,126 failing_strings);127}128 129static void test_simple_grammar() {130 test_schema("min 0",131 R"""({132 "type": "integer",133 "minimum": 0134 })""",135 // Passing strings136 {137 "0",138 "10",139 "12",140 "10000",141 },142 // Failing strings143 {144 "-1",145 "-10",146 "-10000",147 "-100000000000000000000000000000000",148 // "100000000000000000000000000000000",149 "00",150 "01",151 "-0",152 });153 test_schema("min 2",154 // Schema155 R"""({156 "type": "integer",157 "minimum": 2158 })""",159 // Passing strings160 {161 "2",162 "3",163 "4",164 "10",165 "20",166 "1234567890000000",167 },168 // Failing strings169 {170 "0", "1", "-1", "-100", "0", "1", "01", "02",171 // "12345678900000000",172 });173 test_schema("min 456",174 R"""({175 "type": "integer",176 "minimum": 456177 })""",178 // Passing strings179 {180 "456",181 "4560",182 "457",183 "460",184 "500",185 },186 // Failing strings187 {188 "455",189 "356",190 "50",191 "050",192 "-1",193 "-456",194 });195 test_schema("min -123",196 R"""({197 "type": "integer",198 "minimum": -123199 })""",200 // Passing strings201 {202 "-123",203 "-122",204 "-11",205 "-1",206 "0",207 "1",208 "123",209 "1234",210 "2345",211 },212 // Failing strings213 {214 "-1234",215 "-124",216 });217 218 test_schema("max 9999",219 // Schema220 R"""({221 "type": "integer",222 "maximum": 9999223 })""",224 // Passing strings225 {226 "-99999",227 "0",228 "9999",229 },230 // Failing strings231 {232 "10000",233 "99991",234 });235 test_schema("max -9999",236 // Schema237 R"""({238 "type": "integer",239 "maximum": -9999240 })""",241 // Passing strings242 {243 "-10000",244 "-9999",245 },246 // Failing strings247 {248 "-9998",249 "0",250 "9999",251 });252 test_schema("min 5 max 30",253 // Schema254 R"""({255 "type": "integer",256 "minimum": 5,257 "maximum": 30258 })""",259 // Passing strings260 {261 "5",262 "10",263 "30",264 },265 // Failing strings266 {267 "05",268 "4",269 "-1",270 "31",271 "123",272 "0123",273 });274 test_schema("min -1 max 1",275 R"""({276 "type": "integer",277 "minimum": -1,278 "maximum": 1279 })""",280 // Passing strings281 {282 "-1",283 "0",284 "1",285 },286 // Failing strings287 {288 "-11",289 "-10",290 "-2",291 "2",292 "10",293 "11",294 });295 test_schema("min -123 max 42",296 R"""({297 "type": "integer",298 "minimum": -123,299 "maximum": 42300 })""",301 // Passing strings302 {303 "-123",304 "-122",305 "-13",306 "-11",307 "-2",308 "-1",309 "0",310 "1",311 "5",312 "10",313 "39",314 "40",315 "42",316 },317 // Failing strings318 {319 "-0123",320 "-124",321 "-1123",322 "-200",323 "43",324 "123",325 "0123",326 });327 test_schema("exclusive min / max",328 // Schema329 R"""({330 "type": "integer",331 "exclusiveMinimum": 0,332 "exclusiveMaximum": 10000333 })""",334 // Passing strings335 {336 "1",337 "9999",338 },339 // Failing strings340 {341 "0",342 "01",343 "10000",344 "99999",345 });346 347 // Test case for a simple grammar348 test_grammar("simple grammar",349 R"""(350 start: expr351 expr: term ("+" term)*352 term: number353 number: /[0-9]+/ )""",354 // Passing strings355 {356 "42",357 "1+2+3+4+5",358 "123+456",359 },360 // Failing strings361 {362 "+",363 "/ 3",364 "1+2+3+4+5+",365 "12a45",366 });367}368 369static void test_complex_grammar() {370 // Test case for a more complex grammar, with both failure strings and success strings371 test_grammar("medium complexity grammar",372 // Grammar373 R"""(374 start: expression375 expression: term ws (("+"|"-") ws term)*376 term: factor ws (("*"|"/") ws factor)*377 factor: number | variable | "(" expression ")" | function-call378 number: /[0-9]+/379 variable: /[a-zA-Z_][a-zA-Z0-9_]*/380 function-call: variable ws "(" (expression ("," ws expression)*)? ")"381 ws: /[ \t\n\r]?/ )""",382 // Passing strings383 { "42",384 "1*2*3*4*5",385 "x",386 "x+10",387 "x1+y2",388 "(a+b)*(c-d)",389 "func()",390 "func(x,y+2)",391 "a*(b+c)-d/e",392 "f(g(x),h(y,z))",393 "x + 10",394 "x1 + y2",395 "(a + b) * (c - d)",396 "func()",397 "func(x, y + 2)",398 "a * (b + c) - d / e",399 "f(g(x), h(y, z))",400 "123+456",401 "123*456*789-123/456+789*123",402 "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456" },403 // Failing strings404 {405 "+",406 "/ 3x",407 "x + + y",408 "a * / b",409 "func(,)",410 "func(x y)",411 "(a + b",412 "x + y)",413 "a + b * (c - d",414 "42 +",415 "x +",416 "x + 10 +",417 "(a + b) * (c - d",418 "func(",419 "func(x, y + 2",420 "a * (b + c) - d /",421 "f(g(x), h(y, z)",422 "123+456*789-123/456+789*123-456/789+123*456-789/123+456*789-123/456+789*123-456/",423 });424}425 426static void test_special_chars() {427 // A collection of tests to exercise special characters such as "."428 test_grammar("special characters",429 // Grammar430 R"""(431 start: /.../ "abc" /.../432 )""",433 // Passing strings434 { "abcabcabc", "aaaabcccc",435 // NOTE: Also ensures that multi-byte characters still count as a single character436 "🔵🟠✅abc❌🟠🔵" },437 // Failing strings438 { "aaabcccc", "aaaaabcccc", "aaaabccc", "aaaabccccc", "🔵🟠✅❌abc❌✅🟠🔵", "🔵🟠abc🟠🔵" });439}440 441static void test_quantifiers() {442 // A collection of tests to exercise * + and ? quantifiers443 444 test_grammar(445 "* quantifier",446 // Grammar447 R"""(start: "a"*)""",448 // Passing strings449 { "", "a", "aaaaa", "aaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },450 // Failing strings451 { "b", "ab", "aab", "ba", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab" });452 test_grammar(453 "+ quantifier",454 // Grammar455 R"""(start: "a"+)""",456 // Passing strings457 { "a", "aaaaa", "aaaaaaaaaaaaaaaaaa", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" },458 // Failing strings459 { "", "b", "ab", "aab", "ba", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaab" });460 test_grammar("? quantifier",461 // Grammar462 R"""(start: "a"?)""",463 // Passing strings464 { "", "a" },465 // Failing strings466 {467 "b",468 "ab",469 "aa",470 "ba",471 });472 test_grammar("mixed quantifiers",473 // Grammar474 R"""(475 start: cons+ vowel* cons? (vowel cons)*476 vowel: /[aeiouy]/477 cons: /[bcdfghjklmnpqrstvwxyz]/478 )""",479 // Passing strings480 {481 "yes",482 "no",483 "noyes",484 "crwth",485 "four",486 "bryyyy",487 },488 // Failing strings489 {490 "yess",491 "yesno",492 "forty",493 "catyyy",494 });495 test_grammar("simple exact repetition",496 // Grammar497 R"""(498 start: /[ab]{4}/499 )""",500 // Passing strings501 {502 "aaaa",503 "bbbb",504 "abab",505 },506 // Failing strings507 {508 "a",509 "b",510 "aaaaa",511 });512 test_grammar("simple min repetition",513 // Grammar514 R"""(515 start: /[ab]{4,}/516 )""",517 // Passing strings518 {519 "aaaa",520 "aaaaab",521 "bbbb",522 "ababab",523 },524 // Failing strings525 {526 "",527 "aba",528 });529 test_grammar("simple max repetition",530 // Grammar531 R"""(532 start: /[ab]{0,4}/533 )""",534 // Passing strings535 {536 "",537 "a",538 "aa",539 "aaa",540 "aaab",541 },542 // Failing strings543 {544 "aaaaa",545 });546 // test_grammar("min / max repetition",547 // // Grammar548 // R"""(549 // start: ("0x" /[A-F0-9]{2}/ " "?){3,5}550 // )""",551 // // Passing strings552 // {553 // "0xFF 0x12 0xAB",554 // "0xFF 0x12 0xAB 0x00 0x00",555 // },556 // // Failing strings557 // {558 // "",559 // "0xFF",560 // "0xFF 0x12",561 // "0xFF 0x12 0xAB 0x00 0x00 0x00",562 // });563}564 565static void test_json_schema() {566 // Note that this is similar to the regular grammar tests,567 // but we convert each json schema to a grammar before parsing.568 // Otherwise, this test structure is the same.569 570 test_schema("empty schema (object)",571 // Schema572 R"""(573 {"type":"object"}574 )""",575 // Passing strings576 {577 R"""({})""",578 R"""({"foo": "bar"})""",579 },580 // Failing strings581 {582 "",583 "[]",584 "null",585 R"""("")""",586 "true",587 });588 589 test_schema(590 "exotic formats (list)",591 // Schema592 R"""({593 "items": [594 { "format": "date" },595 { "format": "uuid" },596 { "format": "time" },597 { "format": "date-time" }598 ]599 })""",600 // Passing strings601 {602 // "{}", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?603 // "[]", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?604 R"""(["2012-04-23", "12345678-1234-1234-1234-1234567890ab", "18:25:43.511Z", "2012-04-23T18:25:43.511Z"])""",605 //R"""(["2012-04-23","12345678-1234-1234-1234-1234567890ab"])""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?606 //R"""({"foo": "bar"})""", // NOTE: This string passes for this schema on https://www.jsonschemavalidator.net/ -- should it?607 },608 // Failing strings609 {610 R"""(["foo", "bar"])""",611 R"""(["12345678-1234-1234-1234-1234567890ab"])""",612 });613 614 test_schema("string",615 // Schema616 R"""({617 "type": "string"618 })""",619 // Passing strings620 {621 R"""("foo")""",622 R"""("bar")""",623 R"""("")""",624 },625 // Failing strings626 {627 R"""({})""",628 R"""("foo": "bar")""",629 });630 631 test_schema("string w/ min length 1",632 // Schema633 R"""({634 "type": "string",635 "minLength": 1636 })""",637 // Passing strings638 {639 R"""("foo")""",640 R"""("bar")""",641 },642 // Failing strings643 {644 R"""("")""",645 R"""({})""",646 R"""("foo": "bar")""",647 });648 649 test_schema("string w/ min length 3",650 // Schema651 R"""({652 "type": "string",653 "minLength": 3654 })""",655 // Passing strings656 {657 R"""("foo")""",658 R"""("bar")""",659 R"""("foobar")""",660 },661 // Failing strings662 {663 R"""("")""",664 R"""("f")""",665 R"""("fo")""",666 });667 668 test_schema("string w/ max length",669 // Schema670 R"""({671 "type": "string",672 "maxLength": 3673 })""",674 // Passing strings675 {676 R"""("foo")""",677 R"""("bar")""",678 R"""("")""",679 R"""("f")""",680 R"""("fo")""",681 },682 // Failing strings683 {684 R"""("foobar")""",685 });686 687 test_schema("string w/ min & max length",688 // Schema689 R"""({690 "type": "string",691 "minLength": 1,692 "maxLength": 4693 })""",694 // Passing strings695 {696 R"""("foo")""",697 R"""("bar")""",698 R"""("f")""",699 R"""("barf")""",700 },701 // Failing strings702 {703 R"""("")""",704 R"""("barfo")""",705 R"""("foobar")""",706 });707 708 test_schema("boolean",709 // Schema710 R"""({711 "type": "boolean"712 })""",713 // Passing strings714 {715 "true",716 "false",717 },718 // Failing strings719 {720 R"""("")""",721 R"""("true")""",722 R"""(True)""",723 R"""(FALSE)""",724 });725 726 test_schema("integer",727 // Schema728 R"""({729 "type": "integer"730 })""",731 // Passing strings732 {733 R"""(0)""",734 R"""(12345)""",735 R"""(1234567890123456)""",736 },737 // Failing strings738 {739 R"""()""",740 R"""(01)""",741 R"""(007)""",742 R"""(12345678901234567 )""",743 });744 745 test_schema("string const",746 // Schema747 R"""({748 "const": "foo"749 })""",750 // Passing strings751 {752 R"""("foo")""",753 },754 // Failing strings755 {756 R"""(foo)""",757 R"""("bar")""",758 });759 760 test_schema("non-string const",761 // Schema762 R"""({763 "const": true764 })""",765 // Passing strings766 {767 R"""(true)""",768 },769 // Failing strings770 {771 R"""()""",772 R"""(foo)""",773 R"""("true")""",774 });775 776 test_schema("non-string const",777 // Schema778 R"""({779 "enum": ["red", "amber", "green", null, 42, ["foo"]]780 })""",781 // Passing strings782 {783 R"""("red")""",784 R"""(null)""",785 R"""(42)""",786 R"""(["foo"])""",787 },788 // Failing strings789 {790 R"""()""",791 R"""(420)""",792 R"""(true)""",793 R"""(foo)""",794 });795 796 test_schema("simple pattern",797 // Schema798 R"""({799 "pattern": "^[a-zA-Z0-9_-]*$"800 })""",801 // Passing strings802 {803 R"""("")""",804 R"""("He_llo-12")""",805 },806 // Failing strings807 {808 R"""("!")""",809 R"""("Hello World")""",810 });811 812 test_schema("pattern with escapes",813 // Schema814 R"""({815 "pattern": "^a\\^\\$\\.\\[\\]\\(\\)\\|\\{\\}\\*\\+\\?b$"816 })""",817 // Passing strings818 {819 R"""("a^$.[]()|{}*+?b")""",820 },821 // Failing strings822 {823 R"""("ab")""",824 });825 826 test_schema("",827 // Schema828 R"""(829 {830 "type": ["array", "null"],831 "items": { "type": "string" }832 }833 )""",834 // Passing strings835 {836 "null",837 "[]",838 "[\"123\"]",839 "[\"foo\", \"bar\"]",840 },841 // Failing strings842 {843 "",844 "[123]",845 "\"foo\"",846 "[\"foo\", 42]",847 });848 849 test_schema("min+max items",850 // Schema851 R"""({852 "items": {853 "type": ["number", "integer"]854 },855 "minItems": 3,856 "maxItems": 5857 })""",858 // Passing strings859 {860 R"""([1, 2, 3])""",861 R"""([1, 2, 3, 4])""",862 R"""([1, 2, 3, 4, 5])""",863 // this is in fact correct; keyword do not apply if the type is wrong864 R"""(1)""",865 },866 // Failing strings867 {868 R"""([1, 2])""",869 R"""([1, 2, 3, 4, 5, 6])""",870 });871 872 // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)873 test_schema("object properties",874 // Schema875 R"""({876 "type": "object",877 "properties": {878 "number": { "type": "number" },879 "street_name": { "type": "string" },880 "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }881 },882 "additionalProperties": false883 })""",884 // Passing strings885 {886 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",887 // "By default, leaving out properties is valid"888 R"""({ "street_name": "Pennsylvania" })""",889 R"""({ "number": 1600, "street_name": "Pennsylvania" })""",890 // "By extension, even an empty object is valid"891 R"""({})""",892 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",893 },894 // Failing strings895 {896 // Change datatype from number to string897 R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",898 // Reorder properties899 R"""({ "street_name": "Pennsylvania", "number": 1600 })""",900 // Reorder properties901 R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",902 // Additional properties set to false903 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",904 905 });906 907 test_schema("additional properties can't override other properties",908 R"""({909 "properties": {910 "a": {"type": "integer"},911 "b": {"type": "integer"}912 },913 "additionalProperties": true914 })""",915 // Passing strings916 {917 R"""({"a": 42})""",918 R"""({"c": ""})""",919 R"""({"a": 42, "c": ""})""",920 R"""({"a_": ""})""",921 },922 // Failing strings923 {924 R"""()""",925 R"""({"a": ""})""",926 R"""({"a": "", "b": ""})""",927 });928 929 // Properties (from: https://json-schema.org/understanding-json-schema/reference/object#properties)930 test_schema("object properties, additionalProperties: true",931 // Schema932 R"""({933 "type": "object",934 "properties": {935 "number": { "type": "number" },936 "street_name": { "type": "string" },937 "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }938 },939 "additionalProperties": true940 })""",941 // Passing strings942 {943 // "By extension, even an empty object is valid"944 R"""({})""",945 R"""({"number":1600,"street_name":"Pennsylvania","street_type":"Avenue"})""",946 // "By default, leaving out properties is valid"947 R"""({ "street_name": "Pennsylvania" })""",948 R"""({ "number": 1600, "street_name": "Pennsylvania" })""",949 // "By default, providing additional properties is valid"950 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue", "direction":"NW"})""",951 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",952 },953 // Failing strings954 {955 // Change datatype from number to string956 R"""({ "number": "1600", "street_name": "Pennsylvania", "street_type":"Avenue"})""",957 // Reorder properties958 R"""({ "street_name": "Pennsylvania", "number": 1600, "street_type":"Avenue"})""",959 });960 961 // Additional properties: false962 test_schema(963 "required + optional props each in original order",964 // Schema965 R"""({966 "type": "object",967 "properties": {968 "number": { "type": "number" },969 "street_name": { "type": "string" },970 "street_type": { "enum": ["Street", "Avenue", "Boulevard"] }971 },972 "additionalProperties": false973 })""",974 // Passing strings975 {976 R"""({ "street_name": "Pennsylvania" })""",977 R"""({ "number": 1600, "street_type":"Avenue"})""",978 R"""({ "number": 1600, "street_name": "Pennsylvania" })""",979 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type":"Avenue"})""",980 // Spaces are permitted around enum values981 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue" })""",982 },983 // Failing strings984 {985 // Reorder properties986 R"""({ "street_type": "Avenue", "number": 1600 })""",987 // Add "direction"988 R"""({ "number": 1600, "street_name": "Pennsylvania", "street_type": "Avenue", "direction": "NW" })""",989 });990 991 test_schema("required + optional props each in original order",992 // Schema993 R"""({994 "properties": {995 "b": {"type": "string"},996 "a": {"type": "string"},997 "d": {"type": "string"},998 "c": {"type": "string"}999 },1000 "required": ["a", "b"],1001 "additionalProperties": false1002 })""",1003 // Passing strings1004 {1005 R"""({"b": "foo", "a": "bar"})""",1006 R"""({"b":"foo","a":"bar","d":"qux"})""",1007 R"""({"b":"foo", "a":"bar", "d":"qux", "c":"baz"})""",1008 },1009 // Failing strings1010 {1011 R"""({"a": "foo", "b": "bar"})""",1012 R"""({"b": "bar"})""",1013 R"""({"a": "foo", "c": "baz"})""",1014 R"""({"a":"foo", "b":"bar", "c":"baz", "d":"qux"})""",1015 });1016 1017 // NOTE: Example from https://json-schema.org/learn/getting-started-step-by-step#define-required-properties1018 test_schema(1019 "required props",1020 // Schema1021 R"""({1022 "$schema": "https://json-schema.org/draft/2020-12/schema",1023 "$id": "https://example.com/product.schema.json",1024 "title": "Product",1025 "description": "A product from Acme's catalog",1026 "type": "object",1027 "properties": {1028 "productId": {1029 "description": "The unique identifier for a product",1030 "type": "integer"1031 },1032 "productName": {1033 "description": "Name of the product",1034 "type": "string"1035 },1036 "price": {1037 "description": "The price of the product",1038 "type": "number",1039 "exclusiveMinimum": 01040 },1041 "tags": {1042 "description": "Tags for the product",1043 "type": "array",1044 "items": {1045 "type": "string"1046 },1047 "minItems": 1,1048 "DISABLED_uniqueItems": true1049 },1050 "dimensions": {1051 "type": "object",1052 "properties": {1053 "length": {1054 "type": "number"1055 },1056 "width": {1057 "type": "number"1058 },1059 "height": {1060 "type": "number"1061 }1062 },1063 "required": [ "length", "width", "height" ]1064 }1065 },1066 "required": [ "productId", "productName", "price" ]1067 })""",1068 // Passing strings1069 {1070 R"""({"productId": 1, "productName": "A green door", "price": 12.50})""",1071 R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green"]})""",1072 R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green"], "dimensions": {"length": 785, "width": 250.5, "height": -0.359}})""",1073 },1074 // Failing strings1075 {1076 R"""({})""", // Missing all required properties1077 R"""({"productName": "A green door", "price": 12.50, "productId": 1})""", // Out of order properties1078 // `exclusiveMinimum` is OK for llg1079 R"""({"productId": 1, "productName": "A green door", "price": -12.50})""",1080 R"""({"productId": 1, "productName": "A green door"})""", // Missing required property (price)1081 R"""({"productName": "A green door", "price": 12.50})""", // Missing required property (productId)1082 R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": []})""", // tags is empty, but minItems is 11083 R"""({"productId": 1, "productName": "A green door", "price": 12.50, "dimensions": {"length": 785, "width": 250.5, "height": -0.359}, "tags": ["home", "green"]})""", // Tags and dimensions are out of order1084 // TODO: The following line should fail, but currently it passes. `uniqueItems` is not supported, as it would likely be too difficult to implement.1085 // R"""({"productId": 1, "productName": "A green door", "price": 12.50, "tags": ["home", "green", "home"]})""",1086 });1087}1088 1089static void one_hot(llama_token_data_array & tok_arr, llama_token selected) {1090 auto n_vocab = tok_arr.size;1091 1092 tok_arr.selected = -1;1093 tok_arr.sorted = false;1094 for (llama_token token_id = 0; token_id < (llama_token) n_vocab; token_id++) {1095 tok_arr.data[token_id].id = token_id;1096 tok_arr.data[token_id].logit = 0.0f;1097 }1098 1099 tok_arr.data[selected].logit = 100.0f;1100}1101 1102static void test_sampler_chain(void) {1103 auto sparams = llama_sampler_chain_default_params();1104 sparams.no_perf = false;1105 llama_sampler * sampler = llama_sampler_chain_init(sparams);1106 1107 const auto grammar_data = R"(%llguidance {}1108start: /[A-Z ]*/)";1109 1110 llama_sampler_chain_add(sampler, llama_sampler_init_llg(vocab, "lark", grammar_data));1111 llama_sampler_chain_add(sampler, llama_sampler_init_dist(42));1112 1113 auto input = "ALL YOUR BASE ARE BELONG TO US";1114 auto tokens = common_tokenize(vocab, input, false, false);1115 1116 auto n_vocab = llama_vocab_n_tokens(vocab);1117 1118 std::vector<llama_token_data> cur;1119 cur.reserve(n_vocab);1120 for (llama_token token_id = 0; token_id < (llama_token) n_vocab; token_id++) {1121 cur.emplace_back(llama_token_data{ token_id, 0.0f, 0.0f });1122 }1123 auto tok_arr = llama_token_data_array{ cur.data(), cur.size(), -1, false };1124 1125 for (const auto token : tokens) {1126 one_hot(tok_arr, token);1127 1128 fprintf(stderr, "applying token: %d\n", token);1129 llama_sampler_apply(sampler, &tok_arr);1130 1131 auto idx = tok_arr.selected;1132 fprintf(stderr, " -> %d %f\n", cur[idx].id, cur[idx].logit);1133 assert(cur[tok_arr.selected].id == token);1134 llama_sampler_accept(sampler, token);1135 }1136 1137 auto tok_eos = llama_vocab_eot(vocab);1138 if (tok_eos == LLAMA_TOKEN_NULL) {1139 tok_eos = llama_vocab_eos(vocab);1140 }1141 1142 one_hot(tok_arr, tok_eos);1143 1144 llama_sampler_apply(sampler, &tok_arr);1145 assert(cur[tok_arr.selected].id == tok_eos);1146}1147 1148int main(int argc, const char ** argv) {1149 fprintf(stdout, "Running llguidance integration tests...\n");1150 1151 if (argc != 2) {1152 fprintf(stderr, "Usage: %s <vocab-file>\n", argv[0]);1153 return 1;1154 }1155 1156 const char * vocab_file = argv[1];1157 1158 fprintf(stderr, "reading vocab from: '%s'\n", vocab_file);1159 1160 llama_model * model;1161 llama_context * ctx;1162 1163 llama_backend_init();1164 1165 // load the vocab1166 {1167 auto mparams = llama_model_default_params();1168 1169 mparams.vocab_only = true;1170 1171 model = llama_model_load_from_file(vocab_file, mparams);1172 1173 if (model == NULL) {1174 fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file);1175 return 1;1176 }1177 1178 // needed?1179 auto cparams = llama_context_default_params();1180 1181 ctx = llama_init_from_model(model, cparams);1182 1183 if (ctx == NULL) {1184 fprintf(stderr, "%s: error: failed to load vocab '%s'\n", __func__, vocab_file);1185 llama_model_free(model);1186 return 1;1187 }1188 }1189 1190 vocab = llama_model_get_vocab(model);1191 1192 test_simple_grammar();1193 test_complex_grammar();1194 test_special_chars();1195 test_quantifiers();1196 test_json_schema();1197 1198 test_sampler_chain();1199 1200 fprintf(stdout, "All tests passed.\n");