diff --git a/common/json-schema-to-grammar.cpp b/common/json-schema-to-grammar.cpp index 0cfdd54ebd..e2c4d6ce22 100644 --- a/common/json-schema-to-grammar.cpp +++ b/common/json-schema-to-grammar.cpp @@ -233,27 +233,27 @@ struct BuiltinRule { }; static std::unordered_map PRIMITIVE_RULES = { - {"boolean", {"(\"true\" | \"false\")", {}}}, + {"boolean", {"(\"true\" | \"false\") space", {}}}, {"decimal-part", {"[0-9]{1,16}", {}}}, {"integral-part", {"[0] | [1-9] [0-9]{0,15}", {}}}, - {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)?", {"integral-part", "decimal-part"}}}, - {"integer", {"(\"-\"? integral-part)", {"integral-part"}}}, + {"number", {"(\"-\"? integral-part) (\".\" decimal-part)? ([eE] [-+]? integral-part)? space", {"integral-part", "decimal-part"}}}, + {"integer", {"(\"-\"? integral-part) space", {"integral-part"}}}, {"value", {"object | array | string | number | boolean | null", {"object", "array", "string", "number", "boolean", "null"}}}, - {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? space \"}\"", {"string", "value"}}}, - {"array", {"\"[\" space ( value (\",\" space value)* )? space \"]\"", {"value"}}}, - {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\"", {}}}, + {"object", {"\"{\" space ( string \":\" space value (\",\" space string \":\" space value)* )? \"}\" space", {"string", "value"}}}, + {"array", {"\"[\" space ( value (\",\" space value)* )? \"]\" space", {"value"}}}, + {"uuid", {"\"\\\"\" [0-9a-fA-F]{8} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{4} \"-\" [0-9a-fA-F]{12} \"\\\"\" space", {}}}, {"char", {"[^\"\\\\\\x7F\\x00-\\x1F] | [\\\\] ([\"\\\\bfnrt] | \"u\" [0-9a-fA-F]{4})", {}}}, - {"string", {"\"\\\"\" char* \"\\\"\"", {"char"}}}, - {"null", {"\"null\"", {}}}, + {"string", {"\"\\\"\" char* \"\\\"\" space", {"char"}}}, + {"null", {"\"null\" space", {}}}, }; static std::unordered_map STRING_FORMAT_RULES = { {"date", {"[0-9]{4} \"-\" ( \"0\" [1-9] | \"1\" [0-2] ) \"-\" ( \"0\" [1-9] | [1-2] [0-9] | \"3\" [0-1] )", {}}}, {"time", {"([01] [0-9] | \"2\" [0-3]) \":\" [0-5] [0-9] \":\" [0-5] [0-9] ( \".\" [0-9]{3} )? ( \"Z\" | ( \"+\" | \"-\" ) ( [01] [0-9] | \"2\" [0-3] ) \":\" [0-5] [0-9] )", {}}}, {"date-time", {"date \"T\" time", {"date", "time"}}}, - {"date-string", {"\"\\\"\" date \"\\\"\"", {"date"}}}, - {"time-string", {"\"\\\"\" time \"\\\"\"", {"time"}}}, - {"date-time-string", {"\"\\\"\" date-time \"\\\"\"", {"date-time"}}} + {"date-string", {"\"\\\"\" date \"\\\"\" space", {"date"}}}, + {"time-string", {"\"\\\"\" time \"\\\"\" space", {"time"}}}, + {"date-time-string", {"\"\\\"\" date-time \"\\\"\" space", {"date-time"}}} }; static bool is_reserved_name(const std::string & name) { @@ -551,16 +551,16 @@ private: } return join_seq(); }; - return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\""); + return _add_rule(name, "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space"); } /* Returns a rule that matches a JSON string that is none of the provided strings not_strings({"a"}) - -> ["] ( [a] char+ | [^"a] char* )? ["] + -> ["] ( [a] char+ | [^"a] char* )? ["] space not_strings({"and", "also"}) - -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] + -> ["] ( [a] ([l] ([s] ([o] char+ | [^"o] char*) | [^"s] char*) | [n] ([d] char+ | [^"d] char*) | [^"ln] char*) | [^"a] char* )? ["] space */ std::string _not_strings(const std::vector & strings) { @@ -619,7 +619,7 @@ private: if (!trie.is_end_of_string) { out << "?"; } - out << " [\"]"; + out << " [\"] space"; return out.str(); } @@ -725,7 +725,7 @@ private: rule += " )?"; } - rule += " space \"}\""; + rule += " \"}\" space"; return rule; } @@ -858,14 +858,14 @@ public: return _add_rule(rule_name, _generate_union_rule(name, schema_types)); } if (schema.contains("const")) { - return _add_rule(rule_name, _generate_constant_rule(schema["const"])); + return _add_rule(rule_name, _generate_constant_rule(schema["const"]) + " space"); } if (schema.contains("enum")) { std::vector enum_values; for (const auto & v : schema["enum"]) { enum_values.push_back(_generate_constant_rule(v)); } - return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ")"); + return _add_rule(rule_name, "(" + string_join(enum_values, " | ") + ") space"); } if ((schema_type.is_null() || schema_type == "object") && (schema.contains("properties") || @@ -933,7 +933,7 @@ public: } } if (!enum_intersection.empty()) { - return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ")"); + return _add_rule(rule_name, "(" + string_join(enum_intersection, " | ") + ") space"); } } return _add_rule(rule_name, _build_object_rule(properties, required, hybrid_name, json())); @@ -948,7 +948,7 @@ public: } rule += visit(items[i], name + (name.empty() ? "" : "-") + "tuple-" + std::to_string(i)); } - rule += " space \"]\""; + rule += " \"]\" space"; return _add_rule(rule_name, rule); } std::string item_rule_name = visit(items, name + (name.empty() ? "" : "-") + "item"); @@ -956,7 +956,7 @@ public: json max_items_json = schema.contains("maxItems") ? schema["maxItems"] : json(); int max_items = max_items_json.is_number_integer() ? max_items_json.get() : std::numeric_limits::max(); - return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " space \"]\""); + return _add_rule(rule_name, "\"[\" space " + build_repetition(item_rule_name, min_items, max_items, "\",\" space") + " \"]\" space"); } if ((schema_type.is_null() || schema_type == "string") && schema.contains("pattern")) { return _visit_pattern(schema["pattern"], rule_name); @@ -972,7 +972,7 @@ public: std::string char_rule = _add_primitive("char", PRIMITIVE_RULES.at("char")); int min_len = schema.contains("minLength") ? schema["minLength"].get() : 0; int max_len = schema.contains("maxLength") ? schema["maxLength"].get() : std::numeric_limits::max(); - return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\""); + return _add_rule(rule_name, "\"\\\"\" " + build_repetition(char_rule, min_len, max_len) + " \"\\\"\" space"); } if (schema_type == "integer" && (schema.contains("minimum") || schema.contains("exclusiveMinimum") || schema.contains("maximum") || schema.contains("exclusiveMaximum"))) { int64_t min_value = std::numeric_limits::min(); @@ -990,7 +990,7 @@ public: std::stringstream out; out << "("; build_min_max_int(min_value, max_value, out); - out << ")"; + out << ") space"; return _add_rule(rule_name, out.str()); } if (schema.empty() || schema_type == "object") { @@ -1019,78 +1019,8 @@ public: } std::string format_grammar() { - auto is_alpha = [](char c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z'); }; - auto is_alnum = [&](char c) { return is_alpha(c) || (c >= '0' && c <= '9'); }; - - // Collect the rule names referenced in a rule's content, skipping string - // literals ("..."), character classes ([...]) and quantifiers ({m,n}). - auto collect_references = [&](const std::string & content, std::unordered_set & refs) { - size_t i = 0; - size_t n = content.size(); - while (i < n) { - char c = content[i]; - if (c == '"') { - i++; - while (i < n) { - if (content[i] == '\\' && i + 1 < n) { i += 2; } - else if (content[i] == '"') { i++; break; } - else { i++; } - } - } else if (c == '[') { - i++; - while (i < n) { - if (content[i] == '\\' && i + 1 < n) { i += 2; } - else if (content[i] == ']') { i++; break; } - else { i++; } - } - } else if (c == '{') { - while (i < n && content[i] != '}') { i++; } - if (i < n) { i++; } - } else if (is_alnum(c) || (c == '-' && i + 1 < n && is_alnum(content[i + 1]))) { - size_t start = i; - if (c == '-') { i++; } - while (i < n && (is_alnum(content[i]) || content[i] == '-')) { i++; } - refs.insert(content.substr(start, i - start)); - } else { - i++; - } - } - }; - - // Drop rules that nothing references (e.g. the "space" rule when no value - // carries trailing whitespace). Entry rules ("root", "root0", ...) are - // always kept even though no other rule refers to them. - std::unordered_set referenced; - for (const auto & kv : _rules) { - std::unordered_set refs; - collect_references(kv.second, refs); - for (const auto & r : refs) { - if (r != kv.first) { - referenced.insert(r); - } - } - } - - auto is_entry = [](const std::string & name) { - if (name == "root") { - return true; - } - if (name.rfind("root", 0) != 0 || name.size() <= 4) { - return false; - } - for (size_t i = 4; i < name.size(); i++) { - if (name[i] < '0' || name[i] > '9') { - return false; - } - } - return true; - }; - std::stringstream ss; for (const auto & kv : _rules) { - if (!is_entry(kv.first) && referenced.find(kv.first) == referenced.end()) { - continue; - } ss << kv.first << " ::= " << kv.second << '\n'; } return ss.str(); diff --git a/examples/json_schema_to_grammar.py b/examples/json_schema_to_grammar.py index b00541e389..077fcfacac 100755 --- a/examples/json_schema_to_grammar.py +++ b/examples/json_schema_to_grammar.py @@ -198,18 +198,18 @@ class BuiltinRule: SPACE_RULE = '| " " | "\\n"{1,2} [ \\t]{0,20}' PRIMITIVE_RULES = { - 'boolean' : BuiltinRule('("true" | "false")', []), + 'boolean' : BuiltinRule('("true" | "false") space', []), 'decimal-part' : BuiltinRule('[0-9]{1,16}', []), 'integral-part': BuiltinRule('[0] | [1-9] [0-9]{0,15}', []), - 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)?', ['integral-part', 'decimal-part']), - 'integer' : BuiltinRule('("-"? integral-part)', ['integral-part']), + 'number' : BuiltinRule('("-"? integral-part) ("." decimal-part)? ([eE] [-+]? integral-part)? space', ['integral-part', 'decimal-part']), + 'integer' : BuiltinRule('("-"? integral-part) space', ['integral-part']), 'value' : BuiltinRule('object | array | string | number | boolean | null', ['object', 'array', 'string', 'number', 'boolean', 'null']), - 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? space "}"', ['string', 'value']), - 'array' : BuiltinRule('"[" space ( value ("," space value)* )? space "]"', ['value']), - 'uuid' : BuiltinRule(r'"\"" [0-9a-fA-F]{8} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{12} "\""', []), + 'object' : BuiltinRule('"{" space ( string ":" space value ("," space string ":" space value)* )? "}" space', ['string', 'value']), + 'array' : BuiltinRule('"[" space ( value ("," space value)* )? "]" space', ['value']), + 'uuid' : BuiltinRule(r'"\"" [0-9a-fA-F]{8} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{4} "-" [0-9a-fA-F]{12} "\"" space', []), 'char' : BuiltinRule(r'[^"\\\x7F\x00-\x1F] | [\\] (["\\bfnrt] | "u" [0-9a-fA-F]{4})', []), - 'string' : BuiltinRule(r'"\"" char* "\""', ['char']), - 'null' : BuiltinRule('"null"', []), + 'string' : BuiltinRule(r'"\"" char* "\"" space', ['char']), + 'null' : BuiltinRule('"null" space', []), } # TODO: support "uri", "email" string formats @@ -217,9 +217,9 @@ STRING_FORMAT_RULES = { 'date' : BuiltinRule('[0-9]{4} "-" ( "0" [1-9] | "1" [0-2] ) "-" ( \"0\" [1-9] | [1-2] [0-9] | "3" [0-1] )', []), 'time' : BuiltinRule('([01] [0-9] | "2" [0-3]) ":" [0-5] [0-9] ":" [0-5] [0-9] ( "." [0-9]{3} )? ( "Z" | ( "+" | "-" ) ( [01] [0-9] | "2" [0-3] ) ":" [0-5] [0-9] )', []), 'date-time' : BuiltinRule('date "T" time', ['date', 'time']), - 'date-string' : BuiltinRule('"\\"" date "\\""', ['date']), - 'time-string' : BuiltinRule('"\\"" time "\\""', ['time']), - 'date-time-string': BuiltinRule('"\\"" date-time "\\""', ['date-time']), + 'date-string' : BuiltinRule('"\\"" date "\\"" space', ['date']), + 'time-string' : BuiltinRule('"\\"" time "\\"" space', ['time']), + 'date-time-string': BuiltinRule('"\\"" date-time "\\"" space', ['date-time']), } DOTALL = '[\\U00000000-\\U0010FFFF]' @@ -319,7 +319,7 @@ class SchemaConverter: out.append(f'[^"{"".join(rejects)}] {char_rule}*') visit(trie) - out.append(f' ){"" if trie.is_end_of_string else "?"} ["]') + out.append(f' ){"" if trie.is_end_of_string else "?"} ["] space') return ''.join(out) def _add_rule(self, name, rule): @@ -549,7 +549,7 @@ class SchemaConverter: return self._add_rule( name, to_rule(transform()) if self._raw_pattern \ - else "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\"") + else "\"\\\"\" (" + to_rule(transform()) + ") \"\\\"\" space") def _resolve_ref(self, ref): @@ -580,10 +580,10 @@ class SchemaConverter: return self._add_rule(rule_name, self._generate_union_rule(name, [{**schema, 'type': t} for t in schema_type])) elif 'const' in schema: - return self._add_rule(rule_name, self._generate_constant_rule(schema['const'])) + return self._add_rule(rule_name, self._generate_constant_rule(schema['const']) + ' space') elif 'enum' in schema: - rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + ')' + rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in schema['enum'])) + ') space' return self._add_rule(rule_name, rule) elif schema_type in (None, 'object') and \ @@ -624,7 +624,7 @@ class SchemaConverter: enum_intersection &= s if enum_intersection: - rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in sorted(enum_intersection))) + ')' + rule = '(' + ' | '.join((self._generate_constant_rule(v) for v in sorted(enum_intersection))) + ') space' return self._add_rule(rule_name, rule) return self._add_rule(rule_name, self._build_object_rule(properties, required, hybrid_name, additional_properties=None)) @@ -638,12 +638,12 @@ class SchemaConverter: ' "," space '.join( self.visit(item, f'{name}{"-" if name else ""}tuple-{i}') for i, item in enumerate(items)) + - ' space "]"') + ' "]" space') else: item_rule_name = self.visit(items, f'{name}{"-" if name else ""}item') min_items = schema.get("minItems", 0) max_items = schema.get("maxItems") - return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' space "]"') + return self._add_rule(rule_name, '"[" space ' + _build_repetition(item_rule_name, min_items, max_items, separator_rule='"," space') + ' "]" space') elif schema_type in (None, 'string') and 'pattern' in schema: return self._visit_pattern(schema['pattern'], rule_name) @@ -663,7 +663,7 @@ class SchemaConverter: min_len = schema.get('minLength', 0) max_len = schema.get('maxLength') - return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\""') + return self._add_rule(rule_name, r'"\"" ' + _build_repetition(char_rule, min_len, max_len) + r' "\"" space') elif schema_type in (None, 'integer') and \ ('minimum' in schema or 'exclusiveMinimum' in schema or 'maximum' in schema or 'exclusiveMaximum' in schema): @@ -680,7 +680,7 @@ class SchemaConverter: out = ["("] _generate_min_max_int(min_value, max_value, out) - out.append(")") + out.append(") space") return self._add_rule(rule_name, ''.join(out)) elif (schema_type == 'object') or (len(schema) == 0): @@ -765,68 +765,14 @@ class SchemaConverter: rule += ' )' rule += ' )?' - rule += ' space "}"' + rule += ' "}" space' return rule - def _rule_references(self, content): - # Collect rule names referenced in a rule's content, skipping string - # literals ("..."), character classes ([...]) and quantifiers ({m,n}). - refs = set() - i, n = 0, len(content) - while i < n: - c = content[i] - if c == '"': - i += 1 - while i < n: - if content[i] == '\\' and i + 1 < n: - i += 2 - elif content[i] == '"': - i += 1 - break - else: - i += 1 - elif c == '[': - i += 1 - while i < n: - if content[i] == '\\' and i + 1 < n: - i += 2 - elif content[i] == ']': - i += 1 - break - else: - i += 1 - elif c == '{': - while i < n and content[i] != '}': - i += 1 - if i < n: - i += 1 - elif c.isalnum() or (c == '-' and i + 1 < n and content[i + 1].isalnum()): - start = i - if c == '-': - i += 1 - while i < n and (content[i].isalnum() or content[i] == '-'): - i += 1 - refs.add(content[start:i]) - else: - i += 1 - return refs - def format_grammar(self): - # Drop rules that nothing references (e.g. the "space" rule when no value - # carries trailing whitespace). Entry rules ("root", "root0", ...) are - # always kept even though no other rule refers to them. - referenced = set() - for name, rule in self._rules.items(): - referenced |= (self._rule_references(rule) - {name}) - - def is_entry(name): - return name == 'root' or (name.startswith('root') and name[4:].isdigit()) - return '\n'.join( f'{name} ::= {rule}' for name, rule in sorted(self._rules.items(), key=lambda kv: kv[0]) - if is_entry(name) or name in referenced ) diff --git a/tests/peg-parser/test-gbnf-generation.cpp b/tests/peg-parser/test-gbnf-generation.cpp index 222a4a41fa..fe4bbbdd16 100644 --- a/tests/peg-parser/test-gbnf-generation.cpp +++ b/tests/peg-parser/test-gbnf-generation.cpp @@ -25,6 +25,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "hello" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -39,6 +40,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= [a-z] + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -53,6 +55,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "hello" " " "world" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -67,6 +70,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "cat" | "dog" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -81,6 +85,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "a"+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -95,6 +100,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "a"* + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -109,6 +115,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "hello" " world"? + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -123,6 +130,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= ([^<] | "<" [^/] | "])* + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -137,6 +145,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= ("a" | "b")+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -153,6 +162,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( digit ::= [0-9] root ::= digit+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -167,6 +177,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "hello\nworld\n!" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -198,6 +209,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( child ::= " world" root ::= "hello" child + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -212,6 +224,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "a" ("b" | "c") + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -226,6 +239,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "a" "b" | "c" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -240,6 +254,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= ("a" | "b")+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -254,6 +269,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "hello" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -268,6 +284,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "a" "d" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -282,6 +299,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "a" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -296,6 +314,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "a" [a-z]+ + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -310,6 +329,7 @@ void test_gbnf_generation(testing &t) { assert_gbnf_equal(t, R"""( root ::= "x" ("a" | "b") + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); }); @@ -332,6 +352,7 @@ void test_gbnf_generation(testing &t) { rule-2 ::= "b" rule-3 rule-3 ::= "c" rule-4 rule-4 ::= "d" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf); auto gbnf_lazy = build_grammar([&](const common_grammar_builder & builder) { @@ -343,6 +364,7 @@ void test_gbnf_generation(testing &t) { rule-2 ::= "b" rule-3 rule-3 ::= "c" rule-4 rule-4 ::= "d" + space ::= | " " | "\n"{1,2} [ \t]{0,20} )""", gbnf_lazy); }); } diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index c36efc94c8..e65af46c63 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -4949,14 +4949,14 @@ static void test_template_output_peg_parsers(bool detailed_debug) { tst.test("Hello, world!\nWhat's up?").tools({ special_function_tool }).expect(message_assist).expect_reconstruction().run(); tst.test( - "```json\n\"42\"\n```") + "```json\n\"42\" \n```") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .json_schema(const_schema) .expect_content(R"("42")") .run(); tst.test( - "\"42\"\n") + "\"42\" \n") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) .json_schema(const_schema) .expect_content(R"("42")") diff --git a/tests/test-json-schema-to-grammar.cpp b/tests/test-json-schema-to-grammar.cpp index 06ff43fc95..b4362852c3 100755 --- a/tests/test-json-schema-to-grammar.cpp +++ b/tests/test-json-schema-to-grammar.cpp @@ -92,7 +92,8 @@ static void test_all(const std::string & lang, std::function