From f5919bf458ef190468b5c329bb293f8a54a1e69c Mon Sep 17 00:00:00 2001 From: Aldehir Rojas Date: Sun, 2 Aug 2026 04:13:20 -0500 Subject: [PATCH] chat : add qwen3 specialized parser (#26252) * Add tagged thinking tool parser * chat : refactor and add permute helper * cont : add support for omission * cont : update tool delimiters * cont : add comment for qwen3-coder * cont : fix trigger pattern for --- common/chat-peg-parser.cpp | 40 +++++++ common/chat-peg-parser.h | 5 + common/chat.cpp | 174 +++++++++++++++++++++++++++++ tests/test-chat-peg-parser.cpp | 103 +++++++++++++++++ tests/test-chat.cpp | 197 +++++++++++++++++++-------------- 5 files changed, 433 insertions(+), 86 deletions(-) diff --git a/common/chat-peg-parser.cpp b/common/chat-peg-parser.cpp index f786f5ff23..1910b4f1e1 100644 --- a/common/chat-peg-parser.cpp +++ b/common/chat-peg-parser.cpp @@ -6,6 +6,9 @@ #include +#include +#include + using ordered_json = nlohmann::ordered_json; static std::string_view trim_trailing_space(std::string_view sv, int max = -1) { @@ -235,6 +238,43 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri return zero_or_more(choice({ p, content_chunk })); } +common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix, + const std::vector & parsers) { + if (parsers.empty()) { + return eps(); + } + + if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) { + return sequence(parsers); + } + + std::map rules; + std::function remaining_of; + + remaining_of = [&](uint32_t remaining) -> common_peg_parser { + if (remaining == 0) { + return eps(); + } + + auto cached = rules.find(remaining); + if (cached != rules.end()) { + return cached->second; + } + + auto alternatives = choice(); + for (size_t i = 0; i < parsers.size(); i++) { + const uint32_t bit = 1u << i; + if (remaining & bit) { + alternatives |= parsers[i] + remaining_of(remaining & ~bit); + } + } + + return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second; + }; + + return remaining_of((1u << parsers.size()) - 1); +} + std::string & common_chat_peg_mapper::args_target() { return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer; } diff --git a/common/chat-peg-parser.h b/common/chat-peg-parser.h index cd14f2c117..5d764dbaa0 100644 --- a/common/chat-peg-parser.h +++ b/common/chat-peg-parser.h @@ -55,6 +55,8 @@ class common_chat_peg_minimax_m3_mapper : public common_chat_peg_mapper { struct content_structure; struct tool_call_structure; +constexpr size_t COMMON_CHAT_MAX_PERMUTE = 6; + class common_chat_peg_builder : public common_peg_parser_builder { public: // Tag constants (from former common_chat_peg_base_builder) @@ -105,6 +107,9 @@ class common_chat_peg_builder : public common_peg_parser_builder { common_peg_parser tool_arg_json_value(const common_peg_parser & p) { return tag(TOOL_ARG_VALUE, p); } + // Matches every parser exactly once, in any order. + common_peg_parser permute(const std::string & rule_prefix, const std::vector & parsers); + // Return a parser that parses the prefix of a string, up to a given delimiter. common_peg_parser prefix(const std::string & s, const std::string & delimiter = {}); diff --git a/common/chat.cpp b/common/chat.cpp index 0e06fa591f..f7431a6f13 100644 --- a/common/chat.cpp +++ b/common/chat.cpp @@ -1110,6 +1110,172 @@ static common_chat_params common_chat_params_init_ministral_3(const common_chat_ return data; } +static common_chat_params common_chat_params_init_qwen3_coder(const common_chat_template & tmpl, + const autoparser::generation_params & inputs) { + common_chat_params data; + + const std::string GEN_PREFIX = "<|im_start|>assistant\n"; + + data.prompt = common_chat_template_direct_apply_impl(tmpl, inputs); + data.generation_prompt = common_chat_template_generation_prompt_impl(tmpl, inputs); + data.format = COMMON_CHAT_FORMAT_PEG_NATIVE; + + auto supports_reasoning = tmpl.source().find("") != std::string::npos; + + data.supports_thinking = supports_reasoning; + data.preserved_tokens = { + "", + "", + }; + + if (supports_reasoning) { + data.thinking_start_tag = ""; + // Support both and as reasoning end sequences. + // ", "" }; + data.preserved_tokens.insert(data.preserved_tokens.end(), { "", "" }); + } + + data.message_delimiters = { + { COMMON_CHAT_ROLE_ASSISTANT, "<|im_start|>assistant" }, + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>user\n" }, // Qwen3-Coder, Qwen3.5, Nemotron Nano 3 + { COMMON_CHAT_ROLE_TOOL, "<|im_start|>tool_response" }, // StepFun-3.5-Flash + { COMMON_CHAT_ROLE_USER, "<|im_start|>user" }, + { COMMON_CHAT_ROLE_SYSTEM, "<|im_start|>system" }, + }; + + auto has_tools = inputs.tools.is_array() && !inputs.tools.empty(); + auto has_response_format = inputs.json_schema.is_object() && !inputs.json_schema.empty(); + auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE; + auto include_grammar = has_response_format || (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE); + + if (inputs.has_continuation()) { + const auto & msg = inputs.continue_msg; + + data.generation_prompt = GEN_PREFIX; + if (supports_reasoning) { + data.generation_prompt += "\n" + msg.reasoning_content; + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += "\n\n\n"; + } + } + if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) { + data.generation_prompt += msg.render_content(); + } + + data.prompt += data.generation_prompt; + } + + auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) { + auto generation_prompt = p.literal(GEN_PREFIX); + + auto reasoning = p.eps(); + if (supports_reasoning && extract_reasoning) { + reasoning = p.optional("" + p.space() + + p.reasoning(p.until_one_of({ "", "" })) + + (p.literal("") | p.peek(p.literal("")))); + } + + // Response format parser + if (has_response_format) { + return generation_prompt + (reasoning << p.content(p.schema(p.json(), "response-format", inputs.json_schema))); + } + + // Tool call parser + if (has_tools && inputs.tool_choice != COMMON_CHAT_TOOL_CHOICE_NONE) { + auto arg_close = p.tool_arg_close(p.literal("\n\n")); + auto arg_string = p.rule("xml-arg-string", + p.ac(p.tool_arg_string_value(p.until("\n\n")) + arg_close, "\n\n")); + + auto tool_choice = p.choice(); + foreach_function(inputs.tools, [&](const json & tool) { + const auto & function = tool.at("function"); + std::string name = function.at("name"); + auto parameters = function.contains("parameters") ? function.at("parameters") : json::object(); + + auto schema_info = common_schema_info(); + schema_info.resolve_refs(parameters); + + std::vector required_args; + std::vector optional_args; + + foreach_parameter(function, [&](const std::string & param_name, const json & param_schema, bool is_required) { + auto rule_name = "tool-" + name + "-arg-" + param_name; + + auto arg_open = p.tool_arg_open("\n"); + + auto arg_value = schema_info.resolves_to_string(param_schema) ? + arg_string : + p.tool_arg_json_value(p.schema(p.json(), rule_name + "-schema", param_schema)) + arg_close; + + auto arg_rule = p.rule(rule_name, p.tool_arg(arg_open + arg_value)); + + (is_required ? required_args : optional_args).push_back(arg_rule); + }); + + // Accept required arguments in any order, as Qwen does not always adhere to the + // order provided. + auto args = p.permute("tool-" + name + "-args", required_args); + if (!optional_args.empty()) { + args = args + p.zero_or_more(p.choice(optional_args)); + } + + auto func = p.tool(p.tool_open("\n") + + p.tool_args(args) + + p.tool_close(p.literal("\n"))); + + tool_choice |= p.rule("tool-" + name, func); + }); + + auto min_calls = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED ? 1 : 0; + + // Qwen3-Coder models may occasionally omit the token. + auto tool_call_body = tool_choice + "" + p.space(); + auto tool_call_first = p.rule("tool-call-first", p.optional(p.literal("\n")) + tool_call_body); + auto tool_call = p.rule("tool-call", "\n" + tool_call_body); + + auto calls = inputs.parallel_tool_calls ? tool_call_first + p.zero_or_more(tool_call) : tool_call_first; + auto tool_calls = p.trigger_rule("tool-call-root", p.repeat(calls, min_calls, 1)); + + return generation_prompt + + (reasoning << p.content(p.until_one_of({ "", "" }, + // Trigger on " common_chat_try_specialized_template( return common_chat_params_init_minicpm5(tmpl, params); } + // Qwen3-Coder XML tool calls, also used by Nemotron Nano 3, Qwen3.5 and StepFun-3.5-Flash + if (src.find("") != std::string::npos && + src.find(" #include +#include #include #include "nlohmann/json.hpp" @@ -21,6 +22,7 @@ static void test_example_qwen3_non_coder(testing & t); static void test_command7_parser_compare(testing & t); static void test_prefix_tool_names(testing & t); static void test_tagged_peg_parser(testing & t); +static void test_permute(testing & t); int main(int argc, char * argv[]) { testing t(std::cout); @@ -39,6 +41,7 @@ int main(int argc, char * argv[]) { t.test("comparison", test_command7_parser_compare); t.test("prefix tool names", test_prefix_tool_names); t.test("tagged peg parser", test_tagged_peg_parser); + t.test("permute", test_permute); return t.summary(); } @@ -981,3 +984,103 @@ static void test_tagged_peg_parser(testing & t) { t.assert_equal("fun_post should be '>'", ">", result.tags["fun_post"]); }); } + +static void test_permute(testing & t) { + auto accepts = [](const common_peg_arena & parser, const std::string & input) { + common_peg_parse_context ctx(input); + return parser.parse(ctx).success(); + }; + + auto gbnf_of = [](const common_peg_arena & parser) { + return build_grammar([&](const common_grammar_builder & builder) { parser.build_grammar(builder); }); + }; + + auto assert_gbnf_equal = [](testing & t, const std::string & expected, const std::string & actual) { + static const std::regex leading_ws_re = std::regex(R"((^|\n)\s+)"); + t.assert_equal("gbnf are equal", std::regex_replace(expected, leading_ws_re, "$1"), actual); + }; + + auto count_rules = [](const std::string & gbnf, const std::string & prefix) { + size_t count = 0; + for (const auto & line : string_split(gbnf, '\n')) { + if (line.rfind(prefix, 0) == 0) { + count++; + } + } + return count; + }; + + t.test("accepts every ordering", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); + }); + + for (const std::string input : { "abc", "acb", "bac", "bca", "cab", "cba" }) { + t.assert_true("accepts " + input, accepts(parser, input)); + } + }); + + t.test("single element", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("a", { p.literal("a") }) + p.end(); + }); + + t.assert_true("accepts a", accepts(parser, "a")); + t.assert_true("rejects aa", !accepts(parser, "aa")); + }); + + t.test("grammar left-factorizes shared tails", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abc", { p.literal("a"), p.literal("b"), p.literal("c") }) + p.end(); + }); + + // Every rule is one remaining subset, keyed by bitmask: abc-3 is {a,b}, abc-7 is {a,b,c}. + // Each subset is emitted once and shared by every branch that leads into it. + assert_gbnf_equal(t, R"""( + abc-1 ::= "a" + abc-2 ::= "b" + abc-3 ::= "a" abc-2 | "b" abc-1 + abc-4 ::= "c" + abc-5 ::= "a" abc-4 | "c" abc-1 + abc-6 ::= "b" abc-4 | "c" abc-2 + abc-7 ::= "a" abc-6 | "b" abc-5 | "c" abc-3 + root ::= abc-7 + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); + + t.test("grammar emits one rule per remaining subset", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("abcd", { p.literal("a"), p.literal("b"), p.literal("c"), p.literal("d") }) + p.end(); + }); + + // 2^4 - 1 non-empty subsets, one rule each - not the 4! = 24 orderings. + t.assert_equal("permute rule count", 15u, count_rules(gbnf_of(parser), "abcd-")); + }); + + t.test("grammar emits no rules for a single element", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + return p.permute("a", { p.literal("a") }) + p.end(); + }); + + assert_gbnf_equal(t, R"""( + root ::= "a" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); + + t.test("grammar falls back to the given order when too large", [&](testing & t) { + auto parser = build_chat_peg_parser([](common_chat_peg_builder & p) { + std::vector parsers; + for (size_t i = 0; i <= COMMON_CHAT_MAX_PERMUTE; i++) { + parsers.push_back(p.literal(std::string(1, (char) ('a' + i)))); + } + return p.permute("big", parsers) + p.end(); + }); + + assert_gbnf_equal(t, R"""( + root ::= "a" "b" "c" "d" "e" "f" "g" + space ::= | " " | "\n"{1,2} [ \t]{0,20} + )""", gbnf_of(parser)); + }); +} diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index ef02fdde57..8d571d369e 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -2278,46 +2278,39 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); - // tool call segment in reasoning + // a tool call ends the prefilled thinking block, with or without a closing tst.test( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n\n\n" "\n" - "\n" - "\n" - "def hello():\n" - " print(\"Hello, world!\")\n" - "\n" - "hello()\n" + "\n" + "\n" + "pwd\n" "\n" "\n" "") .enable_thinking(true) .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ - python_tool - }) - .expect_reasoning( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "") + .tools({ run_in_terminal_tool }) .expect_tool_calls({ - { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} }, + { "run_in_terminal", R"({"command": "pwd"})", {} }, + }) + .run(); + + // ...including after the model has thought about it + tst.test( + "Need to inspect the current directory.\n" + "\n" + "\n" + "\n" + "pwd\n" + "\n" + "\n" + "") + .enable_thinking(true) + .reasoning_format(COMMON_REASONING_FORMAT_AUTO) + .tools({ run_in_terminal_tool }) + .expect_reasoning("Need to inspect the current directory.") + .expect_tool_calls({ + { "run_in_terminal", R"({"command": "pwd"})", {} }, }) .run(); @@ -2461,17 +2454,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { }) .run(); - tst.test( - "I might call later, but I am still thinking.\n" - "\n\n" - "Final answer without tools.") - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .enable_thinking(true) - .tools({ run_in_terminal_tool }) - .expect_reasoning("I might call later, but I am still thinking.") - .expect_content("Final answer without tools.") - .run(); - // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -2776,49 +2758,6 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_content(R"({"amount": 123.45, "date": "2025-12-03"})") .run(); - // tool call segment in reasoning - tst.test( - "Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n\n" - "\n" - "\n" - "\n" - "def hello():\n" - " print(\"Hello, world!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n" - ) - .enable_thinking(true) - .reasoning_format(COMMON_REASONING_FORMAT_AUTO) - .tools({ - python_tool - }) - .expect_reasoning("Let's call a tool: \n" - "\n" - "\n" - "def hello():\n" - " print(\"Not the real call!\")\n" - "\n" - "hello()\n" - "\n" - "\n" - "\n") - .expect_tool_calls({ - { "python", "{\"code\": \"def hello():\\n print(\\\"Hello, world!\\\")\\n\\nhello()\"}", {} }, - }) - .run(); - // Continuation tests tst.test("world!\nWhat's up?") .reasoning_format(COMMON_REASONING_FORMAT_AUTO) @@ -3572,6 +3511,61 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reconstruction() .run(); + // Some models skip the opening and go straight to + tst.test( + "\n" + "\n" + "1\n" + "\n" + "\n" + "") + .tools({ special_function_tool }) + .expect(message_assist_call) + .run(); + + tst.test( + "Let me call it.\n" + "\n" + "\n" + "1\n" + "\n" + "\n" + "") + .tools({ special_function_tool }) + .expect_content("Let me call it.\n") + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + }) + .run(); + + // Only the first call may omit it, the rest keep the \n separator + tst.test( + "\n" + "\n" + "1\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "\n" + "1\n" + "\n" + "\n" + "2\n" + "\n" + "\n" + "") + .parallel_tool_calls(true) + .tools({ + special_function_tool, special_function_tool_with_optional_param + }) + .expect_tool_calls({ + { "special_function", R"({"arg1": 1})", {} }, + { "special_function_with_opt", R"({"arg1": 1, "arg2": 2})", {} }, + }) + .run(); + tst.test( "\n" "\n" @@ -3680,6 +3674,37 @@ static void test_template_output_peg_parsers(bool detailed_debug) { .expect_reconstruction() .run(); + // Test flexible required argument ordering (required args still come first, in any order) + tst.test( + "\n" + "\n" + "\n#include\n\n" + "\nfoo.c\n\n" + "\n#iclunde\n\n" + "\n" + "") + .tools({ edit_tool }) + .expect_tool_calls({ + { "edit", R"({"newString": "#include", "filename": "foo.c", "oldString": "#iclunde"})", {} }, + }) + .expect_reconstruction() + .run(); + + tst.test( + "\n" + "\n" + "\n42\n\n" + "\nhello\n\n" + "\n200\n\n" + "\n" + "") + .tools({ tool_2req_4opt }) + .expect_tool_calls({ + { "tool_2req_4opt", R"({"req2": 42, "req1": "hello", "opt2": 200})", {} }, + }) + .expect_reconstruction() + .run(); + // Test flexible optional argument ordering (2 required + 4 optional, reversed optional order) tst.test( "\n"