chat : add qwen3 specialized parser (#26252)

* Add tagged thinking tool parser

* chat : refactor and add permute helper

* cont : add support for <tool_call> omission

* cont : update tool delimiters

* cont : add comment for qwen3-coder

* cont : fix trigger pattern for <function

---------

Co-authored-by: Bart de Boer <bart.deboer@gmail.com>
This commit is contained in:
Aldehir Rojas
2026-08-02 04:13:20 -05:00
committed by GitHub
parent 272700b360
commit f5919bf458
5 changed files with 433 additions and 86 deletions

View File

@@ -6,6 +6,9 @@
#include <nlohmann/json.hpp>
#include <cstdint>
#include <functional>
using ordered_json = nlohmann::ordered_json;
static std::string_view trim_trailing_space(std::string_view sv, int max = -1) {
@@ -235,6 +238,43 @@ common_peg_parser common_chat_peg_builder::tag_with_safe_content(const std::stri
return zero_or_more(choice({ p, content_chunk }));
}
common_peg_parser common_chat_peg_builder::permute(const std::string & rule_prefix,
const std::vector<common_peg_parser> & parsers) {
if (parsers.empty()) {
return eps();
}
if (parsers.size() == 1 || parsers.size() > COMMON_CHAT_MAX_PERMUTE) {
return sequence(parsers);
}
std::map<uint32_t, common_peg_parser> rules;
std::function<common_peg_parser(uint32_t)> remaining_of;
remaining_of = [&](uint32_t remaining) -> common_peg_parser {
if (remaining == 0) {
return eps();
}
auto cached = rules.find(remaining);
if (cached != rules.end()) {
return cached->second;
}
auto alternatives = choice();
for (size_t i = 0; i < parsers.size(); i++) {
const uint32_t bit = 1u << i;
if (remaining & bit) {
alternatives |= parsers[i] + remaining_of(remaining & ~bit);
}
}
return rules.emplace(remaining, rule(rule_prefix + "-" + std::to_string(remaining), alternatives)).first->second;
};
return remaining_of((1u << parsers.size()) - 1);
}
std::string & common_chat_peg_mapper::args_target() {
return (current_tool && !current_tool->name.empty()) ? current_tool->arguments : args_buffer;
}