upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-07-23 11:10:55 -05:00
Code Issues Packages Projects Releases Wiki Activity

model: add Hy3 (hy_v3) support with MTP speculative decoding (#25395)

Browse Source 
* model: add Hy3 (hy_v3) architecture support

Adds Tencent Hunyuan 3 (HF architecture HYV3ForCausalLM, GGUF arch
hy_v3): a MoE decoder stack with per-head Q/K RMSNorm, a sigmoid
router with expert selection bias, an always-active ungated shared
expert, and leading dense block(s) (first_k_dense_replace).

The base implementation is ported from charlie12345's fork
(https://github.com/charlie12345/ROCmFPX, src/models/hyv3.cpp),
adapted to current mainline APIs (hparams.n_layer(), build_qkv,
build_moe_ffn with fused gate_up + scale tensors, output_s).

Note: blk.N.exp_probs_b is stored without a .bias suffix for
compatibility with existing hy_v3 GGUFs produced by that fork.

Co-Authored-By: charlie12345 <charlie12345@users.noreply.github.com>
Co-authored-by: Piotr Wilkin <ilintar@gmail.com>
Assisted-by: Claude Fable 5
This commit is contained in:
Satinder Grewal
2026-07-14 10:31:04 +12:00
committed by GitHub
parent 6eddde06a4
commit 2969d6d15d
16 changed files with 882 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
common/chat-auto-parser-generator.cpp
View File

@@ -262,6 +262,10 @@ common_peg_parser analyze_tools::build_func_parser(common_chat_peg_builder & p,
bool matched_atomic = false;
common_peg_parser func_parser = p.eps();
if (!function.args_separator.empty()) {
open = open + p.space() + p.literal(function.args_separator);
}
if (!function.name_suffix.empty()) {
func_parser = open + call_id_section + p.space() + args;
matched_atomic = true;

7
common/chat-auto-parser.h
View File

@@ -192,9 +192,10 @@ struct tool_format_analysis {
};
struct tool_function_analysis {
std::string name_prefix; // e.g., "<function=", "\"name\": \"", "functions."
std::string name_suffix; // e.g., ">", "\"", ":0"
std::string close; // e.g., "</function>", "" (for tag-based)
std::string name_prefix; // e.g., "<function=", "\"name\": \"", "functions."
std::string name_suffix; // e.g., ">", "\"", ":0"
std::string args_separator; // e.g., "<tool_sep>" (marker between function name and arguments)
std::string close; // e.g., "</function>", "" (for tag-based)
};
struct tool_arguments_analysis {

30
common/chat-diff-analyzer.cpp
View File

@@ -259,6 +259,7 @@ void autoparser::analyze_template(const common_chat_template & tmpl) {
LOG_DBG("per_call_end: '%s'\n", tools.format.per_call_end.c_str());
LOG_DBG("func_name_prefix: '%s'\n", tools.function.name_prefix.c_str());
LOG_DBG("func_name_suffix: '%s'\n", tools.function.name_suffix.c_str());
LOG_DBG("func_args_separator: '%s'\n", tools.function.args_separator.c_str());
LOG_DBG("func_close: '%s'\n", tools.function.close.c_str());
LOG_DBG("call_id_prefix: '%s'\n", tools.call_id.prefix.c_str());
LOG_DBG("call_id_suffix: '%s'\n", tools.call_id.suffix.c_str());
@@ -302,6 +303,7 @@ void autoparser::collect_preserved_tokens() {
add_token(tools.format.per_call_end);
add_token(tools.function.name_prefix);
add_token(tools.function.name_suffix);
add_token(tools.function.args_separator);
add_token(tools.function.close);
add_token(tools.arguments.start);
add_token(tools.arguments.end);
@@ -1051,6 +1053,23 @@ void analyze_tools::check_per_call_markers() {
format.section_start.clear();
format.section_end.clear();
}
if (!format.per_call_end.empty()) {
auto count_occurrences = [](const std::string & haystack, const std::string & needle) {
size_t count = 0;
for (size_t pos = haystack.find(needle); pos != std::string::npos;
pos = haystack.find(needle, pos + needle.size())) {
count++;
}
return count;
};
size_t calls_one = count_occurrences(one_vs_two->output_A, format.per_call_end);
size_t calls_two = count_occurrences(one_vs_two->output_B, format.per_call_end);
if (calls_one > 0 && calls_one == calls_two) {
format.section_end = format.per_call_end;
format.per_call_end.clear();
}
}
}
void analyze_tools::extract_function_markers() {
@@ -1132,6 +1151,17 @@ void analyze_tools::extract_function_markers() {
auto suf_result = suffix_parser.parse_and_extract(diff.suffix);
if (suf_result.result.success()) {
function.name_suffix += suf_result.tags["ext"];
auto arg_start = [&](common_peg_parser_builder &p) {
return p.marker() + p.space() + p.choice({ p.literal(ARG_FIRST), p.literal(ARG_SECOND) });
};
auto sep_parser = build_tagged_peg_parser([&](common_peg_parser_builder &p) {
return p.tag("sep", p.zero_or_more(p.negate(arg_start(p)) + p.any())) + arg_start(p);
});
auto sep_result = sep_parser.parse_and_extract(diff.suffix.substr(suf_result.tags["ext"].size()));
if (sep_result.result.success()) {
function.args_separator = trim_whitespace(sep_result.tags["sep"]);
}
}
}

39
common/jinja/value.cpp
View File

@@ -750,11 +750,50 @@ const func_builtins & value_string_t::get_builtins() const {
res->val_str.mark_input_based_on(args.get_pos(0)->val_str);
return res;
}},
{"format", [](const func_args & args) -> value {
value val_input = args.get_pos(0);
if (!is_val<value_string>(val_input)) {
throw raised_exception("format() first argument must be a string");
}
const jinja::string & fmt = val_input->as_string();
const bool fmt_is_input = fmt.all_parts_are_input();
const std::string str = fmt.str();
jinja::string result;
std::string literal;
auto flush_literal = [&]() {
if (!literal.empty()) {
result.parts.push_back({fmt_is_input, literal});
literal.clear();
}
};
size_t arg_idx = 1; // positional args follow the format string
for (size_t i = 0; i < str.size(); ++i) {
if (str[i] != '{') {
literal += str[i];
continue;
}
if (i + 1 >= str.size() || str[i + 1] != '}') {
throw not_implemented_exception("format() only supports simple '{}' placeholders");
}
++i;
flush_literal();
const jinja::string arg_str = args.get_pos(arg_idx++)->as_string();
result.parts.insert(result.parts.end(), arg_str.parts.begin(), arg_str.parts.end());
}
flush_literal();
return mk_val<value_string>(result);
}},
{"int", [](const func_args & args) -> value {
value val_input = args.get_pos(0);
value val_default = args.get_kwarg_or_pos("default", 1);
value val_base = args.get_kwarg_or_pos("base", 2);
const int base = val_base->is_undefined() ? 10 : val_base->as_int();
if (base != 0 && (base < 2 || base > 36)) {
// an out-of-range base makes std::stoi fail fast on the MSVC CRT instead of throwing
throw raised_exception("int() base must be 0 or between 2 and 36");
}
if (is_val<value_string>(val_input) == false) {
throw raised_exception("int() first argument must be a string");
}