upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-02 19:07:25 -05:00
Code Issues Packages Projects Releases Wiki Activity

llama: load qwen4exp (Qwen3.8-Flash-Next) hparams and tensors

Browse Source 
Adds LLM_ARCH_QWEN4EXP with its hparams and tensor loading. The graph
comes in the next commit; this makes the model load and report correct
metadata.

- hyper-connections set n_embd_out_impl = hc_count * n_embd, so the
  residual stream is 4x wide and there is no output_norm: the final
  mixer's hc_norm is the last norm in the model.
- registered as hybrid and given the same recurrent/attention memory
  filters as Qwen3-Next and Qwen3.5.
- reuses the existing indexer, per_layer_token_embd, SSM and
  compress_ratios keys as-is.
- the PLE table row count is read back from the file rather than
  recomputing the vocab padding rule.

llama-model-loader gains UINT64 array support. That branch previously
threw, so no existing caller changes behaviour; it is needed because the
PLE hash multipliers do not fit in int32.
This commit is contained in:
danielhanchen
2026-08-26 13:40:35 +00:00
committed by Daniel Han
parent 6e5b8b7e4d
commit 331fa7e560
10 changed files with 384 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/llama-model.cpp
+5 -1
View File
@@ -319,6 +319,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_qwen35(params);
case LLM_ARCH_QWEN35MOE:
return new llama_model_qwen35moe(params);
case LLM_ARCH_QWEN4EXP:
return new llama_model_qwen4exp(params);
case LLM_ARCH_MISTRAL3:
return new llama_model_mistral3(params);
case LLM_ARCH_EAGLE3:
@@ -927,6 +929,7 @@ const char * llm_type_name(llm_type type) {
case LLM_TYPE_35B_A3B: return "35B.A3B";
case LLM_TYPE_48B_A3B: return "48B.A3B";
case LLM_TYPE_80B_A3B: return "80B.A3B";
case LLM_TYPE_A3B: return "A3B";
case LLM_TYPE_100B_A6B: return "100B.A6B";
case LLM_TYPE_102B_A12B: return "102B.A12B";
case LLM_TYPE_106B_A12B: return "106B.A12B";
@@ -2441,7 +2444,7 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
filter_recr = [&](uint32_t il) {
return hparams.is_recr(il) && hparams.n_ff(il) == 0;
};
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_MINIMAX_01) {
} else if (arch == LLM_ARCH_QWEN3NEXT || arch == LLM_ARCH_QWEN35 || arch == LLM_ARCH_QWEN35MOE || arch == LLM_ARCH_QWEN4EXP || arch == LLM_ARCH_MINIMAX_01) {
filter_attn = [&](uint32_t il) {
return il < hparams.n_layer() && !hparams.is_recr(il);
};
@@ -2891,6 +2894,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_QWEN3VLMOE:
case LLM_ARCH_QWEN35:
case LLM_ARCH_QWEN35MOE:
case LLM_ARCH_QWEN4EXP:
case LLM_ARCH_QWEN3TTS:
return LLAMA_ROPE_TYPE_IMROPE;