diff --git a/LLAMA_CPP_VERSION b/LLAMA_CPP_VERSION index fd41de5c8..1c1f72d52 100644 --- a/LLAMA_CPP_VERSION +++ b/LLAMA_CPP_VERSION @@ -1 +1 @@ -b10069 \ No newline at end of file +b10091 diff --git a/integration/thinking_test.go b/integration/thinking_test.go index 4a94178bf..0bd0e0dd0 100644 --- a/integration/thinking_test.go +++ b/integration/thinking_test.go @@ -11,8 +11,29 @@ import ( "github.com/ollama/ollama/api" ) -// runThinkingEnabled verifies that when thinking is requested, the model -// produces both thinking and content output without leaking raw channel tags. +var rawThinkingProtocolTags = []string{ + "<|channel>", + "", + "", + "", + "", + "", + "", + "", +} + +func rejectRawThinkingProtocolTags(t *testing.T, field, value string) { + t.Helper() + for _, tag := range rawThinkingProtocolTags { + if strings.Contains(value, tag) { + t.Errorf("%s contains raw protocol tag %q: %s", field, tag, value) + } + } +} + +// runThinkingEnabled verifies that thinking-capable models honor an explicit +// thinking request, complete a reasoning trace, and return the final answer +// without leaking raw channel tags. func runThinkingEnabled(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() @@ -33,12 +54,15 @@ func runThinkingEnabled(t *testing.T) { Stream: &stream, Think: &think, Messages: []api.Message{ - {Role: "user", Content: "What is 12 * 15? Think step by step."}, + {Role: "user", Content: "What is 12 multiplied by 15? Give the final answer."}, }, Options: map[string]any{ "temperature": 0, "seed": 42, - "num_predict": 512, + // Deep-thinking models can use several thousand tokens on + // simple problems before producing their final answer. Keep + // this high enough to verify a natural stop, not truncation. + "num_predict": 8192, }, } @@ -61,22 +85,17 @@ func runThinkingEnabled(t *testing.T) { if thinking == "" { t.Error("expected non-empty thinking output when thinking is enabled") } - - // The answer (180) should appear in thinking, content, or both. - // Some models put everything in thinking and leave content empty - // if they hit the token limit while still thinking. - combined := thinking + " " + content - if !strings.Contains(combined, "180") { - t.Errorf("expected '180' in thinking or content, got thinking=%q content=%q", thinking, content) + if content == "" { + t.Error("expected non-empty final content after thinking") + } else if !strings.Contains(content, "180") { + t.Errorf("expected final answer 180, got content=%q", content) + } + if response.DoneReason != "stop" { + t.Errorf("expected completed response, got done reason %q", response.DoneReason) } - // Neither thinking nor content should contain raw channel tags - if strings.Contains(content, "<|channel>") || strings.Contains(content, "") { - t.Errorf("content contains raw channel tags: %s", content) - } - if strings.Contains(thinking, "<|channel>") || strings.Contains(thinking, "") { - t.Errorf("thinking contains raw channel tags: %s", thinking) - } + rejectRawThinkingProtocolTags(t, "content", content) + rejectRawThinkingProtocolTags(t, "thinking", thinking) t.Logf("thinking (%d chars): %.100s...", len(thinking), thinking) t.Logf("content (%d chars): %s", len(content), content) @@ -84,7 +103,7 @@ func runThinkingEnabled(t *testing.T) { } } -// runThinkingSuppressed verifies that when thinking is NOT requested, +// runThinkingSuppressed verifies that when thinking is explicitly disabled, // the model does not leak thinking/channel content into the response. func runThinkingSuppressed(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) @@ -100,10 +119,11 @@ func runThinkingSuppressed(t *testing.T) { pullOrSkip(ctx, t, client, modelName) stream := false + think := api.ThinkValue{Value: false} req := api.ChatRequest{ Model: modelName, Stream: &stream, - // Think is nil — thinking not requested + Think: &think, Messages: []api.Message{ {Role: "user", Content: "What is the capital of Japan? Answer in one word."}, }, @@ -129,24 +149,16 @@ func runThinkingSuppressed(t *testing.T) { content := response.Message.Content thinking := response.Message.Thinking - // The answer should appear in content or thinking - combined := content + " " + thinking - if !strings.Contains(combined, "Tokyo") { - t.Errorf("expected 'Tokyo' in content or thinking, got content=%q thinking=%q", content, thinking) + // With thinking disabled, the answer must be returned as content. + if !strings.Contains(content, "Tokyo") { + t.Errorf("expected 'Tokyo' in content, got content=%q thinking=%q", content, thinking) } - // Content must NOT contain channel/thinking tags - if strings.Contains(content, "<|channel>") || strings.Contains(content, "") { - t.Errorf("content contains leaked channel tags when thinking not requested: %s", content) - } - if strings.Contains(content, "thought") && strings.Contains(content, "") { - t.Errorf("content contains leaked thinking block: %s", content) - } + rejectRawThinkingProtocolTags(t, "content", content) + rejectRawThinkingProtocolTags(t, "thinking", thinking) - // Thinking field should ideally be empty when not requested. - // Some small models may still produce thinking output; log but don't fail. if thinking != "" { - t.Logf("WARNING: model produced thinking output when not requested (%d chars): %.100s...", len(thinking), thinking) + t.Errorf("expected empty thinking when thinking is disabled, got %q", thinking) } t.Logf("content: %s", content) diff --git a/llama/compat/README.md b/llama/compat/README.md index a944c89bf..a6306b2b9 100644 --- a/llama/compat/README.md +++ b/llama/compat/README.md @@ -88,6 +88,7 @@ This table tracks the dispatch surface. Keep it brief; the handler comments in | `deepseekocr` | Maps to `deepseek2-ocr`, injects missing OCR/MoE metadata, and hides embedded SAM/vision/projector tensors. | DeepSeek OCR projector translation. | | `glmocr` | Maps GLM OCR metadata/tensors to the llama.cpp-compatible view. | GLM OCR projector translation. | | `glm4moelite` | Maps GLM-4.7 Flash MLA metadata to the `deepseek2` path and fixes special-token metadata. | n/a | +| `laguna` | Renames legacy attention-gate tensors and SWA RoPE metadata to current llama.cpp names. | n/a | | `nemotron_h_moe` | Fixes latent-FFN variants and hides MTP tensors. | n/a | | `nemotron_h_omni` | Selects the Nemotron text loader and hides audio/vision/projector tensors from the text loader. | Nemotron V2 VL projector translation; audio remains disabled. | | `llama` with Llama 3 markers | Fixes Llama 3 tokenizer metadata. | n/a | diff --git a/llama/compat/llama-ollama-compat.cpp b/llama/compat/llama-ollama-compat.cpp index 3c06c3cb2..ce5abd59e 100644 --- a/llama/compat/llama-ollama-compat.cpp +++ b/llama/compat/llama-ollama-compat.cpp @@ -118,6 +118,43 @@ void fix_glm4moelite_eog_token_ids(gguf_context * meta) { } } +// ========================================================================= +// laguna (text side) +// ========================================================================= + +bool detect_ollama_laguna(const gguf_context * meta, const ggml_context * ctx) { + const int64_t arch_kid = gguf_find_key(meta, "general.architecture"); + if (arch_kid < 0 || std::strcmp(gguf_get_val_str(meta, arch_kid), "laguna") != 0) return false; + + return has_key(meta, "laguna.rope.swa.dimension_count") + || has_key(meta, "laguna.rope.swa.freq_base") + || ggml_get_tensor(const_cast(ctx), "blk.0.attn_g.weight") != nullptr; +} + +void handle_laguna(gguf_context * meta, ggml_context * ctx) { + if (!detect_ollama_laguna(meta, ctx)) return; + + OLLAMA_COMPAT_LOG_INFO("%s: detected Ollama-format laguna GGUF; applying compatibility fixes\n", __func__); + + copy_u32_kv(meta, "laguna.rope.swa.dimension_count", "laguna.rope.dimension_count_swa"); + copy_f32_kv(meta, "laguna.rope.swa.freq_base", "laguna.rope.freq_base_swa"); + + std::vector> renames; + const int64_t n = gguf_get_n_tensors(meta); + constexpr const char * old_suffix = ".attn_g.weight"; + constexpr const char * new_suffix = ".attn_gate.weight"; + for (int64_t i = 0; i < n; ++i) { + const std::string name(gguf_get_tensor_name(meta, i)); + const size_t pos = name.rfind(old_suffix); + if (pos != std::string::npos && pos + std::strlen(old_suffix) == name.size()) { + renames.emplace_back(name, name.substr(0, pos) + new_suffix); + } + } + for (const auto & [from, to] : renames) { + rename_tensor(meta, ctx, from.c_str(), to.c_str()); + } +} + bool get_u32_kv(const gguf_context * meta, const char * key, uint32_t & out) { const int64_t kid = gguf_find_key(meta, key); if (kid < 0) return false; @@ -3221,6 +3258,7 @@ bool translate_metadata(const llama_model_loader * ml, if (arch_name == "qwen35moe") handle_qwen35moe(ml, meta, ctx); if (arch_name == "qwen35") handle_qwen35 (ml, meta, ctx); if (arch_name == "qwen3next") handle_qwen3next(meta, ctx); + if (arch_name == "laguna") handle_laguna (meta, ctx); if (arch_name == "gptoss") handle_gptoss (ml, meta, ctx, arch_name); if (arch_name == "lfm2") handle_lfm2 (ml, meta, ctx); if (arch_name == "olmo3") handle_olmo3 (meta, arch_name); diff --git a/llama/compat/models/003-llama-cpp-laguna-metal.patch b/llama/compat/models/003-llama-cpp-laguna-metal.patch new file mode 100644 index 000000000..689adb93d --- /dev/null +++ b/llama/compat/models/003-llama-cpp-laguna-metal.patch @@ -0,0 +1,42 @@ +diff --git a/src/models/laguna.cpp b/src/models/laguna.cpp +index 1d705440e..7e708b144 100644 +--- a/src/models/laguna.cpp ++++ b/src/models/laguna.cpp +@@ -275,16 +275,35 @@ llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_para + if ((uint32_t)il >= hparams.n_layer_dense_lead) { + // MoE: sigmoid routing + score-correction bias + sum-norm + + // routed_scaling_factor (all handled by build_moe_ffn). ++ ggml_tensor * up_scale = nullptr; ++ float expert_weights_scale = hparams.expert_weights_scale; ++ ++#if defined(GGML_USE_METAL) ++ const ggml_type down_type = model.layers[il].ffn_down_exps->type; ++ if (n_tokens >= 32 && (ggml_is_quantized(down_type) || down_type == GGML_TYPE_F16)) { ++ // At 32 prompt tokens, Metal switches MUL_MAT_ID from its ++ // range-safe matrix-vector kernel to FP16 matrix tiles. ++ // Laguna's routed SwiGLU activations can overflow those tiles. ++ // Generation uses one token and does not need this workaround. ++ constexpr float down_input_scale = 1.0f / 256.0f; ++ up_scale = ggml_fill(ctx0, ++ model.layers[il].ffn_exp_probs_b, down_input_scale); ++ expert_weights_scale /= down_input_scale; ++ } ++#endif ++ + ggml_tensor * moe_out = build_moe_ffn(cur, + model.layers[il].ffn_gate_inp, + model.layers[il].ffn_up_exps, + model.layers[il].ffn_gate_exps, + model.layers[il].ffn_down_exps, + model.layers[il].ffn_exp_probs_b, + n_expert, n_expert_used, + LLM_FFN_SILU, + hparams.expert_weights_norm, +- hparams.expert_weights_scale, ++ expert_weights_scale, + (llama_expert_gating_func_type) hparams.expert_gating_func, +- il); ++ il, ++ nullptr, nullptr, ++ up_scale); + cb(moe_out, "ffn_moe_out", il); diff --git a/llama/compat/models/003-llama-cpp-laguna.patch b/llama/compat/models/003-llama-cpp-laguna.patch deleted file mode 100644 index e6536a731..000000000 --- a/llama/compat/models/003-llama-cpp-laguna.patch +++ /dev/null @@ -1,93 +0,0 @@ -diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp ---- a/src/llama-arch.cpp -+++ b/src/llama-arch.cpp -@@ -136,2 +136,3 @@ static const std::map LLM_ARCH_NAMES = { - { LLM_ARCH_MELLUM, "mellum" }, -+ { LLM_ARCH_LAGUNA, "laguna" }, - { LLM_ARCH_UNKNOWN, "(unknown)" }, -@@ -398,2 +399,3 @@ static const std::map LLM_TENSOR_NAMES = { - { LLM_TENSOR_ATTN_GATE, "blk.%d.attn_gate" }, -+ { LLM_TENSOR_ATTN_GATE_LAGUNA, "blk.%d.attn_g" }, - { LLM_TENSOR_FFN_POST_NORM, "blk.%d.post_ffw_norm" }, -@@ -596,2 +598,3 @@ static const std::map LLM_TENSOR_INFOS = { - {LLM_TENSOR_ATTN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, -+ {LLM_TENSOR_ATTN_GATE_LAGUNA, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_FFN_GATE, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL_MAT}}, -diff --git a/src/llama-arch.h b/src/llama-arch.h ---- a/src/llama-arch.h -+++ b/src/llama-arch.h -@@ -145,2 +145,3 @@ enum llm_arch { - LLM_ARCH_DFLASH, -+ LLM_ARCH_LAGUNA, - LLM_ARCH_UNKNOWN, -@@ -382,2 +383,3 @@ enum llm_tensor { - LLM_TENSOR_ATTN_GATE, -+ LLM_TENSOR_ATTN_GATE_LAGUNA, - LLM_TENSOR_FFN_GATE_INP, -diff --git a/src/llama-model.cpp b/src/llama-model.cpp ---- a/src/llama-model.cpp -+++ b/src/llama-model.cpp -@@ -48,4 +48,6 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params - case LLM_ARCH_TALKIE: - return new llama_model_talkie(params); -+ case LLM_ARCH_LAGUNA: -+ return new llama_model_laguna(params); - case LLM_ARCH_DECI: - return new llama_model_deci(params); -@@ -2525,2 +2527,3 @@ llama_rope_type llama_model_rope_type(const llama_model * model) { - case LLM_ARCH_MELLUM: -+ case LLM_ARCH_LAGUNA: - case LLM_ARCH_DFLASH: -diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp ---- a/src/llama-model-loader.cpp -+++ b/src/llama-model-loader.cpp -@@ -505,2 +505,3 @@ namespace GGUFMeta { - template bool llama_model_loader::get_key_or_arr>(enum llm_kv kid, std::array & result, uint32_t n, bool required); -+ template bool llama_model_loader::get_key_or_arr(const std::string & key, std::array & result, uint32_t n, bool required); - template bool llama_model_loader::get_key_or_arr>(enum llm_kv kid, std::array & result, uint32_t n, bool required); -diff --git a/src/llama-vocab.cpp b/src/llama-vocab.cpp ---- a/src/llama-vocab.cpp -+++ b/src/llama-vocab.cpp -@@ -359,2 +359,8 @@ struct llm_tokenizer_bpe : llm_tokenizer { - break; -+ case LLAMA_VOCAB_PRE_TYPE_LAGUNA: -+ regex_exprs = { -+ "(?:\\r?\\n)+(?!\\r?\\n)", -+ "(?:'[sS]|'[tT]|'[rR][eE]|'[vV][eE]|'[mM]|'[lL][lL]|'[dD])|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+", -+ }; -+ break; - case LLAMA_VOCAB_PRE_TYPE_GPT2: -@@ -2100,2 +2106,4 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { - ignore_merges = true; -+ } else if (tokenizer_pre == "laguna") { -+ pre_type = LLAMA_VOCAB_PRE_TYPE_LAGUNA; - } else if ( -@@ -2773,2 +2781,3 @@ void llama_vocab::impl::load(llama_model_loader & ml, const LLM_KV & kv) { - || t.first == "<｜end▁of▁sentence｜>" // deepseek-ocr -+ || t.first == "" // poolside Laguna (eos_token_ids) - ) { -diff --git a/src/llama-vocab.h b/src/llama-vocab.h ---- a/src/llama-vocab.h -+++ b/src/llama-vocab.h -@@ -65,2 +65,3 @@ enum llama_vocab_pre_type { - LLAMA_VOCAB_PRE_TYPE_MELLUM2 = 55, -+ LLAMA_VOCAB_PRE_TYPE_LAGUNA = 56, - }; -diff --git a/src/models/models.h b/src/models/models.h ---- a/src/models/models.h -+++ b/src/models/models.h -@@ -1657,1 +1657,14 @@ struct llama_model_arcee : public llama_model_base { -+struct llama_model_laguna : public llama_model_base { -+ llama_model_laguna(const struct llama_model_params & params) : llama_model_base(params) {} -+ void load_arch_hparams(llama_model_loader & ml) override; -+ void load_arch_tensors(llama_model_loader & ml) override; -+ -+ struct graph : public llm_graph_context { -+ graph(const llama_model & model, const llm_graph_params & params); -+ }; -+ -+ std::unique_ptr build_arch_graph(const llm_graph_params & params) const override; -+}; -+ -+ - struct llama_model_arcee : public llama_model_base { diff --git a/llama/compat/models/laguna.cpp b/llama/compat/models/laguna.cpp deleted file mode 100644 index 76c8f6c2e..000000000 --- a/llama/compat/models/laguna.cpp +++ /dev/null @@ -1,253 +0,0 @@ -#include "models/models.h" - -void llama_model_laguna::load_arch_hparams(llama_model_loader & ml) { - ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps); - - // MoE - ml.get_key(LLM_KV_LEADING_DENSE_BLOCK_COUNT, hparams.n_layer_dense_lead, false); - ml.get_key(LLM_KV_EXPERT_FEED_FORWARD_LENGTH, hparams.n_ff_exp); - ml.get_key(LLM_KV_EXPERT_SHARED_FEED_FORWARD_LENGTH, hparams.n_ff_shexp, false); - ml.get_key(LLM_KV_EXPERT_SHARED_COUNT, hparams.n_expert_shared, false); - ml.get_key(LLM_KV_EXPERT_WEIGHTS_SCALE, hparams.expert_weights_scale, false); - ml.get_key(LLM_KV_EXPERT_WEIGHTS_NORM, hparams.expert_weights_norm, false); - ml.get_key(LLM_KV_EXPERT_GATING_FUNC, hparams.expert_gating_func, false); - - ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa, false); - hparams.swa_type = LLAMA_SWA_TYPE_STANDARD; - ml.get_key_or_arr("laguna.attention.layer_types", hparams.is_swa_impl, hparams.n_layer(), false); - - ml.get_key("laguna.rope.swa.dimension_count", hparams.n_rot_swa, false); - ml.get_key("laguna.rope.swa.freq_base", hparams.rope_freq_base_train_swa, false); - ml.get_key("laguna.rope.scaling.beta_fast", hparams.yarn_beta_fast, false); - ml.get_key("laguna.rope.scaling.beta_slow", hparams.yarn_beta_slow, false); - - type = LLM_TYPE_UNKNOWN; -} - -void llama_model_laguna::load_arch_tensors(llama_model_loader &) { - LLAMA_LOAD_LOCALS; - - const int64_t n_ff_exp = hparams.n_ff_exp; - const int64_t n_ff_shexp = hparams.n_ff_shexp; - - tok_embd = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, 0); - - output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0); - output = create_tensor(tn(LLM_TENSOR_OUTPUT, "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED); - if (output == NULL) { - output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED); - } - - for (int i = 0; i < n_layer; ++i) { - auto & layer = layers[i]; - - const int64_t n_head_i = hparams.n_head(i); - const int64_t n_head_kv_i = hparams.n_head_kv(i); - const int64_t n_embd_q = n_embd_head_k * n_head_i; - const int64_t n_embd_kv = n_embd_head_k * n_head_kv_i; - - layer.attn_norm = create_tensor(tn(LLM_TENSOR_ATTN_NORM, "weight", i), {n_embd}, 0); - - layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd_q}, 0); - layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_kv}, 0); - layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_kv}, 0); - layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0); - - layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE_LAGUNA, "weight", i), {n_embd, n_head_i}, 0); - - layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0); - layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0); - - layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0); - - if (i < (int) hparams.n_layer_dense_lead) { - layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0); - layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), { n_ff, n_embd}, 0); - layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0); - } else { - layer.ffn_gate_inp = create_tensor(tn(LLM_TENSOR_FFN_GATE_INP, "weight", i), {n_embd, n_expert}, 0); - layer.ffn_exp_probs_b = create_tensor(tn(LLM_TENSOR_FFN_EXP_PROBS_B, "bias", i), {n_expert}, TENSOR_NOT_REQUIRED); - - layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); - layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff_exp, n_embd, n_expert}, 0); - layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), { n_embd, n_ff_exp, n_expert}, 0); - - layer.ffn_gate_shexp = create_tensor(tn(LLM_TENSOR_FFN_GATE_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); - layer.ffn_down_shexp = create_tensor(tn(LLM_TENSOR_FFN_DOWN_SHEXP, "weight", i), {n_ff_shexp, n_embd}, 0); - layer.ffn_up_shexp = create_tensor(tn(LLM_TENSOR_FFN_UP_SHEXP, "weight", i), {n_embd, n_ff_shexp}, 0); - } - } -} - -std::unique_ptr llama_model_laguna::build_arch_graph(const llm_graph_params & params) const { - return std::make_unique(*this, params); -} - -llama_model_laguna::graph::graph(const llama_model & model, const llm_graph_params & params) : - llm_graph_context(params) { - const int64_t n_embd_head = hparams.n_embd_head_v(); - GGML_ASSERT(n_embd_head == hparams.n_embd_head_k()); - - const float kq_scale = 1.0f / sqrtf(float(n_embd_head)); - - ggml_tensor * cur; - ggml_tensor * inpL; - - inpL = build_inp_embd(model.tok_embd); - - ggml_tensor * inp_pos = build_inp_pos(); - - auto * inp_attn = build_attn_inp_kv_iswa(); - - ggml_tensor * inp_out_ids = build_inp_out_ids(); - - for (int il = 0; il < n_layer; ++il) { - ggml_tensor * inpSA = inpL; - - const int64_t n_head_il = hparams.n_head(il); - const int64_t n_head_kv_il = hparams.n_head_kv(il); - const bool is_swa = hparams.is_swa(il); - - const int rope_n_dims = hparams.n_rot(il); - const float rope_base = is_swa ? hparams.rope_freq_base_train_swa : hparams.rope_freq_base_train; - const float rope_scale = is_swa ? hparams.rope_freq_scale_train_swa : hparams.rope_freq_scale_train; - const float rope_ext = is_swa ? 0.0f : 1.0f; - const float rope_bfast = hparams.yarn_beta_fast; - const float rope_bslow = hparams.yarn_beta_slow; - - cur = build_norm(inpL, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il); - cb(cur, "attn_norm", il); - - // self-attention - { - ggml_tensor * Qcur = build_lora_mm(model.layers[il].wq, cur); - cb(Qcur, "Qcur", il); - ggml_tensor * Kcur = build_lora_mm(model.layers[il].wk, cur); - cb(Kcur, "Kcur", il); - ggml_tensor * Vcur = build_lora_mm(model.layers[il].wv, cur); - cb(Vcur, "Vcur", il); - - ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur); - cb(gate, "gate", il); - - Qcur = ggml_reshape_3d(ctx0, Qcur, n_embd_head, n_head_il, n_tokens); - Kcur = ggml_reshape_3d(ctx0, Kcur, n_embd_head, n_head_kv_il, n_tokens); - Vcur = ggml_reshape_3d(ctx0, Vcur, n_embd_head, n_head_kv_il, n_tokens); - - Qcur = build_norm(Qcur, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il); - cb(Qcur, "Qcur_normed", il); - Kcur = build_norm(Kcur, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il); - cb(Kcur, "Kcur_normed", il); - - Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, - rope_n_dims, rope_type, hparams.n_ctx_orig_yarn, rope_base, rope_scale, - rope_ext, hparams.rope_attn_factor, rope_bfast, rope_bslow); - Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, - rope_n_dims, rope_type, hparams.n_ctx_orig_yarn, rope_base, rope_scale, - rope_ext, hparams.rope_attn_factor, rope_bfast, rope_bslow); - - cb(Qcur, "Qcur", il); - cb(Kcur, "Kcur", il); - cb(Vcur, "Vcur", il); - - cur = build_attn(inp_attn, - nullptr, nullptr, nullptr, - Qcur, Kcur, Vcur, nullptr, nullptr, nullptr, kq_scale, il); - cb(cur, "attn_pregate", il); - - gate = ggml_softplus(ctx0, gate); - cur = ggml_reshape_3d(ctx0, cur, n_embd_head, n_head_il, n_tokens); - gate = ggml_reshape_3d(ctx0, gate, 1, n_head_il, n_tokens); - cur = ggml_mul(ctx0, cur, gate); - cur = ggml_reshape_2d(ctx0, cur, n_embd_head * n_head_il, n_tokens); - cb(cur, "attn_gated", il); - - cur = build_lora_mm(model.layers[il].wo, cur, model.layers[il].wo_s); - cb(cur, "attn_out", il); - } - - if (il == n_layer - 1 && inp_out_ids) { - cur = ggml_get_rows(ctx0, cur, inp_out_ids); - inpSA = ggml_get_rows(ctx0, inpSA, inp_out_ids); - } - - ggml_tensor * ffn_inp = ggml_add(ctx0, cur, inpSA); - cb(ffn_inp, "ffn_inp", il); - - // feed-forward - cur = build_norm(ffn_inp, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il); - cb(cur, "ffn_norm", il); - - if ((uint32_t) il < hparams.n_layer_dense_lead) { - cur = build_ffn(cur, - model.layers[il].ffn_up, NULL, NULL, - model.layers[il].ffn_gate, NULL, NULL, - model.layers[il].ffn_down, NULL, NULL, - NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); - cb(cur, "ffn_out", il); - } else { - ggml_tensor * up_scale = nullptr; - float expert_weights_scale = hparams.expert_weights_scale; - -#if defined(GGML_USE_METAL) - if (n_tokens >= 32 && ggml_is_quantized(model.layers[il].ffn_down_exps->type)) { - // ggml-metal switches MUL_MAT_ID from its range-safe - // matrix-vector kernel to FP16 matrix tiles at 32 tokens - // (ne21_mm_id_min in ggml_metal_op_mul_mat_id). Laguna's routed - // SwiGLU activations can overflow those tiles. Scale the linear - // up branch and fold the inverse power-of-two factor into the - // existing routing-weight scale. Revisit this guard if the - // Metal dispatch threshold changes. - constexpr float down_input_scale = 1.0f / 256.0f; - up_scale = ggml_fill(ctx0, - model.layers[il].ffn_exp_probs_b, down_input_scale); - expert_weights_scale /= down_input_scale; - } -#endif - - ggml_tensor * moe_out = build_moe_ffn(cur, - model.layers[il].ffn_gate_inp, - model.layers[il].ffn_up_exps, - model.layers[il].ffn_gate_exps, - model.layers[il].ffn_down_exps, - model.layers[il].ffn_exp_probs_b, - n_expert, n_expert_used, - LLM_FFN_SILU, hparams.expert_weights_norm, - expert_weights_scale, - (llama_expert_gating_func_type) hparams.expert_gating_func, - il, - nullptr, nullptr, - up_scale); - cb(moe_out, "ffn_moe_out", il); - - ggml_tensor * ffn_shexp = build_ffn(cur, - model.layers[il].ffn_up_shexp, NULL, NULL, - model.layers[il].ffn_gate_shexp, NULL, NULL, - model.layers[il].ffn_down_shexp, NULL, NULL, - NULL, LLM_FFN_SILU, LLM_FFN_PAR, il); - cb(ffn_shexp, "ffn_shexp", il); - - cur = ggml_add(ctx0, moe_out, ffn_shexp); - cb(cur, "ffn_out", il); - } - - cur = ggml_add(ctx0, cur, ffn_inp); - - cur = build_cvec(cur, il); - cb(cur, "l_out", il); - - inpL = cur; - } - - cur = inpL; - - cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1); - cb(cur, "result_norm", -1); - res->t_embd = cur; - - cur = build_lora_mm(model.output, cur, model.output_s); - cb(cur, "result_output", -1); - res->t_logits = cur; - - ggml_build_forward_expand(gf, cur); -} diff --git a/model/parsers/laguna_test.go b/model/parsers/laguna_test.go index e0dd055ab..cd97ca88d 100644 --- a/model/parsers/laguna_test.go +++ b/model/parsers/laguna_test.go @@ -535,12 +535,12 @@ func TestLagunaParserNonAssistantLastMessageStillPrimesThinking(t *testing.T) { func TestLagunaV8ParserAssistantHistoryStillPrimesThinking(t *testing.T) { // Laguna v8 closes assistant history and emits a fresh generation prompt, // so an assistant tail message must not switch the parser into prefill mode. - parser := ParserForName("laguna-v8") + parser := ParserForName("poolside-v1") if parser == nil { - t.Fatal("expected laguna-v8 parser") + t.Fatal("expected poolside-v1 parser") } if !parser.HasToolSupport() || !parser.HasThinkingSupport() { - t.Fatal("laguna-v8 parser should advertise tools and thinking") + t.Fatal("poolside-v1 parser should advertise tools and thinking") } parser.Init(nil, &api.Message{Role: "assistant", Content: "Previous."}, &api.ThinkValue{Value: true}) diff --git a/model/parsers/parsers.go b/model/parsers/parsers.go index 17c6025c7..6791d15bd 100644 --- a/model/parsers/parsers.go +++ b/model/parsers/parsers.go @@ -94,7 +94,7 @@ func ParserForName(name string) Parser { return &LFM2Parser{hasThinkingSupport: true} case "laguna": return &LagunaParser{} - case "laguna-v8": + case "poolside-v1": return &LagunaV8Parser{} case "cohere": return &CohereParser{} diff --git a/model/renderers/renderer.go b/model/renderers/renderer.go index 4223806c8..db4d9528d 100644 --- a/model/renderers/renderer.go +++ b/model/renderers/renderer.go @@ -109,7 +109,7 @@ func rendererForName(name string) Renderer { return &LFM2Renderer{IsThinking: true, useImgTags: RenderImgTags} case "laguna": return &LagunaRenderer{} - case "laguna-v8": + case "poolside-v1": return &LagunaV8Renderer{} case "cohere": return &CohereRenderer{} diff --git a/model/renderers/renderer_test.go b/model/renderers/renderer_test.go index 134d1b45f..082949dd0 100644 --- a/model/renderers/renderer_test.go +++ b/model/renderers/renderer_test.go @@ -69,7 +69,7 @@ func TestLeadingBOSForRenderer(t *testing.T) { {name: "lfm2", want: "<|startoftext|>"}, {name: "lfm2-thinking", want: "<|startoftext|>"}, {name: "laguna", want: "〈|EOS|〉"}, - {name: "laguna-v8", want: "〈|EOS|〉"}, + {name: "poolside-v1", want: "〈|EOS|〉"}, {name: "deepseek3.1", want: "<｜begin▁of▁sentence｜>"}, {name: "cogito", want: "<｜begin▁of▁sentence｜>"}, {name: "qwen3-coder", want: ""},