From ff15e5724b91673b846f43fa1ebe71ee6a68ba0b Mon Sep 17 00:00:00 2001 From: Xuan Son Nguyen Date: Fri, 2 Oct 2026 18:56:07 +0200 Subject: [PATCH] clean up --- conversion/clef.py | 49 +++++---- gguf-py/gguf/constants.py | 56 +++++------ src/llama-arch.cpp | 56 +++++------ src/llama-arch.h | 28 +++--- src/llama-batch.cpp | 16 ++- src/llama-batch.h | 4 +- src/llama-kv-cache-dsv4.cpp | 1 + src/models/clef.cpp | 85 ++++++++-------- tools/server/server-context.cpp | 55 +++++----- tools/server/server-decision.cpp | 168 ++++++++++--------------------- tools/server/server-decision.h | 9 +- tools/server/server-task.h | 4 +- 12 files changed, 238 insertions(+), 293 deletions(-) diff --git a/conversion/clef.py b/conversion/clef.py index 58fa9284a2..3cc57a4e35 100644 --- a/conversion/clef.py +++ b/conversion/clef.py @@ -4,7 +4,7 @@ import json import math from pathlib import Path -from typing import Any, Iterable, TYPE_CHECKING +from typing import Any, Iterable, Iterator, TYPE_CHECKING import torch @@ -40,10 +40,8 @@ class ClefModel(Qwen3_5TextModel): "Read the complete state and schema. Decide every field jointly. Each answer " "must be exactly one of that field's allowed options." ) - # the pieces of the prompt are tokenized one by one, this separates them - _PIECE_SEP = "<>" - # start of a piece: state, question span, option span - _PIECE_STATE, _PIECE_QUESTION, _PIECE_OPTION = "<>", "<>", "<>" + # torch.nn.LayerNorm default, used by the head + _HEAD_NORM_EPS = 1e-5 def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -62,25 +60,37 @@ class ClefModel(Qwen3_5TextModel): def text(value: str) -> str: return "{{ " + json.dumps(value) + " }}" - sep = text(cls._PIECE_SEP) - # state, instructions and option text are given as strings + def render(name: str) -> str: + # strings are used as is, other values are compact JSON + return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}" + + # the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep) + # and the text that starts the span of a question or of an option (mark_question, mark_option) + # the keys of JSON objects are given in sorted order + option = ( + "{% set d = o.description %}" + "{% if q.type == 'noul' and d is none %}" + "{% set d = 'The proposition is true or the answer is yes.' if o.key == 'true' else 'The proposition is false or the answer is no.' %}" + "{% endif %}" + "{{ ({'option_id': o.key} if d is none else {'description': d, 'option_id': o.key}) | tojson(separators=[',', ':']) }}" + ) return ( text(f"<|im_start|>system\n{cls._SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n") - + sep + text(cls._PIECE_STATE) + "{{ state }}" - + sep + text("\n\nSCHEMA FIELDS:\n") + + "{{ sep }}" + render("state") + + "{{ sep }}" + text("\n\nSCHEMA FIELDS:\n") + "{% for q in questions %}" - + sep + text("\nFIELD ") + "{{ loop.index }}" + text("\nID: ") + "{{ q.id }}" + + "{{ sep }}" + text("\nFIELD ") + "{{ loop.index }}" + text("\nID: ") + "{{ q.id }}" + text("\nTYPE: ") + "{{ q.type }}" + text("\nINSTRUCTION: ") - + sep + text(cls._PIECE_QUESTION) + "{{ q.instructions }}" - + sep + text("\nALLOWED OPTIONS:\n") + + "{{ sep }}{{ mark_question }}" + render("q.instructions") + + "{{ sep }}" + text("\nALLOWED OPTIONS:\n") + "{% for o in q.options %}" - + sep + text("OPTION ") + "{{ loop.index }}" + text(": ") - + sep + text(cls._PIECE_OPTION) + "{{ o.text }}" - + sep + text("\n") + + "{{ sep }}" + text("OPTION ") + "{{ loop.index }}" + text(": ") + + "{{ sep }}{{ mark_option }}" + option + + "{{ sep }}" + text("\n") + "{% endfor %}" - + sep + text("END FIELD\n") + + "{{ sep }}" + text("END FIELD\n") + "{% endfor %}" - + sep + text("\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJOINT SCHEMA DECISIONS:") + + "{{ sep }}" + text("\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\nJOINT SCHEMA DECISIONS:") ) def set_gguf_parameters(self): @@ -90,9 +100,10 @@ class ClefModel(Qwen3_5TextModel): self.gguf_writer.add_decision_routing_block_count(head["routing_layers"]) self.gguf_writer.add_decision_block_count(head["layers"]) self.gguf_writer.add_decision_head_count(head["heads"]) + self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS) - def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]: - yield from super().generate_extra_tensors() + def get_tensors(self) -> Iterator[tuple[str, Tensor]]: + yield from super().get_tensors() from safetensors.torch import load_file for name, data in load_file(self.dir_model / "joint_head.safetensors").items(): yield "joint_head." + name, data diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 1de4192aca..04b6fa902e 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -864,6 +864,7 @@ class MODEL_TENSOR(IntEnum): DEC_ATTN_OUT = auto() DEC_ATTN_REL_B = auto() DEC_CROSS_ATTN_NORM = auto() + DEC_CROSS_ATTN_NORM_KV = auto() DEC_CROSS_ATTN_Q = auto() DEC_CROSS_ATTN_K = auto() DEC_CROSS_ATTN_V = auto() @@ -874,20 +875,6 @@ class MODEL_TENSOR(IntEnum): DEC_FFN_DOWN = auto() DEC_FFN_UP = auto() DEC_OUTPUT_NORM = auto() - DEC_CROSS_ATTN_NORM_KV = auto() - DECISION_HIDDEN_NORM = auto() - DECISION_PROJ_MEMORY = auto() - DECISION_PROJ_QUESTION = auto() - DECISION_PROJ_OPTION_QUESTION = auto() - DECISION_PROJ_GLOBAL = auto() - DECISION_PROJ_OPTION_CONTEXT = auto() - DECISION_PROJ_OPTION_LEXICAL = auto() - DECISION_OPTION_SUMMARY_NORM = auto() - DECISION_FIELD_NORM = auto() - DECISION_OPTION_NORM = auto() - DECISION_SCALES = auto() - DECISION_SCORER = auto() - DECISION_SCORER_OUT = auto() ENC_ATTN_NORM = auto() ENC_ATTN_Q = auto() ENC_ATTN_K = auto() @@ -902,6 +889,19 @@ class MODEL_TENSOR(IntEnum): CLS = auto() # classifier CLS_OUT = auto() # classifier output projection CLS_NORM = auto() + DECISION_HIDDEN_NORM = auto() + DECISION_PROJ_MEMORY = auto() + DECISION_PROJ_QUESTION = auto() + DECISION_PROJ_OPTION_QUESTION = auto() + DECISION_PROJ_GLOBAL = auto() + DECISION_PROJ_OPTION_CONTEXT = auto() + DECISION_PROJ_OPTION_LEXICAL = auto() + DECISION_OPTION_SUMMARY_NORM = auto() + DECISION_FIELD_NORM = auto() + DECISION_OPTION_NORM = auto() + DECISION_SCALES = auto() + DECISION_SCORER = auto() + DECISION_SCORER_OUT = auto() CONV1D = auto() CONVNEXT_DW = auto() CONVNEXT_NORM = auto() @@ -1642,6 +1642,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.DEC_ATTN_OUT: "dec.blk.{bid}.attn_o", MODEL_TENSOR.DEC_ATTN_REL_B: "dec.blk.{bid}.attn_rel_b", MODEL_TENSOR.DEC_CROSS_ATTN_NORM: "dec.blk.{bid}.cross_attn_norm", + MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: "dec.blk.{bid}.cross_attn_norm_kv", MODEL_TENSOR.DEC_CROSS_ATTN_Q: "dec.blk.{bid}.cross_attn_q", MODEL_TENSOR.DEC_CROSS_ATTN_K: "dec.blk.{bid}.cross_attn_k", MODEL_TENSOR.DEC_CROSS_ATTN_V: "dec.blk.{bid}.cross_attn_v", @@ -1652,20 +1653,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.DEC_FFN_DOWN: "dec.blk.{bid}.ffn_down", MODEL_TENSOR.DEC_FFN_UP: "dec.blk.{bid}.ffn_up", MODEL_TENSOR.DEC_OUTPUT_NORM: "dec.output_norm", - MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: "dec.blk.{bid}.cross_attn_norm_kv", - MODEL_TENSOR.DECISION_HIDDEN_NORM: "decision.hidden_norm", - MODEL_TENSOR.DECISION_PROJ_MEMORY: "decision.proj_memory", - MODEL_TENSOR.DECISION_PROJ_QUESTION: "decision.proj_question", - MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: "decision.proj_option_question", - MODEL_TENSOR.DECISION_PROJ_GLOBAL: "decision.proj_global", - MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: "decision.proj_option_context", - MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: "decision.proj_option_lexical", - MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: "decision.option_summary_norm", - MODEL_TENSOR.DECISION_FIELD_NORM: "decision.field_norm", - MODEL_TENSOR.DECISION_OPTION_NORM: "decision.option_norm", - MODEL_TENSOR.DECISION_SCALES: "decision.scales", - MODEL_TENSOR.DECISION_SCORER: "decision.scorer", - MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out", MODEL_TENSOR.ENC_ATTN_NORM: "enc.blk.{bid}.attn_norm", MODEL_TENSOR.ENC_ATTN_Q: "enc.blk.{bid}.attn_q", MODEL_TENSOR.ENC_ATTN_K: "enc.blk.{bid}.attn_k", @@ -1680,6 +1667,19 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = { MODEL_TENSOR.CLS: "cls", MODEL_TENSOR.CLS_OUT: "cls.output", MODEL_TENSOR.CLS_NORM: "cls.norm", + MODEL_TENSOR.DECISION_HIDDEN_NORM: "decision.hidden_norm", + MODEL_TENSOR.DECISION_PROJ_MEMORY: "decision.proj_memory", + MODEL_TENSOR.DECISION_PROJ_QUESTION: "decision.proj_question", + MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: "decision.proj_option_question", + MODEL_TENSOR.DECISION_PROJ_GLOBAL: "decision.proj_global", + MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: "decision.proj_option_context", + MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: "decision.proj_option_lexical", + MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: "decision.option_summary_norm", + MODEL_TENSOR.DECISION_FIELD_NORM: "decision.field_norm", + MODEL_TENSOR.DECISION_OPTION_NORM: "decision.option_norm", + MODEL_TENSOR.DECISION_SCALES: "decision.scales", + MODEL_TENSOR.DECISION_SCORER: "decision.scorer", + MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out", MODEL_TENSOR.CONV1D: "conv1d", MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw", MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm", diff --git a/src/llama-arch.cpp b/src/llama-arch.cpp index a3f9697466..2af5445dfa 100644 --- a/src/llama-arch.cpp +++ b/src/llama-arch.cpp @@ -493,6 +493,19 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_CLS, "cls" }, { LLM_TENSOR_CLS_OUT, "cls.output" }, { LLM_TENSOR_CLS_NORM, "cls.norm" }, + { LLM_TENSOR_DECISION_HIDDEN_NORM, "decision.hidden_norm" }, + { LLM_TENSOR_DECISION_PROJ_MEMORY, "decision.proj_memory" }, + { LLM_TENSOR_DECISION_PROJ_QUESTION, "decision.proj_question" }, + { LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, "decision.proj_option_question" }, + { LLM_TENSOR_DECISION_PROJ_GLOBAL, "decision.proj_global" }, + { LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, "decision.proj_option_context" }, + { LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, "decision.proj_option_lexical" }, + { LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, "decision.option_summary_norm" }, + { LLM_TENSOR_DECISION_FIELD_NORM, "decision.field_norm" }, + { LLM_TENSOR_DECISION_OPTION_NORM, "decision.option_norm" }, + { LLM_TENSOR_DECISION_SCALES, "decision.scales" }, + { LLM_TENSOR_DECISION_SCORER, "decision.scorer" }, + { LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" }, { LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" }, { LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" }, { LLM_TENSOR_SSM_A_NOSCAN, "blk.%d.ssm_a" }, @@ -602,20 +615,6 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_ATTN_SUB_NORM, "blk.%d.attn_sub_norm" }, { LLM_TENSOR_FFN_SUB_NORM, "blk.%d.ffn_sub_norm" }, { LLM_TENSOR_DEC_OUTPUT_NORM, "dec.output_norm" }, - { LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, "dec.blk.%d.cross_attn_norm_kv" }, - { LLM_TENSOR_DECISION_HIDDEN_NORM, "decision.hidden_norm" }, - { LLM_TENSOR_DECISION_PROJ_MEMORY, "decision.proj_memory" }, - { LLM_TENSOR_DECISION_PROJ_QUESTION, "decision.proj_question" }, - { LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, "decision.proj_option_question" }, - { LLM_TENSOR_DECISION_PROJ_GLOBAL, "decision.proj_global" }, - { LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, "decision.proj_option_context" }, - { LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, "decision.proj_option_lexical" }, - { LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, "decision.option_summary_norm" }, - { LLM_TENSOR_DECISION_FIELD_NORM, "decision.field_norm" }, - { LLM_TENSOR_DECISION_OPTION_NORM, "decision.option_norm" }, - { LLM_TENSOR_DECISION_SCALES, "decision.scales" }, - { LLM_TENSOR_DECISION_SCORER, "decision.scorer" }, - { LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" }, { LLM_TENSOR_DEC_ATTN_NORM, "dec.blk.%d.attn_norm" }, { LLM_TENSOR_DEC_ATTN_Q, "dec.blk.%d.attn_q" }, { LLM_TENSOR_DEC_ATTN_K, "dec.blk.%d.attn_k" }, @@ -623,6 +622,7 @@ static const std::map LLM_TENSOR_NAMES = { { LLM_TENSOR_DEC_ATTN_OUT, "dec.blk.%d.attn_o" }, { LLM_TENSOR_DEC_ATTN_REL_B, "dec.blk.%d.attn_rel_b" }, { LLM_TENSOR_DEC_CROSS_ATTN_NORM, "dec.blk.%d.cross_attn_norm" }, + { LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, "dec.blk.%d.cross_attn_norm_kv" }, { LLM_TENSOR_DEC_CROSS_ATTN_Q, "dec.blk.%d.cross_attn_q" }, { LLM_TENSOR_DEC_CROSS_ATTN_K, "dec.blk.%d.cross_attn_k" }, { LLM_TENSOR_DEC_CROSS_ATTN_V, "dec.blk.%d.cross_attn_v" }, @@ -756,25 +756,24 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_CLS_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_DECISION_HIDDEN_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_DECISION_PROJ_MEMORY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_PROJ_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_PROJ_GLOBAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_DECISION_FIELD_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_DECISION_OPTION_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, + {LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, + {LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output {LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output {LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, {LLM_TENSOR_OUTPUT_NORM_LFM2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, {LLM_TENSOR_DEC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, - {LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, - {LLM_TENSOR_DECISION_HIDDEN_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, - {LLM_TENSOR_DECISION_PROJ_MEMORY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_DECISION_PROJ_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_DECISION_PROJ_GLOBAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, - {LLM_TENSOR_DECISION_FIELD_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, - {LLM_TENSOR_DECISION_OPTION_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, - {LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, - {LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, - {LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, {LLM_TENSOR_ENC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}}, {LLM_TENSOR_ROPE_FREQS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}}, {LLM_TENSOR_ROPE_FACTORS_LONG, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}}, @@ -939,6 +938,7 @@ static const std::map LLM_TENSOR_INFOS = { {LLM_TENSOR_FFN_SUB_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_DEC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_DEC_CROSS_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, + {LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_DEC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ENC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, {LLM_TENSOR_ENC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}}, diff --git a/src/llama-arch.h b/src/llama-arch.h index eddded280f..148d293cee 100644 --- a/src/llama-arch.h +++ b/src/llama-arch.h @@ -639,6 +639,7 @@ enum llm_tensor { LLM_TENSOR_DEC_ATTN_OUT, LLM_TENSOR_DEC_ATTN_REL_B, LLM_TENSOR_DEC_CROSS_ATTN_NORM, + LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, LLM_TENSOR_DEC_CROSS_ATTN_Q, LLM_TENSOR_DEC_CROSS_ATTN_K, LLM_TENSOR_DEC_CROSS_ATTN_V, @@ -649,20 +650,6 @@ enum llm_tensor { LLM_TENSOR_DEC_FFN_DOWN, LLM_TENSOR_DEC_FFN_UP, LLM_TENSOR_DEC_OUTPUT_NORM, - LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, - LLM_TENSOR_DECISION_HIDDEN_NORM, - LLM_TENSOR_DECISION_PROJ_MEMORY, - LLM_TENSOR_DECISION_PROJ_QUESTION, - LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, - LLM_TENSOR_DECISION_PROJ_GLOBAL, - LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, - LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, - LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, - LLM_TENSOR_DECISION_FIELD_NORM, - LLM_TENSOR_DECISION_OPTION_NORM, - LLM_TENSOR_DECISION_SCALES, - LLM_TENSOR_DECISION_SCORER, - LLM_TENSOR_DECISION_SCORER_OUT, LLM_TENSOR_ENC_ATTN_NORM, LLM_TENSOR_ENC_ATTN_Q, LLM_TENSOR_ENC_ATTN_K, @@ -677,6 +664,19 @@ enum llm_tensor { LLM_TENSOR_CLS, LLM_TENSOR_CLS_OUT, LLM_TENSOR_CLS_NORM, + LLM_TENSOR_DECISION_HIDDEN_NORM, + LLM_TENSOR_DECISION_PROJ_MEMORY, + LLM_TENSOR_DECISION_PROJ_QUESTION, + LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, + LLM_TENSOR_DECISION_PROJ_GLOBAL, + LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, + LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, + LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, + LLM_TENSOR_DECISION_FIELD_NORM, + LLM_TENSOR_DECISION_OPTION_NORM, + LLM_TENSOR_DECISION_SCALES, + LLM_TENSOR_DECISION_SCORER, + LLM_TENSOR_DECISION_SCORER_OUT, LLM_TENSOR_CONV1D, LLM_TENSOR_CONVNEXT_DW, LLM_TENSOR_CONVNEXT_NORM, diff --git a/src/llama-batch.cpp b/src/llama-batch.cpp index 651932d54b..6e58af8727 100644 --- a/src/llama-batch.cpp +++ b/src/llama-batch.cpp @@ -168,15 +168,13 @@ bool llama_batch_allocr::init( } } + // kept empty if no entry has one for (int32_t i = 0; i < n_tok; ++i) { if (batch_inp.tokens[i].decision_order != 0) { decision_order.resize(n_tok, 0); - break; + decision_order[i] = batch_inp.tokens[i].decision_order; } } - for (size_t i = 0; i < decision_order.size(); ++i) { - decision_order[i] = batch_inp.tokens[i].decision_order; - } // // set up the internal llama_batch to point to our owned arrays @@ -266,6 +264,7 @@ bool llama_batch_allocr::init( /*.seq_id_unq =*/ this->seq_id_unq.data(), /*.seq_idx =*/ this->seq_idx.data(), /*.output =*/ batch.logits, + /*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(), /*.data =*/ {}, }; @@ -472,6 +471,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t /*.seq_id_unq =*/ udata->seq_id_unq.data(), /*.seq_idx =*/ udata->seq_idx.data(), /*.output =*/ udata->output.data(), + /*.decision_order =*/ nullptr, /*.data =*/ std::move(udata), }; @@ -810,6 +810,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u udata->seq_id_unq.resize(0); udata->seq_idx .resize(LLAMA_MAX_SEQ, -1); udata->output .resize(n_tokens); + udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens); udata->batch_idxs = idxs; udata->seq_id_data.reserve(n_tokens); @@ -838,7 +839,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u udata->output[i] = batch.logits[idxs[i]]; if (!decision_order.empty()) { - udata->decision_order.push_back(decision_order[idxs[i]]); + udata->decision_order[i] = decision_order[idxs[i]]; } for (int s = 0; s < udata->n_seq_id[i]; ++s) { @@ -882,13 +883,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector & idxs, u /*.seq_id_unq =*/ udata->seq_id_unq.data(), /*.seq_idx =*/ udata->seq_idx.data(), /*.output =*/ udata->output.data(), + /*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(), /*.data =*/ std::move(udata), }; - if (!res.data->decision_order.empty()) { - res.decision_order = res.data->decision_order.data(); - } - if (debug > 0) { LLAMA_LOG_DEBUG("%s: added ubatch to split:\n", __func__); diff --git a/src/llama-batch.h b/src/llama-batch.h index 74d0aa3c7f..32f103bfcb 100644 --- a/src/llama-batch.h +++ b/src/llama-batch.h @@ -52,6 +52,7 @@ struct llama_ubatch { llama_seq_id * seq_id_unq; // [n_seqs_unq] | s | seq_id int32_t * seq_idx; // [LLAMA_MAX_SEQ] | - | seq_idx int8_t * output; // [n_tokens] | i | - + int32_t * decision_order; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_order() struct data_t { std::vector token; @@ -70,9 +71,6 @@ struct llama_ubatch { // the llama_ubatch pointers above point to this data if set. otherwise - point to external non-owning data std::shared_ptr data; - - // [n_tokens], see llama_batch_ext_set_decision_order(), NULL if no entry has one - int32_t * decision_order = nullptr; }; struct llama_hparams; diff --git a/src/llama-kv-cache-dsv4.cpp b/src/llama-kv-cache-dsv4.cpp index 948d08146f..4dbcfb8db8 100644 --- a/src/llama-kv-cache-dsv4.cpp +++ b/src/llama-kv-cache-dsv4.cpp @@ -164,6 +164,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) { /*.seq_id_unq =*/ data->seq_id_unq.data(), /*.seq_idx =*/ data->seq_idx.data(), /*.output =*/ data->output.data(), + /*.decision_order =*/ nullptr, /*.data =*/ data, }; diff --git a/src/models/clef.cpp b/src/models/clef.cpp index 055655730e..776bd3f6d3 100644 --- a/src/models/clef.cpp +++ b/src/models/clef.cpp @@ -1,9 +1,8 @@ #include "models.h" -#include +#include "llama-ext.h" -// torch.nn.LayerNorm default -static const float CLEF_HEAD_NORM_EPS = 1e-5f; +#include void llama_model_clef::load_arch_hparams(llama_model_loader & ml) { llama_model_qwen35::load_arch_hparams(ml); @@ -12,11 +11,12 @@ void llama_model_clef::load_arch_hparams(llama_model_loader & ml) { ml.get_key(LLM_KV_DECISION_BLOCK_COUNT, n_layer_joint); ml.get_key(LLM_KV_DECISION_HEAD_COUNT, n_head_decision); - if (n_head_decision == 0) { - throw std::runtime_error("invalid number of heads in the decision head"); + if (n_head_decision == 0 || n_layer_routing > LLAMA_MAX_LAYERS || n_layer_joint > LLAMA_MAX_LAYERS) { + throw std::runtime_error("invalid size of the decision head"); } - hparams.f_norm_eps = CLEF_HEAD_NORM_EPS; + // used by the head + ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps); // the output is one score per token, see llama_batch_ext_set_decision_order() hparams.n_embd_out_impl = 1; @@ -115,45 +115,53 @@ struct clef_spans { }; std::vector questions; std::vector