upstream-archive/llama.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/llama.cpp.git synced 2026-10-03 03:17:32 -05:00
Code Issues Packages Projects Releases Wiki Activity

clean up

Browse Source
This commit is contained in:
Xuan Son Nguyen
2026-10-02 18:56:07 +02:00
parent c290b7f463
commit ff15e5724b
12 changed files with 238 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conversion/clef.py
+30 -19
View File
@@ -4,7 +4,7 @@ import json
import math
from pathlib import Path
from typing import Any, Iterable, TYPE_CHECKING
from typing import Any, Iterable, Iterator, TYPE_CHECKING
import torch
@@ -40,10 +40,8 @@ class ClefModel(Qwen3_5TextModel):
"Read the complete state and schema. Decide every field jointly. Each answer "
"must be exactly one of that field's allowed options."
)
# the pieces of the prompt are tokenized one by one, this separates them
_PIECE_SEP = "<<clef:sep>>"
# start of a piece: state, question span, option span
_PIECE_STATE, _PIECE_QUESTION, _PIECE_OPTION = "<<clef:state>>", "<<clef:question>>", "<<clef:option>>"
# torch.nn.LayerNorm default, used by the head
_HEAD_NORM_EPS = 1e-5
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -62,25 +60,37 @@ class ClefModel(Qwen3_5TextModel):
def text(value: str) -> str:
return "{{ " + json.dumps(value) + " }}"
sep = text(cls._PIECE_SEP)
# state, instructions and option text are given as strings
def render(name: str) -> str:
# strings are used as is, other values are compact JSON
return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}"
# the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep)
# and the text that starts the span of a question or of an option (mark_question, mark_option)
# the keys of JSON objects are given in sorted order
option = (
"{% set d = o.description %}"
"{% if q.type == 'noul' and d is none %}"
"{% set d = 'The proposition is true or the answer is yes.' if o.key == 'true' else 'The proposition is false or the answer is no.' %}"
"{% endif %}"
"{{ ({'option_id': o.key} if d is none else {'description': d, 'option_id': o.key}) | tojson(separators=[',', ':']) }}"
)
return (
text(f"<|im_start|>system\n{cls._SYSTEM_PROMPT}<|im_end|>\n<|im_start|>user\nSTATE:\n")
+ sep + text(cls._PIECE_STATE) + "{{ state }}"
+ sep + text("\n\nSCHEMA FIELDS:\n")
+ "{{ sep }}" + render("state")
+ "{{ sep }}" + text("\n\nSCHEMA FIELDS:\n")
+ "{% for q in questions %}"
+ sep + text("\nFIELD ") + "{{ loop.index }}" + text("\nID: ") + "{{ q.id }}"
+ "{{ sep }}" + text("\nFIELD ") + "{{ loop.index }}" + text("\nID: ") + "{{ q.id }}"
+ text("\nTYPE: ") + "{{ q.type }}" + text("\nINSTRUCTION: ")
+ sep + text(cls._PIECE_QUESTION) + "{{ q.instructions }}"
+ sep + text("\nALLOWED OPTIONS:\n")
+ "{{ sep }}{{ mark_question }}" + render("q.instructions")
+ "{{ sep }}" + text("\nALLOWED OPTIONS:\n")
+ "{% for o in q.options %}"
+ sep + text("OPTION ") + "{{ loop.index }}" + text(": ")
+ sep + text(cls._PIECE_OPTION) + "{{ o.text }}"
+ sep + text("\n")
+ "{{ sep }}" + text("OPTION ") + "{{ loop.index }}" + text(": ")
+ "{{ sep }}{{ mark_option }}" + option
+ "{{ sep }}" + text("\n")
+ "{% endfor %}"
+ sep + text("END FIELD\n")
+ "{{ sep }}" + text("END FIELD\n")
+ "{% endfor %}"
+ sep + text("\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nJOINT SCHEMA DECISIONS:")
+ "{{ sep }}" + text("\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\nJOINT SCHEMA DECISIONS:")
)
def set_gguf_parameters(self):
@@ -90,9 +100,10 @@ class ClefModel(Qwen3_5TextModel):
self.gguf_writer.add_decision_routing_block_count(head["routing_layers"])
self.gguf_writer.add_decision_block_count(head["layers"])
self.gguf_writer.add_decision_head_count(head["heads"])
self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS)
def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
yield from super().generate_extra_tensors()
def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
yield from super().get_tensors()
from safetensors.torch import load_file
for name, data in load_file(self.dir_model / "joint_head.safetensors").items():
yield "joint_head." + name, data
gguf-py/gguf/constants.py
+28 -28
View File
@@ -864,6 +864,7 @@ class MODEL_TENSOR(IntEnum):
DEC_ATTN_OUT = auto()
DEC_ATTN_REL_B = auto()
DEC_CROSS_ATTN_NORM = auto()
DEC_CROSS_ATTN_NORM_KV = auto()
DEC_CROSS_ATTN_Q = auto()
DEC_CROSS_ATTN_K = auto()
DEC_CROSS_ATTN_V = auto()
@@ -874,20 +875,6 @@ class MODEL_TENSOR(IntEnum):
DEC_FFN_DOWN = auto()
DEC_FFN_UP = auto()
DEC_OUTPUT_NORM = auto()
DEC_CROSS_ATTN_NORM_KV = auto()
DECISION_HIDDEN_NORM = auto()
DECISION_PROJ_MEMORY = auto()
DECISION_PROJ_QUESTION = auto()
DECISION_PROJ_OPTION_QUESTION = auto()
DECISION_PROJ_GLOBAL = auto()
DECISION_PROJ_OPTION_CONTEXT = auto()
DECISION_PROJ_OPTION_LEXICAL = auto()
DECISION_OPTION_SUMMARY_NORM = auto()
DECISION_FIELD_NORM = auto()
DECISION_OPTION_NORM = auto()
DECISION_SCALES = auto()
DECISION_SCORER = auto()
DECISION_SCORER_OUT = auto()
ENC_ATTN_NORM = auto()
ENC_ATTN_Q = auto()
ENC_ATTN_K = auto()
@@ -902,6 +889,19 @@ class MODEL_TENSOR(IntEnum):
CLS = auto() # classifier
CLS_OUT = auto() # classifier output projection
CLS_NORM = auto()
DECISION_HIDDEN_NORM = auto()
DECISION_PROJ_MEMORY = auto()
DECISION_PROJ_QUESTION = auto()
DECISION_PROJ_OPTION_QUESTION = auto()
DECISION_PROJ_GLOBAL = auto()
DECISION_PROJ_OPTION_CONTEXT = auto()
DECISION_PROJ_OPTION_LEXICAL = auto()
DECISION_OPTION_SUMMARY_NORM = auto()
DECISION_FIELD_NORM = auto()
DECISION_OPTION_NORM = auto()
DECISION_SCALES = auto()
DECISION_SCORER = auto()
DECISION_SCORER_OUT = auto()
CONV1D = auto()
CONVNEXT_DW = auto()
CONVNEXT_NORM = auto()
@@ -1642,6 +1642,7 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.DEC_ATTN_OUT: "dec.blk.{bid}.attn_o",
MODEL_TENSOR.DEC_ATTN_REL_B: "dec.blk.{bid}.attn_rel_b",
MODEL_TENSOR.DEC_CROSS_ATTN_NORM: "dec.blk.{bid}.cross_attn_norm",
MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: "dec.blk.{bid}.cross_attn_norm_kv",
MODEL_TENSOR.DEC_CROSS_ATTN_Q: "dec.blk.{bid}.cross_attn_q",
MODEL_TENSOR.DEC_CROSS_ATTN_K: "dec.blk.{bid}.cross_attn_k",
MODEL_TENSOR.DEC_CROSS_ATTN_V: "dec.blk.{bid}.cross_attn_v",
@@ -1652,20 +1653,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.DEC_FFN_DOWN: "dec.blk.{bid}.ffn_down",
MODEL_TENSOR.DEC_FFN_UP: "dec.blk.{bid}.ffn_up",
MODEL_TENSOR.DEC_OUTPUT_NORM: "dec.output_norm",
MODEL_TENSOR.DEC_CROSS_ATTN_NORM_KV: "dec.blk.{bid}.cross_attn_norm_kv",
MODEL_TENSOR.DECISION_HIDDEN_NORM: "decision.hidden_norm",
MODEL_TENSOR.DECISION_PROJ_MEMORY: "decision.proj_memory",
MODEL_TENSOR.DECISION_PROJ_QUESTION: "decision.proj_question",
MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: "decision.proj_option_question",
MODEL_TENSOR.DECISION_PROJ_GLOBAL: "decision.proj_global",
MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: "decision.proj_option_context",
MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: "decision.proj_option_lexical",
MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: "decision.option_summary_norm",
MODEL_TENSOR.DECISION_FIELD_NORM: "decision.field_norm",
MODEL_TENSOR.DECISION_OPTION_NORM: "decision.option_norm",
MODEL_TENSOR.DECISION_SCALES: "decision.scales",
MODEL_TENSOR.DECISION_SCORER: "decision.scorer",
MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out",
MODEL_TENSOR.ENC_ATTN_NORM: "enc.blk.{bid}.attn_norm",
MODEL_TENSOR.ENC_ATTN_Q: "enc.blk.{bid}.attn_q",
MODEL_TENSOR.ENC_ATTN_K: "enc.blk.{bid}.attn_k",
@@ -1680,6 +1667,19 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
MODEL_TENSOR.CLS: "cls",
MODEL_TENSOR.CLS_OUT: "cls.output",
MODEL_TENSOR.CLS_NORM: "cls.norm",
MODEL_TENSOR.DECISION_HIDDEN_NORM: "decision.hidden_norm",
MODEL_TENSOR.DECISION_PROJ_MEMORY: "decision.proj_memory",
MODEL_TENSOR.DECISION_PROJ_QUESTION: "decision.proj_question",
MODEL_TENSOR.DECISION_PROJ_OPTION_QUESTION: "decision.proj_option_question",
MODEL_TENSOR.DECISION_PROJ_GLOBAL: "decision.proj_global",
MODEL_TENSOR.DECISION_PROJ_OPTION_CONTEXT: "decision.proj_option_context",
MODEL_TENSOR.DECISION_PROJ_OPTION_LEXICAL: "decision.proj_option_lexical",
MODEL_TENSOR.DECISION_OPTION_SUMMARY_NORM: "decision.option_summary_norm",
MODEL_TENSOR.DECISION_FIELD_NORM: "decision.field_norm",
MODEL_TENSOR.DECISION_OPTION_NORM: "decision.option_norm",
MODEL_TENSOR.DECISION_SCALES: "decision.scales",
MODEL_TENSOR.DECISION_SCORER: "decision.scorer",
MODEL_TENSOR.DECISION_SCORER_OUT: "decision.scorer_out",
MODEL_TENSOR.CONV1D: "conv1d",
MODEL_TENSOR.CONVNEXT_DW: "convnext.{bid}.dw",
MODEL_TENSOR.CONVNEXT_NORM: "convnext.{bid}.norm",
src/llama-arch.cpp
+28 -28
View File
@@ -493,6 +493,19 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_CLS, "cls" },
{ LLM_TENSOR_CLS_OUT, "cls.output" },
{ LLM_TENSOR_CLS_NORM, "cls.norm" },
{ LLM_TENSOR_DECISION_HIDDEN_NORM, "decision.hidden_norm" },
{ LLM_TENSOR_DECISION_PROJ_MEMORY, "decision.proj_memory" },
{ LLM_TENSOR_DECISION_PROJ_QUESTION, "decision.proj_question" },
{ LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, "decision.proj_option_question" },
{ LLM_TENSOR_DECISION_PROJ_GLOBAL, "decision.proj_global" },
{ LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, "decision.proj_option_context" },
{ LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, "decision.proj_option_lexical" },
{ LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, "decision.option_summary_norm" },
{ LLM_TENSOR_DECISION_FIELD_NORM, "decision.field_norm" },
{ LLM_TENSOR_DECISION_OPTION_NORM, "decision.option_norm" },
{ LLM_TENSOR_DECISION_SCALES, "decision.scales" },
{ LLM_TENSOR_DECISION_SCORER, "decision.scorer" },
{ LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" },
{ LLM_TENSOR_ENC_OUTPUT_NORM, "enc.output_norm" },
{ LLM_TENSOR_FFN_GATE_INP_SHEXP, "blk.%d.ffn_gate_inp_shexp" },
{ LLM_TENSOR_SSM_A_NOSCAN, "blk.%d.ssm_a" },
@@ -602,20 +615,6 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_ATTN_SUB_NORM, "blk.%d.attn_sub_norm" },
{ LLM_TENSOR_FFN_SUB_NORM, "blk.%d.ffn_sub_norm" },
{ LLM_TENSOR_DEC_OUTPUT_NORM, "dec.output_norm" },
{ LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, "dec.blk.%d.cross_attn_norm_kv" },
{ LLM_TENSOR_DECISION_HIDDEN_NORM, "decision.hidden_norm" },
{ LLM_TENSOR_DECISION_PROJ_MEMORY, "decision.proj_memory" },
{ LLM_TENSOR_DECISION_PROJ_QUESTION, "decision.proj_question" },
{ LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, "decision.proj_option_question" },
{ LLM_TENSOR_DECISION_PROJ_GLOBAL, "decision.proj_global" },
{ LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, "decision.proj_option_context" },
{ LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, "decision.proj_option_lexical" },
{ LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, "decision.option_summary_norm" },
{ LLM_TENSOR_DECISION_FIELD_NORM, "decision.field_norm" },
{ LLM_TENSOR_DECISION_OPTION_NORM, "decision.option_norm" },
{ LLM_TENSOR_DECISION_SCALES, "decision.scales" },
{ LLM_TENSOR_DECISION_SCORER, "decision.scorer" },
{ LLM_TENSOR_DECISION_SCORER_OUT, "decision.scorer_out" },
{ LLM_TENSOR_DEC_ATTN_NORM, "dec.blk.%d.attn_norm" },
{ LLM_TENSOR_DEC_ATTN_Q, "dec.blk.%d.attn_q" },
{ LLM_TENSOR_DEC_ATTN_K, "dec.blk.%d.attn_k" },
@@ -623,6 +622,7 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
{ LLM_TENSOR_DEC_ATTN_OUT, "dec.blk.%d.attn_o" },
{ LLM_TENSOR_DEC_ATTN_REL_B, "dec.blk.%d.attn_rel_b" },
{ LLM_TENSOR_DEC_CROSS_ATTN_NORM, "dec.blk.%d.cross_attn_norm" },
{ LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, "dec.blk.%d.cross_attn_norm_kv" },
{ LLM_TENSOR_DEC_CROSS_ATTN_Q, "dec.blk.%d.cross_attn_q" },
{ LLM_TENSOR_DEC_CROSS_ATTN_K, "dec.blk.%d.cross_attn_k" },
{ LLM_TENSOR_DEC_CROSS_ATTN_V, "dec.blk.%d.cross_attn_v" },
@@ -756,25 +756,24 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_CLS, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_CLS_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_CLS_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_HIDDEN_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_PROJ_MEMORY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_GLOBAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_FIELD_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_OPTION_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DENSE_2_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
{LLM_TENSOR_DENSE_3_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}}, // Dense layer output
{LLM_TENSOR_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_OUTPUT_NORM_LFM2, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DEC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_HIDDEN_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_PROJ_MEMORY, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_GLOBAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_FIELD_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_OPTION_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_SCALES, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_DECISION_SCORER, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_DECISION_SCORER_OUT, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL_MAT}},
{LLM_TENSOR_ENC_OUTPUT_NORM, {LLM_TENSOR_LAYER_OUTPUT, GGML_OP_MUL}},
{LLM_TENSOR_ROPE_FREQS, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},
{LLM_TENSOR_ROPE_FACTORS_LONG, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_ROPE}},
@@ -939,6 +938,7 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
{LLM_TENSOR_FFN_SUB_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DEC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DEC_CROSS_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_DEC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ENC_ATTN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
{LLM_TENSOR_ENC_FFN_NORM, {LLM_TENSOR_LAYER_REPEATING, GGML_OP_MUL}},
src/llama-arch.h
+14 -14
View File
@@ -639,6 +639,7 @@ enum llm_tensor {
LLM_TENSOR_DEC_ATTN_OUT,
LLM_TENSOR_DEC_ATTN_REL_B,
LLM_TENSOR_DEC_CROSS_ATTN_NORM,
LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV,
LLM_TENSOR_DEC_CROSS_ATTN_Q,
LLM_TENSOR_DEC_CROSS_ATTN_K,
LLM_TENSOR_DEC_CROSS_ATTN_V,
@@ -649,20 +650,6 @@ enum llm_tensor {
LLM_TENSOR_DEC_FFN_DOWN,
LLM_TENSOR_DEC_FFN_UP,
LLM_TENSOR_DEC_OUTPUT_NORM,
LLM_TENSOR_DEC_CROSS_ATTN_NORM_KV,
LLM_TENSOR_DECISION_HIDDEN_NORM,
LLM_TENSOR_DECISION_PROJ_MEMORY,
LLM_TENSOR_DECISION_PROJ_QUESTION,
LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION,
LLM_TENSOR_DECISION_PROJ_GLOBAL,
LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT,
LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL,
LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM,
LLM_TENSOR_DECISION_FIELD_NORM,
LLM_TENSOR_DECISION_OPTION_NORM,
LLM_TENSOR_DECISION_SCALES,
LLM_TENSOR_DECISION_SCORER,
LLM_TENSOR_DECISION_SCORER_OUT,
LLM_TENSOR_ENC_ATTN_NORM,
LLM_TENSOR_ENC_ATTN_Q,
LLM_TENSOR_ENC_ATTN_K,
@@ -677,6 +664,19 @@ enum llm_tensor {
LLM_TENSOR_CLS,
LLM_TENSOR_CLS_OUT,
LLM_TENSOR_CLS_NORM,
LLM_TENSOR_DECISION_HIDDEN_NORM,
LLM_TENSOR_DECISION_PROJ_MEMORY,
LLM_TENSOR_DECISION_PROJ_QUESTION,
LLM_TENSOR_DECISION_PROJ_OPTION_QUESTION,
LLM_TENSOR_DECISION_PROJ_GLOBAL,
LLM_TENSOR_DECISION_PROJ_OPTION_CONTEXT,
LLM_TENSOR_DECISION_PROJ_OPTION_LEXICAL,
LLM_TENSOR_DECISION_OPTION_SUMMARY_NORM,
LLM_TENSOR_DECISION_FIELD_NORM,
LLM_TENSOR_DECISION_OPTION_NORM,
LLM_TENSOR_DECISION_SCALES,
LLM_TENSOR_DECISION_SCORER,
LLM_TENSOR_DECISION_SCORER_OUT,
LLM_TENSOR_CONV1D,
LLM_TENSOR_CONVNEXT_DW,
LLM_TENSOR_CONVNEXT_NORM,
src/llama-batch.cpp
+7 -9
View File
@@ -168,15 +168,13 @@ bool llama_batch_allocr::init(
}
}
// kept empty if no entry has one
for (int32_t i = 0; i < n_tok; ++i) {
if (batch_inp.tokens[i].decision_order != 0) {
decision_order.resize(n_tok, 0);
break;
decision_order[i] = batch_inp.tokens[i].decision_order;
}
}
for (size_t i = 0; i < decision_order.size(); ++i) {
decision_order[i] = batch_inp.tokens[i].decision_order;
}
//
// set up the internal llama_batch to point to our owned arrays
@@ -266,6 +264,7 @@ bool llama_batch_allocr::init(
/*.seq_id_unq =*/ this->seq_id_unq.data(),
/*.seq_idx =*/ this->seq_idx.data(),
/*.output =*/ batch.logits,
/*.decision_order =*/ decision_order.empty() ? nullptr : decision_order.data(),
/*.data =*/ {},
};
@@ -472,6 +471,7 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
/*.seq_id_unq =*/ udata->seq_id_unq.data(),
/*.seq_idx =*/ udata->seq_idx.data(),
/*.output =*/ udata->output.data(),
/*.decision_order =*/ nullptr,
/*.data =*/ std::move(udata),
};
@@ -810,6 +810,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->seq_id_unq.resize(0);
udata->seq_idx .resize(LLAMA_MAX_SEQ, -1);
udata->output .resize(n_tokens);
udata->decision_order.resize(decision_order.empty() ? 0 : n_tokens);
udata->batch_idxs = idxs;
udata->seq_id_data.reserve(n_tokens);
@@ -838,7 +839,7 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
udata->output[i] = batch.logits[idxs[i]];
if (!decision_order.empty()) {
udata->decision_order.push_back(decision_order[idxs[i]]);
udata->decision_order[i] = decision_order[idxs[i]];
}
for (int s = 0; s < udata->n_seq_id[i]; ++s) {
@@ -882,13 +883,10 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
/*.seq_id_unq =*/ udata->seq_id_unq.data(),
/*.seq_idx =*/ udata->seq_idx.data(),
/*.output =*/ udata->output.data(),
/*.decision_order =*/ udata->decision_order.empty() ? nullptr : udata->decision_order.data(),
/*.data =*/ std::move(udata),
};
if (!res.data->decision_order.empty()) {
res.decision_order = res.data->decision_order.data();
}
if (debug > 0) {
LLAMA_LOG_DEBUG("%s: added ubatch to split:\n", __func__);
src/llama-batch.h
+1 -3
View File
@@ -52,6 +52,7 @@ struct llama_ubatch {
llama_seq_id * seq_id_unq; // [n_seqs_unq] | s | seq_id
int32_t * seq_idx; // [LLAMA_MAX_SEQ] | - | seq_idx
int8_t * output; // [n_tokens] | i | -
int32_t * decision_order; // [n_tokens], NULL if no entry has one, see llama_batch_ext_set_decision_order()
struct data_t {
std::vector<llama_token> token;
@@ -70,9 +71,6 @@ struct llama_ubatch {
// the llama_ubatch pointers above point to this data if set. otherwise - point to external non-owning data
std::shared_ptr<data_t> data;
// [n_tokens], see llama_batch_ext_set_decision_order(), NULL if no entry has one
int32_t * decision_order = nullptr;
};
struct llama_hparams;
src/llama-kv-cache-dsv4.cpp
+1
View File
@@ -164,6 +164,7 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) {
/*.seq_id_unq =*/ data->seq_id_unq.data(),
/*.seq_idx =*/ data->seq_idx.data(),
/*.output =*/ data->output.data(),
/*.decision_order =*/ nullptr,
/*.data =*/ data,
};
src/models/clef.cpp
+42 -43
View File
@@ -1,9 +1,8 @@
#include "models.h"
#include <cmath>
#include "llama-ext.h"
// torch.nn.LayerNorm default
static const float CLEF_HEAD_NORM_EPS = 1e-5f;
#include <cmath>
void llama_model_clef::load_arch_hparams(llama_model_loader & ml) {
llama_model_qwen35::load_arch_hparams(ml);
@@ -12,11 +11,12 @@ void llama_model_clef::load_arch_hparams(llama_model_loader & ml) {
ml.get_key(LLM_KV_DECISION_BLOCK_COUNT, n_layer_joint);
ml.get_key(LLM_KV_DECISION_HEAD_COUNT, n_head_decision);
if (n_head_decision == 0) {
throw std::runtime_error("invalid number of heads in the decision head");
if (n_head_decision == 0 || n_layer_routing > LLAMA_MAX_LAYERS || n_layer_joint > LLAMA_MAX_LAYERS) {
throw std::runtime_error("invalid size of the decision head");
}
hparams.f_norm_eps = CLEF_HEAD_NORM_EPS;
// used by the head
ml.get_key(LLM_KV_ATTENTION_LAYERNORM_EPS, hparams.f_norm_eps);
// the output is one score per token, see llama_batch_ext_set_decision_order()
hparams.n_embd_out_impl = 1;
@@ -115,45 +115,53 @@ struct clef_spans {
};
std::vector<question> questions;
std::vector<option> options;
bool valid = false; // one sequence, at least one question, and each question has an option
bool valid = false;
};
// see llama_batch_ext_set_decision_order()
// if the batch has no usable order, returns one empty question with one empty option
static clef_spans clef_get_spans(const llama_ubatch & ubatch) {
const int32_t ORDER_QUESTION_FIRST = 1;
const int32_t ORDER_QUESTION_LAST = 3;
const int32_t ORDER_OPTION = 4;
clef_spans res;
std::vector<bool> has_option;
// TODO: support multiple sequences
if (ubatch.decision_order == nullptr || ubatch.n_seqs_unq != 1) {
return res;
}
bool ok = ubatch.decision_order != nullptr && ubatch.n_seqs_unq == 1;
const int32_t n_tokens = ubatch.n_tokens;
std::vector<bool> has_option;
for (int32_t i = 0; i < n_tokens;) {
for (int32_t i = 0; ok && i < n_tokens;) {
const int32_t order = ubatch.decision_order[i];
int32_t end = i + 1;
while (end < n_tokens && ubatch.decision_order[end] == order) {
end++;
}
if (order >= ORDER_QUESTION_FIRST && order <= ORDER_QUESTION_LAST) {
res.questions.push_back({ order - ORDER_QUESTION_FIRST, i, end });
has_option.push_back(false);
} else if (order == ORDER_OPTION) {
if (res.questions.empty()) {
return res;
}
res.options.push_back({ (int32_t) res.questions.size() - 1, i, end });
has_option.back() = true;
} else if (order != 0) {
return res;
switch (order) {
case LLAMA_DECISION_ORDER_NONE:
break;
case LLAMA_DECISION_ORDER_QUESTION_NOUL:
case LLAMA_DECISION_ORDER_QUESTION_CHOICE:
case LLAMA_DECISION_ORDER_QUESTION_SCORE:
res.questions.push_back({ order - LLAMA_DECISION_ORDER_QUESTION_NOUL, i, end });
has_option.push_back(false);
break;
case LLAMA_DECISION_ORDER_OPTION:
ok = !res.questions.empty();
if (ok) {
res.options.push_back({ (int32_t) res.questions.size() - 1, i, end });
has_option.back() = true;
}
break;
default:
ok = false;
}
i = end;
}
res.valid = !res.questions.empty() && std::find(has_option.begin(), has_option.end(), false) == has_option.end();
// each question needs an option
res.valid = ok && !res.questions.empty() && std::find(has_option.begin(), has_option.end(), false) == has_option.end();
if (!res.valid) {
res.questions = {{ 0, 0, 0 }};
res.options = {{ 0, 0, 0 }};
}
return res;
}
@@ -162,22 +170,15 @@ class llama_model_clef::input_decision : public llm_graph_input_i {
public:
input_decision(const llama_ubatch & ubatch) : n_tokens(ubatch.n_tokens) {
const auto spans = clef_get_spans(ubatch);
if (spans.valid) {
n_questions = spans.questions.size();
n_options = spans.options.size();
}
n_questions = spans.questions.size();
n_options = spans.options.size();
}
void set_input(const llama_ubatch * ubatch) override {
GGML_ASSERT(ubatch->token);
ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens * sizeof(llama_token));
// without valid spans, the head runs on one empty question with one empty option
auto spans = clef_get_spans(*ubatch);
if (!spans.valid) {
spans.questions = {{ 0, 0, 0 }};
spans.options = {{ 0, 0, 0 }};
}
const auto spans = clef_get_spans(*ubatch);
GGML_ASSERT(spans.questions.size() == n_questions && spans.options.size() == n_options);
// the scores are NaN if the batch has a decision order that cannot be used
@@ -216,9 +217,7 @@ public:
bool can_reuse(const llm_graph_params & params) override {
// the values are computed again in set_input(), only the shapes must match
const auto spans = clef_get_spans(params.ubatch);
const size_t n_q = spans.valid ? spans.questions.size() : 1;
const size_t n_o = spans.valid ? spans.options.size() : 1;
return n_q == n_questions && n_o == n_options;
return spans.questions.size() == n_questions && spans.options.size() == n_options;
}
ggml_tensor * tokens = nullptr; // I32 [n_tokens]
@@ -230,8 +229,8 @@ public:
ggml_tensor * status = nullptr; // F32 [1], added to the scores: 0, or NaN on invalid input
const int64_t n_tokens;
size_t n_questions = 1;
size_t n_options = 1;
size_t n_questions;
size_t n_options;
};
// the backbone is copied from llama_model_qwen35::graph, without the memory module
tools/server/server-context.cpp
+28 -27
View File
@@ -2261,6 +2261,16 @@ private:
return i >= 0 && i < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[i]) : nullptr;
};
// joint head (decision model): the scores are the first rows
for (int32_t i = 0; i < decision.n_scores; i++) {
const float * embd = i < (int32_t) idx.size() ? llama_get_embeddings_ith(slot.ctx_tgt, idx[i]) : nullptr;
if (embd == nullptr) {
send_error(slot, "failed to get embeddings", ERROR_TYPE_SERVER);
return;
}
res->scores.push_back(embd[0]);
}
const int32_t n_embd_out = llama_model_n_embd_out(model_tgt);
const int32_t n_pointer = n_embd_out / 2;
const float * embd_q = decision.pointer >= 0 ? get_embd(decision.pointer) : nullptr;
@@ -2273,10 +2283,6 @@ private:
return;
}
if (decision.pointer < 0) {
if (std::isnan(embd[decision.column])) {
send_error(slot, "the model could not evaluate the decision", ERROR_TYPE_SERVER);
return;
}
res->scores.push_back(embd[decision.column]);
continue;
}
@@ -5410,16 +5416,14 @@ void server_routes::init_routes() {
task.id = rd.get_new_id();
decision.fill_task_joint(state, questions, task);
tasks.push_back(std::move(task));
}
for (const auto & question : questions) {
if (decision.is_joint()) {
break;
}
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
task.id = rd.get_new_id();
decision.fill_task(state, questions, question, variant, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
} else {
for (const auto & question : questions) {
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
server_task task = server_task(SERVER_TASK_TYPE_DECISION);
task.id = rd.get_new_id();
decision.fill_task(state, questions, question, variant, files, ctx_server.mctx, ctx_server.init_opt, task);
tasks.push_back(std::move(task));
}
}
}
if (decision.can_share_prompt()) {
@@ -5439,22 +5443,19 @@ void server_routes::init_routes() {
json answers = json::object();
int32_t n_tokens = 0;
if (decision.is_joint()) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[0].get());
GGML_ASSERT(result != nullptr);
const auto scores = decision.split_scores(questions, result->scores);
for (size_t i = 0; i < questions.size(); i++) {
answers[questions[i].id] = decision.format_answer(questions[i], { scores[i] });
}
n_tokens = result->n_tokens;
}
size_t i_result = 0;
size_t i_score = 0;
for (const auto & question : questions) {
if (decision.is_joint()) {
break;
}
std::vector<std::vector<float>> scores;
for (size_t variant = 0; variant < decision.n_variants(question); variant++) {
if (decision.is_joint()) {
// one result with the scores of all the questions, in order
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[0].get());
GGML_ASSERT(result != nullptr && i_score + question.options.size() <= result->scores.size());
scores.emplace_back(result->scores.begin() + i_score, result->scores.begin() + i_score + question.options.size());
i_score += question.options.size();
n_tokens = result->n_tokens;
}
for (size_t variant = 0; !decision.is_joint() && variant < decision.n_variants(question); variant++) {
auto * result = dynamic_cast<server_task_result_decision *>(all_results.results[i_result++].get());
GGML_ASSERT(result != nullptr);
scores.push_back(result->scores);
tools/server/server-decision.cpp
+54 -114
View File
@@ -1,5 +1,7 @@
#include "server-decision.h"
#include "../../src/llama-ext.h" // staging API: llama_decision_order
#include <algorithm>
#include <cmath>
#include <regex>
@@ -119,6 +121,7 @@ void server_decision_context::init(const llama_model * model) {
} else if (model_type == COMMON_DECISION_TYPE_CLEF) {
n_options_max = 255;
noul_true_first = true;
choice_sorted = true;
} else {
throw std::runtime_error("unsupported decision model type: " + type_name);
}
@@ -166,6 +169,11 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
for (const auto & [key, description] : criteria.items()) {
question.options.push_back({key, description});
}
if (choice_sorted) {
std::sort(question.options.begin(), question.options.end(), [](const auto & a, const auto & b) {
return a.key < b.key;
});
}
} else if (type_name == "score") {
question.type = SERVER_DECISION_QUESTION_SCORE;
if (!criteria.is_array() || criteria.size() < 2 || criteria.size() > 10) {
@@ -549,71 +557,39 @@ void server_decision_context::fill_task_laya(llama_tokens & tokens, const server
// joint prompt (clef)
//
// the template separates the pieces of the prompt with these, they are tokenized one by one
static const std::string CLEF_MARKER = "<<clef:";
static const std::string CLEF_PIECE_SEP = "<<clef:sep>>";
static const std::string CLEF_PIECE_STATE = "<<clef:state>>";
static const std::string CLEF_PIECE_QUESTION = "<<clef:question>>";
static const std::string CLEF_PIECE_OPTION = "<<clef:option>>";
// strings are used as is, other values are compact JSON with sorted keys
static std::string clef_render(const json & val) {
return val.is_string() ? val.get<std::string>() : decision_sort_keys(val).dump();
}
std::vector<size_t> server_decision_context::prompt_order(const server_decision_question & question) const {
std::vector<size_t> order(question.options.size());
for (size_t i = 0; i < order.size(); i++) {
order[i] = i;
}
if (type == COMMON_DECISION_TYPE_CLEF && question.type == SERVER_DECISION_QUESTION_CHOICE) {
std::sort(order.begin(), order.end(), [&](size_t a, size_t b) {
return question.options[a].key < question.options[b].key;
});
}
return order;
}
// given to the template: text between the pieces of the prompt, and at the start of the span of a question or of an option
static const std::string CLEF_MARKER = "<<clef:";
static const std::string CLEF_SEP = "<<clef:sep>>";
static const std::string CLEF_MARK_QUESTION = "<<clef:question>>";
static const std::string CLEF_MARK_OPTION = "<<clef:option>>";
void server_decision_context::fill_task_joint(const json & state, const std::vector<server_decision_question> & questions, server_task & task) const {
auto clean = [](const json & val) {
return decision_replace_text(val, CLEF_MARKER, "<<clef ");
};
json inp_questions = json::array();
for (const auto & question : questions) {
json options = json::array();
for (const size_t i : prompt_order(question)) {
const auto & opt = question.options[i];
json description = opt.description;
if (description.is_null() && question.type == SERVER_DECISION_QUESTION_NOUL) {
description = opt.key == "true"
? "The proposition is true or the answer is yes."
: "The proposition is false or the answer is no.";
}
// keys in sorted order
json semantics = json::object();
if (!description.is_null()) {
semantics["description"] = decision_sort_keys(description);
}
semantics["option_id"] = opt.key;
options.push_back(json{{"text", semantics.dump()}});
for (const auto & opt : question.options) {
options.push_back(json{
{"key", opt.key},
{"description", opt.description},
});
}
inp_questions.push_back(json{
{"id", question.id},
{"type", decision_question_type_name(question.type)},
{"instructions", clef_render(question.instructions)},
{"instructions", question.instructions},
{"options", options},
});
}
const json inp = clean(json{
{"state", clef_render(state)},
// the template is given raw JSON values with sorted keys, and no marker in the input
json inp = json{
{"state", state},
{"questions", inp_questions},
});
};
inp = decision_replace_text(decision_sort_keys(inp), CLEF_MARKER, "<<clef ");
inp["sep"] = CLEF_SEP;
inp["mark_question"] = CLEF_MARK_QUESTION;
inp["mark_option"] = CLEF_MARK_OPTION;
jinja::context ctx(tmpl->source());
jinja::global_from_json(ctx, inp, false);
@@ -623,84 +599,44 @@ void server_decision_context::fill_task_joint(const json & state, const std::vec
// the model was trained with the pieces tokenized one by one
llama_tokens tokens;
int32_t i_question = -1;
size_t n_questions = 0;
size_t n_options = 0;
for (std::string piece : string_split(prompt, CLEF_PIECE_SEP)) {
enum { PIECE_TEXT, PIECE_QUESTION, PIECE_OPTION } kind = PIECE_TEXT;
if (string_starts_with(piece, CLEF_PIECE_QUESTION)) {
piece = piece.substr(CLEF_PIECE_QUESTION.size());
kind = PIECE_QUESTION;
} else if (string_starts_with(piece, CLEF_PIECE_OPTION)) {
piece = piece.substr(CLEF_PIECE_OPTION.size());
kind = PIECE_OPTION;
} else if (string_starts_with(piece, CLEF_PIECE_STATE)) {
piece = piece.substr(CLEF_PIECE_STATE.size());
size_t i_question = 0;
for (std::string piece : string_split(prompt, CLEF_SEP)) {
int32_t order = LLAMA_DECISION_ORDER_NONE;
if (string_starts_with(piece, CLEF_MARK_QUESTION)) {
piece = piece.substr(CLEF_MARK_QUESTION.size());
if (i_question >= questions.size()) {
throw std::runtime_error("unexpected layout of the decision prompt");
}
switch (questions[i_question++].type) {
case SERVER_DECISION_QUESTION_NOUL: order = LLAMA_DECISION_ORDER_QUESTION_NOUL; break;
case SERVER_DECISION_QUESTION_CHOICE: order = LLAMA_DECISION_ORDER_QUESTION_CHOICE; break;
case SERVER_DECISION_QUESTION_SCORE: order = LLAMA_DECISION_ORDER_QUESTION_SCORE; break;
}
} else if (string_starts_with(piece, CLEF_MARK_OPTION)) {
piece = piece.substr(CLEF_MARK_OPTION.size());
order = LLAMA_DECISION_ORDER_OPTION;
task.decision.n_scores++;
}
const int32_t start = tokens.size();
const llama_tokens piece_tokens = common_tokenize(vocab, piece, false, true);
tokens.insert(tokens.end(), piece_tokens.begin(), piece_tokens.end());
const int32_t end = tokens.size();
if (kind == PIECE_TEXT) {
continue;
}
if (start == end) {
if (order != LLAMA_DECISION_ORDER_NONE && piece_tokens.empty()) {
throw std::invalid_argument("the instructions and the options of a question must not be empty");
}
// see llama_batch_ext_set_decision_order(): question of type noul, choice, score, or option
int32_t order = 4;
if (kind == PIECE_QUESTION) {
i_question++;
switch (questions.at(i_question).type) {
case SERVER_DECISION_QUESTION_NOUL: order = 1; break;
case SERVER_DECISION_QUESTION_CHOICE: order = 2; break;
case SERVER_DECISION_QUESTION_SCORE: order = 3; break;
}
n_questions++;
} else {
// the score of option i is returned at row i
task.decision.markers.push_back(n_options);
n_options++;
}
task.decision.order.resize(tokens.size(), 0);
std::fill(task.decision.order.begin() + start, task.decision.order.end(), order);
tokens.insert(tokens.end(), piece_tokens.begin(), piece_tokens.end());
task.decision.order.resize(tokens.size(), order);
}
task.decision.order.resize(tokens.size(), 0);
size_t n_options_exp = 0;
size_t n_options = 0;
for (const auto & question : questions) {
n_options_exp += question.options.size();
n_options += question.options.size();
}
if (n_questions != questions.size() || n_options != n_options_exp) {
if (i_question != questions.size() || (size_t) task.decision.n_scores != n_options) {
throw std::runtime_error("unexpected layout of the decision prompt");
}
task.decision.column = 0;
task.tokens = server_tokens(tokens, false);
}
std::vector<std::vector<float>> server_decision_context::split_scores(
const std::vector<server_decision_question> & questions,
const std::vector<float> & scores) const {
std::vector<std::vector<float>> result;
size_t offset = 0;
for (const auto & question : questions) {
const auto order = prompt_order(question);
if (offset + order.size() > scores.size()) {
throw std::runtime_error("decision result does not match the number of options");
}
std::vector<float> cur(order.size());
for (size_t i = 0; i < order.size(); i++) {
cur[order[i]] = scores[offset + i];
}
offset += order.size();
result.push_back(std::move(cur));
}
return result;
}
//
// answer
//
@@ -768,6 +704,10 @@ json server_decision_context::format_answer(const server_decision_question & que
if (s.size() != n) {
throw std::runtime_error("decision result does not match the number of options");
}
// a joint head returns NaN if it could not use the decision order
if (std::any_of(s.begin(), s.end(), [](float v) { return std::isnan(v); })) {
throw std::runtime_error("the model could not evaluate the decision");
}
const float score_max = *std::max_element(s.begin(), s.end());
std::vector<double> p(n);
double sum = 0.0;
tools/server/server-decision.h
+2 -7
View File
@@ -86,12 +86,9 @@ struct server_decision_context {
const mtmd_helper_init_opt & init_opt,
server_task & task) const;
// set the prompt of all the questions, and where to read their results
// set the prompt of all the questions, the result has the scores of all their options, in order
void fill_task_joint(const json & state, const std::vector<server_decision_question> & questions, server_task & task) const;
// result of fill_task_joint() -> scores of each question, in the order of its options
std::vector<std::vector<float>> split_scores(const std::vector<server_decision_question> & questions, const std::vector<float> & scores) const;
// scores: the raw model outputs of each variant
json format_answer(const server_decision_question & question, const std::vector<std::vector<float>> & scores) const;
@@ -102,6 +99,7 @@ private:
std::map<std::string, float> temperatures; // "<type>" or "<type>.<n_options bucket>"
size_t n_options_max = 0;
bool noul_true_first = false; // noul options are [true, false] instead of [false, true]
bool choice_sorted = false; // choice options are in the order of their keys
// OPENJEV, LEV, NIMBLE
std::vector<llama_token> labels;
@@ -124,9 +122,6 @@ private:
size_t n_outputs(const server_decision_question & question) const;
void fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const;
// indices of the options of a question, in the order they have in the prompt
std::vector<size_t> prompt_order(const server_decision_question & question) const;
float get_temperature(const server_decision_question & question) const;
};
tools/server/server-task.h
+3 -1
View File
@@ -194,7 +194,9 @@ struct server_task {
}
// for a joint head: one value per prompt token, see llama_batch_ext_set_decision_order()
// the scores are the first n_scores rows of the embeddings
std::vector<int32_t> order;
int32_t n_scores = 0;
};
decision decision;
@@ -215,7 +217,7 @@ struct server_task {
case SERVER_TASK_TYPE_RERANK:
return true;
case SERVER_TASK_TYPE_DECISION:
return !decision.markers.empty();
return !decision.markers.empty() || !decision.order.empty();
default:
return false;
}