vendor: apply deep nested json patch from upstream

CODEOWNERS

@@ -97,7 +97,6 @@
 /src/models/                            @CISC
 /tests/                                 @ggerganov
 /tests/test-chat.*                      @pwilkin
-/tests/test-server-component.cpp        @ggml-org/llama-server
 /tools/batched-bench/                   @ggerganov
 /tools/cli/                             @ngxson
 /tools/completion/                      @ggerganov

common/arg.cpp

@@ -1730,7 +1730,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
         [](common_params & params, bool value) {
             params.kv_unified = value;
         }
-    ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL, LLAMA_EXAMPLE_SPECULATIVE}));
+    ).set_env("LLAMA_ARG_KV_UNIFIED").set_examples({LLAMA_EXAMPLE_SERVER, LLAMA_EXAMPLE_PERPLEXITY, LLAMA_EXAMPLE_BATCHED, LLAMA_EXAMPLE_BENCH, LLAMA_EXAMPLE_PARALLEL}));
     add_opt(common_arg(
         {"--cache-idle-slots"},
         {"--no-cache-idle-slots"},
@@ -3643,7 +3643,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
                 params.slot_save_path += DIRECTORY_SEPARATOR;
             }
         }
-    ).set_examples({LLAMA_EXAMPLE_SERVER}).set_env("LLAMA_ARG_SLOT_SAVE_PATH"));
+    ).set_examples({LLAMA_EXAMPLE_SERVER}));
     add_opt(common_arg(
         {"--media-path"}, "PATH",
         "directory for loading local media files; files can be accessed via file:// URLs using relative paths (default: disabled)",

common/common.cpp

@@ -1170,10 +1170,8 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_LAYA,           "laya"          },
     { COMMON_DECISION_TYPE_CLEF,           "clef"          },
     { COMMON_DECISION_TYPE_PPLX_DECIDER,   "pplx-decider"  },
-    { COMMON_DECISION_TYPE_VLLM_DECISION2, "vllm-decision2" },
     { COMMON_DECISION_TYPE_LFM2_D1,        "lfm2-d1"       },
     { COMMON_DECISION_TYPE_LFM2_D1_OMNI,   "lfm2-d1-omni"  },
-    // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 
 static common_decision_type common_decision_type_from_string(const std::string & str) {
@@ -1291,7 +1289,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)
     // TODO: maybe improve this in the future
     const auto decision_type = common_get_decision_type(model);
     if (decision_type == COMMON_DECISION_TYPE_LAYA || decision_type == COMMON_DECISION_TYPE_KEV || decision_type == COMMON_DECISION_TYPE_CLEF ||
-        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || decision_type == COMMON_DECISION_TYPE_VLLM_DECISION2) {
+        decision_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
         params.embedding    = true;
         params.pooling_type = LLAMA_POOLING_TYPE_NONE;
 
@@ -1405,14 +1403,6 @@ std::vector<llama_adapter_lora_ptr> & common_init_result::lora() {
     return pimpl->lora;
 }
 
-// only for warmup and probe decodes, fill zeros as dummy input
-static void common_batch_set_zero_state(common_batch & batch, const llama_model * model, std::vector<float> & zeros) {
-    zeros.assign(llama_model_n_embd_out(model), 0.0f);
-    for (int32_t i = 0; i < batch.size(); ++i) {
-        batch.set_embd_state(i, { zeros.data(), 1, zeros.size() });
-    }
-}
-
 common_init_result_ptr common_init_from_params(common_params & params, bool model_only) {
     common_init_result_ptr res(new common_init_result(params, model_only));
 
@@ -1519,8 +1509,6 @@ common_init_result_ptr common_init_from_params(common_params & params, bool mode
         if (llama_model_has_decoder(model)) {
             tmp.resize(std::min(tmp.size(), (size_t) params.n_batch));
             common_batch batch = common_batch_get_one(lctx, tmp);
-            std::vector<float> zeros;
-            common_batch_set_zero_state(batch, model, zeros);
             llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
         }
         llama_memory_clear(llama_get_memory(lctx), true);
@@ -1588,8 +1576,6 @@ common_context_seq_rm_type common_context_can_seq_rm(llama_context * ctx) {
     int ret;
     {
         common_batch batch = common_batch_get_one(ctx, tmp);
-        std::vector<float> zeros;
-        common_batch_set_zero_state(batch, llama_get_model(ctx), zeros);
         ret = llama_process(ctx, LLAMA_PROCESS_TYPE_DECODE, batch.get());
     }
     if (ret != 0) {
@@ -2175,7 +2161,7 @@ void common_batch::clear() {
 }
 
 int32_t common_batch::add(llama_token id, llama_pos pos, llama_seq_id seq_id, bool output) {
-    tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, { nullptr, 0, 0 }, {} });
+    tokens.push_back({ id, { pos, 0, 0, 0 }, seq_id, output, { nullptr, 0, 0 }, {} });
     return size() - 1;
 }
 
@@ -2213,16 +2199,8 @@ bool common_batch::set_embd(int32_t idx, llama_embd embd) {
     return true;
 }
 
-bool common_batch::set_embd_state(int32_t idx, llama_embd state) {
-    if (idx < 0 || idx >= size() || tokens[idx].state.data != nullptr) {
-        return false;
-    }
-    tokens[idx].state = state;
-    return true;
-}
-
 int32_t common_batch::add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output) {
-    token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, { nullptr, 0, 0 }, {} };
+    token t = { LLAMA_TOKEN_NULL, { 0, 0, 0, 0 }, seq_id, output, embd, {} };
     for (int32_t j = 0; j < n_pos; ++j) {
         t.pos[j] = pos[j];
     }
@@ -2267,9 +2245,6 @@ llama_batch_ext * common_batch::get_sub_batch(int32_t off, int32_t n) {
         if (t.output) {
             llama_batch_ext_set_output_logits(res, idx, true);
         }
-        if (t.state.data) {
-            llama_batch_ext_set_embd_state(res, idx, t.state); // contexts without a state input ignore it
-        }
         if (t.decision_order != 0) {
             llama_batch_ext_set_decision_order(res, idx, (llama_decision_order) t.decision_order);
         }

common/common.h

@@ -963,7 +963,6 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_LAYA,    // score of one marker token per option, read from the embeddings output
     COMMON_DECISION_TYPE_CLEF,    // all questions in one prompt, score of option i read from the embeddings output at row i
     COMMON_DECISION_TYPE_PPLX_DECIDER, // same as openjev, label codes of 1 or 2 letters
-    COMMON_DECISION_TYPE_VLLM_DECISION2, // score of each option read from the embeddings output, the options of a question are marked in the batch
     COMMON_DECISION_TYPE_LFM2_D1, // same as openjev, the labels depend on the question type
     COMMON_DECISION_TYPE_LFM2_D1_OMNI, // same as laya, other prompt layout
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
@@ -1075,7 +1074,6 @@ struct common_batch {
         llama_seq_id seq_id; // the first sequence id, see add_seq()
         bool         output;
         llama_embd   embd; // non-owning view of the data passed to add_embd()/set_embd(), data == NULL if none
-        llama_embd   state; // non-owning view of the data passed to set_embd_state(), data == NULL if none
         std::vector<llama_seq_id> seq_ids_extra; // see add_seq()
         int32_t      decision_order = 0; // see llama_batch_ext_set_decision_order()
     };
@@ -1113,9 +1111,6 @@ struct common_batch {
     // attach a token embedding to the entry at idx, can only be set once per entry
     bool set_embd(int32_t idx, llama_embd embd);
 
-    // attach a state embedding (e.g. the target hidden state for MTP) to the entry at idx, can only be set once per entry
-    bool set_embd_state(int32_t idx, llama_embd state);
-
     // add an embedding-only entry (no token id)
     // pos points to n_pos positions
     int32_t add_embd(llama_embd embd, const llama_pos * pos, llama_seq_id seq_id, bool output);

common/sampling.cpp

@@ -589,7 +589,6 @@ void common_perf_print(const struct llama_context * ctx, const struct common_sam
         LOG_INF("%s:       total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
         LOG_INF("%s: unaccounted time = %10.2f ms / %5.1f %%      (total - sampling - prompt eval - eval) / (total)\n", __func__, t_unacc_ms, t_unacc_pc);
         LOG_INF("%s:    graphs reused = %10d\n", __func__, data.n_reused);
-        LOG_INF("%s:   graph reallocs = %10d    / %5d        (same size) / (changed size)\n", __func__, data.n_reallocs_same, data.n_reallocs_diff);
 
         common_memory_breakdown_print(ctx);
     }

common/speculative.cpp

@@ -1541,7 +1541,8 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             return true;
         }
 
-        if (!batch_in.has_token() && !batch_in.has_embd()) {
+        // TODO: how to make it work with vision tokens?
+        if (!batch_in.has_token() || batch_in.has_embd()) {
             return true;
         }
 
@@ -1580,20 +1581,15 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             const float * h_tgt = llama_get_embeddings_nextn(ctx_tgt);
 
             for (int k = 0; k < n_tokens; ++k) {
-                const auto & t = batch_in.tokens[k];
+                const llama_seq_id seq_id = batch_in.tokens[k].seq_id;
 
-                const llama_seq_id seq_id = t.seq_id;
-
-                // vision tokens carry an embedding instead of an id
-                const int32_t idx = t.id != LLAMA_TOKEN_NULL
-                    ? batch.add(t.id, t.pos[0], seq_id, false)
-                    : batch.add_embd(t.embd, t.pos.data(), seq_id, false);
+                const int32_t idx = batch.add(batch_in.tokens[k].id, batch_in.tokens[k].pos[0], seq_id, false);
 
                 const float * h_row = k == i_batch_beg[seq_id]
                     ? pending_h[seq_id].data()
                     : h_tgt + (size_t) (k - 1) * n_embd;
 
-                batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd });
+                batch.set_embd(idx, { h_row, 1, (size_t) n_embd });
             }
 
             auto * mem_dft = llama_get_memory(ctx_dft);
@@ -1683,7 +1679,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
             }
 
             const int32_t idx = batch.add(dp.id_last, dp.pos0, seq_id, true);
-            batch.set_embd_state(idx, { pending_h[seq_id].data(), 1, (size_t) n_embd });
+            batch.set_embd(idx, { pending_h[seq_id].data(), 1, (size_t) n_embd });
 
             i_last[seq_id] = idx;
 
@@ -1776,18 +1772,18 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
                     for (int t = 0; t < n_rows; ++t) {
                         const llama_token tok = (t == 0) ? dp.id_last : result[t - 1];
                         const int32_t idx = batch.add(tok, dp.pos0 + t, seq_id, t == n_rows - 1);
-                        batch.set_embd_state(idx, { chain_h[seq_id].data() + (size_t) t * n_embd, 1, (size_t) n_embd });
+                        batch.set_embd(idx, { chain_h[seq_id].data() + (size_t) t * n_embd, 1, (size_t) n_embd });
                         i_last[seq_id] = idx;
                     }
                 } else if (is_mem_shared) {
                     // note: with shared memory (e.g. Gemma4 assistants) we use the same position for all draft tokens
                     // ref: https://github.com/huggingface/transformers/blob/effde20942e3f82a1b97449f60b3a48c5ff96145/docs/source/en/model_doc/gemma4_assistant.md?plain=1#L36-L37
                     const int32_t idx = batch.add(id, dp.pos0, seq_id, true);
-                    batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd });
+                    batch.set_embd(idx, { h_row, 1, (size_t) n_embd });
                     i_last[seq_id] = idx;
                 } else {
                     const int32_t idx = batch.add(id, dp.pos0 + i + 1, seq_id, true);
-                    batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd });
+                    batch.set_embd(idx, { h_row, 1, (size_t) n_embd });
                     i_last[seq_id] = idx;
                 }
             }

conversion/__init__.py

@@ -189,7 +189,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "MiniCPM3ForCausalLM": "minicpm",
     "MiniCPMForCausalLM": "minicpm",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
-    "MiniCPMV4_7ForConditionalGeneration": "minicpm",
     "MiniMaxText01ForCausalLM": "minimax",
     "MiniMaxM1ForCausalLM": "minimax",
     "MiniMaxM2ForCausalLM": "minimax",
@@ -291,7 +290,6 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "UltravoxModel": "ultravox",
     "UnlimitedOCRForCausalLM": "deepseek",
     "VLlama3ForCausalLM": "llama",
-    "VllmDecision2Model": "vllm_decision2",
     "VoxtralForConditionalGeneration": "llama",
     "WavTokenizerDec": "wavtokenizer",
     "XLMRobertaForSequenceClassification": "bert",
@@ -348,7 +346,6 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "MiMoV2ForCausalLM": "mimo",
     "MiniMaxM3SparseForConditionalGeneration": "minimax",
     "MiniCPMV4_6ForConditionalGeneration": "minicpm",
-    "MiniCPMV4_7ForConditionalGeneration": "minicpm",
     "Mistral3ForConditionalGeneration": "llava",
     "NemotronH_Nano_VL_V2": "nemotron",
     "MuseGlimmerForConditionalGeneration": "muse_glimmer",

conversion/base.py

@@ -673,171 +673,6 @@ class ModelBase:
         if algo == "W4A16_NVFP4":
             self._prec_a4[gguf_name] = False
 
-    def hadamard_folded_names(self) -> set[str]:
-        """Source-tensor names folded under a Hadamard manifest, or empty."""
-        cached = getattr(self, "_hadamard_folded_names", None)
-        if cached is not None:
-            return cached
-        names: set[str] = set()
-        manifest_path = self.dir_model / "hadamard_packing.json"
-        if manifest_path.is_file():
-            with manifest_path.open("r", encoding="utf-8") as f:
-                for record in json.load(f).get("tensors", []):
-                    if isinstance(record, dict) and isinstance(record.get("name"), str):
-                        names.add(record["name"])
-        self._hadamard_folded_names = names
-        return names
-
-    def add_hadamard_metadata(self) -> None:
-        """Transfer a packed-checkpoint transform contract into GGUF metadata."""
-        manifest_path = self.dir_model / "hadamard_packing.json"
-        if not manifest_path.is_file():
-            return
-
-        with manifest_path.open("r", encoding="utf-8") as f:
-            manifest = json.load(f)
-
-        schema_version = manifest.get("schema_version")
-        if schema_version not in (1, 2, 3) or manifest.get("kind") != "hadamard-weight-fold":
-            raise ValueError(f"Unsupported Hadamard manifest: {manifest_path}")
-        if manifest.get("status") != "requires-matching-runtime":
-            raise ValueError(f"Unexpected Hadamard manifest status: {manifest.get('status')!r}")
-
-        transform = manifest.get("transform")
-        if not isinstance(transform, dict):
-            raise ValueError("Hadamard manifest is missing transform metadata")
-        block_size = transform.get("block_size")
-        if not isinstance(block_size, int) or block_size <= 0 or block_size & (block_size - 1):
-            raise ValueError(f"Invalid Hadamard block size: {block_size!r}")
-        if transform.get("name") != "normalized-signed-sylvester-walsh-hadamard":
-            raise ValueError(f"Unsupported Hadamard transform: {transform.get('name')!r}")
-        sign_mode = transform.get("sign_mode")
-        if sign_mode not in ("identity", "explicit"):
-            raise ValueError(f"Unsupported Hadamard sign mode: {sign_mode!r}")
-        sign_widths: list[int] = []
-        sign_values: list[int] = []
-        if sign_mode == "explicit":
-            signs = manifest.get("signs")
-            if not isinstance(signs, dict) or not signs:
-                raise ValueError("explicit sign mode requires a signs table")
-            for width_str, vec in sorted(signs.items(), key=lambda kv: int(kv[0])):
-                width = int(width_str)
-                # same width rule as the runtime, so a manifest that converts also loads
-                if width <= 0 or width % block_size != 0:
-                    raise ValueError(
-                        f"sign width {width} must be positive and a multiple of block size {block_size}"
-                    )
-                if len(vec) != width or any(v not in (-1, 1) for v in vec):
-                    raise ValueError(f"invalid sign vector for width {width}")
-                sign_widths.append(width)
-                sign_values.extend(int(v) for v in vec)
-
-        tensor_records = manifest.get("tensors")
-        if not isinstance(tensor_records, list) or not tensor_records:
-            raise ValueError("Hadamard manifest has no folded tensors")
-
-        # only build_lora_mm/build_lora_mm_id apply the transform: refuse archs and tensor kinds that can skip them
-        _HADAMARD_ARCHS = {
-            gguf.MODEL_ARCH.LLAMA,
-            gguf.MODEL_ARCH.QWEN3,
-            gguf.MODEL_ARCH.QWEN3MOE,
-            gguf.MODEL_ARCH.QWEN35,
-            gguf.MODEL_ARCH.QWEN35MOE,
-            gguf.MODEL_ARCH.QWEN3NEXT,
-        }
-        if self.model_arch not in _HADAMARD_ARCHS:
-            raise ValueError(
-                f"Hadamard folding is not verified for arch {self.model_arch.name}; "
-                "the runtime would load the GGUF without applying the activation transform"
-            )
-        _HADAMARD_KINDS = re.compile(
-            r"output\.weight|"
-            r"blk\.\d+\.("
-            r"attn_q|attn_k|attn_v|attn_qkv|attn_gate|attn_output"
-            r"|ffn_gate|ffn_up|ffn_down"
-            r"|ffn_gate_exps|ffn_up_exps|ffn_down_exps|ffn_gate_up_exps"
-            r"|ffn_gate_shexp|ffn_up_shexp|ffn_down_shexp"
-            r"|ssm_out"
-            r")\.weight"
-        )
-        weight_names: list[str] = []
-        inverse_weight_names: list[str] = []
-        for record in tensor_records:
-            if not isinstance(record, dict) or not isinstance(record.get("name"), str):
-                raise ValueError("Hadamard manifest has an invalid tensor record")
-            if record.get("axis") != -1:
-                raise ValueError(f"Unsupported Hadamard tensor axis for {record['name']!r}")
-            role = record.get("role", "fold-before-matmul")
-            if role not in ("fold-before-matmul", "inverse-after-lookup"):
-                raise ValueError(f"Unsupported Hadamard tensor role for {record['name']!r}: {role!r}")
-            filtered = self.filter_tensors((record["name"], lambda: torch.empty(0)))
-            if filtered is None:
-                raise ValueError(f"Hadamard tensor is filtered out: {record['name']!r}")
-            mapped = self.map_tensor_name(filtered[0])
-            if role == "inverse-after-lookup":
-                # the runtime applies the inverse only after the token-embedding lookup, other latent tables stay rotated
-                if mapped != "token_embd.weight":
-                    raise ValueError(
-                        f"Hadamard tensor {record['name']!r} maps to {mapped!r}, which is not a "
-                        "verified inverse-after-lookup table"
-                    )
-                inverse_weight_names.append(mapped)
-            else:
-                if not _HADAMARD_KINDS.fullmatch(mapped):
-                    raise ValueError(
-                        f"Hadamard tensor {record['name']!r} maps to {mapped!r}, which is not on a "
-                        "verified Hadamard-aware matmul path"
-                    )
-                weight_names.append(mapped)
-
-        # --fuse-qkv writes one attn_qkv per layer, so Q, K and V must all be folded and get one name
-        for bid in sorted(self._fusable_qkv_weight_layers):
-            qkv = [
-                self.format_tensor_name(t, bid)
-                for t in (gguf.MODEL_TENSOR.ATTN_Q, gguf.MODEL_TENSOR.ATTN_K, gguf.MODEL_TENSOR.ATTN_V)
-            ]
-            n_folded = sum(name in weight_names for name in qkv)
-            if n_folded == 0:
-                continue
-            if n_folded != len(qkv):
-                raise ValueError(f"--fuse-qkv needs all of Q, K and V folded in layer {bid}, or none of them")
-            weight_names = [name for name in weight_names if name not in qkv]
-            weight_names.append(self.format_tensor_name(gguf.MODEL_TENSOR.ATTN_QKV, bid))
-
-        tied_output = manifest.get("tied_output", False)
-        if not isinstance(tied_output, bool) or (schema_version == 3) != tied_output:
-            raise ValueError("Hadamard schema 3 requires tied_output=true; older schemas forbid it")
-        if tied_output:
-            if inverse_weight_names != ["token_embd.weight"]:
-                raise ValueError("Tied Hadamard output requires one latent token embedding")
-            if not self.hparams.get("tie_word_embeddings", False):
-                raise ValueError("Tied Hadamard output requires tie_word_embeddings=true")
-            if "output.weight" in weight_names or any(
-                self.tensor_map.get_name(name, try_suffixes=(".weight", ".bias")) == "output.weight"
-                for name in self.model_tensors
-            ):
-                raise ValueError("Tied Hadamard output must not carry a separate output head")
-            self.gguf_writer.add_prism_hadamard_tied_output(True)
-        elif "token_embd.weight" in inverse_weight_names and self.hparams.get("tie_word_embeddings", False):
-            raise ValueError("A tied latent embedding requires Hadamard schema 3 and tied_output=true")
-
-        self.gguf_writer.add_prism_hadamard_version(2 if tied_output else 1)
-        self.gguf_writer.add_prism_hadamard_block_size(block_size)
-        self.gguf_writer.add_prism_hadamard_transform("normalized-sylvester-walsh-hadamard")
-        self.gguf_writer.add_prism_hadamard_axis("input-last-dimension")
-        self.gguf_writer.add_prism_hadamard_sign_mode(sign_mode)
-        self.gguf_writer.add_prism_hadamard_weight_names(weight_names)
-        if sign_mode == "explicit":
-            self.gguf_writer.add_prism_hadamard_sign_widths(sign_widths)
-            self.gguf_writer.add_prism_hadamard_sign_values(sign_values)
-        if inverse_weight_names:
-            self.gguf_writer.add_prism_hadamard_inverse_weight_names(inverse_weight_names)
-        if getattr(self, "_hadamard_gdn_v_grouped", False):
-            self.gguf_writer.add_prism_hadamard_gdn_v_grouped(True)
-            logger.info("GGUF Hadamard: linear-attention out_proj kept in grouped V order")
-        logger.info("GGUF Hadamard contract: H%d, sign_mode=%s, %d folded weight(s), %d inverse-lookup",
-                    block_size, sign_mode, len(weight_names), len(inverse_weight_names))
-
     def set_gguf_parameters(self):
         raise NotImplementedError("set_gguf_parameters() must be implemented in subclasses")
 
@@ -1385,8 +1220,6 @@ class ModelBase:
         logger.info("Set model quantization version")
         self.gguf_writer.add_quantization_version(gguf.GGML_QUANT_VERSION)
 
-        self.add_hadamard_metadata()
-
         if self._prec_a4:
             names = sorted(self._prec_a4.keys())
             values = [self._prec_a4[n] for n in names]
@@ -1556,17 +1389,14 @@ class TextModel(ModelBase):
         name, gen = item
 
         # Skip multimodal tensors
-        # strip the "model." wrapper so the prefixes below match (name is not returned)
-        if name.startswith("model."):
-            name = name[len("model."):]
-        if name.startswith(("mlp", "vit.", "vpm.", "siglip2.", "conformer.", "connector.", "merger.", "resampler.", "sound_encoder.", "sound_projection.", "speech_embeddings.")) \
+        if name.startswith(("mlp", "vit.", "vpm.", "siglip2.", "conformer.", "merger.", "resampler.", "sound_encoder.", "sound_projection.", "speech_embeddings.")) \
                 or "visual." in name or "vision." in name or "audio." in name or "talker." in name \
                 or "vision_" in name or "audio_" in name \
                 or "token2wav." in name or "code2wav." in name \
                 or "projector." in name or "pre_mm_projector_norm" in name \
                 or "image_newline" in name or "view_seperator" in name \
                 or "patch_embed" in name or "patch_embedding" in name \
-                or "patch_merger." in name or "patch_merge_mlp." in name:
+                or "patch_merger." in name or "patch_merge_mlp." in name or "model.connector." in name:
             return None
 
         return super().filter_tensors(item)

conversion/minicpm.py

@@ -139,16 +139,9 @@ class MiniCPMV4_6TextModel(Qwen3_5TextModel):
 @ModelBase.register("MiniCPMV4_6ForConditionalGeneration")
 @ModelBase.example("openbmb/MiniCPM-V-4_6")
 class MiniCPMV4_6VisionModel(MmprojModel):
-    projector_type = gguf.VisionProjectorType.MINICPMV4_6
-    # fallback for checkpoints whose preprocessor config omits `scale_resolution`
-    default_scale_resolution: int | None = None
-
-    def get_downsample_mode(self) -> str:
-        return self.preprocessor_config.get("downsample_mode", "16x")
-
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.downsample_mode = self.get_downsample_mode()
+        self.downsample_mode = self.preprocessor_config.get("downsample_mode", "16x")
         if self.downsample_mode not in {"4x", "16x"}:
             raise ValueError(f"Unsupported downsample mode: {self.downsample_mode}")
         if self.downsample_mode == "4x":
@@ -164,8 +157,7 @@ class MiniCPMV4_6VisionModel(MmprojModel):
             # The CLIP loader in tools/mtmd/clip.cpp consumes `clip.vision.image_size`
             # as the slice size and warmup resolution, so report `scale_resolution` there
             # to match the upstream MiniCPMV4_6ImageProcessorPil slicing rules.
-            scale_resolution = self.preprocessor_config.get(
-                "scale_resolution", self.default_scale_resolution)
+            scale_resolution = self.preprocessor_config.get("scale_resolution")
             if scale_resolution is not None:
                 self.hparams_vision["image_size"] = int(scale_resolution)
 
@@ -174,15 +166,12 @@ class MiniCPMV4_6VisionModel(MmprojModel):
         assert self.hparams_vision is not None
 
         # projector type string is consumed by clip_projector_type_from_string() in clip.cpp
-        self.gguf_writer.add_clip_projector_type(self.projector_type)
+        # (mapped to PROJECTOR_TYPE_MINICPMV4_6).
+        self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.MINICPMV4_6)
 
         self.gguf_writer.add_vision_projector_scale_factor(
             2 if self.downsample_mode == "4x" else 4)
 
-        max_slice_nums = self.preprocessor_config.get("max_slice_nums")
-        if max_slice_nums is not None:
-            self.gguf_writer.add_vision_max_slice_nums(int(max_slice_nums))
-
         # borrow wa_layer_indexes for vit_merger insertion point
         insert_layer_id = int(self.global_config.get(
             "insert_layer_id", self.hparams_vision.get("insert_layer_id", 6)))
@@ -202,73 +191,3 @@ class MiniCPMV4_6VisionModel(MmprojModel):
             return None
 
         return super().filter_tensors(item)
-
-
-# MiniCPM-V 4.7 shares the v4.6 stack: the same Qwen3.5 text tower (MoE variant when the checkpoint says so) and the same SigLIP + vit_merger + merger vision tower.
-
-@ModelBase.register("MiniCPMV4_7ForConditionalGeneration")
-@ModelBase.example("openbmb/MiniCPM-V-4.7")
-class MiniCPMV4_7TextModel(Qwen3_5TextModel):
-    model_arch = gguf.MODEL_ARCH.QWEN35
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        # mtmd puts the time of the image canvas in slot z, slot t stays the KV cache position
-        self.gguf_writer.add_rope_section_order(gguf.RopeSectionOrder.ZYXT)
-
-    def __init__(self, dir_model, ftype, fname_out, *, hparams: dict | None = None, **kwargs):
-        if hparams is None:
-            hparams = ModelBase.load_hparams(dir_model, is_mistral_format=False)
-        text_config = hparams.get("text_config", {})
-        if text_config.get("model_type") == "qwen3_5_moe_text":
-            self.model_arch = gguf.MODEL_ARCH.QWEN35MOE
-        else:
-            self.model_arch = gguf.MODEL_ARCH.QWEN35
-        super().__init__(dir_model, ftype, fname_out, hparams=hparams, **kwargs)
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        name, gen = item
-
-        # MTP tensors are not used yet
-        if name.startswith("mtp"):
-            return None
-
-        return super().filter_tensors(item)
-
-
-@ModelBase.register("MiniCPMV4_7ForConditionalGeneration")
-@ModelBase.example("openbmb/MiniCPM-V-4.7")
-class MiniCPMV4_7VisionModel(MiniCPMV4_6VisionModel):
-    projector_type = gguf.VisionProjectorType.MINICPMV4_7
-    # MiniCPMV4_7ImageProcessorPil default
-    default_scale_resolution = 448
-    # rows of v.tok_embd_sep, the order must match clip_suffix_rows() in clip-impl.h
-    tok_embd_sep = ["</image>", "<slice>", "</slice>", "\n"]
-
-    def get_downsample_mode(self) -> str:
-        # 4.7 moved downsample_mode to the model config; preprocessor value takes priority
-        return self.preprocessor_config.get(
-            "downsample_mode", self.global_config.get("downsample_mode", "16x"))
-
-    @classmethod
-    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
-        # keep the text tok_embd, the separator rows are taken from it in modify_tensors
-        if item[0] == "model.language_model.embed_tokens.weight":
-            return item
-        return super().filter_tensors(item)
-
-    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
-        if name == "model.language_model.embed_tokens.weight":
-            # the tile separators are text tokens; clip appends their embeddings so that one chunk holds the whole image
-            from transformers import AutoTokenizer
-            tokenizer = AutoTokenizer.from_pretrained(self.dir_model)
-            ids = []
-            for text in self.tok_embd_sep:
-                tok = tokenizer.encode(text, add_special_tokens=False)
-                if len(tok) != 1:
-                    raise ValueError(f"separator {text!r} must be a single token, got {tok}")
-                ids.append(tok[0])
-            yield self.format_tensor_name(gguf.MODEL_TENSOR.V_TOK_EMBD_SEP, suffix=""), data_torch[ids]
-            return
-        yield from super().modify_tensors(data_torch, name, bid)

conversion/qwen.py

@@ -570,15 +570,11 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
         elif name.endswith((".linear_attn.in_proj_a.weight", ".linear_attn.in_proj_b.weight")):
             weight, scale = reorder_rows(weight, scale, 1)
         elif name.endswith(".linear_attn.out_proj.weight"):
-            if self._hadamard_folds_tensor(name):
-                # folded weight: keep the grouped V order, the runtime permutes the activation instead
-                self._hadamard_gdn_v_grouped = True
-            else:
-                col_perm = self._reorder_v_heads(
-                    torch.arange(num_v_heads * head_v_dim, dtype=torch.long).unsqueeze(0),
-                    1, num_k_heads, num_v_per_k, head_v_dim,
-                ).squeeze(0)
-                weight, scale = apply_col_perm(weight, scale, col_perm)
+            col_perm = self._reorder_v_heads(
+                torch.arange(num_v_heads * head_v_dim, dtype=torch.long).unsqueeze(0),
+                1, num_k_heads, num_v_per_k, head_v_dim,
+            ).squeeze(0)
+            weight, scale = apply_col_perm(weight, scale, col_perm)
 
         return weight, scale
 
@@ -586,10 +582,6 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
         weight, scale = self._transform_nvfp4_weight(name, weight, scale)
         super()._repack_nvfp4(name, weight, scale, scale2, input_scale)
 
-    def _hadamard_folds_tensor(self, name: str) -> bool:
-        # a manifest name and `name` can differ only by leading wrapper prefixes
-        return any(name.endswith(n) or n.endswith(name) for n in self.hadamard_folded_names())
-
     def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
         num_k_heads = self.hparams.get("linear_num_key_heads", 0)
         num_v_heads = self.hparams.get("linear_num_value_heads", 0)
@@ -636,12 +628,8 @@ class _LinearAttentionVReorderBase(Qwen3NextModel):
                 data_torch = torch.cat([qk_part, v_part], dim=0)
 
             elif ".out_proj." in name:
-                if self._hadamard_folds_tensor(name):
-                    # folded weight: keep the grouped V order, the runtime permutes the activation instead
-                    self._hadamard_gdn_v_grouped = True
-                else:
-                    # Out projection weight: reorder columns (input dimension)
-                    data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
+                # Out projection weight: reorder columns (input dimension)
+                data_torch = self._reorder_v_heads(data_torch, 1, num_k_heads, num_v_per_k, head_v_dim)
 
         yield from super().modify_tensors(data_torch, name, bid)
 

conversion/vllm_decision2.py

@@ -1,107 +0,0 @@
-from __future__ import annotations
-
-import json
-
-from pathlib import Path
-from typing import Any, Callable, Iterator, TYPE_CHECKING
-
-if TYPE_CHECKING:
-    from torch import Tensor
-
-from .base import ModelBase, gguf, logger
-from .qwen import Qwen3Model
-
-
-def _is_vllm_decision2_checkpoint(dir_model: Path) -> bool:
-    if not (dir_model / "config.json").is_file():
-        return False
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        return json.load(f).get("decision_format") == "vllm-sr-decision"
-
-
-@ModelBase.register_hparams_loader(_is_vllm_decision2_checkpoint)
-def _load_vllm_decision2_hparams(dir_model: Path) -> dict[str, Any]:
-    logger.info("gguf: detected vllm-sr Decision 2.0 checkpoint")
-    with open(dir_model / "config.json", encoding="utf-8") as f:
-        package = json.load(f)
-    hparams = ModelBase.load_hparams((dir_model / package["backbone"]["config"]).parent, False, guess=False)
-    hparams["architectures"] = ["VllmDecision2Model"]
-    with open(dir_model / package["model_config"], encoding="utf-8") as f:
-        hparams["decision"] = json.load(f)
-    with open(dir_model / "score_bias.json", encoding="utf-8") as f:
-        hparams["decision"]["score_bias"] = json.load(f)["offsets"]
-    return hparams
-
-
-@ModelBase.register("VllmDecision2Model")
-@ModelBase.example("vllm-sr/Decision-2.0-Kai-0.6B")
-class VllmDecision2Model(Qwen3Model):
-    model_arch = gguf.MODEL_ARCH.VLLM_DECISION2
-
-    # torch.nn.LayerNorm default, used by the head
-    _HEAD_NORM_EPS = 1e-5
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        head = self.hparams["decision"]
-        if head["architecture"] != "qwen3-text-endpoints-global-query-shared-bilinear-mlp" or head["head_variant"] != "shared":
-            raise ValueError(f"unsupported decision head: {head['architecture']}")
-
-    def index_tensors(self, *args, **kwargs) -> dict[str, Callable[[], Tensor]]:
-        # the backbone weights are in their own directory
-        dir_model, self.dir_model = self.dir_model, self.dir_model / "backbone"
-        try:
-            return super().index_tensors(*args, **kwargs)
-        finally:
-            self.dir_model = dir_model
-
-    def get_tensors(self) -> Iterator[tuple[str, Tensor]]:
-        yield from super().get_tensors()
-        from safetensors.torch import load_file
-        for name, data in load_file(self.dir_model / "decision_head.safetensors").items():
-            yield "decision_head." + name, data
-
-    def tensor_force_quant(self, name: str, new_name: str, bid: int | None, n_dims: int) -> gguf.GGMLQuantizationType | bool:
-        # the head is computed in F32
-        if new_name.startswith("decision."):
-            return gguf.GGMLQuantizationType.F32
-        return super().tensor_force_quant(name, new_name, bid, n_dims)
-
-    def set_vocab(self):
-        super().set_vocab()
-        self.gguf_writer.add_chat_template([{"name": "systemone", "template": self._systemone_template()}])
-
-    @classmethod
-    def _systemone_template(cls) -> str:
-        # prompt follows decision2/_vendor/dev2model/decision_model.py (segments) of the model repo
-        # the pieces of the prompt are tokenized one by one, the server gives the text that separates them (sep)
-        # and the text that starts the span of a question or of an option (mark_question, mark_option)
-        # the head reads the last token of each option and the last token of the question
-        # the start of an option is a piece of its own, so that the spans of two options are not joined
-        # the keys of JSON objects are given in sorted order
-        def text(value: str) -> str:
-            return "{{ " + json.dumps(value) + " }}"
-
-        def render(name: str) -> str:
-            return "{{ " + name + " if " + name + " is string else " + name + " | tojson(separators=[',', ':']) }}"
-
-        option = (
-            "{% set d = o.description %}"
-            "{% if q.type == 'noul' and d is none %}{% set d = 'Yes' if o.key == 'true' else 'No' %}{% endif %}"
-            "{{ {'description': d, 'key': o.key} | tojson(separators=[',', ':']) }}"
-        )
-        return (
-            "{% for q in questions %}"
-            + text("Context:\n") + render("state")
-            + text("\n\nTask type: ") + "{{ q.type }}" + text("\nQuestion:\n") + render("q.instructions") + text("\nOptions:")
-            + "{% for o in q.options %}{{ sep }}" + text("\n<option>\n") + "{{ sep }}{{ mark_option }}" + option + text("\n</option>") + "{% endfor %}"
-            + "{{ sep }}{{ mark_question }}" + text("\n\nSelect the single option best supported by the context and instructions.\nDecision:")
-            + "{% endfor %}"
-        )
-
-    def set_gguf_parameters(self):
-        super().set_gguf_parameters()
-        self.gguf_writer.add_decision_type(gguf.DecisionType.VLLM_DECISION2)
-        self.gguf_writer.add_layer_norm_eps(self._HEAD_NORM_EPS)
-        for offsets in self.hparams["decision"]["score_bias"].values():
-            self.gguf_writer.add_decision_score_bias(offsets)

docs/multimodal/minicpmv4.7.md

@@ -1,55 +0,0 @@
-## MiniCPM-V 4.7
-
-### Prepare models and code
-
-Download [MiniCPM-V-4.7](https://huggingface.co/openbmb/MiniCPM-V-4.7) PyTorch model from huggingface to "MiniCPM-V-4.7" folder.
-
-The model must be the standard `transformers` checkpoint (no `trust_remote_code` for the text and vision graph used here); the architecture in `config.json` is `MiniCPMV4_7ForConditionalGeneration` with a `qwen3_5_text` (or `qwen3_5_moe_text`) text model and a SigLIP-based vision tower plus a window-attention `vit_merger`, same as MiniCPM-V 4.6.
-
-If the checkpoint ships no MTP weights, pass `--no-mtp` to skip the nextn layers.
-
-### Build llama.cpp
-
-If there are differences in usage, please refer to the official build [documentation](https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md)
-
-Clone llama.cpp:
-```bash
-git clone https://github.com/ggml-org/llama.cpp
-cd llama.cpp
-```
-
-Build llama.cpp using `CMake`:
-```bash
-cmake -B build
-cmake --build build --config Release
-```
-
-
-### Usage of MiniCPM-V 4.7
-
-MiniCPM-V 4.7 is converted directly through `convert_hf_to_gguf.py`. The same script is invoked twice on the original Hugging Face directory: once to produce the language-model GGUF and once with `--mmproj` to produce the multimodal projector GGUF.
-
-```bash
-# language model
-python ./convert_hf_to_gguf.py ../MiniCPM-V-4.7 --outfile ../MiniCPM-V-4.7/ggml-model-f16.gguf --no-mtp
-
-# multimodal projector (vision tower + window-attention vit_merger + DownsampleMLP merger)
-python ./convert_hf_to_gguf.py ../MiniCPM-V-4.7 --mmproj --outfile ../MiniCPM-V-4.7/mmproj-model-f16.gguf
-
-# optional: quantize to Q4_K_M
-./build/bin/llama-quantize ../MiniCPM-V-4.7/ggml-model-f16.gguf ../MiniCPM-V-4.7/ggml-model-Q4_K_M.gguf Q4_K_M
-```
-
-The default projector merges 16x (4x4 patches into one token). To keep 4x more visual tokens, copy the model dir and set `"downsample_mode": "4x"` in the copy's `preprocessor_config.json` before running the `--mmproj` conversion; the loader reads `clip.vision.projector.scale_factor` to pick the graph.
-
-
-Inference on Linux or Mac
-```bash
-# run in single-turn mode
-./build/bin/llama-mtmd-cli -m ../MiniCPM-V-4.7/ggml-model-f16.gguf --mmproj ../MiniCPM-V-4.7/mmproj-model-f16.gguf -c 4096 --jinja --image xx.jpg -p "What is in the image?"
-
-# run in conversation mode
-./build/bin/llama-mtmd-cli -m ../MiniCPM-V-4.7/ggml-model-Q4_K_M.gguf --mmproj ../MiniCPM-V-4.7/mmproj-model-f16.gguf --jinja
-```
-
-The chat template enables thinking by default. Pass `--chat-template-kwargs '{"enable_thinking": false}'` to `llama-server` to turn it off.

examples/speculative/speculative.cpp

@@ -50,11 +50,6 @@ int main(int argc, char ** argv) {
         return 1;
     }
 
-    if (params.n_parallel > 1 && !params.kv_unified) {
-        LOG_ERR("%s: --kv-unified/-kvu is required when --parallel/-np > 1\n", __func__);
-        return 1;
-    }
-
     if (params.speculative.draft.mparams.path.empty()) {
         LOG_ERR("%s: --model-draft is required\n", __func__);
         return 1;
@@ -272,7 +267,7 @@ int main(int argc, char ** argv) {
             // for stochastic sampling, attempt to match the token with the drafted tokens
             {
                 bool accept = false;
-                if (params.sampling.temp > 0 && n_seq_dft == 1) {
+                if (params.sampling.temp > 0) {
                     // stochastic verification
                     common_sampler_sample(smpl, ctx_tgt, drafts[s_keep].i_batch_tgt[i_dft], true);
 
@@ -346,17 +341,13 @@ int main(int argc, char ** argv) {
 
                             float sum_probs = 0.0f;
 
-                            for (size_t i = 0, j = 0; i < dist_tgt.size; i++) {
-                                // try to find the current target token id in the draft distribution
-                                while (j < dist_dft.size && dist_dft.data[j].id < dist_tgt.data[i].id) {
-                                    ++j;
+                            for (size_t i = 0; i < dist_tgt.size; i++) {
+                                if (i < dist_dft.size) {
+                                    dist_tgt.data[i].p = std::max(0.0f, dist_tgt.data[i].p - dist_dft.data[i].p);
+                                } else {
+                                    dist_tgt.data[i].p = std::max(0.0f, dist_tgt.data[i].p);
                                 }
 
-                                // tokens absent from the draft distribution have zero probability
-                                float p_dft = j < dist_dft.size && dist_dft.data[j].id == dist_tgt.data[i].id ?
-                                    dist_dft.data[j].p : 0.0f;
-
-                                dist_tgt.data[i].p = std::max(0.0f, dist_tgt.data[i].p - p_dft);
                                 sum_probs += dist_tgt.data[i].p;
                             }
 
@@ -379,7 +370,7 @@ int main(int argc, char ** argv) {
                                 // synchronize active status for sequences with the same drafted token
                                 drafts[i].active = drafts[i].active && accept;
                                 if (!drafts[i].active) {
-                                    active_seqs.erase(i);
+                                    active_seqs.erase(s);
                                 }
                             }
                         }
@@ -403,7 +394,7 @@ int main(int argc, char ** argv) {
                         token_str = common_token_to_piece(ctx_tgt, token_id);
                     }
                 } else {
-                    // verify drafts by matching tokens sampled from the target
+                    // greedy verification
 
                     // sample from the target model
                     LOG_DBG("sampling target: s_keep = %3d, i_dft = %3d, i_batch_tgt = %3d\n", s_keep, i_dft, drafts[s_keep].i_batch_tgt[i_dft]);
@@ -527,7 +518,7 @@ int main(int argc, char ** argv) {
                     continue;
                 }
 
-                const llama_token sampled_token = common_sampler_sample(drafts[s].smpl, ctx_dft, drafts[s].i_batch_dft, true);
+                common_sampler_sample(drafts[s].smpl, ctx_dft, drafts[s].i_batch_dft, true);
 
                 const auto * cur_p = common_sampler_get_candidates(drafts[s].smpl, true);
 
@@ -576,7 +567,7 @@ int main(int argc, char ** argv) {
 
                 // add drafted token for each sequence
                 for (int is = 0; is < (int) sa.size(); ++is) {
-                    const llama_token id = n_seq_dft == 1 ? sampled_token : cur_p->data[is].id;
+                    const llama_token id = cur_p->data[is].id;
 
                     const int s = sa[is];
 

ggml/include/ggml-backend.h

@@ -326,14 +326,6 @@ extern "C" {
     // `graph` is the compute graph of the split
     typedef bool (*ggml_backend_sched_copy_callback)(ggml_backend_t backend, const struct ggml_tensor * src, struct ggml_tensor * dst, struct ggml_cgraph * graph, void * user_data);
 
-    // Callback when the buffers reserved for a graph are not sufficient and have to be reserved again
-    // it happens on the graph allocation path and costs a synchronization of the backends
-    // `sched` is the scheduler
-    // `graph` is the compute graph being allocated
-    // `backend_ids_changed` is true if the backend assignment of the graph nodes changed since the last allocation
-    // `graph_size_changed` is true if the graph size changed since the last allocation
-    typedef void (*ggml_backend_sched_realloc_callback)(ggml_backend_sched_t sched, struct ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data);
-
     // Initialize a backend scheduler, backends with low index are given priority over backends with high index
     GGML_API ggml_backend_sched_t ggml_backend_sched_new(ggml_backend_t * backends, ggml_backend_buffer_type_t * bufts, int n_backends, size_t graph_size, bool parallel, bool op_offload);
     GGML_API void                 ggml_backend_sched_free(ggml_backend_sched_t sched);
@@ -375,9 +367,6 @@ extern "C" {
     // Set a callback to be called when the inputs weights of a split are being copied
     GGML_API void                 ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backend_sched_copy_callback callback, void * user_data);
 
-    // Set a callback to be called when the compute buffers of a graph are being reallocated
-    GGML_API void                 ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data);
-
     //
     // Meta backend
     //

ggml/src/CMakeLists.txt

@@ -462,8 +462,6 @@ function(ggml_add_cpu_backend_variant tag_name)
             set(GGML_INTERNAL_${feat} ON)
         endforeach()
     elseif (GGML_SYSTEM_ARCH STREQUAL "s390x")
-        set(GGML_NATIVE OFF)
-
         foreach (feat VXE2 NNPA)
             set(GGML_INTERNAL_${feat} OFF)
         endforeach()
@@ -571,14 +569,6 @@ if (GGML_CPU_ALL_VARIANTS)
         if (CMAKE_SYSTEM_NAME MATCHES "Linux")
             ggml_add_cpu_backend_variant(z15    Z15 VXE2)
             ggml_add_cpu_backend_variant(z16    Z16 VXE2 NNPA)
-
-            # check if compiler supports "-march=z17" codename
-            check_cxx_compiler_flag("-march=arch15" GGML_CXX_SUPPORTS_Z17)
-            if (GGML_CXX_SUPPORTS_Z17)
-                ggml_add_cpu_backend_variant(arch15 Z17 VXE2 NNPA)
-            else()
-                message(WARNING "Skipping z17 target: compiler must be GCC 15.1 and later")
-            endif()
         else()
             message(FATAL_ERROR "Unsupported s390x target OS: ${CMAKE_SYSTEM_NAME}")
         endif()
@@ -637,11 +627,7 @@ endif()
 
 if (BUILD_SHARED_LIBS)
     foreach (target ggml-base ggml)
-        set_target_properties(${target} PROPERTIES
-            POSITION_INDEPENDENT_CODE ON
-            C_VISIBILITY_PRESET hidden
-            CXX_VISIBILITY_PRESET hidden
-        )
+        set_target_properties(${target} PROPERTIES POSITION_INDEPENDENT_CODE ON)
         target_compile_definitions(${target} PRIVATE GGML_BUILD)
         target_compile_definitions(${target} PUBLIC  GGML_SHARED)
     endforeach()

ggml/src/ggml-backend.cpp

@@ -969,9 +969,6 @@ struct ggml_backend_sched {
     ggml_backend_sched_copy_callback callback_copy;
     void * callback_copy_user_data;
 
-    ggml_backend_sched_realloc_callback callback_realloc;
-    void * callback_realloc_user_data;
-
     char * context_buffer;
     size_t context_buffer_size;
 
@@ -979,13 +976,11 @@ struct ggml_backend_sched {
 
     int debug;
 
-    // graph size of the current and the previous scheduling, used by the realloc callback and for debugging
-    // graph reallocations [GGML_SCHED_DEBUG_REALLOC]
+    // used for debugging graph reallocations [GGML_SCHED_DEBUG_REALLOC]
     // ref: https://github.com/ggml-org/llama.cpp/pull/17617
-    int graph_size;
-    int prev_graph_size;
-
     int debug_realloc;
+    int debug_graph_size;
+    int debug_prev_graph_size;
 };
 
 #define hash_id(tensor) ggml_hash_find_or_insert(&sched->hash_set, tensor)
@@ -1638,9 +1633,9 @@ void ggml_backend_sched_split_graph(ggml_backend_sched_t sched, struct ggml_cgra
     }
     int graph_size = std::max(graph->n_nodes, graph->n_leafs) + total_inputs * 2 * sched->n_copies + n_dep_nodes;
 
-    // remember the actual graph_size for performing reallocation checks later
-    sched->prev_graph_size = sched->graph_size;
-    sched->graph_size = graph_size;
+    // remember the actual graph_size for performing reallocation checks later [GGML_SCHED_DEBUG_REALLOC]
+    sched->debug_prev_graph_size = sched->debug_graph_size;
+    sched->debug_graph_size = graph_size;
 
     if (sched->graph.size < graph_size) {
         sched->graph.size = graph_size;
@@ -1777,21 +1772,14 @@ static bool ggml_backend_sched_alloc_splits(ggml_backend_sched_t sched) {
         GGML_LOG_DEBUG("%s: failed to allocate graph, reserving (backend_ids_changed = %d)\n", __func__, backend_ids_changed);
 #endif
 
-        // a reallocation with an unchanged graph size means that the previous reservation was not sufficient
-        const bool graph_size_changed = sched->prev_graph_size != sched->graph_size;
-
-        if (sched->callback_realloc != NULL) {
-            sched->callback_realloc(sched, &sched->graph, backend_ids_changed, graph_size_changed, sched->callback_realloc_user_data);
-        }
-
         if (sched->debug_realloc > 0) {
             // we are interested only in situations where the graph was reallocated even though its size remained the same [GGML_SCHED_DEBUG_REALLOC]
             // example: https://github.com/ggml-org/llama.cpp/pull/17143
-            const bool unexpected = !backend_ids_changed && !graph_size_changed;
+            const bool unexpected = !backend_ids_changed && sched->debug_prev_graph_size == sched->debug_graph_size;
 
             if (unexpected || sched->debug_realloc > 1) {
                 GGML_ABORT("%s: unexpected graph reallocation (graph size = %d, nodes = %d, leafs = %d), debug_realloc = %d\n", __func__,
-                        sched->graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc);
+                        sched->debug_graph_size, sched->graph.n_nodes, sched->graph.n_leafs, sched->debug_realloc);
             }
         }
 
@@ -1985,8 +1973,8 @@ ggml_backend_sched_t ggml_backend_sched_new(
     sched->prev_node_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_node_backend_ids[0]));
     sched->prev_leaf_backend_ids = (int *) calloc(nodes_size, sizeof(sched->prev_leaf_backend_ids[0]));
 
-    sched->graph_size      = 0;
-    sched->prev_graph_size = 0;
+    sched->debug_graph_size = 0;
+    sched->debug_prev_graph_size = 0;
 
     sched->context_buffer_size = ggml_sched_max_splits*GGML_SCHED_MAX_SPLIT_INPUTS*2*sizeof(struct ggml_tensor) + ggml_graph_overhead_custom(graph_size, false);
     sched->context_buffer = (char *) malloc(sched->context_buffer_size);
@@ -2155,12 +2143,6 @@ void ggml_backend_sched_set_copy_callback(ggml_backend_sched_t sched, ggml_backe
     sched->callback_copy_user_data = user_data;
 }
 
-void ggml_backend_sched_set_realloc_callback(ggml_backend_sched_t sched, ggml_backend_sched_realloc_callback callback, void * user_data) {
-    GGML_ASSERT(sched);
-    sched->callback_realloc = callback;
-    sched->callback_realloc_user_data = user_data;
-}
-
 int ggml_backend_sched_get_n_splits(ggml_backend_sched_t sched) {
     GGML_ASSERT(sched);
     return sched->n_splits;

ggml/src/ggml-cpu/CMakeLists.txt

@@ -593,12 +593,7 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
             foreach (ZHW RANGE 15 17)
                 if(DEFINED GGML_INTERNAL_Z${ZHW})
                     message(STATUS "z${ZHW} cross-compile target")
-                    if (ZHW EQUAL 17)
-                        # z17 is an alias of arch15, use the arch level for wider toolchain support
-                        list(APPEND ARCH_FLAGS -march=arch15)
-                    else()
-                        list(APPEND ARCH_FLAGS -march=z${ZHW})
-                    endif()
+                    list(APPEND ARCH_FLAGS -march=z${ZHW})
                 endif()
             endforeach()
         endif()

ggml/src/ggml-cpu/ggml-cpu-impl.h

@@ -390,16 +390,17 @@ typedef unsigned char uchar8x16_t __attribute__((vector_size(16)));
 typedef int8_t  int8x16_t __attribute__((vector_size(16)));
 typedef int16_t int16x8_t __attribute__((vector_size(16)));
 typedef int32_t int32x4_t __attribute__((vector_size(16)));
-typedef int64_t int64x2_t __attribute__((vector_size(16)));
 
 typedef uint8_t  uint8x16_t __attribute__((vector_size(16)));
 typedef uint16_t uint16x8_t __attribute__((vector_size(16)));
 typedef uint32_t uint32x4_t __attribute__((vector_size(16)));
-typedef uint64_t uint64x2_t __attribute__((vector_size(16)));
 
 typedef float  float32x4_t  __attribute__((vector_size(16)));
 typedef double double64x2_t __attribute__((vector_size(16)));
 
+typedef signed   long long long64x2_t  __attribute__((vector_size(16)));
+typedef unsigned long long ulong64x2_t __attribute__((vector_size(16)));
+
 typedef struct ggml_uint8x16x2_t {
     uint8x16_t val[2];
 } ggml_uint8x16x2_t;

ggml/src/ggml-cpu/ggml-cpu.c

@@ -3504,12 +3504,6 @@ void ggml_cpu_fp32_to_fp16(const float * x, ggml_fp16_t * y, int64_t n) {
         vfloat16m1_t vy = __riscv_vfncvt_f_f_w_f16m1(vx, vl);
         __riscv_vse16_v_f16m1((_Float16 *)&y[i], vy, vl);
     }
-#elif defined(__VXE__) || defined(__VXE2__)
-    for (; i + 7 < n; i += 8) {
-        const uint32x4_t v_yl = __lzs_f32cx4_to_f16(vec_xl(0, x + i + 0));
-        const uint32x4_t v_yh = __lzs_f32cx4_to_f16(vec_xl(0, x + i + 4));
-        vec_xst(vec_pack(v_yl, v_yh), 0, (uint16_t *)(y + i));
-    }
 #endif
     for (; i < n; ++i) {
         y[i] = GGML_CPU_FP32_TO_FP16(x[i]);

ggml/src/ggml-cpu/simd-mappings.h

@@ -140,14 +140,6 @@ extern float ggml_table_f32_ue4m3[1 << 8];
 #define GGML_CPU_UE4M3_TO_FP32(x) ggml_ue4m3_to_fp32(x)
 #endif
 
-// Pre AVX512 CPUs (AVX2 with F16C, such as i5-8th gen or i7-4790)
-// do not produce codegen as checks below don't test for __F16C__
-// but this check does.
-#if defined(__F16C__)
-#define GGML_CPU_FP16_TO_FP32(x) GGML_CPU_COMPUTE_FP16_TO_FP32(x)
-#define GGML_CPU_FP32_TO_FP16(x) GGML_CPU_COMPUTE_FP32_TO_FP16(x)
-#endif
-
 // On ARM NEON, it's quicker to directly convert x -> x instead of calling into ggml_lookup_fp16_to_fp32,
 // so we define GGML_CPU_FP16_TO_FP32 and GGML_CPU_FP32_TO_FP16 elsewhere for NEON.
 // This is also true for POWER9.
@@ -1231,24 +1223,6 @@ static inline void __lsx_f16x4_store(ggml_fp16_t * x, __m128 y) {
 #define GGML_F16_STEP GGML_F32_STEP
 #define GGML_F16_EPR  GGML_F32_EPR
 
-static inline uint32x4_t __lzs_f32cx4_to_f16(float32x4_t v_f) {
-    float32x4_t v_base = vec_mul(vec_mul(vec_abs(v_f), vec_splats(0x1.0p+112f)), vec_splats(0x1.0p-110f));
-
-    const uint32x4_t v_w      = (uint32x4_t)v_f;
-    const uint32x4_t v_shl1_w = vec_add(v_w, v_w);
-    const uint32x4_t v_sign   = vec_and(v_w, vec_splats(UINT32_C(0x80000000)));
-    const uint32x4_t v_bias   = vec_max(vec_and(v_shl1_w, vec_splats(UINT32_C(0xFF000000))), vec_splats(UINT32_C(0x71000000)));
-
-    v_base = vec_add((float32x4_t)vec_add(vec_sr(v_bias, 1), vec_splats(UINT32_C(0x07800000))), v_base);
-
-    const uint32x4_t v_bits    = (uint32x4_t)v_base;
-    const uint32x4_t v_nonsign = vec_add(vec_and(vec_sr(v_bits, 13), vec_splats(UINT32_C(0x00007C00))),
-                                         vec_and(v_bits, vec_splats(UINT32_C(0x00000FFF))));
-    const uint32x4_t v_is_nan  = (uint32x4_t)vec_cmpgt(v_shl1_w, vec_splats(UINT32_C(0xFF000000)));
-
-    return vec_or(vec_sr(v_sign, 16), vec_sel(v_nonsign, vec_splats(UINT32_C(0x7E00)), v_is_nan));
-}
-
 static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) {
     float tmp[4];
 
@@ -1262,9 +1236,15 @@ static inline float32x4_t __lzs_f16cx4_load(const ggml_fp16_t * x) {
 }
 
 static inline void __lzs_f16cx4_store(ggml_fp16_t * x, float32x4_t v_y) {
-    const uint32x4_t v_h = __lzs_f32cx4_to_f16(v_y);
-    const uint64_t   tmp = ((uint64x2_t)vec_pack(v_h, v_h))[0];
-    memcpy(x, &tmp, sizeof(tmp));
+    float arr[4];
+
+    // note: keep type-cast here to prevent compiler bugs
+    // see: https://github.com/ggml-org/llama.cpp/issues/12846
+    vec_xst(v_y, 0, (float *)(arr));
+
+    for (int i = 0; i < 4; i++) {
+        x[i] = GGML_CPU_FP32_TO_FP16(arr[i]);
+    }
 }
 
 #define GGML_F16_VEC                GGML_F32x4

ggml/src/ggml-cuda/fattn-mma-f16.cuh

@@ -1772,7 +1772,7 @@ static __device__ __forceinline__ void flash_attn_ext_f16_process_tile(
                 }
             }
         }
-        if (np > 1 || nbatch_combine != DKQ/2) {
+        if (np > 1) {
             __syncthreads();
         }
     }

ggml/src/ggml-cuda/mmvq.cu

@@ -283,9 +283,9 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type
 
 // Host function: returns the max batch size for the current arch+type at runtime.
 int get_mmvq_mmid_max_batch(ggml_type type, int cc) {
-    // NVIDIA: P100, Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID.
+    // NVIDIA: Volta, Ada Lovelace, and Blackwell always use MMVQ for MUL_MAT_ID.
     if (GGML_CUDA_CC_IS_NVIDIA(cc)) {
-        if (cc == GGML_CUDA_CC_PASCAL || cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) {
+        if (cc == GGML_CUDA_CC_VOLTA || cc >= GGML_CUDA_CC_ADA_LOVELACE) {
             return MMVQ_MAX_BATCH_SIZE;
         }
         if (cc >= GGML_CUDA_CC_TURING) {
@@ -440,7 +440,7 @@ static constexpr __device__ int get_mmvq_mmid_max_batch_for_device() {
     return get_mmvq_mmid_max_batch_cdna(type);
 #elif defined(GCN)
     return get_mmvq_mmid_max_batch_gcn(type);
-#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_PASCAL || __CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE)
+#elif !defined(GGML_USE_MUSA) && (__CUDA_ARCH__ == GGML_CUDA_CC_VOLTA || __CUDA_ARCH__ >= GGML_CUDA_CC_ADA_LOVELACE)
     return MMVQ_MAX_BATCH_SIZE;
 #elif !defined(GGML_USE_MUSA) && __CUDA_ARCH__ >= GGML_CUDA_CC_TURING
     return get_mmvq_mmid_max_batch_turing_plus(type);

ggml/src/ggml-hexagon/ggml-hexagon.cpp

@@ -74,7 +74,6 @@
 #include "htp/argsort-ops.h"
 #include "htp/concat-ops.h"
 #include "htp/cpy-ops.h"
-#include "htp/im2col-ops.h"
 #include "htp_iface.h"
 #include "htp-drv.h"
 
@@ -288,8 +287,8 @@ static inline bool ggml_hexagon_is_repack_type(enum ggml_type type) {
            type == GGML_TYPE_Q3_K || type == GGML_TYPE_Q2_K;
 }
 
-// Size of one repacked row in the DSP tiled layout. The Q3_K..Q6_K tiles store uncompressed scales/mins,
-// so they are larger than the ggml blocks. For the other repack types (Q2_K included) the tile has the same size.
+// Size of one repacked row in the DSP tiled layout. The K-quant tiles store uncompressed scales/mins,
+// so they are larger than the ggml blocks. For the other repack types the tile has the same size as the ggml blocks.
 static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne0) {
     if (type == GGML_TYPE_Q6_K) {
         return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q6_K / 32);
@@ -303,6 +302,9 @@ static inline size_t ggml_hexagon_tiled_row_size(enum ggml_type type, int64_t ne
     if (type == GGML_TYPE_Q3_K) {
         return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q3_K / 32);
     }
+    if (type == GGML_TYPE_Q2_K) {
+        return (size_t) (ne0 / 32) * (HTP_MM_WEIGHT_TILE_SIZE_Q2_K / 32);
+    }
     return ggml_row_size(type, ne0);
 }
 
@@ -416,18 +418,6 @@ static bool ggml_hexagon_precompute_cpy_params(
     struct htp_copy_kernel_params * kparams
 );
 
-static bool ggml_hexagon_precompute_im2col_params(
-    const struct ggml_hexagon_session * sess,
-    const struct ggml_tensor * op,
-    struct htp_im2col_kernel_params * kparams
-);
-
-static bool ggml_hexagon_precompute_im2col_3d_params(
-    const struct ggml_hexagon_session * sess,
-    const struct ggml_tensor * op,
-    struct htp_im2col_kernel_params * kparams
-);
-
 static void ggml_hexagon_precompute_fused_mmnx_params(
     const struct ggml_hexagon_session * sess,
     const struct ggml_tensor * src0,
@@ -1906,11 +1896,13 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset,
                 for (int kt = 0; kt < n_k_tiles; kt++) {
                     const int kt_local = kt % 8;  // k-tile within the super-block
                     const block_q2_K * b = &src_row[kt / 8];
+                    const float d    = GGML_FP16_TO_FP32(b->d);
+                    const float dmin = GGML_FP16_TO_FP32(b->dmin);
 
                     uint8_t * tile = matrix_dst + ((size_t) ct * n_k_tiles + kt) * tile_size;
                     uint8_t * lo_pl = tile;
-                    uint8_t * sc_pl = tile + 256;
-                    ggml_half * dd_pl = (ggml_half *) (tile + 320);
+                    ggml_half * sc_pl = (ggml_half *) (tile + 256);
+                    ggml_half * m_pl  = (ggml_half *) (tile + 384);
 
                     for (int lk = 0; lk < 32; lk++) {
                         const int g   = lk >> 2;
@@ -1918,10 +1910,9 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset,
                         lo_pl[(g >> 2) * 128 + pos] |= (uint8_t) (q2_3_K_get_low2(b->qs, kt_local * 32 + lk) << ((g & 3) * 2));
                     }
                     for (int sub = 0; sub < 2; sub++) {
-                        sc_pl[sub * 32 + row] = b->scales[kt_local * 2 + sub];
-                    }
-                    if (row / 8 == kt_local % 4) {
-                        dd_pl[row % 8] = kt_local < 4 ? b->d : (ggml_half) (b->dmin ^ 0x8000);  // -dmin
+                        const uint8_t sc = b->scales[kt_local * 2 + sub];
+                        sc_pl[sub * 32 + row] = GGML_FP32_TO_FP16( d    * (float) (sc & 0xF));
+                        m_pl [sub * 32 + row] = GGML_FP32_TO_FP16(-dmin * (float) (sc >> 4));
                     }
                 }
             }
@@ -1931,7 +1922,7 @@ static void repack_q2_K_tiled(ggml_tensor * t, const void * data, size_t offset,
     GGML_UNUSED(size);
 }
 
-// Reverse of repack_q2_K_tiled. Lossless. Read-back only.
+// Reverse of repack_q2_K_tiled. Unpacks quants losslessly and normalizes scales/mins. Read-back only.
 static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset, size_t size) {
     GGML_ASSERT(offset == 0);
 
@@ -1966,12 +1957,14 @@ static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset,
                     block_q2_K * b = &dst_row[sb];
                     memset(b, 0, sizeof(block_q2_K));
 
+                    ggml_half sub_scales[16];
+                    ggml_half sub_mins[16];
                     for (int kt_local = 0; kt_local < 8; kt_local++) {
                         const int kt = sb * 8 + kt_local;
                         const uint8_t *   tile  = matrix_src + ((size_t) ct * n_k_tiles + kt) * tile_size;
                         const uint8_t *   lo_pl = tile;
-                        const uint8_t *   sc_pl = tile + 256;
-                        const ggml_half * dd_pl = (const ggml_half *) (tile + 320);
+                        const ggml_half * sc_pl = (const ggml_half *) (tile + 256);
+                        const ggml_half * m_pl  = (const ggml_half *) (tile + 384);
 
                         for (int lk = 0; lk < 32; lk++) {
                             const int e   = kt_local * 32 + lk;
@@ -1982,15 +1975,36 @@ static void repack_tiled_q2_K(void * data, const ggml_tensor * t, size_t offset,
                         }
 
                         for (int sub = 0; sub < 2; sub++) {
-                            b->scales[kt_local * 2 + sub] = sc_pl[sub * 32 + row];
+                            const float D = GGML_FP16_TO_FP32(sc_pl[sub * 32 + row]);
+                            const float M = GGML_FP16_TO_FP32(m_pl[sub * 32 + row]);
+                            sub_scales[kt_local * 2 + sub] = GGML_FP32_TO_FP16((D > 0.0f) ? D : 0.0f);
+                            sub_mins[kt_local * 2 + sub]   = GGML_FP32_TO_FP16((-M > 0.0f) ? -M : 0.0f);
                         }
-                        if (row / 8 == kt_local % 4) {
-                            if (kt_local < 4) {
-                                b->d = dd_pl[row % 8];
-                            } else {
-                                b->dmin = (ggml_half) (dd_pl[row % 8] ^ 0x8000);
-                            }
+                    }
+
+                    int ls[16];
+                    int lm[16];
+                    ggml_half * const dd[2] = { &b->d, &b->dmin };
+                    const ggml_half * const prod[2] = { sub_scales, sub_mins };
+                    int * const ll[2] = { ls, lm };
+                    for (int w = 0; w < 2; w++) {
+                        if (hexagon_recover_k_scales(prod[w], 16, 0, 15, dd[w], ll[w])) {
+                            continue;
                         }
+                        float max_val = 0.0f;
+                        for (int j = 0; j < 16; j++) {
+                            max_val = (std::max)(max_val, GGML_FP16_TO_FP32(prod[w][j]));
+                        }
+                        *dd[w] = GGML_FP32_TO_FP16(max_val / 15.0f);
+                        const float d_actual = GGML_FP16_TO_FP32(*dd[w]);
+                        const float inv_d    = (d_actual > 0.0f) ? (1.0f / d_actual) : 0.0f;
+                        for (int j = 0; j < 16; j++) {
+                            ll[w][j] = (std::min)(15, (int) roundf(inv_d * GGML_FP16_TO_FP32(prod[w][j])));
+                        }
+                    }
+
+                    for (int j = 0; j < 16; j++) {
+                        b->scales[j] = (uint8_t) (ls[j] | (lm[j] << 4));
                     }
                 }
             }
@@ -7682,397 +7696,21 @@ static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session *
     return true;
 }
 
-static bool im2col_3d_calc_output_size(uint32_t input, uint32_t kernel, int32_t stride, int32_t padding,
-                                       int32_t dilation, uint32_t * output) {
-    if (stride <= 0 || padding < 0 || dilation <= 0 || kernel == 0) {
-        return false;
-    }
-
-    const int64_t numerator = (int64_t) input + 2LL * padding - (int64_t) dilation * (kernel - 1) - 1;
-    if (numerator < 0) {
-        return false;
-    }
-
-    const uint64_t size = (uint64_t) (numerator / stride) + 1;
-    if (size == 0 || size > UINT32_MAX) {
-        return false;
-    }
-
-    *output = (uint32_t) size;
-    return true;
-}
-
-static bool ggml_hexagon_precompute_im2col_params(
-    const struct ggml_hexagon_session * sess,
-    const struct ggml_tensor * op,
-    struct htp_im2col_kernel_params * kparams
-) {
-    memset(kparams, 0, sizeof(*kparams));
-    kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;
-
-    const struct ggml_tensor * src0 = op->src[0];
-    const struct ggml_tensor * src1 = op->src[1];
-    const struct ggml_tensor * dst  = op;
-
-    if (!src0 || !src1 || !dst) {
-        return false;
-    }
-
-    if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {
-        return false;
-    }
-
-    if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {
-        return false;
-    }
-
-    const int32_t * op_params = (const int32_t *) op->op_params;
-    const int32_t s0 = op_params[0];
-    const int32_t s1 = op_params[1];
-    const int32_t p0 = op_params[2];
-    const int32_t p1 = op_params[3];
-    const int32_t d0 = op_params[4];
-    const int32_t d1 = op_params[5];
-    const bool is_2D = (op_params[6] == 1);
-
-    if (s0 <= 0 || d0 <= 0 || p0 < 0) {
-        return false;
-    }
-    if (is_2D && (s1 <= 0 || d1 <= 0 || p1 < 0)) {
-        return false;
-    }
-
-    const uint32_t N  = is_2D ? (uint32_t) src1->ne[3] : (uint32_t) src1->ne[2];
-    const uint32_t IC = is_2D ? (uint32_t) src1->ne[2] : (uint32_t) src1->ne[1];
-    const uint32_t IH = is_2D ? (uint32_t) src1->ne[1] : 1;
-    const uint32_t IW = (uint32_t) src1->ne[0];
-    const uint32_t KH = is_2D ? (uint32_t) src0->ne[1] : 1;
-    const uint32_t KW = (uint32_t) src0->ne[0];
-    const uint32_t OH = is_2D ? (uint32_t) dst->ne[2] : 1;
-    const uint32_t OW = (uint32_t) dst->ne[1];
-
-    if (KW == 0 || KH == 0 || IW == 0 || IH == 0 || IC == 0 || OW == 0 || OH == 0 || N == 0) {
-        return false;
-    }
-
-    const uint64_t patch_stride_64 = (uint64_t) IC * KH * KW;
-    const uint64_t total_rows_64   = (uint64_t) N * OH;
-    if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX) {
-        return false;
-    }
-    const uint32_t patch_stride = (uint32_t) patch_stride_64;
-    const uint32_t total_rows   = (uint32_t) total_rows_64;
-
-    if (dst->ne[0] != patch_stride || dst->ne[1] != OW) {
-        return false;
-    }
-    if (is_2D && dst->ne[2] != OH) {
-        return false;
-    }
-
-    const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
-
-    bool is_patchembed = ggml_is_contiguous(src1) && (s0 == (int32_t) KW && p0 == 0 && d0 == 1);
-    if (is_2D) {
-        is_patchembed = is_patchembed && (s1 == (int32_t) KH && p1 == 0 && d1 == 1);
-    }
-
-    const uint32_t max_threads = (sess && sess->n_threads > 0) ? (uint32_t) sess->n_threads : 8;
-    const uint32_t n_threads   = (total_rows < max_threads) ? total_rows : max_threads;
-    const uint32_t vtcm_size   = (sess && sess->vtcm_size > 0) ? sess->vtcm_size : 0;
-
-    if (is_patchembed) {
-        const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * IW * sizeof(float);
-        const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;
-        if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {
-            const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
-            const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
-            struct htp_im2col_vtcm_layout layout;
-            htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
-            if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
-                kparams->kernel_type          = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
-                kparams->is_2d                = is_2D ? 1 : 0;
-                kparams->is_3d                = 0;
-                kparams->n_threads            = (uint8_t) n_threads;
-                kparams->vtcm_size            = layout.total_bytes;
-                kparams->off_src              = layout.off_src;
-                kparams->off_dst              = layout.off_dst;
-                kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
-                kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
-                kparams->src_row_bytes        = src_row_bytes;
-                kparams->dst_row_bytes        = dst_row_bytes;
-                kparams->owb                  = OW;
-                kparams->wb                   = IW;
-                kparams->nbpr                 = 1;
-                kparams->N                    = N;
-                kparams->IC                   = IC;
-                kparams->ID                   = 1;
-                kparams->IH                   = IH;
-                kparams->IW                   = IW;
-                kparams->KD                   = 1;
-                kparams->KH                   = KH;
-                kparams->KW                   = KW;
-                kparams->OD                   = 1;
-                kparams->OH                   = OH;
-                kparams->OW                   = OW;
-                kparams->patch_stride         = patch_stride;
-                kparams->total_rows           = total_rows;
-                return true;
-            }
-        }
-    }
-
-    for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {
-        const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;
-        if (Wb > UINT32_MAX) {
-            if (owb == 1) break;
-            continue;
-        }
-
-        const uint64_t src_row_bytes_raw = (uint64_t) IC * KH * Wb * sizeof(float);
-        const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;
-        if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {
-            if (owb == 1) break;
-            continue;
-        }
-
-        const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
-        const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
-        struct htp_im2col_vtcm_layout layout;
-        htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
-        if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
-            kparams->kernel_type          = HTP_IM2COL_KERNEL_BLOCKED_DMA;
-            kparams->is_2d                = is_2D ? 1 : 0;
-            kparams->is_3d                = 0;
-            kparams->n_threads            = (uint8_t) n_threads;
-            kparams->vtcm_size            = layout.total_bytes;
-            kparams->off_src              = layout.off_src;
-            kparams->off_dst              = layout.off_dst;
-            kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
-            kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
-            kparams->src_row_bytes        = src_row_bytes;
-            kparams->dst_row_bytes        = dst_row_bytes;
-            kparams->owb                  = owb;
-            kparams->wb                   = (uint32_t) Wb;
-            kparams->nbpr                 = (OW + owb - 1) / owb;
-            kparams->N                    = N;
-            kparams->IC                   = IC;
-            kparams->ID                   = 1;
-            kparams->IH                   = IH;
-            kparams->IW                   = IW;
-            kparams->KD                   = 1;
-            kparams->KH                   = KH;
-            kparams->KW                   = KW;
-            kparams->OD                   = 1;
-            kparams->OH                   = OH;
-            kparams->OW                   = OW;
-            kparams->patch_stride         = patch_stride;
-            kparams->total_rows           = total_rows;
-            return true;
-        }
-        if (owb == 1) break;
-    }
-
-    return false;
-}
-
-static bool ggml_hexagon_precompute_im2col_3d_params(
-    const struct ggml_hexagon_session * sess,
-    const struct ggml_tensor * op,
-    struct htp_im2col_kernel_params * kparams
-) {
-    memset(kparams, 0, sizeof(*kparams));
-    kparams->kernel_type = HTP_IM2COL_KERNEL_UNSUPPORTED;
-
-    const struct ggml_tensor * src0 = op->src[0];
-    const struct ggml_tensor * src1 = op->src[1];
-    const struct ggml_tensor * dst  = op;
-
-    if (!src0 || !src1 || !dst) {
-        return false;
-    }
-
-    if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {
-        return false;
-    }
-
-    if (src1->nb[0] != sizeof(float) || !ggml_is_contiguous(dst)) {
-        return false;
-    }
-
-    const int32_t * params = (const int32_t *) op->op_params;
-    const int32_t s0 = params[0], s1 = params[1], s2 = params[2];
-    const int32_t p0 = params[3], p1 = params[4], p2 = params[5];
-    const int32_t d0 = params[6], d1 = params[7], d2 = params[8];
-    const int32_t ic = params[9];
-
-    if (s0 <= 0 || s1 <= 0 || s2 <= 0 ||
-        p0 < 0  || p1 < 0  || p2 < 0  ||
-        d0 <= 0 || d1 <= 0 || d2 <= 0 || ic <= 0) {
-        return false;
-    }
-
-    if (src1->ne[3] % ic != 0) {
-        return false;
-    }
-
-    const uint32_t IC = (uint32_t) ic;
-    const uint32_t N  = (uint32_t) (src1->ne[3] / IC);
-    if (N == 0) {
-        return false;
-    }
-
-    uint32_t OW, OH, OD;
-    if (!im2col_3d_calc_output_size(src1->ne[0], src0->ne[0], s0, p0, d0, &OW) ||
-        !im2col_3d_calc_output_size(src1->ne[1], src0->ne[1], s1, p1, d1, &OH) ||
-        !im2col_3d_calc_output_size(src1->ne[2], src0->ne[2], s2, p2, d2, &OD)) {
-        return false;
-    }
-
-    const uint32_t KD = (uint32_t) src0->ne[2];
-    const uint32_t KH = (uint32_t) src0->ne[1];
-    const uint32_t KW = (uint32_t) src0->ne[0];
-    const uint32_t ID = (uint32_t) src1->ne[2];
-    const uint32_t IH = (uint32_t) src1->ne[1];
-    const uint32_t IW = (uint32_t) src1->ne[0];
-
-    const uint64_t patch_stride_64 = (uint64_t) IC * KD * KH * KW;
-    const uint64_t total_rows_64   = (uint64_t) N * OD * OH;
-    const uint64_t dst_ne3_64      = (uint64_t) OD * N;
-
-    if (patch_stride_64 > UINT32_MAX || total_rows_64 > UINT32_MAX || dst_ne3_64 > UINT32_MAX) {
-        return false;
-    }
-    const uint32_t patch_stride = (uint32_t) patch_stride_64;
-    const uint32_t total_rows   = (uint32_t) total_rows_64;
-    const uint32_t dst_ne3      = (uint32_t) dst_ne3_64;
-
-    if (dst->ne[0] != patch_stride || dst->ne[1] != OW || dst->ne[2] != OH || dst->ne[3] != dst_ne3) {
-        return false;
-    }
-
-    const uint32_t dst_elem = (dst->type == GGML_TYPE_F16) ? sizeof(ggml_fp16_t) : sizeof(float);
-
-    const bool is_patchembed = ggml_is_contiguous(src1) &&
-                               (s0 == (int32_t) KW && s1 == (int32_t) KH && s2 == (int32_t) KD &&
-                                p0 == 0 && p1 == 0 && p2 == 0 &&
-                                d0 == 1 && d1 == 1 && d2 == 1);
-
-    const uint32_t max_threads = (sess && sess->n_threads > 0) ? (uint32_t) sess->n_threads : 8;
-    const uint32_t n_threads   = (total_rows < max_threads) ? total_rows : max_threads;
-    const uint32_t vtcm_size   = (sess && sess->vtcm_size > 0) ? sess->vtcm_size : 0;
-
-    if (is_patchembed) {
-        const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * IW * sizeof(float);
-        const uint64_t dst_row_bytes_raw = (uint64_t) OW * patch_stride * dst_elem;
-        if (src_row_bytes_raw <= UINT32_MAX && dst_row_bytes_raw <= UINT32_MAX) {
-            const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
-            const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
-            struct htp_im2col_vtcm_layout layout;
-            htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
-            if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
-                kparams->kernel_type          = HTP_IM2COL_KERNEL_PATCHEMBED_DMA;
-                kparams->is_2d                = 0;
-                kparams->is_3d                = 1;
-                kparams->n_threads            = (uint8_t) n_threads;
-                kparams->vtcm_size            = layout.total_bytes;
-                kparams->off_src              = layout.off_src;
-                kparams->off_dst              = layout.off_dst;
-                kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
-                kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
-                kparams->src_row_bytes        = src_row_bytes;
-                kparams->dst_row_bytes        = dst_row_bytes;
-                kparams->owb                  = OW;
-                kparams->wb                   = IW;
-                kparams->nbpr                 = 1;
-                kparams->N                    = N;
-                kparams->IC                   = IC;
-                kparams->ID                   = ID;
-                kparams->IH                   = IH;
-                kparams->IW                   = IW;
-                kparams->KD                   = KD;
-                kparams->KH                   = KH;
-                kparams->KW                   = KW;
-                kparams->OD                   = OD;
-                kparams->OH                   = OH;
-                kparams->OW                   = OW;
-                kparams->patch_stride         = patch_stride;
-                kparams->total_rows           = total_rows;
-                return true;
-            }
-        }
-    }
-
-    for (uint32_t owb = (OW < 256 ? OW : 256); owb >= 1; owb--) {
-        const uint64_t Wb = (uint64_t) (owb - 1) * s0 + (uint64_t) (KW - 1) * d0 + 1;
-        if (Wb > UINT32_MAX) {
-            if (owb == 1) break;
-            continue;
-        }
-
-        const uint64_t src_row_bytes_raw = (uint64_t) IC * KD * KH * Wb * sizeof(float);
-        const uint64_t dst_row_bytes_raw = (uint64_t) owb * patch_stride * dst_elem;
-        if (src_row_bytes_raw > UINT32_MAX || dst_row_bytes_raw > UINT32_MAX) {
-            if (owb == 1) break;
-            continue;
-        }
-
-        const uint32_t src_row_bytes = hex_round_up((uint32_t) src_row_bytes_raw, 256);
-        const uint32_t dst_row_bytes = hex_round_up((uint32_t) dst_row_bytes_raw, 256);
-
-        struct htp_im2col_vtcm_layout layout;
-        htp_im2col_vtcm_layout_build(&layout, src_row_bytes, dst_row_bytes, n_threads);
-
-        if (vtcm_size == 0 || layout.total_bytes <= vtcm_size) {
-            kparams->kernel_type          = HTP_IM2COL_KERNEL_BLOCKED_DMA;
-            kparams->is_2d                = 0;
-            kparams->is_3d                = 1;
-            kparams->n_threads            = (uint8_t) n_threads;
-            kparams->vtcm_size            = layout.total_bytes;
-            kparams->off_src              = layout.off_src;
-            kparams->off_dst              = layout.off_dst;
-            kparams->src_bytes_per_thread = layout.src_bytes_per_thread;
-            kparams->dst_bytes_per_thread = layout.dst_bytes_per_thread;
-            kparams->src_row_bytes        = src_row_bytes;
-            kparams->dst_row_bytes        = dst_row_bytes;
-            kparams->owb                  = owb;
-            kparams->wb                   = (uint32_t) Wb;
-            kparams->nbpr                 = (OW + owb - 1) / owb;
-            kparams->N                    = N;
-            kparams->IC                   = IC;
-            kparams->ID                   = ID;
-            kparams->IH                   = IH;
-            kparams->IW                   = IW;
-            kparams->KD                   = KD;
-            kparams->KH                   = KH;
-            kparams->KW                   = KW;
-            kparams->OD                   = OD;
-            kparams->OH                   = OH;
-            kparams->OW                   = OW;
-            kparams->patch_stride         = patch_stride;
-            kparams->total_rows           = total_rows;
-            return true;
-        }
-        if (owb == 1) break;
-    }
-
-    return false;
-}
-
 static bool ggml_hexagon_supported_im2col(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
-    struct htp_im2col_kernel_params kparams;
-    return ggml_hexagon_precompute_im2col_params(sess, op, &kparams);
-}
+    const struct ggml_tensor * src1 = op->src[1];
+    const struct ggml_tensor * dst  = op;
 
-static bool ggml_hexagon_supported_im2col_3d(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
-    struct htp_im2col_kernel_params kparams;
-    return ggml_hexagon_precompute_im2col_3d_params(sess, op, &kparams);
+    // For now support F32->F32 and F32->F16 only.
+    if (src1->type != GGML_TYPE_F32 || (dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_F32)) {
+        return false;
+    }
+
+    if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(dst)) {
+        return false;
+    }
+
+    GGML_UNUSED(sess);
+    return true;
 }
 
 static bool ggml_hexagon_supported_pad(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -8237,7 +7875,6 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
         case GGML_OP_TRI:             return HTP_OP_TRI;
         case GGML_OP_PAD:             return HTP_OP_PAD;
         case GGML_OP_IM2COL:          return HTP_OP_IM2COL;
-        case GGML_OP_IM2COL_3D:       return HTP_OP_IM2COL_3D;
         case GGML_OP_ROLL:            return HTP_OP_ROLL;
 
         case GGML_OP_UNARY:
@@ -8520,16 +8157,6 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
                 if (ok && node.opcode == HTP_OP_CPY && kparams->total_elems == 0) {
                     continue;
                 }
-            } else if (node.opcode == HTP_OP_IM2COL) {
-                ggml_hexagon_precompute_im2col_params(sess,
-                    node.node,
-                    (struct htp_im2col_kernel_params *) node.kernel_params
-                );
-            } else if (node.opcode == HTP_OP_IM2COL_3D) {
-                ggml_hexagon_precompute_im2col_3d_params(sess,
-                    node.node,
-                    (struct htp_im2col_kernel_params *) node.kernel_params
-                );
             }
             computed_nodes.push_back(std::move(node));
         }
@@ -9391,10 +9018,6 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
             supp = ggml_hexagon_supported_im2col(sess, op);
             break;
 
-        case GGML_OP_IM2COL_3D:
-            supp = ggml_hexagon_supported_im2col_3d(sess, op);
-            break;
-
         case GGML_OP_GATED_DELTA_NET:
             supp = ggml_hexagon_supported_gated_delta_net(sess, op);
             break;

ggml/src/ggml-hexagon/htp/hmx-mm-kernels-tiled.h

@@ -701,29 +701,20 @@ static void dequantize_tiled_weight_to_fp16_task_q3_k(
     }
 }
 
-// Q2_K stores 2-bit weights, the ggml scale bytes and d / -dmin per super-block, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K.
+// Q2_K stores 2-bit weights and one fp16 scale and offset per 16 k, see HTP_MM_WEIGHT_TILE_SIZE_Q2_K.
 static void dequantize_tiled_weight_to_fp16_task_q2_k(
         const tiled_dequantize_state_t *state,
         uint32_t start_tile, uint32_t end_tile) {
 
     const HVX_Vector mask_03 = Q6_Vb_vsplat_R(0x03);
 
-    HVX_Vector v_d = Q6_V_vzero();
-    HVX_Vector v_dm = Q6_V_vzero();
-
     for (uint32_t t = start_tile; t < end_tile; t++) {
-        const uint8_t * tile = state->src + t * state->aligned_tile_size;
-        const HVX_Vector * vptr = (const HVX_Vector *) tile;
+        const HVX_Vector * vptr = (const HVX_Vector *) (state->src + t * state->aligned_tile_size);
         __fp16 * dst_ptr = state->dst + t * HTP_MM_HMX_TILE_N_ELMS;
 
-        // rows hold all k tiles and n_k_tiles is a multiple of 8, so t % 8 is the tile within its super-block
-        if (t == start_tile || (t & 7) == 0) {
-            q2_k_load_dmin(tile - (t & 7) * state->aligned_tile_size, &v_d, &v_dm);
-        }
-
-        HVX_Vector v_sc, v_m;
-        q2_k_unpack_scales(tile, v_d, v_dm, &v_sc, &v_m);
+        HVX_Vector v_sc       = vptr[2];
         HVX_Vector v_sc_k16   = Q6_V_vror_VR(v_sc, 64);
+        HVX_Vector v_m        = vptr[3];
         HVX_Vector v_m_k16    = Q6_V_vror_VR(v_m, 64);
         HVX_Vector v_scale_k0   = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc, v_sc, -2));
         HVX_Vector v_scale_k16  = Q6_V_lo_W(Q6_W_vshuff_VVR(v_sc_k16, v_sc_k16, -2));

ggml/src/ggml-hexagon/htp/htp-ctx.h

@@ -173,7 +173,6 @@ int op_solve_tri(struct htp_ops_context * octx);
 int op_gated_delta_net(struct htp_ops_context * octx);
 int op_pad(struct htp_ops_context * octx);
 int op_im2col(struct htp_ops_context * octx);
-int op_im2col_3d(struct htp_ops_context * octx);
 int op_pool_2d(struct htp_ops_context * octx);
 int op_pool_1d(struct htp_ops_context * octx);
 int op_allreduce(struct htp_ops_context * octx);

ggml/src/ggml-hexagon/htp/htp-ops.h

@@ -118,7 +118,6 @@ enum htp_op_code {
     HTP_OP_POOL_2D,
     HTP_OP_POOL_1D,
     HTP_OP_UNARY_GELU_QUICK,
-    HTP_OP_IM2COL_3D,
 
     HTP_OP_INVALID
 };

ggml/src/ggml-hexagon/htp/hvx-mm-kernels-tiled.h

@@ -1,31 +1,19 @@
 // Dynamic quantizers that produce tiled activations
 
-// q8_1: vector 9 holds d * sum(q) of the 32 k, replicated.
-// q8_1_s16 (Q2_K, sums16): one fp16 d (vector 8) for the 256 k super-block starting at x_sb, and vector 9 holds the
-// integer sums of k 0..15 (low halfword) and k 16..31 (high halfword) in every word
+// vector 9: d * sum(q) of the 32 k, replicated (q8_1), or per 16 k for Q2_K (q8_1_s16): k 0..15 in lanes 0..31,
+// k 16..31 in lanes 32..63, with d the fp16 scale of vector 8
 __attribute__((always_inline))
-static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16,
-                                                      const float * restrict x_sb) {
+static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_t * restrict y_block, const bool sums16) {
     assert((unsigned long) x % 128 == 0);
     assert((unsigned long) y_block % 128 == 0);
 
     HVX_Vector * vx = (HVX_Vector *) x;
     HVX_Vector zero = Q6_V_vzero();
 
-    HVX_Vector vmax0_sf, vmax1_sf, vmax2_sf, vmax3_sf;
-    if (sums16) {
-        const HVX_Vector * vsb = (const HVX_Vector *) x_sb;
-        HVX_Vector vm = hvx_vec_abs_f32(vsb[0]);
-        for (int i = 1; i < 8; i++) {
-            vm = Q6_Vsf_vmax_VsfVsf(vm, hvx_vec_abs_f32(vsb[i]));
-        }
-        vmax0_sf = vmax1_sf = vmax2_sf = vmax3_sf = hvx_vec_reduce_max_f32(vm);
-    } else {
-        vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
-        vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
-        vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
-        vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
-    }
+    HVX_Vector vmax0_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[0]));
+    HVX_Vector vmax1_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[1]));
+    HVX_Vector vmax2_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[2]));
+    HVX_Vector vmax3_sf = hvx_vec_reduce_max_f32(hvx_vec_abs_f32(vx[3]));
 
     HVX_Vector vx0_qf = Q6_Vqf32_vsub_VsfVsf(vx[0], zero);
     HVX_Vector vx1_qf = Q6_Vqf32_vsub_VsfVsf(vx[1], zero);
@@ -73,6 +61,15 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
     HVX_Vector vd3_sf = hvx_vec_mul_f32_f32(vmax3_sf, v_inv127);
 
     HVX_Vector v_sums_sf = Q6_Vsf_equals_Vw(v_sums);
+    if (sums16) {
+        // the fp16 d of vector 8
+        HVX_VectorPair vd01_sf = hvx_vec_f16_to_f32(vd01_hf);
+        HVX_VectorPair vd23_sf = hvx_vec_f16_to_f32(vd23_hf);
+        vd0_sf = Q6_V_lo_W(vd01_sf);
+        vd1_sf = Q6_V_hi_W(vd01_sf);
+        vd2_sf = Q6_V_lo_W(vd23_sf);
+        vd3_sf = Q6_V_hi_W(vd23_sf);
+    }
     HVX_Vector voff0_sf = hvx_vec_mul_f32_f32(vd0_sf, v_sums_sf);
     HVX_Vector voff1_sf = hvx_vec_mul_f32_f32(vd1_sf, Q6_V_vror_VR(v_sums_sf, 32));
     HVX_Vector voff2_sf = hvx_vec_mul_f32_f32(vd2_sf, Q6_V_vror_VR(v_sums_sf, 64));
@@ -94,12 +91,12 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
         hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 64)),
     };
     if (sums16) {
-        // |sum of 16 int8| <= 2032 fits a halfword
-        int32_t __attribute__((aligned(128))) ws[32];
-        *(HVX_Vector *) ws = v_sums;
-        for (int b = 0; b < 4; b++) {
-            r_offset[b] = Q6_V_vsplat_R(((uint32_t) ws[8 * b + 4] << 16) | ((uint32_t) ws[8 * b] & 0xFFFF));
-        }
+        // the k 16..31 sums sit 4 halfwords after the k 0..15 sums
+        const HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
+        r_offset[0] = Q6_V_vmux_QVV(q_lo, r_offset[0], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 8)));
+        r_offset[1] = Q6_V_vmux_QVV(q_lo, r_offset[1], hvx_vec_repl_f16(Q6_V_vror_VR(voff01_hf, 72)));
+        r_offset[2] = Q6_V_vmux_QVV(q_lo, r_offset[2], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 8)));
+        r_offset[3] = Q6_V_vmux_QVV(q_lo, r_offset[3], hvx_vec_repl_f16(Q6_V_vror_VR(voff23_hf, 72)));
     }
 
     static const uint8_t __attribute__((aligned(128))) repl[128] = {
@@ -141,11 +138,11 @@ static inline void quantize_block_f32_q8_1_tiled_impl(float * restrict x, uint8_
 }
 
 static inline void quantize_block_f32_q8_1_tiled(float * restrict x, uint8_t * restrict y_block) {
-    quantize_block_f32_q8_1_tiled_impl(x, y_block, false, NULL);
+    quantize_block_f32_q8_1_tiled_impl(x, y_block, false);
 }
 
-static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block, const float * restrict x_sb) {
-    quantize_block_f32_q8_1_tiled_impl(x, y_block, true, x_sb);
+static inline void quantize_block_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restrict y_block) {
+    quantize_block_f32_q8_1_tiled_impl(x, y_block, true);
 }
 
 static inline void quantize_block_f32_q8_0_tiled(float * restrict x, uint8_t * restrict y_block) {
@@ -270,7 +267,7 @@ static void quantize_row_f32_q8_1_s16_tiled(float * restrict x, uint8_t * restri
 
     for (uint32_t i = 0; i < nb; i++) {
         uint8_t * restrict y_block = y + i * 4 * 1280;
-        quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block, x + (i & ~1u) * qk);  // k % 256 == 0 for Q2_K
+        quantize_block_f32_q8_1_s16_tiled(x + i * qk, y_block);
     }
 }
 
@@ -658,7 +655,7 @@ static inline void accum_q3_k_32x2(
     *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);
 }
 
-// Q2_K: the Q3_K dot products without the -4 flags, scales and mins are applied in q2_k_int_accum
+// Q2_K (x = D * q + M): the Q3_K dot products without the -4 flags, M uses the q8_1_s16 sums
 static inline HVX_VectorPair accum_q2_k_32x1(
     const HVX_Vector * restrict vptr,
     const HVX_Vector * restrict v_act
@@ -705,57 +702,11 @@ static inline void accum_q2_k_32x2(
     *v_sums1 = Q6_W_vcombine_VV(v_sum1_hi, v_sum1_lo);
 }
 
-// Q2_K: fp16 d (*v_d) and -dmin (*v_m) of the 32 rows of a super-block (8 tiles, VTCM stride 384), each in both vector halves
-static inline void q2_k_load_dmin(const uint8_t * restrict sb, HVX_Vector * restrict v_d, HVX_Vector * restrict v_m) {
-    HVX_Vector v_dd = Q6_V_vzero();  // lanes 0..31: d, lanes 32..63: -dmin
-
-    #pragma unroll
-    for (int j = 0; j < 8; j++) {
-        HVX_Vector v = hvx_vmem(sb + j * 384 + 256);  // the slice is at bytes 64..79
-        v_dd = Q6_V_vmux_QVV(Q6_Q_vsetq_R(16 * j), v_dd, Q6_V_vror_VR(v, (64 - 16 * j) & 127));
-    }
-
-    HVX_VectorPred q_lo = Q6_Q_vsetq_R(64);
-    HVX_Vector     v_r  = Q6_V_vror_VR(v_dd, 64);
-    *v_d = Q6_V_vmux_QVV(q_lo, v_dd, v_r);
-    *v_m = Q6_V_vmux_QVV(q_lo, v_r, v_dd);
-}
-
-// Q2_K: fp16 D = d * scale and M = -dmin * min of a tile from its scale bytes, in the lanes of the Q6_K scales
-static inline void q2_k_unpack_scales(const uint8_t * restrict tile, HVX_Vector v_d, HVX_Vector v_m,
-                                      HVX_Vector * restrict v_D, HVX_Vector * restrict v_M) {
-    HVX_Vector v_sc = hvx_vmem(tile + 256);
-    HVX_Vector v_l  = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_V_vand_VV(v_sc, Q6_Vb_vsplat_R(0x0F))));
-    HVX_Vector v_mn = Q6_V_lo_W(Q6_Wuh_vunpack_Vub(Q6_Vub_vlsr_VubR(v_sc, 4)));
-    // qf32 product: rounds like GGML_FP32_TO_FP16(d * scale) (qf16 is off by 1 ulp for many values on v73)
-    *v_D = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_l), v_d));
-    *v_M = Q6_Vhf_equals_Wqf32(Q6_Wqf32_vmpy_VhfVhf(Q6_Vhf_equals_Vh(v_mn), v_m));
-}
-
-// Q2_K scale and min nibbles of a tile as halfword pairs: word r = sub-block 1 << 16 | sub-block 0 of row r
-static inline void q2_k_scale_pairs(const uint8_t * restrict tile, HVX_Vector * restrict v_sc2, HVX_Vector * restrict v_mn2) {
-    HVX_Vector v_h = Q6_Vh_vshuff_Vh(Q6_V_lo_W(Q6_Wuh_vunpack_Vub(hvx_vmem(tile + 256))));
-    *v_sc2 = Q6_V_vand_VV(v_h, Q6_Vh_vsplat_R(0x000F));
-    *v_mn2 = Q6_Vuh_vlsr_VuhR(v_h, 4);
-}
-
-// Q2_K with q8_1_s16 activations, one tile: isum += scale * (q . a), msum += min * sum(a), per row, in int32.
-// |sum of 16 q * a| <= 3 * 128 * 16 fits a halfword, so both sub-blocks go through one vdmpy (no saturation)
-static inline void q2_k_int_accum(HVX_VectorPair v_sums, HVX_Vector v_sc2, HVX_Vector v_mn2, HVX_Vector v_bsum,
-                                  HVX_Vector * restrict v_isum, HVX_Vector * restrict v_msum) {
-    HVX_Vector v_s2 = Q6_Vh_vshuffe_VhVh(Q6_V_hi_W(v_sums), Q6_V_lo_W(v_sums));
-    *v_isum = Q6_Vw_vadd_VwVw(*v_isum, Q6_Vw_vdmpy_VhVh_sat(v_s2, v_sc2));
-    *v_msum = Q6_Vw_vadd_VwVw(*v_msum, Q6_Vw_vdmpy_VhVh_sat(v_mn2, v_bsum));
-}
-
-// Q2_K super-block result: d_a * (d * isum - dmin * msum), with v_d / v_m the fp16 d / -dmin of q2_k_load_dmin
-static inline HVX_Vector q2_k_int_scale(HVX_Vector v_isum, HVX_Vector v_msum, HVX_Vector v_d, HVX_Vector v_m, HVX_Vector v_act8) {
-    HVX_Vector v_d32  = Q6_V_lo_W(hvx_vec_f16_to_f32(v_d));
-    HVX_Vector v_m32  = Q6_V_lo_W(hvx_vec_f16_to_f32(v_m));
-    HVX_Vector v_da32 = Q6_V_lo_W(hvx_vec_f16_to_f32(v_act8));
-    HVX_Vector v_t = hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(v_d32, Q6_Vsf_equals_Vw(v_isum)),
-                                         hvx_vec_mul_f32_f32(v_m32, Q6_Vsf_equals_Vw(v_msum)));
-    return hvx_vec_mul_f32_f32(v_t, v_da32);
+// D as the Q6_K scales, plus M (vector 3) times the per-16 activation sums in v_act[9]
+static inline HVX_Vector scale_q2_k_32x1(HVX_VectorPair v_sums, const HVX_Vector * restrict vptr, const HVX_Vector * restrict v_act) {
+    HVX_VectorPair v_m = hvx_vec_mul_f16_f16_to_f32_pair(vptr[3], v_act[9]);
+    HVX_Vector     v_d = scale_q6_k_32x1(v_sums, vptr[2], v_act[8]);
+    return hvx_vec_add_f32_f32(v_d, hvx_vec_add_f32_f32(Q6_V_lo_W(v_m), Q6_V_hi_W(v_m)));
 }
 
 static void tiled_vec_dot_q4_0_32x1(const uint32_t n, float * restrict s, const void * restrict vx, const void * restrict vy, uint32_t valid_rows, const float * restrict sz) {
@@ -1223,22 +1174,12 @@ static void tiled_vec_dot_q2_k_32x1(const uint32_t n, float * restrict s, const
     HVX_Vector v_sum_float = Q6_V_vzero();
 
     uint32_t n_k_tiles = n / 32;
-    for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) {
-        HVX_Vector v_d, v_m;
-        q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m);
+    for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512);
+        const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280);
 
-        HVX_Vector v_isum = Q6_V_vzero();
-        HVX_Vector v_msum = Q6_V_vzero();
-        for (uint32_t kt = kb; kt < kb + 8; kt++) {
-            const uint8_t * restrict tile = tile_ptr + kt * 384;
-            const HVX_Vector * restrict v_act = (const HVX_Vector *) (y_q + kt * 1280);
-
-            HVX_Vector v_sc2, v_mn2;
-            q2_k_scale_pairs(tile, &v_sc2, &v_mn2);
-            q2_k_int_accum(accum_q2_k_32x1((const HVX_Vector *) tile, v_act), v_sc2, v_mn2, v_act[9], &v_isum, &v_msum);
-        }
-        const HVX_Vector * restrict v_act_sb = (const HVX_Vector *) (y_q + kb * 1280);
-        v_sum_float = hvx_vec_add_f32_f32(v_sum_float, q2_k_int_scale(v_isum, v_msum, v_d, v_m, v_act_sb[8]));
+        HVX_VectorPair v_sums = accum_q2_k_32x1(vptr, v_act);
+        v_sum_float = hvx_vec_add_f32_f32(v_sum_float, scale_q2_k_32x1(v_sums, vptr, v_act));
     }
 
     if (sz) {
@@ -1257,31 +1198,16 @@ static void tiled_vec_dot_q2_k_32x2(const uint32_t n, float * restrict s0, float
     HVX_Vector v_sum_float_c1 = Q6_V_vzero();
 
     uint32_t n_k_tiles = n / 32;
-    for (uint32_t kb = 0; kb < n_k_tiles; kb += 8) {
-        HVX_Vector v_d, v_m;
-        q2_k_load_dmin(tile_ptr + kb * 384, &v_d, &v_m);
+    for (uint32_t kt = 0; kt < n_k_tiles; kt++) {
+        const HVX_Vector * restrict vptr = (const HVX_Vector *) (tile_ptr + kt * 512);
+        const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280);
+        const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280);
 
-        HVX_Vector v_isum0 = Q6_V_vzero();
-        HVX_Vector v_msum0 = Q6_V_vzero();
-        HVX_Vector v_isum1 = Q6_V_vzero();
-        HVX_Vector v_msum1 = Q6_V_vzero();
-        for (uint32_t kt = kb; kt < kb + 8; kt++) {
-            const uint8_t * restrict tile = tile_ptr + kt * 384;
-            const HVX_Vector * restrict v_act0 = (const HVX_Vector *) (y0_q + kt * 1280);
-            const HVX_Vector * restrict v_act1 = (const HVX_Vector *) (y1_q + kt * 1280);
+        HVX_VectorPair v_sums0, v_sums1;
+        accum_q2_k_32x2(vptr, v_act0, v_act1, &v_sums0, &v_sums1);
 
-            HVX_Vector v_sc2, v_mn2;
-            q2_k_scale_pairs(tile, &v_sc2, &v_mn2);
-
-            HVX_VectorPair v_sums0, v_sums1;
-            accum_q2_k_32x2((const HVX_Vector *) tile, v_act0, v_act1, &v_sums0, &v_sums1);
-            q2_k_int_accum(v_sums0, v_sc2, v_mn2, v_act0[9], &v_isum0, &v_msum0);
-            q2_k_int_accum(v_sums1, v_sc2, v_mn2, v_act1[9], &v_isum1, &v_msum1);
-        }
-        const HVX_Vector * restrict v_act0_sb = (const HVX_Vector *) (y0_q + kb * 1280);
-        const HVX_Vector * restrict v_act1_sb = (const HVX_Vector *) (y1_q + kb * 1280);
-        v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, q2_k_int_scale(v_isum0, v_msum0, v_d, v_m, v_act0_sb[8]));
-        v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, q2_k_int_scale(v_isum1, v_msum1, v_d, v_m, v_act1_sb[8]));
+        v_sum_float_c0 = hvx_vec_add_f32_f32(v_sum_float_c0, scale_q2_k_32x1(v_sums0, vptr, v_act0));
+        v_sum_float_c1 = hvx_vec_add_f32_f32(v_sum_float_c1, scale_q2_k_32x1(v_sums1, vptr, v_act1));
     }
 
     if (sz0) {
@@ -1613,7 +1539,7 @@ static inline void quantize_f32_q8_1_s16_tiled_block_kernel(
         const float * restrict src_ptr = (const float *) ((const uint8_t *) src + r * src_row_size + c * qk * sizeof(float));
         uint8_t * restrict dst_ptr = dst + r * dst_row_size + c * 4 * 1280;
 
-        quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr, src_ptr - (c & 1) * qk);  // super-block of 256 k
+        quantize_block_f32_q8_1_s16_tiled((float *) src_ptr, dst_ptr);
 
         c++;
         if (c == nb) {

ggml/src/ggml-hexagon/htp/im2col-ops.h

@@ -1,68 +0,0 @@
-#ifndef HTP_IM2COL_OPS_H
-#define HTP_IM2COL_OPS_H
-
-#include "hex-common.h"
-#include <stdint.h>
-
-enum htp_im2col_kernel_type {
-    HTP_IM2COL_KERNEL_UNSUPPORTED    = 0,
-    HTP_IM2COL_KERNEL_PATCHEMBED_DMA = 1,
-    HTP_IM2COL_KERNEL_BLOCKED_DMA    = 2,
-};
-
-struct htp_im2col_vtcm_layout {
-    uint32_t off_src;
-    uint32_t off_dst;
-    uint32_t src_bytes_per_thread;
-    uint32_t dst_bytes_per_thread;
-    uint32_t total_bytes;
-};
-
-static inline void htp_im2col_vtcm_layout_build(
-    struct htp_im2col_vtcm_layout * layout,
-    uint32_t src_row_bytes,
-    uint32_t dst_row_bytes,
-    uint32_t n_threads) {
-
-    layout->src_bytes_per_thread = 2 * src_row_bytes + 256;
-    layout->dst_bytes_per_thread = 2 * dst_row_bytes;
-
-    layout->off_src     = 0;
-    layout->off_dst     = layout->off_src + layout->src_bytes_per_thread * n_threads;
-    layout->total_bytes = layout->off_dst + layout->dst_bytes_per_thread * n_threads;
-}
-
-struct htp_im2col_kernel_params {
-    uint8_t  kernel_type;
-    uint8_t  is_2d;
-    uint8_t  is_3d;
-    uint8_t  n_threads;
-
-    uint32_t vtcm_size;
-    uint32_t off_src;
-    uint32_t off_dst;
-    uint32_t src_bytes_per_thread;
-    uint32_t dst_bytes_per_thread;
-    uint32_t src_row_bytes;
-    uint32_t dst_row_bytes;
-
-    uint32_t owb;
-    uint32_t wb;
-    uint32_t nbpr;
-
-    uint32_t N;
-    uint32_t IC;
-    uint32_t ID, IH, IW;
-    uint32_t KD, KH, KW;
-    uint32_t OD, OH, OW;
-    uint32_t patch_stride;
-    uint32_t total_rows;
-};
-
-#if defined(__cplusplus)
-static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob");
-#else
-_Static_assert(sizeof(struct htp_im2col_kernel_params) <= 128, "htp_im2col_kernel_params is too large for kernel_params blob");
-#endif
-
-#endif // HTP_IM2COL_OPS_H

ggml/src/ggml-hexagon/htp/main.c

@@ -929,9 +929,6 @@ static int execute_op(struct htp_ops_context * octx) {
         case HTP_OP_IM2COL:
             return op_im2col(octx);
 
-        case HTP_OP_IM2COL_3D:
-            return op_im2col_3d(octx);
-
         case HTP_OP_ROLL:
             return op_roll(octx);
 

ggml/src/ggml-hexagon/htp/matmul-ops.c

@@ -543,7 +543,7 @@ MATMUL_2D_REPACKED_IMPL(q8_0,       1088, tiled_vec_dot_q8_0_32x2,  tiled_vec_do
 MATMUL_2D_REPACKED_IMPL(q6_k,       896,  tiled_vec_dot_q6_k_32x2,  tiled_vec_dot_q6_k_32x1)
 MATMUL_2D_REPACKED_IMPL(q5_k,       768,  tiled_vec_dot_q5_k_32x2,  tiled_vec_dot_q5_k_32x1)
 MATMUL_2D_REPACKED_IMPL(q3_k,       512,  tiled_vec_dot_q3_k_32x2,  tiled_vec_dot_q3_k_32x1)
-MATMUL_2D_REPACKED_IMPL(q2_k,       336,  tiled_vec_dot_q2_k_32x2,  tiled_vec_dot_q2_k_32x1)
+MATMUL_2D_REPACKED_IMPL(q2_k,       512,  tiled_vec_dot_q2_k_32x2,  tiled_vec_dot_q2_k_32x1)
 MATMUL_2D_REPACKED_IMPL(iq4nl,      576,  tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
 MATMUL_2D_REPACKED_IMPL(mxfp4,      544,  tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
 
@@ -761,7 +761,7 @@ MATVEC_2D_REPACKED_IMPL(q8_0,       1088, tiled_vec_dot_q8_0_32x1)
 MATVEC_2D_REPACKED_IMPL(q5_k,       768,  tiled_vec_dot_q5_k_32x1)
 MATVEC_2D_REPACKED_IMPL(q6_k,       896,  tiled_vec_dot_q6_k_32x1)
 MATVEC_2D_REPACKED_IMPL(q3_k,       512,  tiled_vec_dot_q3_k_32x1)
-MATVEC_2D_REPACKED_IMPL(q2_k,       336,  tiled_vec_dot_q2_k_32x1)
+MATVEC_2D_REPACKED_IMPL(q2_k,       512,  tiled_vec_dot_q2_k_32x1)
 MATVEC_2D_REPACKED_IMPL(iq4nl,      576,  tiled_vec_dot_iq4nl_32x1)
 MATVEC_2D_REPACKED_IMPL(mxfp4,      544,  tiled_vec_dot_mxfp4_32x1)
 
@@ -772,7 +772,7 @@ MATMUL_NX_2D_REPACKED_IMPL(iq4nl,   576,  tiled_vec_dot_iq4nl_32x2, tiled_vec_do
 MATMUL_NX_2D_REPACKED_IMPL(mxfp4,   544,  tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
 MATMUL_NX_2D_REPACKED_IMPL(q5_k,    768,  tiled_vec_dot_q5_k_32x2,  tiled_vec_dot_q5_k_32x1)
 MATMUL_NX_2D_REPACKED_IMPL(q3_k,    512,  tiled_vec_dot_q3_k_32x2,  tiled_vec_dot_q3_k_32x1)
-MATMUL_NX_2D_REPACKED_IMPL(q2_k,    336,  tiled_vec_dot_q2_k_32x2,  tiled_vec_dot_q2_k_32x1)
+MATMUL_NX_2D_REPACKED_IMPL(q2_k,    512,  tiled_vec_dot_q2_k_32x2,  tiled_vec_dot_q2_k_32x1)
 
 #define MATMUL_4D_REPACKED_IMPL(SUFFIX, TILE_SIZE, DOT_2X2, DOT_2X1)                                                                                        \
 static void hvx_mm_4d_repacked_##SUFFIX(unsigned int nth, unsigned int ith, void * data) {                                                                  \
@@ -907,7 +907,7 @@ MATMUL_4D_REPACKED_IMPL(q8_0,       1088, tiled_vec_dot_q8_0_32x2,  tiled_vec_do
 MATMUL_4D_REPACKED_IMPL(q6_k,       896,  tiled_vec_dot_q6_k_32x2,  tiled_vec_dot_q6_k_32x1)
 MATMUL_4D_REPACKED_IMPL(q5_k,       768,  tiled_vec_dot_q5_k_32x2,  tiled_vec_dot_q5_k_32x1)
 MATMUL_4D_REPACKED_IMPL(q3_k,       512,  tiled_vec_dot_q3_k_32x2,  tiled_vec_dot_q3_k_32x1)
-MATMUL_4D_REPACKED_IMPL(q2_k,       336,  tiled_vec_dot_q2_k_32x2,  tiled_vec_dot_q2_k_32x1)
+MATMUL_4D_REPACKED_IMPL(q2_k,       512,  tiled_vec_dot_q2_k_32x2,  tiled_vec_dot_q2_k_32x1)
 MATMUL_4D_REPACKED_IMPL(iq4nl,      576,  tiled_vec_dot_iq4nl_32x2, tiled_vec_dot_iq4nl_32x1)
 MATMUL_4D_REPACKED_IMPL(mxfp4,      544,  tiled_vec_dot_mxfp4_32x2, tiled_vec_dot_mxfp4_32x1)
 

ggml/src/ggml-hexagon/htp/matmul-ops.h

@@ -38,11 +38,11 @@ extern "C" {
 //   vector 2: bit g set where the hmask bit of group g is clear (quant = low 2 bits - 4)
 //   vector 3: fp16 scales per row, d * (scales[] - 32): k 0..15 in lanes 0..31, k 16..31 in lanes 32..63
 #define HTP_MM_WEIGHT_TILE_SIZE_Q3_K   512
-// Q2_K native 2-bit tile with the ggml scale bytes, vrmpy-ready like Q6_K (same bits per weight as block_q2_K)
-//   bytes   0..255: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
-//   bytes 256..319: scales[] bytes (min << 4 | scale), sub-block s of row r at 32 * s + r
-//   bytes 320..335: 8 fp16, tile j of a super-block: d (j < 4) or -dmin (j >= 4) of rows 8 * (j % 4) .. 8 * (j % 4) + 7
-#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K   336
+// Q2_K native 2-bit tile, vrmpy-ready like Q6_K
+//   vectors 0..1: unsigned 2-bit quants, vector m holds groups 4m..4m+3 at bit offsets 0,2,4,6
+//   vector 2: fp16 scales per row, d * (scales[] & 0xF), same lanes as Q3_K vector 3
+//   vector 3: fp16 offsets per row, -dmin * (scales[] >> 4), same lanes
+#define HTP_MM_WEIGHT_TILE_SIZE_Q2_K   512
 
 // --- Weight Repacked Aligned Tile Sizes ---
 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q4_0   640
@@ -53,7 +53,7 @@ extern "C" {
 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q5_K   768
 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q6_K   896
 #define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q3_K   512
-#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K   384
+#define HTP_MM_WEIGHT_ALIGNED_TILE_SIZE_Q2_K   512
 
 // --- Activation Tiled Block Sizes (including padding) ---
 #define HTP_MM_ACT_TILE_SIZE_Q8_0      1152

ggml/src/ggml-metal/ggml-metal-common.cpp

@@ -43,10 +43,9 @@ bool ggml_metal_op_mul_mat_use_mm(const struct ggml_tensor * op, bool has_simdgr
 
 bool ggml_metal_op_mul_mat_id_use_mm(const struct ggml_tensor * op, bool has_simdgroup_mm) {
     const int64_t ne00 = op->src[0]->ne[0];
-    const int64_t ne02 = op->src[0]->ne[2];
     const int64_t ne21 = op->src[2]->ne[1];
 
-    return has_simdgroup_mm && ne00 >= 64 && ne02 <= 1024 && ne21 >= 32;
+    return has_simdgroup_mm && ne00 >= 64 && ne21 >= 32;
 }
 
 // the most src1 rows of the few-row MMA kernels

ggml/src/ggml-metal/ggml-metal-device.cpp

@@ -1268,16 +1268,10 @@ ggml_metal_pipeline_with_params ggml_metal_library_get_pipeline_mul_mm_id_map0(g
 
     ggml_metal_pipeline_with_params res = ggml_metal_library_get_pipeline(lib, name);
     if (!res.pipeline) {
-        ggml_metal_cv_t cv = ggml_metal_cv_init();
-        ggml_metal_cv_set_int32(cv, ne02, FC_MUL_MM + 7);
-
-        res = ggml_metal_library_compile_pipeline(lib, base, name, cv);
-
-        ggml_metal_cv_free(cv);
+        res = ggml_metal_library_compile_pipeline(lib, base, name, nullptr);
     }
 
-    // per expert: entry count + base offset, followed by the staging of the ids of ntg tokens
-    res.smem = 2*(size_t) ne02*sizeof(uint32_t) + (size_t) ne02*ne20*sizeof(uint16_t);
+    res.smem = (size_t) ne02*ne20*sizeof(uint16_t);
     res.smem = GGML_PAD(res.smem, 16);
 
     return res;

ggml/src/ggml-metal/ggml-metal-impl.h

@@ -560,6 +560,7 @@ typedef struct {
 } ggml_metal_kargs_mul_mv_ext;
 
 typedef struct {
+    int32_t  ne02;
     int32_t  ne10;
     int32_t  ne11;  // n_expert_used (bcast)
     uint64_t nb11;
@@ -589,6 +590,7 @@ typedef struct {
     uint64_t nb12;
     uint64_t nb13;
     int32_t  ne20;
+    int32_t  ne21;
     int32_t  ne0;
     int32_t  ne1;
     int16_t  r2;

ggml/src/ggml-metal/ggml-metal-ops.cpp

@@ -2772,39 +2772,21 @@ int ggml_metal_op_mul_mat(ggml_metal_op_t ctx, int idx) {
     return 1;
 }
 
-// upper bound on the number of entry tiles emitted by kernel_mul_mm_id_map0: the full tiles
-// over all entries plus at most one partial tile per expert
-static int64_t ggml_metal_op_mul_mat_id_max_tiles(const ggml_tensor * op) {
-    const int64_t ne02 = op->src[0]->ne[2]; // n_expert
-    const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used
-    const int64_t ne21 = op->src[2]->ne[1]; // n_token
-
-    const int64_t n_rows = ne20*ne21;
-    const int64_t tiles  = (n_rows + 31)/32 + (ne02 < n_rows ? ne02 : n_rows) - 1;
-
-    return n_rows < tiles ? n_rows : tiles;
-}
-
 size_t ggml_metal_op_mul_mat_id_extra_tpe(const ggml_tensor * op) {
     assert(op->op == GGML_OP_MUL_MAT_ID);
 
-    // row map of kernel_mul_mm_id: total tile count + one
-    // (expert, first entry, valid rows) descriptor per tile
-    const int64_t max_tiles = ggml_metal_op_mul_mat_id_max_tiles(op);
+    const int64_t ne02 = op->src[0]->ne[2]; // n_expert
 
-    return ggml_type_size(GGML_TYPE_I32)*(1 + 3*max_tiles);
+    return ggml_type_size(GGML_TYPE_I32)*ne02;
 }
 
 size_t ggml_metal_op_mul_mat_id_extra_ids(const ggml_tensor * op) {
     assert(op->op == GGML_OP_MUL_MAT_ID);
 
     const int64_t ne02 = op->src[0]->ne[2]; // n_expert
-    const int64_t ne20 = op->src[2]->ne[0]; // n_expert_used
     const int64_t ne21 = op->src[2]->ne[1]; // n_token
 
-    // one entry per (expert used, token) pair of ids plus one padding slot per expert
-    // (see kernel_mul_mm_id_map0)
-    return ggml_type_size(GGML_TYPE_I32)*(ne20*ne21 + ne02);
+    return ggml_type_size(GGML_TYPE_I32)*ne02*ne21;
 }
 
 size_t ggml_metal_op_mul_mat_id_extra_amax(const ggml_tensor * op) {
@@ -2902,6 +2884,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
 
         {
             ggml_metal_kargs_mul_mm_id_map0 args = {
+                ne02,
                 ne10,
                 ne11, // n_expert_used (bcast)
                 nb11,
@@ -2959,6 +2942,7 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
                 /*.nb12  =*/ nb12,
                 /*.nb13  =*/ nb13,
                 /*.ne20  =*/ ne20, // n_expert_used
+                /*.ne21  =*/ ne21, // n_tokens
                 /*.ne0   =*/ ne0,
                 /*.ne1   =*/ ne1,
                 /*.r2    =*/ r2,
@@ -2976,13 +2960,9 @@ int ggml_metal_op_mul_mat_id(ggml_metal_op_t ctx, int idx) {
 
             const size_t smem = pipeline.smem;
 
-            GGML_ASSERT(smem <= props_dev->max_theadgroup_memory_size);
-
             ggml_metal_encoder_set_threadgroup_memory_size(enc, smem, 0);
 
-            const int max_tiles = (int) ggml_metal_op_mul_mat_id_max_tiles(op);
-
-            ggml_metal_encoder_dispatch_threadgroups(enc, max_tiles, (ne01 + 63)/64, 1, 128, 1, 1);
+            ggml_metal_encoder_dispatch_threadgroups(enc, (ne21 + 31)/32, (ne01 + 63)/64, ne02, 128, 1, 1);
         }
     } else {
         auto pipeline = ggml_metal_library_get_pipeline_mul_mv_id(lib, op);

ggml/src/ggml-metal/kernels/mul_mm.metal

@@ -8,7 +8,6 @@ constant short FC_mul_mm_ne13  [[function_constant(FC_MUL_MM + 3)]];
 constant short FC_mul_mm_r2    [[function_constant(FC_MUL_MM + 4)]];
 constant short FC_mul_mm_r3    [[function_constant(FC_MUL_MM + 5)]];
 constant bool FC_mul_mm_id_amax [[function_constant(FC_MUL_MM + 6)]];
-constant int32_t FC_mul_mm_id_map0_ne02 [[function_constant(FC_MUL_MM + 7)]];
 
 // each block_q contains 16*nl weights
 #ifdef GGML_METAL_HAS_TENSOR
@@ -372,23 +371,19 @@ kernel void kernel_mul_mm_id_map0(
         ushort   ntg[[threads_per_threadgroup]]) {
     const short ide = tpitg; // expert id
 
-    // per expert: entry count (pass 1), replaced by the base offset of the slice (pass 2)
-    threadgroup uint32_t * cnt = (threadgroup uint32_t *) (shmem);
-    threadgroup uint32_t * bas = cnt + FC_mul_mm_id_map0_ne02;
-
-    // staging of the ids of ntg tokens
-    threadgroup uint16_t * sids = (threadgroup uint16_t *) (bas + FC_mul_mm_id_map0_ne02);
-
-    // pass 1: count the entries of this expert - every occurrence of this expert within the
-    // ids is an entry (duplicate ids produce multiple entries for the same token)
     uint32_t n_all = 0;
 
+    device int32_t * ids_i32 = (device int32_t *) hids + ide*args.ne21;
+
     for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens
         if (i21 + tpitg < args.ne21) {
             device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21);
 
-            FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
-                sids[tpitg*ne20 + i20] = src2_i32[i20];
+            threadgroup uint16_t * sids = (threadgroup uint16_t *) shmem + tpitg*ne20;
+
+            #pragma unroll(ne20)
+            for (short i20 = 0; i20 < ne20; i20++) {
+                sids[i20] = src2_i32[i20];
             }
         }
 
@@ -399,107 +394,24 @@ kernel void kernel_mul_mm_id_map0(
                 break;
             }
 
-            threadgroup const uint16_t * sids_t = sids + t*ne20;
+            threadgroup const uint16_t * sids = (threadgroup const uint16_t *) shmem + t*ne20;
 
-            FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
-                n_all += sids_t[i20] == ide;
+            short sel = 0;
+            #pragma unroll(ne20)
+            for (short i20 = 0; i20 < ne20; i20++) {
+                sel += (sids[i20] == ide)*(i20 + 1);
             }
+
+            ids_i32[n_all] = (i21 + t)*ne20 + sel - 1;
+
+            n_all += sel > 0;
         }
 
         threadgroup_barrier(mem_flags::mem_threadgroup);
     }
 
-    cnt[ide] = n_all;
-
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    // exclusive prefix sum over the counts: the entries of this expert start at bas[ide] -
-    // every slice is padded by one entry for the uniform dummy store of pass 2 - and emit the
-    // entry tile descriptors of kernel_mul_mm_id: (expert, first entry, valid rows) per tile
-    if (tpitg == 0) {
-        constexpr uint32_t NTILE = 32; // must match NR1 of kernel_mul_mm_id
-
-        uint32_t base = 0;
-        uint32_t tile = 0;
-
-        device uint32_t * tpe_u32 = (device uint32_t *) (htpe);
-
-        for (int e = 0; e < FC_mul_mm_id_map0_ne02; ++e) {
-            const uint32_t n = cnt[e];
-
-            bas[e] = base;
-
-            for (uint32_t k = 0; k < n; k += NTILE) {
-                tpe_u32[1 + 3*tile    ] = e;
-                tpe_u32[1 + 3*tile + 1] = base + k;
-                tpe_u32[1 + 3*tile + 2] = n - k < NTILE ? n - k : NTILE;
-
-                tile++;
-            }
-
-            base += n + 1;
-        }
-
-        tpe_u32[0] = tile;
-    }
-
-    threadgroup_barrier(mem_flags::mem_threadgroup);
-
-    // pass 2: write the entries of this expert into its slice of the compact list
-    device int32_t * ids_i32 = (device int32_t *) (hids);
-
-    uint32_t ient = bas[ide];
-
-    for (int i21 = 0; i21 < args.ne21; i21 += ntg) { // n_tokens
-        if (i21 + tpitg < args.ne21) {
-            device const int32_t * src2_i32 = (device const int32_t *) (src2 + (i21 + tpitg)*args.nb21);
-
-            FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
-                sids[tpitg*ne20 + i20] = src2_i32[i20];
-            }
-        }
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-
-        for (short t = 0; t < ntg; t++) {
-            if (i21 + t >= args.ne21) {
-                break;
-            }
-
-            threadgroup const uint16_t * sids_t = sids + t*ne20;
-
-            short first = -1;
-            short n_ents = 0;
-
-            FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
-                const short hit = sids_t[i20] == ide;
-
-                first = hit && first < 0 ? i20 : first;
-                n_ents += hit;
-            }
-
-            // one store per token for every thread, regardless of the match - conditional
-            // stores of the divergent lanes are very slow - the dummy value of the tokens
-            // without a match lands on the padding slot of this slice and is never read
-            ids_i32[ient] = first >= 0 ? (i21 + t)*ne20 + first : 0;
-
-            if (n_ents > 1) {
-                // duplicate ids: the remaining matches of this token
-                short j = 1;
-
-                FOR_UNROLL (short i20 = 0; i20 < ne20; i20++) {
-                    if (sids_t[i20] == ide && i20 != first) {
-                        ids_i32[ient + j] = (i21 + t)*ne20 + i20;
-                        j += 1;
-                    }
-                }
-            }
-
-            ient += n_ents;
-        }
-
-        threadgroup_barrier(mem_flags::mem_threadgroup);
-    }
+    device uint32_t * tpe_u32 = (device uint32_t *) (htpe);
+    tpe_u32[ide] = n_all;
 }
 
 kernel void kernel_mul_mm_id_amax_part_f32(
@@ -593,7 +505,6 @@ template [[host_name("kernel_mul_mm_id_map0_ne20_10")]] kernel kernel_mul_mm_id_
 template [[host_name("kernel_mul_mm_id_map0_ne20_16")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<16>;
 template [[host_name("kernel_mul_mm_id_map0_ne20_22")]] kernel kernel_mul_mm_id_map0_t kernel_mul_mm_id_map0<22>;
 
-// TODO: refactor into two seperate functions to avoid ifdef branching: with and without tensor support
 template<typename S0, typename S0_4x4, typename S0_8x8, typename S1, typename S1_2x4, typename S1_8x8, typename block_q, short nl, void (*dequantize_func)(device const block_q *, short, thread S0_4x4 &), typename T0, typename T0_4x4, typename T1, typename T1_2x4>
 kernel void kernel_mul_mm_id(
         constant ggml_metal_kargs_mul_mm_id & args,
@@ -622,42 +533,55 @@ kernel void kernel_mul_mm_id(
     constexpr int NL0 = NK/16;
     constexpr int NL1 = NK/8;
 
-    const int tile = tgpig.x; // entry tile, described by the prepass
-    const int r0   = tgpig.y*NR0;
+    const int im = tgpig.z; // expert
+    const int r0 = tgpig.y*NR0;
+    const int r1 = tgpig.x*NR1;
 
     device const uint32_t * tpe_u32 = (device const uint32_t *) (htpe);
     device const int32_t  * ids_i32 = (device const int32_t  *) (hids);
 
-    // the prepass emits one (expert, first entry, valid rows) descriptor per tile that has
-    // work - the grid is sized by a static upper bound, so the tail early exits
-    if (tile >= (int) tpe_u32[0]) {
+    const int32_t neh1 = tpe_u32[im];
+
+    if (r1 >= neh1) {
         return;
     }
 
-    const int im  = tpe_u32[1 + 3*tile];     // expert
-    const int idb = tpe_u32[1 + 3*tile + 1]; // first entry of this tile in hids
-
-    // number of entries in this tile, always <= NR1
-    const short nr1 = (short) ((tpe_u32[1 + 3*tile + 2] < NR1) ? tpe_u32[1 + 3*tile + 2] : NR1);
-
     // if this block is of 64x32 shape or smaller
     const short nr0 = (args.ne0 - r0 < NR0) ? (args.ne0 - r0) : NR0;
+    const short nr1 = (    neh1 - r1 < NR1) ? (    neh1 - r1) : NR1;
 
     // a thread shouldn't load data outside of the matrix
     const short lr0 = ((short)tiitg/NL0) < nr0 ? ((short)tiitg/NL0) : nr0 - 1; // 0 .. 63
+    const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31
 
     const short il0 = (tiitg % NL0);
 
+    short il = il0;
+
+    const int id = ids_i32[im*args.ne21 + r1 + lr1];
+
+    const short i11 = (id % args.ne20) % args.ne11;
+    const short i12 = (id / args.ne20);
     const short i13 = 0;
 
     const uint64_t offset0 = im*args.nb02 + i13*args.nb03;
     const short    offset1 = il0/nl;
 
+    device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
+
     const short iy = 8*(tiitg % NL1);
 
+    device const T1 * y = (device const T1 *)(src1
+        + args.nb13*i13
+        + args.nb12*i12
+        + args.nb11*i11
+        + args.nb10*iy);
+
     // skip the upper half of the token tile when the expert did not fill it
     constexpr short NR1H = NR1/2;
 
+    const bool has_hi = nr1 > NR1H;
+
     // power-of-two rescaling
     const float s1_inv   = FC_mul_mm_id_amax ? ((device const float *) amax)[0] : 1.0f;
     const float s1_scale = FC_mul_mm_id_amax ? ((device const float *) amax)[1] : 1.0f;
@@ -667,6 +591,13 @@ kernel void kernel_mul_mm_id(
     S1_8x8 mb[2];
 
     simdgroup_float8x8 mc[8];
+
+    for (short i = 0; i < 8; i++){
+        mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
+    }
+
+    // simdgroups 2,3 own rows NR1H..NR1-1
+    const bool sg_active = has_hi || sgitg < 2;
 #else
     auto tA  = tensor<threadgroup S0, dextents<int32_t, 2>, tensor_inline>(sa, dextents<int32_t, 2>(NK, NR0));
 
@@ -678,35 +609,6 @@ kernel void kernel_mul_mm_id(
         mpp::tensor_ops::matmul2d_descriptor(NR1H, NR0, NK, false, true, false, mpp::tensor_ops::matmul2d_descriptor::mode::multiply_accumulate),
         execution_simdgroups<4>> mm;
 
-#endif
-
-    const short lr1 = ((short)tiitg/NL1) < nr1 ? ((short)tiitg/NL1) : nr1 - 1; // 0 .. 31
-
-    short il = il0;
-
-    const int id = ids_i32[idb + lr1];
-
-    const short i11 = (id % args.ne20) % args.ne11;
-    const short i12 = (id / args.ne20);
-
-    device const block_q * x = (device const block_q *)(src0 + args.nb01*(r0 + lr0) + offset0) + offset1;
-
-    device const T1 * y = (device const T1 *)(src1
-        + args.nb13*i13
-        + args.nb12*i12
-        + args.nb11*i11
-        + args.nb10*iy);
-
-    const bool has_hi = nr1 > NR1H;
-
-#ifndef GGML_METAL_HAS_TENSOR
-    for (short i = 0; i < 8; i++){
-        mc[i] = make_filled_simdgroup_matrix<float, 8>(0.f);
-    }
-
-    // simdgroups 2,3 own rows NR1H..NR1-1
-    const bool sg_active = has_hi || sgitg < 2;
-#else
     auto cT0 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB0), float>();
     auto cT1 = mm.get_destination_cooperative_tensor<decltype(tA), decltype(tB1), float>();
 #endif
@@ -917,7 +819,7 @@ kernel void kernel_mul_mm_id(
     threadgroup_barrier(mem_flags::mem_threadgroup);
 
     for (short j = sgitg; j < nr1; j += 4) {
-        const int id = ids_i32[idb + j];
+        const int id = ids_i32[im*args.ne21 + r1 + j];
 
         const short ide = id % args.ne20;
         const short idt = id / args.ne20;

ggml/src/ggml-opencl/ggml-opencl.cpp

@@ -48,7 +48,6 @@ typedef const void * (*get_adreno_bin_kernel_func_t)(
 #include <mutex>
 #include <regex>
 #include <set>
-#include <tuple>
 #include <unordered_set>
 
 #undef MIN
@@ -495,13 +494,6 @@ struct ggml_opencl_fa_kernels {
     std::map<std::pair<int, int>, cl_kernel> f32_f16_q1_split;       // flash-decoding K-split
     // vec decode
     std::map<std::pair<int, int>, cl_kernel> f32_f16_q1_vec;
-    bool f32_f16_vec_512_attempted = false;
-    std::map<std::tuple<int, int, int>, cl_kernel> f32_f16_mq_decode;
-    std::map<std::tuple<int, int, int>, size_t>    f32_f16_mq_decode_wg;
-    std::map<std::tuple<int, int, int>, int>       f32_f16_mq_decode_hs;
-    std::set<std::tuple<int, int, int>>           f32_f16_mq_decode_attempted;
-    ggml_cl_buffer fd_partial;
-    cl_uint compute_units = 0;
     // kv-head-coalesced vec decode
     std::map<std::pair<int, int>, cl_kernel> f32_f16_q1_vec_mq;
     // kv-head-coalesced + flash-decoding split
@@ -1032,8 +1024,6 @@ struct ggml_backend_opencl_context {
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a = nullptr;    // dp4a (int8) q4_0 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_mxfp4_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) mxfp4 MoE prefill GEMM
     cl_kernel kernel_gemm_moe_q4_0_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q4_0 MoE prefill GEMM
-    cl_kernel kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr;   // binary dp4a (int8) q4_k MoE prefill GEMM
-    cl_kernel kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr;    // binary dp4a (int8) q6_k MoE prefill GEMM
     cl_kernel kernel_moe_reorder_b;
     cl_kernel kernel_moe_histogram, kernel_moe_scan, kernel_moe_fill, kernel_moe_scatter;
     cl_kernel kernel_moe_scatter_stable = nullptr;   // deterministic slot assignment
@@ -1297,11 +1287,6 @@ struct ggml_backend_opencl_context {
 
         ref_count--;
         if (ref_count == 0) {
-            if (fa.fd_partial.buffer) {
-                CL_CHECK(clReleaseMemObject(fa.fd_partial.buffer));
-                fa.fd_partial.buffer = nullptr;
-                fa.fd_partial.size = 0;
-            }
 #ifdef GGML_OPENCL_PROFILING
             flush_profiling_batch();
             write_profiling_info();
@@ -1486,7 +1471,14 @@ static bool use_adreno_bin_kernels(ggml_backend_opencl_context * backend_ctx) {
 #endif // GGML_OPENCL_USE_ADRENO_BIN_KERNELS
 }
 
-static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_context * backend_ctx) {
+static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
+    if (backend_ctx->kernels_loaded) {
+        return;
+    }
+
+    cl_int err;
+
+    // compiler options for general kernels
     auto opencl_c_std =
         std::string("CL") + std::to_string(backend_ctx->opencl_c_version.major) + "." + std::to_string(backend_ctx->opencl_c_version.minor);
     std::string compile_opts = std::string("-cl-std=") + opencl_c_std +
@@ -1497,18 +1489,7 @@ static std::string ggml_opencl_make_compile_opts(const ggml_backend_opencl_conte
         compile_opts += " -qcom-enable-large-buffer ";
     }
 
-    return compile_opts;
-}
-
-static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
-    if (backend_ctx->kernels_loaded) {
-        return;
-    }
-
-    cl_int err;
-    const std::string & compile_opts = backend_ctx->kernel_compile_opts;
-    const std::string opencl_c_std = "CL" + std::to_string(backend_ctx->opencl_c_version.major) +
-                                   "." + std::to_string(backend_ctx->opencl_c_version.minor);
+    backend_ctx->kernel_compile_opts = compile_opts;
 
     GGML_LOG_INFO("ggml_opencl: loading OpenCL kernels");
 
@@ -4849,24 +4830,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
-    // gemm_moe_q4_k_q8_1_dp4a_bin (dp4a prefill GEMM)
-    if (backend_ctx->has_integer_dot) {
-        size_t bin_size = 0;
-        backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = nullptr;
-
-        if (use_adreno_bin_kernels(backend_ctx)) {
-            const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q4_k_q8_1_dp4a_ila", &bin_size);
-            if (kernel_bin && bin_size > 0) {
-                cl_program prog =
-                    build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);
-
-                CL_CHECK((backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q4_k_q8_1_dp4a_ila", &err), err));
-                CL_CHECK(clReleaseProgram(prog));
-                GGML_LOG_CONT(".");
-            }
-        }
-    }
-
     // gemm_moe_mxfp4_q8_1_dp4a (dp4a prefill GEMM)
     if (backend_ctx->has_integer_dot) {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -5086,24 +5049,6 @@ static void load_cl_kernels(ggml_backend_opencl_context *backend_ctx) {
         GGML_LOG_CONT(".");
     }
 
-    // gemm_moe_q6_k_q8_1_dp4a_bin (dp4a prefill GEMM)
-    if (backend_ctx->has_integer_dot) {
-        size_t bin_size = 0;
-        backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = nullptr;
-
-        if (use_adreno_bin_kernels(backend_ctx)) {
-            const char * kernel_bin = (const char *)backend_ctx->get_adreno_bin_kernel("gemm_moe_q6_k_q8_1_dp4a_ila", &bin_size);
-            if (kernel_bin && bin_size > 0) {
-                cl_program prog =
-                    build_program_from_binary(backend_ctx->context, backend_ctx->device, kernel_bin, CL_moe_compile_opts, bin_size);
-
-                CL_CHECK((backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin = clCreateKernel(prog, "kernel_gemm_moe_q6_k_q8_1_dp4a_ila", &err), err));
-                CL_CHECK(clReleaseProgram(prog));
-                GGML_LOG_CONT(".");
-            }
-        }
-    }
-
     // gemv_moe_mxfp4_f32_ns
     {
 #ifdef GGML_OPENCL_EMBED_KERNELS
@@ -5623,127 +5568,6 @@ static void ggml_opencl_ensure_fa_pre_kernels(ggml_backend_opencl_context * back
     clReleaseProgram(prog_pre_f16);
 }
 
-static bool ggml_opencl_ensure_fa_f32_f16_vec_512(ggml_backend_opencl_context * backend_ctx) {
-    const std::pair<int, int> key = {512, 512};
-    auto & fa = backend_ctx->fa;
-    if (fa.f32_f16_q1_vec.count(key) > 0) {
-        return true;
-    }
-    if (fa.f32_f16_vec_512_attempted || backend_ctx->kernel_compile_opts.empty()) {
-        return false;
-    }
-    fa.f32_f16_vec_512_attempted = true;
-
-    const ggml_opencl_fa_dim * cfg = nullptr;
-    for (const auto & d : g_opencl_fa_dims) {
-        if (d.dk == 512 && d.dv == 512) {
-            cfg = &d;
-            break;
-        }
-    }
-    if (cfg == nullptr) {
-        return false;
-    }
-
-    // Compile only vec decode and merge to stay within the Adreno compiler's memory limit.
-    const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) +
-                             " -D FA_DECODE_ONLY -D FA_VEC_ONLY";
-    cl_program prog = build_program_from_source_ex(
-        backend_ctx->context, backend_ctx->device,
-        ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16).c_str(), opts,
-        /*fatal=*/false, "fa f32_f16 decode512 vec", backend_ctx->queue);
-    if (!prog) {
-        return false;
-    }
-    cl_int err;
-    cl_kernel vec = clCreateKernel(prog, "flash_attn_f32_f16_q1_vec", &err);
-    if (err != CL_SUCCESS) {
-        clReleaseProgram(prog);
-        return false;
-    }
-    cl_kernel merge = clCreateKernel(prog, "flash_attn_f32_merge", &err);
-    clReleaseProgram(prog);
-    if (err != CL_SUCCESS) {
-        clReleaseKernel(vec);
-        return false;
-    }
-    if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, vec, 256, "flash_attn_f32_f16_q1_vec", 512, 512) ||
-        !ggml_opencl_fa_kernel_fits_wg(backend_ctx, merge, 128, "flash_attn_f32_merge", 512, 512)) {
-        clReleaseKernel(vec);
-        clReleaseKernel(merge);
-        return false;
-    }
-    fa.f32_f16_q1_vec[key] = vec;
-    if (fa.f32_merge.count(key) > 0) {
-        clReleaseKernel(merge);
-    } else {
-        fa.f32_merge[key] = merge;
-    }
-    return true;
-}
-
-static void ggml_opencl_ensure_fa_f32_f16_mq_decode(ggml_backend_opencl_context * backend_ctx, int dk, int dv, int gqa) {
-    const std::tuple<int, int, int> key = {dk, dv, gqa};
-    auto & fa = backend_ctx->fa;
-    if (fa.f32_f16_mq_decode.count(key) > 0 || fa.f32_f16_mq_decode_attempted.count(key) > 0 ||
-        backend_ctx->kernel_compile_opts.empty()) {
-        return;
-    }
-    if (gqa == 4 && dk != 128 && fa.f32_f16_q1_vec_mq_split.count({dk, dv}) > 0) {
-        fa.f32_f16_mq_decode[key] = fa.f32_f16_q1_vec_mq_split.at({dk, dv});
-        fa.f32_f16_mq_decode_wg[key] = 256;
-        fa.f32_f16_mq_decode_hs[key] = 1;
-        return;
-    }
-    fa.f32_f16_mq_decode_attempted.insert(key);
-    const ggml_opencl_fa_dim * cfg = nullptr;
-    for (const auto & d : g_opencl_fa_dims) {
-        if (d.dk == dk && d.dv == dv) {
-            cfg = &d;
-            break;
-        }
-    }
-    if (cfg == nullptr) {
-        return;
-    }
-
-    const bool cluster = dk == 64 || dk == 128;
-    const int head_sub = cluster ? 2 : (gqa == 8 ? (dk == 512 ? 4 : 2) : 1);
-    const int nsg_max = dk == 64 ? 1 : (dk == 128 || (gqa == 8 && dk == 256) ? 2 : 4);
-    const char * kernel_name = cluster ? "flash_attn_f32_f16_q1_vec_mq_split_c8" : "flash_attn_f32_f16_q1_vec_mq_split";
-    const std::string src = ggml_opencl_fa_kernel_src(FA_VARIANT_F32_F16);
-    const std::string opts = ggml_opencl_fa_compile_opts(backend_ctx, cfg, FA_VARIANT_F32_F16) +
-                             " -D FA_MQ_ONLY -D MQ_GQA=" + std::to_string(gqa / head_sub) +
-                             " -D FA_HEAD_SUB=" + std::to_string(head_sub) +
-                             (cluster ? " -D MQ_NSG=" + std::to_string(nsg_max) + " -D FA_CL_C=16" : " -D FA_MQ_SPLIT_ONLY") +
-                             (dk == 64 ? " -D FA_CL_MHRED -D FA_CL_MASK_BCAST" : "") +
-                             (gqa == 8 && dk == 256 ? " -D FA_Q1_Q_REG" : "");
-    for (int nsg = nsg_max; nsg >= 1; nsg /= 2) {
-        const size_t wg = 64 * nsg;
-        cl_program prog = build_program_from_source_ex(
-            backend_ctx->context, backend_ctx->device, src.c_str(),
-            opts + " -D MQ_NSG_SPLIT=" + std::to_string(nsg),
-            /*fatal=*/false, "fa f32_f16 mq decode", backend_ctx->queue);
-        if (!prog) {
-            continue;
-        }
-        cl_int err;
-        cl_kernel kernel = clCreateKernel(prog, kernel_name, &err);
-        clReleaseProgram(prog);
-        if (err != CL_SUCCESS) {
-            continue;
-        }
-        if (!ggml_opencl_fa_kernel_fits_wg(backend_ctx, kernel, wg, kernel_name, dk, dv)) {
-            clReleaseKernel(kernel);
-            continue;
-        }
-        fa.f32_f16_mq_decode[key] = kernel;
-        fa.f32_f16_mq_decode_wg[key] = wg;
-        fa.f32_f16_mq_decode_hs[key] = head_sub;
-        return;
-    }
-}
-
 // DK=512 prefill BM-tile
 static bool ggml_opencl_ensure_fa_f32_f16_prefill_512(ggml_backend_opencl_context * backend_ctx, bool split) {
     const int dk = 512, dv = 512;
@@ -6972,10 +6796,6 @@ static ggml_backend_opencl_context * ggml_cl_init(ggml_backend_dev_t dev) {
     backend_ctx->adreno_use_large_buffer = getenv("GGML_OPENCL_ADRENO_USE_LARGE_BUFFER") != nullptr &&
                                            backend_ctx->gpu_family == GPU_FAMILY::ADRENO;
 
-    backend_ctx->kernel_compile_opts = ggml_opencl_make_compile_opts(backend_ctx.get());
-    CL_CHECK(clGetDeviceInfo(device, CL_DEVICE_MAX_COMPUTE_UNITS,
-                            sizeof(backend_ctx->fa.compute_units), &backend_ctx->fa.compute_units, NULL));
-
     // ragged moe, unspecified or non-zero means enabled, set to 0 to disable
     static const char * ragged_fp16_env = getenv("GGML_OPENCL_MOE_RAGGED_FP16");
     backend_ctx->adreno_use_moe_ragged = (ragged_fp16_env == NULL) ? 1 : (atoi(ragged_fp16_env) != 0);
@@ -8725,11 +8545,6 @@ inline bool use_adreno_kernels(const ggml_backend_opencl_context *backend_ctx, c
     bool threashold_ok = tensor->ne[0] >= threshold_ne0 && tensor->ne[1] >= threshold_ne1 &&
             tensor->ne[2] == 1 && tensor->ne[3] == 1;
 
-    // the transposed layout needs K % 32 == 0 and M % 4 == 0
-    if (tensor->ne[0] % 32 != 0 || tensor->ne[1] % 4 != 0) {
-        return false;
-    }
-
     // The noshuffle layout packs 2 rows per 32-bit texel and the GEMV reads it at an
     // ne1/2 texel stride with an exact-cover dispatch, so it is only addressable when
     // ne1 is a multiple of 64; an unaligned ne1 truncates the stride and the weight is
@@ -8878,10 +8693,7 @@ inline bool use_q4_0_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
         !backend_ctx->kernel_gemm_noshuffle_q4_0_f32_32b_trans_ila_a8_bin) {
         return false;
     }
-
-    // bin kernels require ne2 == 1 and ne3 == 1 for weights
-    return tensor->ne[2] == 1 && tensor->ne[3] == 1 &&
-           (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0);
+    return (tensor->ne[0] % 32 == 0) && (tensor->ne[1] % 64 == 0);
 #else
     GGML_UNUSED(backend_ctx);
     GGML_UNUSED(tensor);
@@ -8929,30 +8741,22 @@ static inline bool flat_large_m_enabled() {
     return en;
 }
 
-// The noshuffle Q4_K weight image stores eight weights per uint texel.
-static inline bool q4_K_weight_image_fits(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
-    const size_t texels = (size_t) ggml_nelements(tensor) / 8;
-    return texels != 0 && texels <= backend_ctx->image_max_buffer_size;
-}
-
 static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
     if (tensor->ne[1] % 4 != 0 && tensor->ne[2] == 1 && tensor->ne[3] == 1) {
         return true;
     }
 
-    if (tensor->ne[2] != 1 || tensor->ne[3] != 1 || use_q4k_tiled(backend_ctx, tensor)) {
+    if (!flat_large_m_enabled()) {
         return false;
     }
-
-    // The image limit is a correctness guard, independent of the large-M opt-in.
-    if (!q4_K_weight_image_fits(backend_ctx, tensor)) {
-        return true;
-    }
-
     // gemv_noshuffle variant perf drops for large M, use flat variant for large M.
     // threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
     // note that this forces large M weights to use LM GEMM.
-    return flat_large_m_enabled() && tensor->ne[1] >= 32768;
+    // EXCEPT when this branch's tiled-canonical lm_head/embed layout is active: the
+    // weight is converted to the 64-row tiled layout, which the flat gemv would
+    // misread as garbage. use_q4k_tiled owns these large-M weights, so defer to it.
+    return tensor->ne[1] >= 32768 && tensor->ne[2] == 1 && tensor->ne[3] == 1
+           && !use_q4k_tiled(backend_ctx, tensor);
 }
 
 static inline bool use_flat_gemv_for_large_m_q6_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
@@ -9004,9 +8808,7 @@ inline bool use_q6_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
         !backend_ctx->kernel_gemm_noshuffle_q6_k_f32_32b_trans_ila_a8_bin) {
         return false;
     }
-    // bin kernels require ne2 == 1 and ne3 == 1 for weights
-    return tensor->ne[2] == 1 && tensor->ne[3] == 1 &&
-           (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
+    return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
            !use_q6k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q6_K(backend_ctx, tensor);
 #else
     GGML_UNUSED(backend_ctx);
@@ -9021,9 +8823,7 @@ inline bool use_q4_k_bin_kernels(const ggml_backend_opencl_context *backend_ctx,
         !backend_ctx->kernel_gemm_noshuffle_q4_k_f32_32b_trans_ila_a8_bin) {
         return false;
     }
-    // bin kernels require ne2 == 1 and ne3 == 1 for weights
-    return tensor->ne[2] == 1 && tensor->ne[3] == 1 &&
-           (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
+    return (tensor->ne[0] % 256 == 0) && (tensor->ne[1] % 64 == 0) &&
            !use_q4k_tiled(backend_ctx, tensor) && !use_flat_gemv_for_large_m_q4_K(backend_ctx, tensor);
 #else
     GGML_UNUSED(backend_ctx);
@@ -9489,13 +9289,10 @@ static bool ggml_opencl_supports_op(ggml_backend_dev_t dev, const struct ggml_te
                     return false;
                 }
                 if (q->ne[1] == 1) {
-                    const char * decode_env = getenv("GGML_OPENCL_FA_DK512_DECODE");
-                    if ((decode_env && decode_env[0] == '0') ||
-                        backend_ctx->gpu_family != ADRENO || k->ne[2] <= 0 ||
-                        (q->ne[2] / k->ne[2] != 4 && q->ne[2] / k->ne[2] != 8) || q->ne[2] % k->ne[2] != 0 ||
-                        !ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx)) {
-                        return false;
-                    }
+                    // DK=512 decode is bandwidth-bound and slower on the GPU
+                    // than on the CPU; decline it here so it runs on the CPU.
+                    // Prefill (n_q > 1) stays on the GPU.
+                    return false;
                 } else {
                     // prefill, BM-tile in its own FA_PREFILL_ONLY program
                     if (!ggml_opencl_ensure_fa_f32_f16_prefill_512(backend_ctx, /*split=*/false)) {
@@ -15236,7 +15033,7 @@ static void ggml_cl_rms_norm(ggml_backend_t backend, const ggml_tensor * src0, c
 
     GGML_ASSERT(ne00 % 4 == 0);
 
-    const int nth = 64;
+    const int nth = MIN(64, ne00);
 
     size_t global_work_size[] = {(size_t)ne01*nth, (size_t)ne02, (size_t)ne03};
     size_t local_work_size[] = {(size_t)nth, 1, 1};
@@ -18083,7 +17880,10 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
     }
 #endif
 
-    // Compile DK512 decode separately from the prefill programs.
+    // DK=512 (Gemma-4 global layers) runs decode-only (q1 / q1_split) on
+    // Adreno - it never uses the BM-tile path, and the prepass + split-tile
+    // programs OOM the compiler at DK=512; supports_op only admits
+    // n_q==1 here and prefill goes to CPU
     const bool fa_decode_only_512 = (d_head_q == 512);
 
     // per-variant lazy compile for this (dk, dv)
@@ -18112,11 +17912,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
     if (is_f16) {
         ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F16);
     } else if (is_mixed) {
-        if (fa_decode_only_512 && n_q == 1) {
-            GGML_ASSERT(ggml_opencl_ensure_fa_f32_f16_vec_512(backend_ctx));
-        } else {
-            ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16);
-        }
+        ggml_opencl_ensure_fa_variant(backend_ctx, d_head_q, d_head_v, FA_VARIANT_F32_F16);
         if (fa_decode_only_512) {
             // DK=512: the BM-tile prefill kernels are specifically compiled from
             // FA_PREFILL_ONLY
@@ -18150,16 +17946,6 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
     }
 
     const std::pair<int, int> dk_dv = {d_head_q, d_head_v};
-    const int mq_gqa = n_head_kv > 0 ? n_head / n_head_kv : 0;
-    const std::tuple<int, int, int> mq_decode_key = {d_head_q, d_head_v, mq_gqa};
-    const bool mq_decode_shape = backend_ctx->gpu_family == ADRENO && is_mixed && n_q == 1 &&
-                                 d_head_q == d_head_v && n_head_kv > 0 && n_head % n_head_kv == 0 &&
-                                 ((((d_head_q == 64 && mq_gqa == 8) || (d_head_q == 128 && mq_gqa == 4)) &&
-                                   backend_ctx->has_subgroup_shuffle) ||
-                                  ((d_head_q == 256 || d_head_q == 512) && (mq_gqa == 4 || mq_gqa == 8)));
-    if (mq_decode_shape && n_kv >= 32) {
-        ggml_opencl_ensure_fa_f32_f16_mq_decode(backend_ctx, d_head_q, d_head_v, mq_gqa);
-    }
     const bool use_native_q8_0_q1 = is_q8_0 && n_q == 1 &&
                                     backend_ctx->fa.f32_q8_0_q1.count(dk_dv) > 0;
     // Native q8_0 prefill — reads q8_0 directly, wg_size = cfg->bm.
@@ -18418,8 +18204,6 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
     const int fd_max_n_q = (d_head_q <= FD_MAX_DK_MULTI) ? FD_MAX_N_Q_MULTI : 1;
     cl_kernel fd_k_split = NULL;
     bool use_fd_mq = false;
-    bool use_fd_mq_decode = false;
-    int fd_head_sub = 1;
     size_t fd_mq_wg = 256;  // MQ_GQA=4 kernel: Q1_WG_SIZE(64) * MQ_NSG_SPLIT(4)
     bool use_fa_k_img = false;  // K bound as image1d_buffer_t instead of (buf, offset)
 
@@ -18452,15 +18236,9 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
         if (mq_enabled && mq_kv_ok && nq_in_vec_range && !is_causal &&
             backend_ctx->gpu_family != INTEL &&
             !use_local_tile &&
-            n_kv >= (mq_decode_shape ? 32 : FD_MIN_N_KV) &&
+            n_kv >= FD_MIN_N_KV &&
             backend_ctx->fa.f32_merge.count(dk_dv) > 0) {
-            if (mq_decode_shape && backend_ctx->fa.f32_f16_mq_decode.count(mq_decode_key) > 0) {
-                fd_k_split = backend_ctx->fa.f32_f16_mq_decode.at(mq_decode_key);
-                fd_mq_wg = backend_ctx->fa.f32_f16_mq_decode_wg.at(mq_decode_key);
-                fd_head_sub = backend_ctx->fa.f32_f16_mq_decode_hs.at(mq_decode_key);
-                use_fd_mq = true;
-                use_fd_mq_decode = true;
-            } else if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 &&
+            if (nq1_only && lmq_on && is_mixed && d_head_q == 128 && d_head_v == 128 &&
                 gqa_ratio_dispatch == 8 &&
                 backend_ctx->fa.f32_f16_q1_local_mq_split_g8.count(dk_dv) > 0) {
                 fd_k_split = backend_ctx->fa.f32_f16_q1_local_mq_split_g8.at(dk_dv);
@@ -18739,14 +18517,6 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
         int n_splits = (n_kv + fd_kv_per_split - 1) / fd_kv_per_split;
         if (n_splits < FD_MIN_SPLITS) { n_splits = FD_MIN_SPLITS; }
         if (n_splits > fd_max_splits) { n_splits = fd_max_splits; }
-        if (use_fd_mq_decode) {
-            const size_t wg_per_split = (size_t) n_head_kv * n_batch;
-            const size_t wg_target = 4 * (size_t) backend_ctx->fa.compute_units;
-            while (wg_per_split * n_splits < wg_target && n_splits < fd_max_splits &&
-                   n_kv / (n_splits + 1) >= 32) {
-                n_splits++;
-            }
-        }
         const int kv_per_split = (n_kv + n_splits - 1) / n_splits;
 
         const int fa_partial_floats = 2 + d_head_v;
@@ -18754,26 +18524,15 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
             (size_t) n_batch * n_head * n_q * n_splits * fa_partial_floats * sizeof(float);
 
         ggml_cl_flash_attn_temp_buffer temp_partial;
-        cl_mem partial_buffer;
         cl_int err;
-        if (use_fd_mq_decode) {
-            auto & pool = backend_ctx->fa.fd_partial;
-            if (partial_size_bytes > pool.size) {
-                CL_CHECK(clFinish(backend_ctx->queue));
-                pool.allocate(backend_ctx->context, partial_size_bytes);
-            }
-            partial_buffer = pool.buffer;
-        } else {
+        temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE,
+                                           partial_size_bytes, NULL, &err);
+        if (err != CL_SUCCESS) {
+            CL_CHECK(clFinish(backend_ctx->queue));
             temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE,
                                                partial_size_bytes, NULL, &err);
-            if (err != CL_SUCCESS) {
-                CL_CHECK(clFinish(backend_ctx->queue));
-                temp_partial.data = clCreateBuffer(backend_ctx->context, CL_MEM_READ_WRITE,
-                                                   partial_size_bytes, NULL, &err);
-            }
-            CL_CHECK(err);
-            partial_buffer = temp_partial.data;
         }
+        CL_CHECK(err);
 
         cl_kernel k_split = fd_k_split;
         int argi = 0;
@@ -18841,7 +18600,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
         CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_ulong), &mask_nb3));
         CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int),      &mask_ne2));
         CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int),      &mask_ne3));
-        CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem),   &partial_buffer));
+        CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(cl_mem),   &temp_partial.data));
         CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int),      &n_splits));
         CL_CHECK(clSetKernelArg(k_split, argi++, sizeof(int),      &kv_per_split));
 
@@ -18849,7 +18608,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
         // matches Q1_WG_SIZE * NSG (MQ_GQA=4 -> 256; MQ_GQA=8 -> 192)
         const size_t fd_wg = use_fd_mq ? fd_mq_wg : 64;
         const size_t fd_head_dim = use_fd_mq
-            ? (size_t)(n_head_kv * fd_head_sub * n_batch)
+            ? (size_t)(n_head_kv * n_batch)
             : (size_t)(n_head     * n_batch);
         size_t fd_lws[3] = { fd_wg, 1, 1 };
         // gid(2) packs q_idx * n_splits + split_idx.
@@ -18858,7 +18617,7 @@ static void ggml_cl_flash_attn(ggml_backend_t backend, const ggml_tensor * q, co
 
         cl_kernel k_merge = backend_ctx->fa.f32_merge.at(dk_dv);
         argi = 0;
-        CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem),   &partial_buffer));
+        CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem),   &temp_partial.data));
         CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_mem),   &extra_o->data_device));
         CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(cl_ulong), &offset_o));
         CL_CHECK(clSetKernelArg(k_merge, argi++, sizeof(int),      &n_head));
@@ -23656,17 +23415,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
     // quant kv without FA
     // used for non-contiguous src0 (the usual head-major permuted K view when n_head_kv>1)
     // AND for the contiguous case that occurs when n_head_kv==1 (e.g. Gemma-4 E2B)
-    // Q4_0 bin kernels use a special weight layout, which the restore below does not support.
-    bool q4_0_bin_layout = false;
-#ifdef GGML_OPENCL_USE_ADRENO_KERNELS
-    q4_0_bin_layout = src0t == GGML_TYPE_Q4_0 && src0->view_src == nullptr &&
-                      ggml_is_contiguous(src0) &&
-                      src0->ne[2] == 1 && src0->ne[3] == 1 &&
-                      use_adreno_kernels(backend_ctx, src0) &&
-                      !use_adreno_moe_kernels(backend_ctx, src0) &&
-                      use_q4_0_bin_kernels(backend_ctx, src0);
-#endif
-    if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) && !q4_0_bin_layout &&
+    if ((src0t == GGML_TYPE_Q4_0 || src0t == GGML_TYPE_Q8_0) &&
         (!ggml_is_contiguous(src0) || src1->ne[2] > src0->ne[2])) {
         cl_mem f16_buf = ggml_cl_mul_mat_dequant_quant_to_f16(backend_ctx, src0, nullptr);
 
@@ -27436,10 +27185,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                                          : (backend_ctx->adreno_gen == ADRENO_GPU_GEN::X2E || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);
                     // dot prod has to be available
                     use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;
-                    // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel
-                    if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin == nullptr) {
-                        use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;
-                    }
+                    // bin kernel takes precedence
+                    use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q4_k_f32_ns_bin == nullptr;
 
                     cl_buffer_region region;
                     region.origin = 0;
@@ -27541,10 +27288,6 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
 
                         // dp4a GEMM
                         cl_kernel dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a;
-                        if (backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin) {
-                            dk = backend_ctx->kernel_gemm_moe_q4_k_q8_1_dp4a_bin;
-                        }
-
                         int aidx = 0;
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->q_img));
                         CL_CHECK(clSetKernelArg(dk, aidx++, sizeof(cl_mem), &extra0_q4_K->d));
@@ -27952,10 +27695,8 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                                                             || backend_ctx->adreno_gen == ADRENO_GPU_GEN::X1E);
                     // dot prod has to be available
                     use_moe_dp4a = backend_ctx->has_integer_dot && use_moe_dp4a;
-                    // bin kernel takes precedence, dp4a bin kernel has higher priority than normal bin kernel
-                    if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin == nullptr) {
-                        use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr;
-                    }
+                    // bin kernel takes precedence
+                    use_moe_dp4a = use_moe_dp4a && backend_ctx->kernel_gemm_moe_q6_k_f32_ns_bin == nullptr;
 
                     cl_buffer_region region;
                     region.origin = 0;
@@ -28057,10 +27798,6 @@ static void ggml_cl_mul_mat_id(ggml_backend_t backend, const ggml_tensor * src0,
                         backend_ctx->enqueue_ndrange_kernel(rq, 2, rq_global, rq_local, dst);
 
                         cl_kernel dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a;
-                        if (backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin) {
-                            dk = backend_ctx->kernel_gemm_moe_q6_k_q8_1_dp4a_bin;
-                        }
-
                         int qi = 0;
                         CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->ql_img));
                         CL_CHECK(clSetKernelArg(dk, qi++, sizeof(cl_mem), &extra0_q6_K->qh));

ggml/src/ggml-opencl/kernels/flash_attn_f32_f16.cl

@@ -665,7 +665,7 @@ __kernel void FA_TILE_NAME(
 
 // allow bypassing decode kernels to avoid compiler crash for DK=512 on Adreno GPUs
 #ifndef FA_PREFILL_ONLY
-#if !defined(FA_MQ_ONLY) && !defined(FA_VEC_ONLY)
+#ifndef FA_MQ_ONLY  // q1 excluded from the MQ-only (g8) program
 REQD_FA_SG
 __kernel void flash_attn_f32_f16_q1(
     const global void * q_void, ulong q_offset,
@@ -932,14 +932,14 @@ __kernel void flash_attn_f32_f16_q1_vec(
         }
         ACC_TYPE dot_partial = dot4.s0 + dot4.s1 + dot4.s2 + dot4.s3;
         ACC_TYPE score = sub_group_reduce_add(dot_partial) * scale;
-        if (logit_softcap > 0.0f) {
-            score = logit_softcap * tanh(score / logit_softcap);
-        }
 
         if (mask_base != NULL) {
             const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base;
             score += slope * (ACC_TYPE) mask_ptr[k_idx];
         }
+        if (logit_softcap > 0.0f) {
+            score = logit_softcap * tanh(score / logit_softcap);
+        }
 
         // FA-2 online update. All threads in the subgroup see the same score,
         // so m_i and l_i evolve identically across lanes within the subgroup.
@@ -1385,7 +1385,6 @@ __kernel void flash_attn_f32_f16_q1_local_mq_split(
 #endif
 #define MQ_WG_SIZE (Q1_WG_SIZE * MQ_NSG)
 
-#ifndef FA_MQ_SPLIT_ONLY
 REQD_SUBGROUP_SIZE_64
 __kernel void flash_attn_f32_f16_q1_vec_mq(
     const global void * q_void, ulong q_offset,
@@ -1607,8 +1606,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq(
     }
 }
 
-#endif  // !FA_MQ_SPLIT_ONLY
-
 #ifndef MQ_NSG_SPLIT
 #define MQ_NSG_SPLIT 4
 #endif
@@ -1618,10 +1615,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq(
 #define FA_PARTIAL_FLOATS (2 + DV)
 #endif
 
-#ifndef FA_HEAD_SUB
-#define FA_HEAD_SUB 1
-#endif
-
 REQD_SUBGROUP_SIZE_64
 __kernel void flash_attn_f32_f16_q1_vec_mq_split(
     const global void * q_void, ulong q_offset,
@@ -1659,12 +1652,8 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
     const int split_idx        = split_q_idx % n_splits;
     const int q_idx            = split_q_idx / n_splits;
 
-    const int hgroups     = n_head_kv * FA_HEAD_SUB;
-    const int batch_idx   = kvhead_batch_idx / hgroups;
-    const int hg          = kvhead_batch_idx % hgroups;
-    const int head_kv_idx = hg / FA_HEAD_SUB;
-    const int head_sub    = hg % FA_HEAD_SUB;
-#define FA_MQS_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h))
+    const int batch_idx   = kvhead_batch_idx / n_head_kv;
+    const int head_kv_idx = kvhead_batch_idx % n_head_kv;
 
     const int kv_start = split_idx * kv_per_split;
     const int kv_end   = min(kv_start + kv_per_split, n_kv);
@@ -1677,7 +1666,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
         if (tid == 0) {
             #pragma unroll
             for (int h = 0; h < MQ_GQA; ++h) {
-                const int head_idx = FA_MQS_HEAD_IDX(h);
+                const int head_idx = head_kv_idx * MQ_GQA + h;
                 const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
                                        * n_splits + split_idx);
                 global float * rec = partial_void + rec_idx * record_stride;
@@ -1692,33 +1681,22 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
     const global char * k_base = (const global char *) k_void + k_offset;
     const global char * v_base = (const global char *) v_void + v_offset;
 
-#ifdef FA_Q1_Q_REG
-    ACC_TYPE4 q_reg[MQ_GQA];
-    #pragma unroll
-    for (int h = 0; h < MQ_GQA; ++h) {
-        const int head_idx = FA_MQS_HEAD_IDX(h);
-        const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
-        const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
-        q_reg[h] = (tid_sg < DK_VEC) ? CONVERT_Q_ACC4(q_ptr[tid_sg]) : (ACC_TYPE4)(0.0f);
-    }
-#else
     // stage MQ_GQA Q rows in __local once (uniform across WG)
     __local ACC_TYPE4 q_shared[MQ_GQA * DK_VEC];
     for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) {
         const int h        = i / DK_VEC;
         const int k        = i % DK_VEC;
-        const int head_idx = FA_MQS_HEAD_IDX(h);
+        const int head_idx = head_kv_idx * MQ_GQA + h;
         const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
         const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
         q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
     }
     barrier(CLK_LOCAL_MEM_FENCE);
-#endif
 
     float slope[MQ_GQA];
     #pragma unroll
     for (int h = 0; h < MQ_GQA; ++h) {
-        slope[h] = get_alibi_slope(max_bias, FA_MQS_HEAD_IDX(h), n_head_log2, m0, m1);
+        slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
     }
 
     const global char * mask_base[MQ_GQA];
@@ -1729,7 +1707,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
                                           (ulong) q_idx * mask_nb1;
         #pragma unroll
         for (int h = 0; h < MQ_GQA; ++h) {
-            const int head_idx      = FA_MQS_HEAD_IDX(h);
+            const int head_idx      = head_kv_idx * MQ_GQA + h;
             const int mask_head_idx = head_idx % mask_ne2;
             mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
         }
@@ -1764,15 +1742,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
         ACC_TYPE4 dot4[MQ_GQA];
         #pragma unroll
         for (int h = 0; h < MQ_GQA; ++h) dot4[h] = (ACC_TYPE4)(0.0f);
-#ifdef FA_Q1_Q_REG
-        if (tid_sg < DK_VEC) {
-            const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[tid_sg]);
-            #pragma unroll
-            for (int h = 0; h < MQ_GQA; ++h) {
-                dot4[h] = mad(q_reg[h], k_vec, dot4[h]);
-            }
-        }
-#else
         for (int k = tid_sg; k < DK_VEC; k += Q1_WG_SIZE) {
             const ACC_TYPE4 k_vec = CONVERT_KV_ACC4(k_ptr[k]);
             #pragma unroll
@@ -1780,20 +1749,19 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
                 dot4[h] = mad(q_shared[h * DK_VEC + k], k_vec, dot4[h]);
             }
         }
-#endif
 
         ACC_TYPE score[MQ_GQA];
         #pragma unroll
         for (int h = 0; h < MQ_GQA; ++h) {
             const ACC_TYPE dot_partial = dot4[h].s0 + dot4[h].s1 + dot4[h].s2 + dot4[h].s3;
             ACC_TYPE s = sub_group_reduce_add(dot_partial) * scale;
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
             if (mask_base[h] != NULL) {
                 const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                 s += slope[h] * (ACC_TYPE) mask_ptr[k_idx];
             }
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
             score[h] = s;
         }
 
@@ -1842,7 +1810,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
         barrier(CLK_LOCAL_MEM_FENCE);
 
         if (sgid == 0) {
-            const int head_idx = FA_MQS_HEAD_IDX(h);
+            const int head_idx = head_kv_idx * MQ_GQA + h;
 
             // fold per-subgroup (m, l) into split-level (m_c, l_c)
             ACC_TYPE m_c = sg_m[h][0];
@@ -1880,9 +1848,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split(
     }
 }
 
-#undef FA_MQS_HEAD_IDX
-
-#ifndef FA_MQ_SPLIT_ONLY
 // Cluster-parallel variant of _q1_vec_mq_split
 //
 // Tthe baseline keeps one 256B K row in flight per subgroup (32 lanes cooperate
@@ -1971,12 +1936,8 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
     const int split_idx        = split_q_idx % n_splits;
     const int q_idx            = split_q_idx / n_splits;
 
-    const int hgroups     = n_head_kv * FA_HEAD_SUB;
-    const int batch_idx   = kvhead_batch_idx / hgroups;
-    const int hg          = kvhead_batch_idx % hgroups;
-    const int head_kv_idx = hg / FA_HEAD_SUB;
-    const int head_sub    = hg % FA_HEAD_SUB;
-#define FA_HEAD_IDX(h) (head_kv_idx * (MQ_GQA * FA_HEAD_SUB) + head_sub * MQ_GQA + (h))
+    const int batch_idx   = kvhead_batch_idx / n_head_kv;
+    const int head_kv_idx = kvhead_batch_idx % n_head_kv;
 
     const int kv_start = split_idx * kv_per_split;
     const int kv_end   = min(kv_start + kv_per_split, n_kv);
@@ -1987,7 +1948,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
         if (tid == 0) {
             #pragma unroll
             for (int h = 0; h < MQ_GQA; ++h) {
-                const int head_idx = FA_HEAD_IDX(h);
+                const int head_idx = head_kv_idx * MQ_GQA + h;
                 const ulong rec_idx = ((((ulong) batch_idx * n_head + head_idx) * n_q + q_idx)
                                        * n_splits + split_idx);
                 global float * rec = partial_void + rec_idx * record_stride;
@@ -2007,7 +1968,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
     for (int i = tid; i < MQ_GQA * DK_VEC; i += MQ_SPLIT_WG_SIZE) {
         const int h        = i / DK_VEC;
         const int k        = i % DK_VEC;
-        const int head_idx = FA_HEAD_IDX(h);
+        const int head_idx = head_kv_idx * MQ_GQA + h;
         const ulong q_row_offset = batch_idx * q_nb3 + head_idx * q_nb2 + (ulong) q_idx * q_nb1;
         const global Q_DATA_TYPE4 * q_ptr = (const global Q_DATA_TYPE4 *) (q_base + q_row_offset);
         q_shared[h * DK_VEC + k] = CONVERT_Q_ACC4(q_ptr[k]);
@@ -2017,17 +1978,9 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
     float slope[MQ_GQA];
     #pragma unroll
     for (int h = 0; h < MQ_GQA; ++h) {
-        slope[h] = get_alibi_slope(max_bias, FA_HEAD_IDX(h), n_head_log2, m0, m1);
+        slope[h] = get_alibi_slope(max_bias, head_kv_idx * MQ_GQA + h, n_head_log2, m0, m1);
     }
 
-#ifdef FA_CL_MASK_BCAST
-    const global char * mask_base_b = NULL;
-    if (mask_void != NULL) {
-        mask_base_b = (const global char *) mask_void + mask_offset +
-                      (batch_idx % mask_ne3) * mask_nb3 + (ulong) q_idx * mask_nb1;
-    }
-    const int mask_bcast = mask_base_b != NULL && mask_ne2 == 1;
-#else
     const global char * mask_base[MQ_GQA];
     if (mask_void != NULL) {
         const int mask_batch_idx = batch_idx % mask_ne3;
@@ -2036,7 +1989,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
                                           (ulong) q_idx * mask_nb1;
         #pragma unroll
         for (int h = 0; h < MQ_GQA; ++h) {
-            const int head_idx      = FA_HEAD_IDX(h);
+            const int head_idx      = head_kv_idx * MQ_GQA + h;
             const int mask_head_idx = head_idx % mask_ne2;
             mask_base[h] = mask_base_b + mask_head_idx * mask_nb2;
         }
@@ -2044,7 +1997,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
         #pragma unroll
         for (int h = 0; h < MQ_GQA; ++h) mask_base[h] = NULL;
     }
-#endif
 
     // Per-CLUSTER online-softmax state (uniform across the cluster's lanes);
     // o_acc holds this lane's DV slice {lic + FA_CL_C*i}.
@@ -2079,73 +2031,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
         const global KV_DATA_TYPE4 * k_ptr = (const global KV_DATA_TYPE4 *) (k_base + kv_row_base + (ulong) k_safe * k_nb1);
         const global KV_DATA_TYPE4 * v_ptr = (const global KV_DATA_TYPE4 *) (v_base + v_row_base  + (ulong) k_safe * v_nb1);
 
-#if defined(FA_CL_MHRED) && MQ_GQA == 4 && FA_CL_C == 16 && FA_CL_DK == 1 && FA_CL_DV == 1
-        ACC_TYPE mask_val = 0.0f;
-        if (mask_bcast) {
-            mask_val = (ACC_TYPE) ((const global MASK_DATA_TYPE *) mask_base_b)[k_safe];
-        }
-        const ACC_TYPE4 k_vec_1 = CONVERT_KV_ACC4(k_ptr[lic]);
-        const ACC_TYPE4 v_vec_1 = CONVERT_KV_ACC4(v_ptr[lic]);
-
-        // Reduce four heads with eight shuffles and keep each head's summation order.
-        const int mh_b0 = lic & 1;
-        const int mh_b1 = lic & 2;
-
-        ACC_TYPE mh_p[MQ_GQA];
-        #pragma unroll
-        for (int h = 0; h < MQ_GQA; ++h) {
-            const ACC_TYPE4 d4 = mad(q_shared[h * DK_VEC + lic], k_vec_1, (ACC_TYPE4)(0.0f));
-            mh_p[h] = d4.s0 + d4.s1 + d4.s2 + d4.s3;
-        }
-
-        ACC_TYPE mh_r2[2];
-        #pragma unroll
-        for (int j = 0; j < 2; ++j) {
-            const ACC_TYPE keep = mh_b0 ? mh_p[j + 2] : mh_p[j];
-            const ACC_TYPE send = mh_b0 ? mh_p[j]     : mh_p[j + 2];
-            mh_r2[j] = keep + sub_group_shuffle_xor(send, 1);
-        }
-        ACC_TYPE mh_r1 = (mh_b1 ? mh_r2[1] : mh_r2[0]) +
-                         sub_group_shuffle_xor(mh_b1 ? mh_r2[0] : mh_r2[1], 2);
-        mh_r1 += sub_group_shuffle_xor(mh_r1, 4);
-        mh_r1 += sub_group_shuffle_xor(mh_r1, 8);
-
-        ACC_TYPE mh_e2[2];
-        {
-            const ACC_TYPE other = sub_group_shuffle_xor(mh_r1, 2);
-            mh_e2[0] = mh_b1 ? other  : mh_r1;
-            mh_e2[1] = mh_b1 ? mh_r1  : other;
-        }
-        ACC_TYPE mh_s[MQ_GQA];
-        #pragma unroll
-        for (int j = 0; j < 2; ++j) {
-            const ACC_TYPE other = sub_group_shuffle_xor(mh_e2[j], 1);
-            mh_s[j]     = mh_b0 ? other     : mh_e2[j];
-            mh_s[j + 2] = mh_b0 ? mh_e2[j]  : other;
-        }
-
-        #pragma unroll
-        for (int h = 0; h < MQ_GQA; ++h) {
-            ACC_TYPE s = mh_s[h] * scale;
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
-            if (mask_bcast) {
-                s += slope[h] * mask_val;
-            } else if (mask_base_b != NULL) {
-                const int mask_head_idx = FA_HEAD_IDX(h) % mask_ne2;
-                const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) (mask_base_b + mask_head_idx * mask_nb2);
-                s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];
-            }
-            const ACC_TYPE sc    = valid ? s : FA_M_INIT;
-            const ACC_TYPE m_new = max(m_i[h], sc);
-            const ACC_TYPE sp    = native_exp(m_i[h] - m_new);
-            const ACC_TYPE p     = native_exp(sc - m_new);
-            l_i[h] = l_i[h] * sp + p;
-            m_i[h] = m_new;
-            o_acc[h][0] = mad(p, v_vec_1, o_acc[h][0] * sp);
-        }
-#else
         // Dot: this lane covers DK elements {lic + FA_CL_C*i} of the cluster's row.
         ACC_TYPE4 dot4[MQ_GQA];
         #pragma unroll
@@ -2170,13 +2055,13 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
                 s += sub_group_shuffle_xor(s, step);
             }
             s *= scale;
-            if (logit_softcap > 0.0f) {
-                s = logit_softcap * tanh(s / logit_softcap);
-            }
             if (mask_base[h] != NULL) {
                 const global MASK_DATA_TYPE * mask_ptr = (const global MASK_DATA_TYPE *) mask_base[h];
                 s += slope[h] * (ACC_TYPE) mask_ptr[k_safe];
             }
+            if (logit_softcap > 0.0f) {
+                s = logit_softcap * tanh(s / logit_softcap);
+            }
             score[h] = valid ? s : FA_M_INIT;
         }
 
@@ -2202,7 +2087,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
                 o_acc[h][i] = mad(p_h[h], v_vec, o_acc[h][i] * sp_h[h]);
             }
         }
-#endif
     }
 
     // Merge stage 1: fold the FA_CL_NCL cluster partials inside the subgroup.
@@ -2264,7 +2148,7 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
         barrier(CLK_LOCAL_MEM_FENCE);
 
         if (sgid == 0) {
-            const int head_idx = FA_HEAD_IDX(h);
+            const int head_idx = head_kv_idx * MQ_GQA + h;
 
             ACC_TYPE m_c = sg_m[h][0];
             #pragma unroll
@@ -2300,8 +2184,6 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_c8(
     }
 }
 
-#undef FA_HEAD_IDX
-
 #endif  // DK_VEC/DV_VEC divisible by FA_CL_C
 #endif  // HAS_SUBGROUP_SHUFFLE (q1_vec_mq_split_c8)
 
@@ -2537,11 +2419,9 @@ __kernel void flash_attn_f32_f16_q1_vec_mq_split_k_img(
         barrier(CLK_LOCAL_MEM_FENCE);
     }
 }
-#endif  // !FA_MQ_SPLIT_ONLY
 #endif  // !FA_DECODE_ONLY
 
 #ifndef FA_MQ_ONLY  // q1_split + merge excluded from the MQ-only (g8) program
-#ifndef FA_VEC_ONLY
 __kernel void flash_attn_f32_f16_q1_split(
     const global void * q_void, ulong q_offset,
     const global void * k_void, ulong k_offset,
@@ -2698,8 +2578,6 @@ __kernel void flash_attn_f32_f16_q1_split(
     }
 }
 
-#endif  // !FA_VEC_ONLY
-
 // FD Pass 2: merge per-split partials into final O
 // empty splits drop via exp(-INF)=0.
 __kernel void flash_attn_f32_merge(

ggml/src/ggml-rpc/ggml-rpc.cpp

@@ -3,7 +3,6 @@
 #include "ggml-backend-impl.h"
 #include "ggml-cpp.h"
 #include "transport.h"
-#include "log.h"
 
 #include <array>
 #include <chrono>
@@ -25,6 +24,12 @@
 #include <algorithm>
 #include <atomic>
 
+static const char * RPC_DEBUG = std::getenv("GGML_RPC_DEBUG");
+
+#define LOG_DBG(...) \
+    do { if (RPC_DEBUG) GGML_LOG_DEBUG(__VA_ARGS__); } while (0)
+
+
 namespace fs = std::filesystem;
 
 // macro for nicer error messages on server crash
@@ -84,31 +89,6 @@ enum rpc_cmd {
 
 static_assert(RPC_CMD_HELLO == 14, "RPC_CMD_HELLO must be always 14");
 
-static const char * rpc_cmd_name(enum rpc_cmd cmd) {
-    switch (cmd) {
-        case RPC_CMD_ALLOC_BUFFER:      return "ALLOC_BUFFER";
-        case RPC_CMD_GET_ALIGNMENT:     return "GET_ALIGNMENT";
-        case RPC_CMD_GET_MAX_SIZE:      return "GET_MAX_SIZE";
-        case RPC_CMD_BUFFER_GET_BASE:   return "BUFFER_GET_BASE";
-        case RPC_CMD_FREE_BUFFER:       return "FREE_BUFFER";
-        case RPC_CMD_BUFFER_CLEAR:      return "BUFFER_CLEAR";
-        case RPC_CMD_SET_TENSOR:        return "SET_TENSOR";
-        case RPC_CMD_SET_TENSOR_HASH:   return "SET_TENSOR_HASH";
-        case RPC_CMD_GET_TENSOR:        return "GET_TENSOR";
-        case RPC_CMD_COPY_TENSOR:       return "COPY_TENSOR";
-        case RPC_CMD_GRAPH_COMPUTE:     return "GRAPH_COMPUTE";
-        case RPC_CMD_GET_DEVICE_MEMORY: return "GET_DEVICE_MEMORY";
-        case RPC_CMD_INIT_TENSOR:       return "INIT_TENSOR";
-        case RPC_CMD_GET_ALLOC_SIZE:    return "GET_ALLOC_SIZE";
-        case RPC_CMD_HELLO:             return "HELLO";
-        case RPC_CMD_DEVICE_COUNT:      return "DEVICE_COUNT";
-        case RPC_CMD_GRAPH_RECOMPUTE:   return "GRAPH_RECOMPUTE";
-        case RPC_CMD_MEMSET_TENSOR:     return "MEMSET_TENSOR";
-        case RPC_CMD_NONE:              return "NONE";
-        default:                        return "UNKNOWN";
-    }
-}
-
 // Try RPC_CMD_SET_TENSOR_HASH first when data size is larger than this threshold
 const size_t HASH_THRESHOLD = 10 * 1024 * 1024;
 
@@ -344,7 +324,7 @@ static bool recv_msg(socket_ptr sock, std::vector<uint8_t> & input) {
     try {
         input.resize(size);
     } catch (const std::bad_alloc & e) {
-        LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size);
+        GGML_LOG_ERROR("Failed to allocate input buffer of size %" PRIu64 "\n", size);
         return false;
     }
     return sock->recv_data(input.data(), size);
@@ -425,14 +405,12 @@ static bool negotiate_hello(const std::shared_ptr<socket_t> & sock) {
     RPC_STATUS_ASSERT(status);
 
     if (response.major != RPC_PROTO_MAJOR_VERSION || response.minor > RPC_PROTO_MINOR_VERSION) {
-        LOG_ERROR("RPC server version mismatch: %d.%d.%d\n",
+        GGML_LOG_ERROR("RPC server version mismatch: %d.%d.%d\n",
                        response.major, response.minor, response.patch);
         return false;
     }
 
     sock->update_caps(response.conn_caps);
-    LOG_DBG("[%s] handshake ok: server version %d.%d.%d, transport: %s\n",
-            __func__, response.major, response.minor, response.patch, sock->transport_name());
     return true;
 }
 
@@ -668,9 +646,6 @@ void rpc_dispatcher::work() {
             break;
         }
         if (msg_ptr->cmd != RPC_CMD_NONE) {
-            LOG_DBG2("[%s] %s (in: %zu, out: %zu)\n", __func__, rpc_cmd_name(msg_ptr->cmd), msg_ptr->input_size, msg_ptr->output_size);
-            const bool trace = rpc_debug_level() >= 3;
-            const auto t0 = std::chrono::steady_clock::now();
             if (msg_ptr->output) {
                 bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->output, msg_ptr->output_size);
                 RPC_STATUS_ASSERT(status);
@@ -678,12 +653,6 @@ void rpc_dispatcher::work() {
                 bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size);
                 RPC_STATUS_ASSERT(status);
             }
-            if (trace) {
-                const std::chrono::duration<double, std::milli> elapsed = std::chrono::steady_clock::now() - t0;
-                LOG_DBG3("[%s] %s done in %.3f ms\n", __func__, rpc_cmd_name(msg_ptr->cmd), elapsed.count());
-            }
-        } else {
-            LOG_DBG3("[%s] sync barrier\n", __func__);
         }
         msg_ptr->completion.set_value();
         last_cmd = std::chrono::steady_clock::now();
@@ -721,7 +690,6 @@ static void ggml_backend_rpc_buffer_free_buffer(ggml_backend_buffer_t buffer) {
     ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
     auto request = std::make_shared<rpc_msg_free_buffer_req>();
     request->remote_ptr = ctx->remote_ptr;
-    LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request->remote_ptr);
     ctx->dispatcher->send(RPC_CMD_FREE_BUFFER, request, sizeof(*request));
     delete ctx;
 }
@@ -733,7 +701,6 @@ static void * ggml_backend_rpc_buffer_get_base(ggml_backend_buffer_t buffer) {
     }
     auto request = std::make_shared<rpc_msg_buffer_get_base_req>();
     request->remote_ptr = ctx->remote_ptr;
-    LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, ctx->remote_ptr);
     rpc_msg_buffer_get_base_rsp response;
     ctx->dispatcher->send(RPC_CMD_BUFFER_GET_BASE, request, sizeof(*request), &response, sizeof(response));
     ctx->base_ptr = reinterpret_cast<void *>(response.base_ptr);
@@ -798,7 +765,6 @@ static enum ggml_status ggml_backend_rpc_buffer_init_tensor(ggml_backend_buffer_
     // Due to bandwidth constraints, we only call the server init tensor functions if necessary.
     // In particular, only quantized tensors need padding
     if (ggml_is_quantized(tensor->type) && (tensor->ne[0] % 512 != 0) && (tensor->view_src == nullptr)) {
-        LOG_DBG2("[%s] tensor: %s\n", __func__, tensor->name);
         auto request = std::make_shared<rpc_msg_init_tensor_req>();
         request->tensor = serialize_tensor(tensor);
         ctx->dispatcher->send(RPC_CMD_INIT_TENSOR, request, sizeof(*request));
@@ -814,7 +780,6 @@ static void ggml_backend_rpc_buffer_memset_tensor(
     request->offset = offset;
     request->size   = size;
     request->value  = value;
-    LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %zu, value: %u\n", __func__, tensor->name, offset, size, value);
     ctx->dispatcher->send(RPC_CMD_MEMSET_TENSOR, request, sizeof(*request));
 }
 
@@ -842,7 +807,6 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm
     ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
     rpc_tensor rpc_tensor = serialize_tensor(tensor);
     uint8_t cache_flag = 0;
-    LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %zu\n", __func__, tensor->name, offset, size);
     if (rpc_use_hash_cache(tensor, size)) {
         auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
         request->tensor = rpc_tensor;
@@ -850,7 +814,6 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm
         request->hash = fnv_hash((const uint8_t*)data, size);
         rpc_msg_set_tensor_hash_rsp response;
         ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response));
-        LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, tensor->name, request->hash, response.result ? "hit" : "miss");
         if (response.result) {
             // the server has the same data, no need to send it
             return;
@@ -905,7 +868,6 @@ static void ggml_backend_rpc_buffer_get_tensor_2d(ggml_backend_buffer_t buffer,
 
 static void ggml_backend_rpc_buffer_get_tensor(ggml_backend_buffer_t buffer, const ggml_tensor * tensor, void * data, size_t offset, size_t size) {
     ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
-    LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %zu\n", __func__, tensor->name, offset, size);
     auto request = std::make_shared<rpc_msg_get_tensor_req>();
     request->tensor = serialize_tensor(tensor);
     request->offset = offset;
@@ -921,11 +883,9 @@ static bool ggml_backend_rpc_buffer_cpy_tensor(ggml_backend_buffer_t buffer, con
         ggml_backend_buffer_t dst_buffer = dst->buffer;
         ggml_backend_rpc_buffer_context * dst_ctx = (ggml_backend_rpc_buffer_context *)dst_buffer->context;
         if (src_ctx->dispatcher != dst_ctx->dispatcher) {
-            LOG_DBG2("[%s] src and dst are on different servers, falling back to a host copy\n", __func__);
             return false;
         }
         ggml_backend_rpc_buffer_context * ctx = (ggml_backend_rpc_buffer_context *)buffer->context;
-        LOG_DBG2("[%s] src: %s -> dst: %s\n", __func__, src->name, dst->name);
         auto request = std::make_shared<rpc_msg_copy_tensor_req>();
         request->src = serialize_tensor(src);
         request->dst = serialize_tensor(dst);
@@ -941,7 +901,6 @@ static void ggml_backend_rpc_buffer_clear(ggml_backend_buffer_t buffer, uint8_t
     auto request = std::make_shared<rpc_msg_buffer_clear_req>();
     request->remote_ptr = ctx->remote_ptr;
     request->value = value;
-    LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 ", value: %u\n", __func__, request->remote_ptr, request->value);
     ctx->dispatcher->send(RPC_CMD_BUFFER_CLEAR, request, sizeof(*request));
 }
 
@@ -974,16 +933,12 @@ static ggml_backend_buffer_t ggml_backend_rpc_buffer_type_alloc_buffer(ggml_back
     auto dispatcher = get_dispatcher(buft_ctx->endpoint);
     dispatcher->send(RPC_CMD_ALLOC_BUFFER, request, sizeof(*request), &response, sizeof(response));
     if (response.remote_ptr != 0) {
-        LOG_DBG("[%s] endpoint: %s, device: %u, size: %" PRIu64 " -> remote_ptr: 0x%" PRIx64 ", remote_size: %" PRIu64 "\n",
-                __func__, buft_ctx->endpoint.c_str(), buft_ctx->device, request->size, response.remote_ptr, response.remote_size);
         ggml_backend_buffer_t buffer = ggml_backend_buffer_init(buft,
             ggml_backend_rpc_buffer_interface,
             new ggml_backend_rpc_buffer_context{dispatcher, nullptr, response.remote_ptr},
             response.remote_size);
         return buffer;
     } else {
-        LOG_WARN("[%s] endpoint: %s, device: %u, size: %" PRIu64 " -> failed\n",
-                __func__, buft_ctx->endpoint.c_str(), buft_ctx->device, request->size);
         return nullptr;
     }
 }
@@ -993,7 +948,6 @@ static size_t get_alignment(const std::shared_ptr<rpc_dispatcher> & dispatcher,
     request->device = device;
     rpc_msg_get_alignment_rsp response;
     dispatcher->send(RPC_CMD_GET_ALIGNMENT, request, sizeof(*request), &response, sizeof(response));
-    LOG_DBG2("[%s] device: %u, alignment: %" PRIu64 "\n", __func__, device, response.alignment);
     return response.alignment;
 }
 
@@ -1007,7 +961,6 @@ static size_t get_max_size(const std::shared_ptr<rpc_dispatcher> & dispatcher, u
     request->device = device;
     rpc_msg_get_max_size_rsp response;
     dispatcher->send(RPC_CMD_GET_MAX_SIZE, request, sizeof(*request), &response, sizeof(response));
-    LOG_DBG2("[%s] device: %u, max_size: %" PRIu64 "\n", __func__, device, response.max_size);
     return response.max_size;
 }
 
@@ -1073,7 +1026,6 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty
             std::lock_guard<std::mutex> lock(cache_mutex);
             auto it = cache.find(cache_hash);
             if (it != cache.end()) {
-                LOG_DBG3("[%s] cache hit: %s [%s] -> %zu\n", __func__, ggml_op_name(tensor->op), tensor->name, it->second);
                 return std::max<size_t>(it->second, min_size);
             }
         }
@@ -1087,7 +1039,6 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty
             request->srcs[i] = serialize_tensor(tensor->src[i]);
         }
 
-        LOG_DBG2("[%s] cache miss: %s [%s], querying server\n", __func__, ggml_op_name(tensor->op), tensor->name);
         rpc_msg_get_alloc_size_rsp response;
         auto dispatcher = get_dispatcher(buft_ctx->endpoint);
         dispatcher->send(RPC_CMD_GET_ALLOC_SIZE, request, sizeof(*request), &response, sizeof(response));
@@ -1097,8 +1048,6 @@ static size_t ggml_backend_rpc_buffer_type_get_alloc_size(ggml_backend_buffer_ty
             cache[cache_hash] = response.alloc_size;
         }
 
-        LOG_DBG2("[%s] %s [%s] -> alloc_size: %" PRIu64 "\n", __func__, ggml_op_name(tensor->op), tensor->name, response.alloc_size);
-
         return std::max<size_t>(response.alloc_size, min_size);
     }
 
@@ -1132,7 +1081,6 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso
     ggml_backend_rpc_context * ctx = (ggml_backend_rpc_context *)backend->context;
     rpc_tensor rpc_tensor = serialize_tensor(tensor);
     uint8_t cache_flag = 0;
-    LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %zu\n", __func__, tensor->name, offset, size);
     if (rpc_use_hash_cache(tensor, size)) {
         auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
         request->tensor = rpc_tensor;
@@ -1141,7 +1089,6 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso
         rpc_msg_set_tensor_hash_rsp response;
         // TODO: make this async
         ctx->dispatcher->send(RPC_CMD_SET_TENSOR_HASH, request, sizeof(*request), &response, sizeof(response));
-        LOG_DBG2("[%s] tensor: %s, hash: 0x%" PRIx64 ", cache: %s\n", __func__, tensor->name, request->hash, response.result ? "hit" : "miss");
         if (response.result) {
             // the server has the same data, no need to send it
             return;
@@ -1223,7 +1170,6 @@ void rpc_dispatcher::graph_compute(uint32_t device, const ggml_cgraph * cgraph)
     GGML_ASSERT(cgraph->n_nodes > 0);
     auto & device_graph_uids = graph_uids[device];
     bool reuse = cgraph->uid != 0 && device_graph_uids.count(cgraph->uid) > 0;
-    LOG_DBG("[%s] device: %u, uid: %" PRIu64 ", n_nodes: %u, reuse: %s\n", __func__, device, cgraph->uid, cgraph->n_nodes, reuse ? "yes" : "no");
     if (reuse) {
         auto request = std::make_shared<rpc_msg_graph_recompute_req>();
         request->device = device;
@@ -1232,15 +1178,12 @@ void rpc_dispatcher::graph_compute(uint32_t device, const ggml_cgraph * cgraph)
     } else {
         if (cgraph->uid != 0) {
             if (device_graph_uids.size() >= GRAPH_CACHE_MAX) {
-                LOG_DBG("[%s] clearing graph cache for device %u (%zu entries)\n",
-                        __func__, device, device_graph_uids.size());
                 device_graph_uids.clear();
             }
             device_graph_uids.insert(cgraph->uid);
         }
         size_t input_size = 0;
         uint8_t * input = serialize_graph(device, cgraph, this, &input_size);
-        LOG_DBG2("[%s] serialized graph: %zu bytes\n", __func__, input_size);
         std::shared_ptr<uint8_t> input_ptr(input, std::default_delete<uint8_t[]>());
         send_async(RPC_CMD_GRAPH_COMPUTE, input_ptr, input_size);
     }
@@ -1321,7 +1264,6 @@ ggml_backend_buffer_type_t ggml_backend_rpc_buffer_type(const char * endpoint, u
         /* .context = */ buft_ctx
     };
     buft_map[buft_name] = buft;
-    LOG_DBG("[%s] created %s (alignment: %zu, max_size: %zu)\n", __func__, buft_name.c_str(), alignment, max_size);
     return buft;
 }
 
@@ -1335,7 +1277,6 @@ ggml_backend_t ggml_backend_rpc_init(const char * endpoint, uint32_t device) {
         /* .name       = */ dev_name,
     };
     auto reg = ggml_backend_rpc_add_server(endpoint);
-    LOG_DBG("[%s] %s\n", __func__, dev_name.c_str());
     ggml_backend_t backend = new ggml_backend {
         /* .guid    = */ ggml_backend_rpc_guid(),
         /* .iface   = */ ggml_backend_rpc_interface,
@@ -1355,7 +1296,6 @@ void ggml_backend_rpc_get_device_memory(const char * endpoint, uint32_t device,
     request->device = device;
     rpc_msg_get_device_memory_rsp response;
     dispatcher->send(RPC_CMD_GET_DEVICE_MEMORY, request, sizeof(*request), &response, sizeof(response));
-    LOG_DBG2("[%s] device: %u, free: %" PRIu64 ", total: %" PRIu64 "\n", __func__, device, response.free_mem, response.total_mem);
     *free = response.free_mem;
     *total = response.total_mem;
 }
@@ -1453,7 +1393,7 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_
 
     ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
     if (tensor == nullptr) {
-        LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n");
+        GGML_LOG_ERROR("Null tensor pointer passed to server get_alloc_size function.\n");
         return false;
     }
     for (int i = 0; i < GGML_MAX_SRC; i++) {
@@ -1462,7 +1402,7 @@ bool rpc_server::get_alloc_size(const rpc_msg_get_alloc_size_req & request, rpc_
         }
     }
 
-    LOG_DBG2("[%s] device: %u, buffer: %p, data: %p\n", __func__, dev_id, (void*)tensor->buffer, tensor->data);
+    LOG_DBG("[%s] device: %d, buffer: %p, data: %p\n", __func__, dev_id, (void*)tensor->buffer, tensor->data);
     if (tensor->buffer == nullptr) {
         //No buffer allocated.
         buft = ggml_backend_get_default_buffer_type(backends[dev_id]);
@@ -1487,11 +1427,11 @@ bool rpc_server::alloc_buffer(const rpc_msg_alloc_buffer_req & request, rpc_msg_
     if (buffer != nullptr) {
         response.remote_ptr = reinterpret_cast<uint64_t>(buffer);
         response.remote_size = buffer->size;
-        LOG_DBG("[%s] device: %u, size: %" PRIu64 " -> remote_ptr: 0x%" PRIx64 ", remote_size: %" PRIu64 "\n",
-                __func__, dev_id, request.size, response.remote_ptr, response.remote_size);
+        LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> remote_ptr: %" PRIx64 ", remote_size: %" PRIu64 "\n",
+            __func__, dev_id, request.size, response.remote_ptr, response.remote_size);
         buffers.insert(buffer);
     } else {
-        LOG_WARN("[%s] device: %u, size: %" PRIu64 " -> failed\n", __func__, dev_id, request.size);
+        LOG_DBG("[%s] device: %d, size: %" PRIu64 " -> failed\n", __func__, dev_id, request.size);
     }
     return true;
 }
@@ -1503,7 +1443,7 @@ bool rpc_server::get_alignment(const rpc_msg_get_alignment_req & request, rpc_ms
     }
     ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);
     size_t alignment = ggml_backend_buft_get_alignment(buft);
-    LOG_DBG2("[%s] device: %u, alignment: %zu\n", __func__, dev_id, alignment);
+    LOG_DBG("[%s] device: %d, alignment: %lu\n", __func__, dev_id, alignment);
     response.alignment = alignment;
     return true;
 }
@@ -1515,16 +1455,16 @@ bool rpc_server::get_max_size(const rpc_msg_get_max_size_req & request, rpc_msg_
     }
     ggml_backend_buffer_type_t buft = ggml_backend_get_default_buffer_type(backends[dev_id]);
     size_t max_size = ggml_backend_buft_get_max_size(buft);
-    LOG_DBG2("[%s] device: %u, max_size: %zu\n", __func__, dev_id, max_size);
+    LOG_DBG("[%s] device: %d, max_size: %lu\n", __func__, dev_id, max_size);
     response.max_size = max_size;
     return true;
 }
 
 bool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rpc_msg_buffer_get_base_rsp & response) {
-    LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request.remote_ptr);
+    LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr);
     ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr);
     if (buffers.find(buffer) == buffers.end()) {
-        LOG_ERROR("[%s] buffer not found\n", __func__);
+        GGML_LOG_ERROR("[%s] buffer not found\n", __func__);
         return false;
     }
     void * base = ggml_backend_buffer_get_base(buffer);
@@ -1534,15 +1474,14 @@ bool rpc_server::buffer_get_base(const rpc_msg_buffer_get_base_req & request, rp
 
 bool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) {
     sync_all_backends();
-    LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 "\n", __func__, request.remote_ptr);
+    LOG_DBG("[%s] remote_ptr: %" PRIx64 "\n", __func__, request.remote_ptr);
     ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr);
     if (buffers.find(buffer) == buffers.end()) {
-        LOG_ERROR("[%s] buffer not found\n", __func__);
+        GGML_LOG_ERROR("[%s] buffer not found\n", __func__);
         return false;
     }
     // Discard all cached graphs to avoid use-after-free in graph_recompute,
-    // since their nodes may hold pointers to the buffer being freed. The client
-    // will have to re-serialize its graphs from now on.
+    // since their nodes may hold pointers to the buffer being freed.
     for (auto & sgs : stored_graphs) {
         for (auto & sg : sgs) {
             sg.second.graph = nullptr;
@@ -1555,10 +1494,10 @@ bool rpc_server::free_buffer(const rpc_msg_free_buffer_req & request) {
 
 bool rpc_server::buffer_clear(const rpc_msg_buffer_clear_req & request) {
     sync_all_backends();
-    LOG_DBG2("[%s] remote_ptr: 0x%" PRIx64 ", value: %u\n", __func__, request.remote_ptr, request.value);
+    LOG_DBG("[%s] remote_ptr: %" PRIx64 ", value: %u\n", __func__, request.remote_ptr, request.value);
     ggml_backend_buffer_t buffer = reinterpret_cast<ggml_backend_buffer_t>(request.remote_ptr);
     if (buffers.find(buffer) == buffers.end()) {
-        LOG_ERROR("[%s] buffer not found\n", __func__);
+        GGML_LOG_ERROR("[%s] buffer not found\n", __func__);
         return false;
     }
     ggml_backend_buffer_clear(buffer, request.value);
@@ -1577,13 +1516,13 @@ bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) {
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
     if (tensor == nullptr || tensor->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensor\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
         return false;
     }
 
     const uint64_t tensor_size = ggml_nbytes(tensor);
     if (request.offset > tensor_size || request.size > tensor_size - request.offset) {
-        LOG_ERROR("[%s] tensor region (offset=%" PRIu64 ", size=%" PRIu64 ") out of tensor bounds [0, %" PRIu64 ")\n",
+        GGML_LOG_ERROR("[%s] tensor region (offset=%" PRIu64 ", size=%" PRIu64 ") out of tensor bounds [0, %" PRIu64 ")\n",
                        __func__, request.offset, request.size, tensor_size);
         return false;
     }
@@ -1591,22 +1530,22 @@ bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) {
     const uint64_t buffer_start = (uint64_t) ggml_backend_buffer_get_base(tensor->buffer);
     const uint64_t buffer_size = ggml_backend_buffer_get_size(tensor->buffer);
     if (request.tensor.data < buffer_start) {
-        LOG_ERROR("[%s] tensor data before buffer start\n", __func__);
+        GGML_LOG_ERROR("[%s] tensor data before buffer start\n", __func__);
         return false;
     }
     const uint64_t data_offset = request.tensor.data - buffer_start;
     if (data_offset > buffer_size ||
         request.offset > buffer_size - data_offset ||
         request.size > buffer_size - data_offset - request.offset) {
-        LOG_ERROR("[%s] tensor region out of buffer bounds\n", __func__);
+        GGML_LOG_ERROR("[%s] tensor region out of buffer bounds\n", __func__);
         return false;
     }
     if (tensor->buffer->iface.memset_tensor == nullptr) {
-        LOG_ERROR("[%s] memset not implemented by backend buffer\n", __func__);
+        GGML_LOG_ERROR("[%s] memset not implemented by backend buffer\n", __func__);
         return false;
     }
 
-    LOG_DBG2("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", value: %u\n",
+    LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", value: %u\n",
             __func__, (void *) tensor->buffer, tensor->data, request.offset, request.size, request.value);
     ggml_backend_tensor_memset(tensor, request.value, request.offset, request.size);
     return true;
@@ -1615,13 +1554,13 @@ bool rpc_server::memset_tensor(const rpc_msg_memset_tensor_req & request) {
 ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rpc_tensor * tensor) {
     // Validate tensor type before using it
     if (tensor->type >= GGML_TYPE_COUNT) {
-        LOG_ERROR("[%s] invalid tensor type received: %u\n", __func__, tensor->type);
+        GGML_LOG_ERROR("[%s] invalid tensor type received: %u\n", __func__, tensor->type);
         return nullptr;
     }
 
     // Fix: Prevent division by zero if blck_size is 0 (e.g., deprecated types)
     if (ggml_blck_size((enum ggml_type)tensor->type) == 0) {
-        LOG_ERROR("[%s] invalid tensor type received (blck_size is 0): %u\n", __func__, tensor->type);
+        GGML_LOG_ERROR("[%s] invalid tensor type received (blck_size is 0): %u\n", __func__, tensor->type);
         return nullptr;
     }
 
@@ -1630,7 +1569,7 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp
 
     // ggml_new_tensor_4d might fail if dimensions are invalid, although less likely to crash than invalid type
     if (result == nullptr) {
-        LOG_ERROR("[%s] ggml_new_tensor_4d failed for type %u\n", __func__, tensor->type);
+        GGML_LOG_ERROR("[%s] ggml_new_tensor_4d failed for type %u\n", __func__, tensor->type);
         return nullptr;
     }
 
@@ -1649,11 +1588,11 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp
         uint64_t buffer_size = (uint64_t) ggml_backend_buffer_get_size(result->buffer);
         if (tensor->data + tensor_size < tensor->data ||
             tensor->data < buffer_start || tensor->data + tensor_size > buffer_start + buffer_size) {
-            LOG_ERROR("[%s] tensor '%s' (op %s, type %s, ne [%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "]) "
-                      "data [0x%" PRIx64 ", 0x%" PRIx64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n",
-                      __func__, tensor->name, ggml_op_name((ggml_op) tensor->op), ggml_type_name(result->type),
-                      result->ne[0], result->ne[1], result->ne[2], result->ne[3],
-                      tensor->data, tensor->data + tensor_size, buffer_start, buffer_start + buffer_size);
+            GGML_LOG_ERROR("[%s] tensor '%s' (op %s, type %s, ne [%" PRId64 ", %" PRId64 ", %" PRId64 ", %" PRId64 "]) "
+                           "data [0x%" PRIx64 ", 0x%" PRIx64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n",
+                           __func__, tensor->name, ggml_op_name((ggml_op) tensor->op), ggml_type_name(result->type),
+                           result->ne[0], result->ne[1], result->ne[2], result->ne[3],
+                           tensor->data, tensor->data + tensor_size, buffer_start, buffer_start + buffer_size);
             return nullptr;
         }
     }
@@ -1693,10 +1632,10 @@ bool rpc_server::set_tensor(const std::vector<uint8_t> & input) {
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor);
     if (tensor == nullptr || tensor->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensor\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
         return false;
     }
-    LOG_DBG2("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu\n", __func__, (void*)tensor->buffer, tensor->data, offset, size);
+    LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu\n", __func__, (void*)tensor->buffer, tensor->data, offset, size);
 
     // sanitize tensor->data
     {
@@ -1704,7 +1643,7 @@ bool rpc_server::set_tensor(const std::vector<uint8_t> & input) {
         const size_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);
 
         if (in_tensor->data + offset < p0 || in_tensor->data + offset >= p1 || size > (p1 - in_tensor->data - offset)) {
-            LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\n",
+            GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu) out of buffer bounds [0x%zx, 0x%zx)\n",
                            __func__, in_tensor->data, offset, size, p0, p1);
             return false;
         }
@@ -1719,7 +1658,7 @@ bool rpc_server::set_tensor(const std::vector<uint8_t> & input) {
         fs::path cache_file = fs::path(cache_dir) / hash_str;
         std::ofstream ofs(cache_file, std::ios::binary);
         ofs.write((const char *)data, size);
-        LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str());
+        GGML_LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str());
     }
     ggml_backend_tensor_set(tensor, data, offset, size);
     return true;
@@ -1754,10 +1693,10 @@ bool rpc_server::set_tensor_2d(const std::vector<uint8_t> & input) {
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * tensor = deserialize_tensor(ctx, in_tensor);
     if (tensor == nullptr || tensor->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensor\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
         return false;
     }
-    LOG_DBG2("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", n_copies: %" PRIu64 ", stride: %" PRIu64 "\n",
+    LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", n_copies: %" PRIu64 ", stride: %" PRIu64 "\n",
             __func__, (void*)tensor->buffer, tensor->data, offset, size, n_copies, stride);
 
     // sanitize tensor->data
@@ -1770,13 +1709,13 @@ bool rpc_server::set_tensor_2d(const std::vector<uint8_t> & input) {
         const uint64_t p1 = p0 + ggml_backend_buffer_get_size(tensor->buffer);
 
         if (in_tensor->data < p0 || in_tensor->data > p1 || offset > p1 - in_tensor->data || span > p1 - in_tensor->data - offset) {
-            LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n",
-                      __func__, in_tensor->data, offset, span, p0, p1);
+            GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n",
+                           __func__, in_tensor->data, offset, span, p0, p1);
             return false;
         }
         if (offset > ggml_nbytes(tensor) || span > ggml_nbytes(tensor) - offset) {
-            LOG_ERROR("[%s] tensor write region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n",
-                      __func__, offset, span, ggml_nbytes(tensor));
+            GGML_LOG_ERROR("[%s] tensor write region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n",
+                           __func__, offset, span, ggml_nbytes(tensor));
             return false;
         }
     }
@@ -1811,12 +1750,10 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp
     sync_all_backends();
     std::vector<uint8_t> cached_file;
     if (!get_cached_file(request.hash, cached_file)) {
-        LOG_DBG2("[%s] hash: 0x%" PRIx64 ", cache miss\n", __func__, request.hash);
         response.result = 0;
         return true;
     }
     size_t size = cached_file.size();
-    LOG_DBG2("[%s] hash: 0x%" PRIx64 ", cache hit (%zu bytes)\n", __func__, request.hash, size);
     struct ggml_init_params params {
         /*.mem_size   =*/ ggml_tensor_overhead(),
         /*.mem_buffer =*/ NULL,
@@ -1827,10 +1764,10 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
     if (tensor == nullptr || tensor->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensor\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
         return false;
     }
-    LOG_DBG2("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu, hash: %" PRIx64 "\n",
+    LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %zu, hash: %" PRIx64 "\n",
             __func__, (void*)tensor->buffer, tensor->data, request.offset, size, request.hash);
 
     // sanitize tensor->data
@@ -1841,7 +1778,7 @@ bool rpc_server::set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rp
         if (request.tensor.data + request.offset < p0
          || request.tensor.data + request.offset >= p1
          || size > (p1 - request.tensor.data - request.offset)) {
-            LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu, hash=0x%" PRIx64 ") out of buffer bounds [0x%zx, 0x%zx)\n",
+            GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%zu, hash=0x%" PRIx64 ") out of buffer bounds [0x%zx, 0x%zx)\n",
                            __func__, request.tensor.data, request.offset, size, request.hash, p0, p1);
             return false;
         }
@@ -1863,24 +1800,24 @@ bool rpc_server::init_tensor(const rpc_msg_init_tensor_req & request) {
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
     if (tensor == nullptr) {
-        LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n");
+        GGML_LOG_ERROR("Null tensor pointer passed to server init_tensor function.\n");
         return false;
     }
-    LOG_DBG2("[%s] buffer: %p, data: %p\n", __func__, (void*)tensor->buffer, tensor->data);
+    LOG_DBG("[%s] buffer: %p, data: %p\n", __func__, (void*)tensor->buffer, tensor->data);
     // Call the backend's buffer_init_tensor function
     ggml_backend_buffer_t buffer = tensor->buffer;
     if (buffer && buffer->iface.init_tensor) {
         buffer->iface.init_tensor(buffer, tensor);
     } else {
         if (!buffer) {
-            LOG_ERROR("Tensor with null buffer passed to init_tensor function\n");
+            GGML_LOG_ERROR("Tensor with null buffer passed to init_tensor function\n");
         }
     }
 
     if (tensor->extra != nullptr) {
         // This pointer can either be passed around client/server, or probably better stored server-side and kept track of.
         // Currently unimplemented.
-        LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n");
+        GGML_LOG_ERROR("tensor->extra populated by the backend, this is currently unsupported.\n");
         return false;
     }
 
@@ -1899,10 +1836,10 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector<
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
     if (tensor == nullptr || tensor->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensor\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
         return false;
     }
-    LOG_DBG2("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 "\n", __func__, (void*)tensor->buffer, tensor->data, request.offset, request.size);
+    LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 "\n", __func__, (void*)tensor->buffer, tensor->data, request.offset, request.size);
 
     // sanitize tensor->data
     {
@@ -1912,7 +1849,7 @@ bool rpc_server::get_tensor(const rpc_msg_get_tensor_req & request, std::vector<
         if (request.tensor.data + request.offset < p0 ||
             request.tensor.data + request.offset >= p1 ||
             request.size > (p1 - request.tensor.data - request.offset)) {
-                LOG_ERROR("[%s] requested tensor region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%" PRIu64 ") out of buffer bounds [0x%zx, 0x%zx)\n",
+                GGML_LOG_ERROR("[%s] requested tensor region (data=0x%" PRIx64 ", offset=%" PRIu64 ", size=%" PRIu64 ") out of buffer bounds [0x%zx, 0x%zx)\n",
                                __func__, request.tensor.data, request.offset, request.size, p0, p1);
                 return false;
         }
@@ -1935,10 +1872,10 @@ bool rpc_server::get_tensor_2d(const rpc_msg_get_tensor_2d_req & request, std::v
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * tensor = deserialize_tensor(ctx, &request.tensor);
     if (tensor == nullptr || tensor->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensor\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
         return false;
     }
-    LOG_DBG2("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", n_copies: %" PRIu64 ", stride: %" PRIu64 "\n",
+    LOG_DBG("[%s] buffer: %p, data: %p, offset: %" PRIu64 ", size: %" PRIu64 ", n_copies: %" PRIu64 ", stride: %" PRIu64 "\n",
             __func__, (void*)tensor->buffer, tensor->data, request.offset, request.size, request.n_copies, request.stride);
 
     // sanitize tensor->data
@@ -1955,13 +1892,13 @@ bool rpc_server::get_tensor_2d(const rpc_msg_get_tensor_2d_req & request, std::v
 
         if (request.tensor.data < p0 || request.tensor.data > p1 || request.offset > p1 - request.tensor.data ||
                 span > p1 - request.tensor.data - request.offset) {
-            LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n",
-                      __func__, request.tensor.data, request.offset, span, p0, p1);
+            GGML_LOG_ERROR("[%s] tensor data region (data=0x%" PRIx64 ", offset=%" PRIu64 ", span=%" PRIu64 ") out of buffer bounds [0x%" PRIx64 ", 0x%" PRIx64 ")\n",
+                           __func__, request.tensor.data, request.offset, span, p0, p1);
             return false;
         }
         if (request.offset > ggml_nbytes(tensor) || span > ggml_nbytes(tensor) - request.offset) {
-            LOG_ERROR("[%s] tensor read region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n",
-                      __func__, request.offset, span, ggml_nbytes(tensor));
+            GGML_LOG_ERROR("[%s] tensor read region (offset=%" PRIu64 ", span=%" PRIu64 ") out of tensor bounds (%zu)\n",
+                           __func__, request.offset, span, ggml_nbytes(tensor));
             return false;
         }
     }
@@ -1985,7 +1922,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co
     ggml_tensor * src = deserialize_tensor(ctx, &request.src);
     ggml_tensor * dst = deserialize_tensor(ctx, &request.dst);
     if (src == nullptr || dst == nullptr || src->buffer == nullptr || dst->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensors\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensors\n", __func__);
         return false;
     }
 
@@ -1995,7 +1932,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co
     uint64_t dst_buf_sz = (uint64_t) ggml_backend_buffer_get_size(dst->buffer);
 
     if (dst_data + src_size > dst_base + dst_buf_sz) {
-        LOG_ERROR("[%s] out-of-bounds write in rpc_server::copy_tensor:\n"
+        GGML_LOG_ERROR("[%s] out-of-bounds write in rpc_server::copy_tensor:\n"
                          "    write range : [0x%" PRIx64 ", 0x%" PRIx64 "]\n"
                          "    buffer base: [0x%" PRIx64 ", 0x%" PRIx64 "]\n",
                          __func__,
@@ -2006,7 +1943,7 @@ bool rpc_server::copy_tensor(const rpc_msg_copy_tensor_req & request, rpc_msg_co
         return false;
     }
 
-    LOG_DBG2("[%s] src->buffer: %p, dst->buffer: %p\n",
+    LOG_DBG("[%s] src->buffer: %p, dst->buffer: %p\n",
             __func__, (void*) src->buffer, (void*) dst->buffer);
 
     response.result = ggml_backend_buffer_copy_tensor(src, dst);
@@ -2032,7 +1969,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id,
         return nullptr;
     }
     if (result->buffer == nullptr && result->data != nullptr) {
-        LOG_ERROR("[%s] invalid data ptr\n", __func__);
+        GGML_LOG_ERROR("[%s] invalid data ptr", __func__);
         return nullptr;
     }
     tensor_map[id] = result;
@@ -2044,7 +1981,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id,
             result->src[i] = create_node(tensor->src[i], ctx, tensor_ptrs, tensor_map);
             // If the recursive call failed for a non-zero ID, propagate the error
             if (result->src[i] == nullptr) {
-                LOG_ERROR("[%s] failed to create source node %d (src_id=%" PRIu64 ") for node id %" PRIu64 "\n",
+                GGML_LOG_ERROR("[%s] failed to create source node %d (src_id=%" PRIu64 ") for node id %" PRIu64 "\n",
                                __func__, i, tensor->src[i], id);
                 // Must return nullptr to signal failure up the call stack
                 return nullptr;
@@ -2059,7 +1996,7 @@ ggml_tensor * rpc_server::create_node(uint64_t id,
         result->view_src = create_node(tensor->view_src, ctx, tensor_ptrs, tensor_map);
         // If the recursive call failed for a non-zero ID, propagate the error
         if (result->view_src == nullptr) {
-            LOG_ERROR("[%s] failed to create view_src node (view_src_id=%" PRIu64 ") for node id %" PRIu64 "\n",
+            GGML_LOG_ERROR("[%s] failed to create view_src node (view_src_id=%" PRIu64 ") for node id %" PRIu64 "\n",
                            __func__, tensor->view_src, id);
             // Must return nullptr to signal failure up the call stack
             return nullptr;
@@ -2104,8 +2041,6 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) {
 
     // graphs with uid == 0 are not cached, see GRAPH_CACHE_MAX for the eviction policy
     if (uid != 0 && stored_graphs[device].size() >= GRAPH_CACHE_MAX) {
-        LOG_DBG("[%s] clearing graph cache for device %u (%zu entries)\n",
-                __func__, device, stored_graphs[device].size());
         stored_graphs[device].clear();
     }
     stored_graph sg_tmp;
@@ -2132,9 +2067,6 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) {
     }
     std::unordered_map<uint64_t, ggml_tensor*> tensor_map;
     tensor_map.reserve(n_nodes);
-    // the compute itself is asynchronous, so building the graph is the only server-side cost
-    // observable here - the round trip is timed on the client
-    const auto t_deserialize = std::chrono::steady_clock::now();
     for (uint32_t i = 0; i < n_nodes; i++) {
         int64_t id;
         memcpy(&id, &nodes[i], sizeof(id));
@@ -2143,21 +2075,17 @@ bool rpc_server::graph_compute(const std::vector<uint8_t> & input) {
         // If id was 0, create_node returning nullptr is expected.
         // If id was non-zero and create_node returned nullptr, it indicates a deserialization error.
         if (graph->nodes[i] == nullptr && id != 0) {
-            LOG_ERROR("[%s] failed to create graph node %d (id=%" PRId64 ")\n", __func__, i, id);
+            GGML_LOG_ERROR("[%s] failed to create graph node %d (id=%" PRId64 ")\n", __func__, i, id);
             return false;
         }
         if (graph->nodes[i] != nullptr) {
-            LOG_DBG3("[%s] node %u: id=%" PRId64 ", op=%s, name=%s\n", __func__, i, id, ggml_op_name(graph->nodes[i]->op), graph->nodes[i]->name);
             const size_t hash_pos = ggml_hash_insert(&graph->visited_hash_set, graph->nodes[i]);
             graph->use_counts[hash_pos] = tensor_ptrs.at(id)->use_count;
         }
     }
-    const std::chrono::duration<double, std::milli> deserialize_ms = std::chrono::steady_clock::now() - t_deserialize;
-
     ggml_status status = ggml_backend_graph_compute_async(backends[device], graph);
     GGML_ASSERT(status == GGML_STATUS_SUCCESS && "Unsuccessful graph computations are not supported with RPC");
     sg.graph = graph;
-    LOG_DBG3("[%s] device: %u, graph build: %.3f ms\n", __func__, device, deserialize_ms.count());
     return true;
 }
 
@@ -2168,7 +2096,7 @@ bool rpc_server::graph_recompute(const rpc_msg_graph_recompute_req & request) {
     }
     auto it = stored_graphs[device].find(request.uid);
     if (it == stored_graphs[device].end() || it->second.graph == nullptr) {
-        LOG_ERROR("[%s] device: %u, graph with uid %" PRIu64 " not found\n", __func__, device, request.uid);
+        GGML_LOG_ERROR("[%s] device: %u, graph with uid %" PRIu64 " not found\n", __func__, device, request.uid);
         return false;
     }
     ggml_cgraph * graph = it->second.graph;
@@ -2202,7 +2130,7 @@ bool rpc_server::comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_i
     if (request.rank == 0) {
         socket_ptr srv = socket_t::create_server("0.0.0.0", request.port);
         if (srv == nullptr) {
-            LOG_ERROR("[%s] failed to listen on comm port %u\n", __func__, request.port);
+            GGML_LOG_ERROR("[%s] failed to listen on comm port %u\n", __func__, request.port);
             return true;
         }
         state.peer = srv->accept();
@@ -2229,7 +2157,7 @@ bool rpc_server::comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_i
             }
         }
         if (state.peer == nullptr) {
-            LOG_ERROR("[%s] failed to connect to peer %s:%u\n", __func__, host.c_str(), request.port);
+            GGML_LOG_ERROR("[%s] failed to connect to peer %s:%u\n", __func__, host.c_str(), request.port);
             return true;
         }
         state.peer->get_caps(local_caps);
@@ -2242,7 +2170,7 @@ bool rpc_server::comm_init(const rpc_msg_comm_init_req & request, rpc_msg_comm_i
     }
     state.rank  = request.rank;
     state.world = request.world;
-    LOG_INFO("[%s] device %u joined pairwise comm as rank %u\n", __func__, request.device, request.rank);
+    GGML_LOG_INFO("[%s] device %u joined pairwise comm as rank %u\n", __func__, request.device, request.rank);
     response.ok = 1;
     return true;
 }
@@ -2253,7 +2181,7 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) {
     }
     comm_state & state = comm_states[request.device];
     if (state.peer == nullptr) {
-        LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device);
+        GGML_LOG_ERROR("[%s] no communicator for device %u\n", __func__, request.device);
         return false;
     }
     ggml_backend_t backend = backends[request.device];
@@ -2269,7 +2197,7 @@ bool rpc_server::comm_allreduce(const rpc_msg_comm_allreduce_req & request) {
     ggml_context * ctx = ctx_ptr.get();
     ggml_tensor * t_dst = deserialize_tensor(ctx, &request.tensor);
     if (t_dst == nullptr || t_dst->buffer == nullptr) {
-        LOG_ERROR("[%s] error deserializing tensor\n", __func__);
+        GGML_LOG_ERROR("[%s] error deserializing tensor\n", __func__);
         return false;
     }
     const size_t  nbytes = ggml_nbytes(t_dst);
@@ -2386,7 +2314,7 @@ bool rpc_server::get_device_memory(const rpc_msg_get_device_memory_req & request
     ggml_backend_dev_memory(dev, &free, &total);
     response.free_mem = free;
     response.total_mem = total;
-    LOG_DBG2("[%s] device: %u, free: %" PRIu64 ", total: %" PRIu64 "\n", __func__, dev_id, response.free_mem, response.total_mem);
+    LOG_DBG("[%s] device: %u, free_mem: %" PRIu64 ", total_mem: %" PRIu64 "\n", __func__, dev_id, response.free_mem, response.total_mem);
     return true;
 }
 
@@ -2404,7 +2332,7 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const
         return;
     }
     if (cmd != RPC_CMD_HELLO) {
-        LOG_ERROR("Expected HELLO command, update client\n");
+        GGML_LOG_ERROR("Expected HELLO command, update client\n");
         return;
     }
 
@@ -2415,7 +2343,7 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const
     }
 
     if (hello_input_size != sizeof(rpc_msg_hello_req)) {
-        LOG_ERROR("HELLO request size mismatch (%zu vs %zu) — client needs upgrade to protocol v%d.x\n",
+        GGML_LOG_ERROR("HELLO request size mismatch (%zu vs %zu) — client needs upgrade to protocol v%d.x\n",
                        (size_t)hello_input_size, sizeof(rpc_msg_hello_req), RPC_PROTO_MAJOR_VERSION);
         return;
     }
@@ -2435,17 +2363,15 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const
 
     // Activate transport upgrade using client's caps
     sock->update_caps(req.conn_caps);
-    LOG_DBG("[%s] client connected, transport: %s\n", __func__, sock->transport_name());
     while (true) {
         if (!sock->recv_data(&cmd, 1)) {
             break;
         }
         if (cmd >= RPC_CMD_COUNT) {
             // fail fast if the command is invalid
-            LOG_ERROR("Unknown command: %d\n", cmd);
+            GGML_LOG_ERROR("Unknown command: %d\n", cmd);
             break;
         }
-        LOG_DBG2("[%s] %s\n", __func__, rpc_cmd_name((enum rpc_cmd)cmd));
         switch (cmd) {
             case RPC_CMD_HELLO: {
                 // HELLO command is handled above
@@ -2717,7 +2643,7 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const
                 break;
             }
             default: {
-                LOG_ERROR("Unhandled command: %s\n", rpc_cmd_name((enum rpc_cmd)cmd));
+                GGML_LOG_ERROR("Unknown command: %d\n", cmd);
                 return;
             }
         }
@@ -2727,26 +2653,26 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const
 void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir,
                                    size_t n_threads, size_t n_devices, ggml_backend_dev_t * devices) {
     if (n_devices == 0 || devices == nullptr) {
-        LOG_ERROR("Invalid arguments to ggml_backend_rpc_start_server\n");
+        fprintf(stderr, "Invalid arguments to ggml_backend_rpc_start_server\n");
         return;
     }
     std::vector<ggml_backend_t> backends;
-    LOG_INFO("Starting RPC server v%d.%d.%d\n",
+    printf("Starting RPC server v%d.%d.%d\n",
         RPC_PROTO_MAJOR_VERSION,
         RPC_PROTO_MINOR_VERSION,
         RPC_PROTO_PATCH_VERSION);
-    LOG_INFO("  endpoint       : %s\n", endpoint);
-    LOG_INFO("  local cache    : %s\n", cache_dir ? cache_dir : "n/a");
-    LOG_INFO("Devices:\n");
+    printf("  endpoint       : %s\n", endpoint);
+    printf("  local cache    : %s\n", cache_dir ? cache_dir : "n/a");
+    printf("Devices:\n");
     for (size_t i = 0; i < n_devices; i++) {
         auto dev = devices[i];
         size_t free, total;
         ggml_backend_dev_memory(dev, &free, &total);
-        LOG_INFO("  %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),
-                 total / 1024 / 1024, free / 1024 / 1024);
+        printf("  %s: %s (%zu MiB, %zu MiB free)\n", ggml_backend_dev_name(dev), ggml_backend_dev_description(dev),
+               total / 1024 / 1024, free / 1024 / 1024);
         auto backend = ggml_backend_dev_init(dev, nullptr);
         if (!backend) {
-            LOG_ERROR("Failed to create backend for device %s\n", dev->iface.get_name(dev));
+            fprintf(stderr, "Failed to create backend for device %s\n", dev->iface.get_name(dev));
             return;
         }
         backends.push_back(backend);
@@ -2766,28 +2692,30 @@ void ggml_backend_rpc_start_server(const char * endpoint, const char * cache_dir
     }
 
 #ifdef GGML_RPC_RDMA
-    LOG_INFO("  transport      : TCP (RDMA auto-negotiate enabled)\n");
+    printf("  transport      : TCP (RDMA auto-negotiate enabled)\n");
 #else
-    LOG_INFO("  transport      : TCP\n");
+    printf("  transport      : TCP\n");
 #endif // GGML_RPC_RDMA
     if (!rpc_transport_init()) {
-        LOG_ERROR("Failed to initialize RPC transport\n");
+        fprintf(stderr, "Failed to initialize RPC transport\n");
         return;
     }
     auto server_socket = socket_t::create_server(host.c_str(), port);
     if (server_socket == nullptr) {
-        LOG_ERROR("Failed to create server socket\n");
+        fprintf(stderr, "Failed to create server socket\n");
         return;
     }
     while (true) {
         auto client_socket = server_socket->accept();
         if (client_socket == nullptr) {
-            LOG_ERROR("Failed to accept client connection\n");
+            fprintf(stderr, "Failed to accept client connection\n");
             return;
         }
-        LOG_INFO("Accepted client connection\n");
+        printf("Accepted client connection\n");
+        fflush(stdout);
         rpc_serve_client(backends, cache_dir, client_socket);
-        LOG_INFO("Client connection closed\n");
+        printf("Client connection closed\n");
+        fflush(stdout);
     }
     rpc_transport_shutdown();
     for (auto backend : backends) {
@@ -2970,7 +2898,7 @@ static void * ggml_backend_rpc_comm_init(ggml_backend_t * backends, size_t n_bac
         // COMM_INIT on the same connection would deadlock behind the first
         for (const auto & rank : ranks) {
             if (rank.endpoint == rpc_ctx->endpoint) {
-                LOG_WARN("%s: multiple ranks on endpoint %s are not supported\n", __func__, rpc_ctx->endpoint.c_str());
+                GGML_LOG_WARN("%s: multiple ranks on endpoint %s are not supported\n", __func__, rpc_ctx->endpoint.c_str());
                 return nullptr;
             }
         }
@@ -3014,7 +2942,7 @@ static void * ggml_backend_rpc_comm_init(ggml_backend_t * backends, size_t n_bac
     bool ok = true;
     for (size_t i = 0; i < n_backends; i++) {
         if (!responses[i].ok) {
-            LOG_WARN("%s: rank %zu (%s) failed to initialize\n", __func__, i, ranks[i].endpoint.c_str());
+            GGML_LOG_WARN("%s: rank %zu (%s) failed to initialize\n", __func__, i, ranks[i].endpoint.c_str());
             ok = false;
         }
     }
@@ -3024,8 +2952,8 @@ static void * ggml_backend_rpc_comm_init(ggml_backend_t * backends, size_t n_bac
         }
         return nullptr;
     }
-    LOG_INFO("%s: pairwise communicator initialized (%s <-> %s)\n", __func__,
-             ranks[0].endpoint.c_str(), ranks[1].endpoint.c_str());
+    GGML_LOG_INFO("%s: pairwise communicator initialized (%s <-> %s)\n", __func__,
+                  ranks[0].endpoint.c_str(), ranks[1].endpoint.c_str());
     return new ggml_backend_rpc_comm_context{std::move(ranks)};
 }
 
@@ -3118,11 +3046,9 @@ ggml_backend_reg_t ggml_backend_rpc_add_server(const char * endpoint) {
     static uint32_t dev_id = 0;
     std::lock_guard<std::mutex> lock(mutex);
     if (reg_map.find(endpoint) != reg_map.end()) {
-        LOG_DBG3("[%s] endpoint %s already registered\n", __func__, endpoint);
         return reg_map[endpoint];
     }
     uint32_t dev_count = ggml_backend_rpc_get_device_count(endpoint);
-    LOG_DBG2("[%s] endpoint: %s, device count: %u\n", __func__, endpoint, dev_count);
     if (dev_count == 0) {
         return nullptr;
     }

ggml/src/ggml-rpc/log.h

@@ -1,48 +0,0 @@
-#pragma once
-
-// Log infrastructure for the RPC backend: all RPC logging goes through this header.
-//
-// LOG_ERROR / LOG_WARN / LOG_INFO are unconditional severity logs.
-//
-// LOG_DBG* are debug logs gated by the GGML_RPC_DEBUG verbosity variable:
-//   unset / 0 - disabled (only warnings and errors are printed)
-//   1         - high-level events: connections, handshake, buffer allocations, graph computes, cache evictions
-//   2         - per-command trace: every RPC message sent/received, tensor transfers, cache hits/misses
-//   3         - fine detail: per-command timings, graph node dumps, transport byte counts
-// non-numeric values fall back to 1
-
-#include "ggml-impl.h"
-
-#include <cstdlib>
-
-static inline int rpc_debug_level() {
-    static const int level = [] {
-        const char * env = std::getenv("GGML_RPC_DEBUG");
-        if (env == nullptr) {
-            return 0;
-        }
-        if (env[0] == '\0') {
-            return 1;
-        }
-        int value = 0;
-        for (const char * p = env; *p != '\0'; p++) {
-            if (*p < '0' || *p > '9') {
-                return 1;
-            }
-            value = value * 10 + (*p - '0');
-            if (value > 3) {
-                value = 3;
-            }
-        }
-        return value;
-    }();
-    return level;
-}
-
-#define LOG_INFO(...)  GGML_LOG_INFO(__VA_ARGS__)
-#define LOG_WARN(...)  GGML_LOG_WARN(__VA_ARGS__)
-#define LOG_ERROR(...) GGML_LOG_ERROR(__VA_ARGS__)
-
-#define LOG_DBG(...)  do { if (rpc_debug_level() >= 1) GGML_LOG_DEBUG(__VA_ARGS__); } while (0)
-#define LOG_DBG2(...) do { if (rpc_debug_level() >= 2) GGML_LOG_DEBUG(__VA_ARGS__); } while (0)
-#define LOG_DBG3(...) do { if (rpc_debug_level() >= 3) GGML_LOG_DEBUG(__VA_ARGS__); } while (0)

ggml/src/ggml-rpc/transport-apple.cpp

@@ -1,6 +1,6 @@
 #include "transport-apple.h"
 #include "transport.h"
-#include "log.h"
+#include "ggml-impl.h"
 
 #include <infiniband/verbs.h>
 
@@ -199,7 +199,6 @@ static bool rdma_library_present() {
 // address, i.e. the one cabled to the peer.
 std::unique_ptr<apple_rdma> apple_rdma::probe(int fd, const uint8_t * target_gid, uint8_t * caps) {
     if (!rdma_library_present()) {
-        LOG_DBG2("[RDMA(Apple)] librdma.dylib not present, continuing with TCP\n");
         return nullptr;
     }
     int ndev = 0;
@@ -225,10 +224,7 @@ std::unique_ptr<apple_rdma> apple_rdma::probe(int fd, const uint8_t * target_gid
         break;
     }
     ibv_free_device_list(devs);
-    if (!ctx) {
-        LOG_DBG2("[RDMA(Apple)] no RDMA device matched the connection address, continuing with TCP\n");
-        return nullptr;
-    }
+    if (!ctx) return nullptr;
 
     std::unique_ptr<impl> c(new impl());
     c->fd  = fd;
@@ -290,7 +286,7 @@ std::unique_ptr<apple_rdma> apple_rdma::probe(int fd, const uint8_t * target_gid
     memcpy(rc.gid, gid.raw, RDMA_GID_SIZE);
     memcpy(caps, &rc, sizeof(rc));
 
-    LOG_INFO("RDMA(Apple/UC) probed: dev=%s port=%u gid=%d qpn=%u lid=%u mtu=%d ring=%d x %zu KiB\n",
+    GGML_LOG_INFO("RDMA(Apple/UC) probed: dev=%s port=%u gid=%d qpn=%u lid=%u mtu=%d ring=%d x %zu KiB\n",
                   matched.c_str(), port, gid_idx, c->qpn, (unsigned)pa.lid, 128 << c->path_mtu,
                   RDMA_NBUF, RDMA_STRIDE / 1024);
     return std::unique_ptr<apple_rdma>(new apple_rdma(std::move(c)));
@@ -321,7 +317,7 @@ bool apple_rdma::activate(const uint8_t * caps) {
         memcpy(&a.ah_attr.grh.dgid, rc.gid, RDMA_GID_SIZE);
         if (ibv_modify_qp(c->qp, &a,
                 IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU | IBV_QP_DEST_QPN | IBV_QP_RQ_PSN) != 0) {
-            LOG_ERROR("RDMA(Apple/UC) RTR failed: %s\n", strerror(errno));
+            GGML_LOG_ERROR("RDMA(Apple/UC) RTR failed: %s\n", strerror(errno));
             ok = false;
         }
     }
@@ -330,7 +326,7 @@ bool apple_rdma::activate(const uint8_t * caps) {
         a.qp_state = IBV_QPS_RTS;
         a.sq_psn   = RDMA_PSN;
         if (ibv_modify_qp(c->qp, &a, IBV_QP_STATE | IBV_QP_SQ_PSN) != 0) {
-            LOG_ERROR("RDMA(Apple/UC) RTS failed: %s\n", strerror(errno));
+            GGML_LOG_ERROR("RDMA(Apple/UC) RTS failed: %s\n", strerror(errno));
             ok = false;
         }
     }
@@ -338,7 +334,7 @@ bool apple_rdma::activate(const uint8_t * caps) {
     // Recvs are posted only now: the controller starts processing them at RTR.
     for (int i = 0; ok && i < RDMA_NBUF; i++) {
         if (!c->post_recv(i)) {
-            LOG_ERROR("RDMA(Apple/UC) post_recv %d/%d failed\n", i, RDMA_NBUF);
+            GGML_LOG_ERROR("RDMA(Apple/UC) post_recv %d/%d failed\n", i, RDMA_NBUF);
             ok = false;
         }
     }
@@ -354,7 +350,7 @@ bool apple_rdma::activate(const uint8_t * caps) {
         return false;
     }
 
-    LOG_INFO("RDMA(Apple/UC) activated: qpn=%u->%u mtu=%d rx_depth=%d\n",
+    GGML_LOG_INFO("RDMA(Apple/UC) activated: qpn=%u->%u mtu=%d rx_depth=%d\n",
                   c->qpn, rc.qpn, 128 << c->path_mtu, RDMA_NBUF);
     return true;
 }
@@ -364,20 +360,20 @@ bool apple_rdma::activate(const uint8_t * caps) {
 int apple_rdma::impl::progress() {
     struct ibv_wc wc[RDMA_NBUF * 2];
     int n = ibv_poll_cq(cq, RDMA_NBUF * 2, wc);
-    if (n < 0) { LOG_ERROR("RDMA(Apple/UC) poll_cq failed\n"); broken = true; return -1; }
+    if (n < 0) { GGML_LOG_ERROR("RDMA(Apple/UC) poll_cq failed\n"); broken = true; return -1; }
     for (int j = 0; j < n; j++) {
         uint64_t id = wc[j].wr_id;
         bool is_recv = (id & RDMA_RECV_WR) != 0;
         if (wc[j].status != IBV_WC_SUCCESS) {
-            LOG_ERROR("RDMA(Apple/UC) %s wc error: status=%d\n", is_recv ? "recv" : "send", wc[j].status);
+            GGML_LOG_ERROR("RDMA(Apple/UC) %s wc error: status=%d\n", is_recv ? "recv" : "send", wc[j].status);
             broken = true;
             return -1;
         }
         if (is_recv) {
             int b = (int)(id & RDMA_WR_IDX_MASK);
             const rdma_seg_hdr * h = (const rdma_seg_hdr *)(recv_mem + (size_t)b * RDMA_STRIDE);
-            if (h->magic != RDMA_SEG_MAGIC) { LOG_ERROR("RDMA(Apple/UC) bad frame magic\n"); broken = true; return -1; }
-            if (h->len > RDMA_PAYLOAD) { LOG_ERROR("RDMA(Apple/UC) frame len %u exceeds payload\n", h->len); broken = true; return -1; }
+            if (h->magic != RDMA_SEG_MAGIC) { GGML_LOG_ERROR("RDMA(Apple/UC) bad frame magic\n"); broken = true; return -1; }
+            if (h->len > RDMA_PAYLOAD) { GGML_LOG_ERROR("RDMA(Apple/UC) frame len %u exceeds payload\n", h->len); broken = true; return -1; }
             int slot = (inq_head + inq_count) % RDMA_NBUF;
             inq[slot].buf  = b;
             inq[slot].off  = 0;

ggml/src/ggml-rpc/transport.cpp

@@ -1,5 +1,5 @@
 #include "transport.h"
-#include "log.h"
+#include "ggml-impl.h"
 
 #ifdef _WIN32
 #  define WIN32_LEAN_AND_MEAN
@@ -43,6 +43,11 @@ using ssize_t = __int64;
 typedef int sockfd_t;
 #endif
 
+static const char * RPC_DEBUG = std::getenv("GGML_RPC_DEBUG");
+
+#define LOG_DBG(...) \
+    do { if (RPC_DEBUG) GGML_LOG_DEBUG(__VA_ARGS__); } while (0)
+
 #ifdef GGML_RPC_RDMA
 static constexpr size_t RDMA_GID_SIZE = 16;            // RoCE GID / IB GID is always 16 bytes
 using rdma_gid_t = std::array<uint8_t, RDMA_GID_SIZE>;
@@ -337,7 +342,7 @@ bool socket_t::impl::rdma_probe() {
     } else if (gid_version == IBV_GID_TYPE_ROCE_V1) {
         ver_str = " RoCEv1";
     }
-    LOG_INFO("RDMA probed: dev=%s gid=%d%s qpn=%u inline=%u\n",
+    GGML_LOG_INFO("RDMA probed: dev=%s gid=%d%s qpn=%u inline=%u\n",
                   matched_dev, gid_idx, ver_str, rdma_local.qpn, rdma->max_inline);
     return true;
 }
@@ -402,7 +407,7 @@ bool socket_t::impl::rdma_activate(uint32_t remote_qpn, uint32_t remote_psn, con
 
     rdma->last_active = std::chrono::steady_clock::now();
 
-    LOG_INFO("RDMA activated: qpn=%u->%u mtu=%d rx_depth=%d\n",
+    GGML_LOG_INFO("RDMA activated: qpn=%u->%u mtu=%d rx_depth=%d\n",
                   rdma_local.qpn, remote_qpn, 128 << rdma_local.path_mtu, RDMA_RX_DEPTH);
     return true;
 }
@@ -440,7 +445,7 @@ bool socket_t::impl::rdma_poll(struct ibv_cq * cq, struct ibv_wc * wc) {
         if (n > 0) {
             c->last_active = std::chrono::steady_clock::now();
             if (wc->status != IBV_WC_SUCCESS) {
-                LOG_ERROR("RDMA CQ wc error: status=%d (%s) vendor_err=0x%x\n",
+                GGML_LOG_ERROR("RDMA CQ wc error: status=%d (%s) vendor_err=0x%x\n",
                     wc->status, ibv_wc_status_str(wc->status), wc->vendor_err);
             }
             return wc->status == IBV_WC_SUCCESS;
@@ -532,7 +537,6 @@ bool socket_t::impl::rdma_recv(void * data, size_t size) {
 #endif // GGML_RPC_RDMA
 
 bool socket_t::impl::send_data(const void * data, size_t size) {
-    LOG_DBG3("[%s] transport: %s, size: %zu\n", __func__, use_rdma ? "RDMA" : "TCP", size);
 #ifdef GGML_RPC_RDMA_APPLE
     if (use_rdma) {
         return rdma->send(data, size);
@@ -547,7 +551,7 @@ bool socket_t::impl::send_data(const void * data, size_t size) {
         size_t size_to_send = std::min(size - bytes_sent, MAX_CHUNK_SIZE);
         ssize_t n = send(fd, (const char *)data + bytes_sent, size_to_send, 0);
         if (n < 0) {
-            LOG_ERROR("send failed (bytes_sent=%zu, size_to_send=%zu)\n",
+            GGML_LOG_ERROR("send failed (bytes_sent=%zu, size_to_send=%zu)\n",
                            bytes_sent, size_to_send);
             return false;
         }
@@ -557,7 +561,6 @@ bool socket_t::impl::send_data(const void * data, size_t size) {
 }
 
 bool socket_t::impl::recv_data(void * data, size_t size) {
-    LOG_DBG3("[%s] transport: %s, size: %zu\n", __func__, use_rdma ? "RDMA" : "TCP", size);
 #ifdef GGML_RPC_RDMA_APPLE
     if (use_rdma) {
         return rdma->recv(data, size);
@@ -572,12 +575,12 @@ bool socket_t::impl::recv_data(void * data, size_t size) {
         size_t size_to_recv = std::min(size - bytes_recv, MAX_CHUNK_SIZE);
         ssize_t n = recv(fd, (char *)data + bytes_recv, size_to_recv, 0);
         if (n < 0) {
-            LOG_ERROR("recv failed (bytes_recv=%zu, size_to_recv=%zu)\n",
+            GGML_LOG_ERROR("recv failed (bytes_recv=%zu, size_to_recv=%zu)\n",
                            bytes_recv, size_to_recv);
             return false;
         }
         if (n == 0) {
-            LOG_WARN("recv returned 0 (peer closed?)\n");
+            LOG_DBG("recv returned 0 (peer closed?)\n");
             return false;
         }
         bytes_recv += (size_t)n;
@@ -596,9 +599,6 @@ void socket_t::impl::get_caps(uint8_t * local_caps) {
     if (target_gid) {
         rdma = apple_rdma::probe(fd, target_gid->data(), local_caps);
     }
-    if (!rdma) {
-        LOG_DBG2("[%s] RDMA probe failed, continuing with TCP\n", __func__);
-    }
 #  else
     rdma_local = {};
     if (rdma_probe()) {
@@ -609,7 +609,6 @@ void socket_t::impl::get_caps(uint8_t * local_caps) {
         memcpy(local_caps, &rc, sizeof(rc));
     } else {
         rdma.reset();
-        LOG_DBG2("[%s] RDMA probe failed, continuing with TCP\n", __func__);
     }
 #  endif
 #endif // GGML_RPC_RDMA
@@ -624,9 +623,6 @@ void socket_t::impl::update_caps(const uint8_t * remote_caps) {
         remote_rdma |= remote_caps[i] != 0;
     }
     if (!rdma || !remote_rdma) {
-        if (rdma && !remote_rdma) {
-            LOG_WARN("[%s] peer does not support RDMA, continuing with TCP\n", __func__);
-        }
         rdma.reset();
         return;
     }
@@ -640,7 +636,7 @@ void socket_t::impl::update_caps(const uint8_t * remote_caps) {
     if (activated) {
         use_rdma = true;
     } else {
-        LOG_WARN("RDMA activate failed, staying on TCP\n");
+        GGML_LOG_ERROR("RDMA activate failed, staying on TCP\n");
         rdma.reset();
     }
 #else
@@ -683,10 +679,6 @@ void socket_t::update_caps(const uint8_t * remote_caps) {
     return pimpl->update_caps(remote_caps);
 }
 
-const char * socket_t::transport_name() const {
-    return pimpl->use_rdma ? "RDMA" : "TCP";
-}
-
 static bool is_valid_fd(sockfd_t sockfd) {
 #ifdef _WIN32
     return sockfd != INVALID_SOCKET;
@@ -714,10 +706,9 @@ socket_ptr socket_t::accept() {
         return nullptr;
     }
     if (!set_no_delay(client_socket_fd)) {
-        LOG_ERROR("Failed to set TCP_NODELAY\n");
+        GGML_LOG_ERROR("Failed to set TCP_NODELAY\n");
         return nullptr;
     }
-    LOG_DBG("[%s] accepted client connection\n", __func__);
     return socket_ptr(new socket_t(std::make_unique<impl>(client_socket_fd)));
 }
 
@@ -727,11 +718,11 @@ socket_ptr socket_t::create_server(const char * host, int port) {
         return nullptr;
     }
     if (!set_reuse_addr(sockfd)) {
-        LOG_ERROR("Failed to set SO_REUSEADDR\n");
+        GGML_LOG_ERROR("Failed to set SO_REUSEADDR\n");
         return nullptr;
     }
     if (inet_addr(host) == INADDR_NONE) {
-        LOG_ERROR("Invalid host address: %s\n", host);
+        GGML_LOG_ERROR("Invalid host address: %s\n", host);
         return nullptr;
     }
     struct sockaddr_in serv_addr;
@@ -745,7 +736,6 @@ socket_ptr socket_t::create_server(const char * host, int port) {
     if (listen(sockfd, 1) < 0) {
         return nullptr;
     }
-    LOG_DBG("[%s] listening on %s:%d\n", __func__, host, port);
     return socket_ptr(new socket_t(std::make_unique<impl>(sockfd)));
 }
 
@@ -755,7 +745,7 @@ socket_ptr socket_t::connect(const char * host, int port) {
         return nullptr;
     }
     if (!set_no_delay(sockfd)) {
-        LOG_ERROR("Failed to set TCP_NODELAY\n");
+        GGML_LOG_ERROR("Failed to set TCP_NODELAY\n");
         return nullptr;
     }
     struct sockaddr_in addr;
@@ -763,14 +753,13 @@ socket_ptr socket_t::connect(const char * host, int port) {
     addr.sin_port = htons(port);
     struct hostent * server = gethostbyname(host);
     if (server == NULL) {
-        LOG_ERROR("Cannot resolve host '%s'\n", host);
+        GGML_LOG_ERROR("Cannot resolve host '%s'\n", host);
         return nullptr;
     }
     memcpy(&addr.sin_addr.s_addr, server->h_addr, server->h_length);
     if (::connect(sockfd, (struct sockaddr *)&addr, sizeof(addr)) < 0) {
         return nullptr;
     }
-    LOG_DBG("[%s] connected to %s:%d\n", __func__, host, port);
     return socket_ptr(new socket_t(std::make_unique<impl>(sockfd)));
 }
 

ggml/src/ggml-rpc/transport.h

@@ -24,7 +24,6 @@ struct socket_t {
 
     void get_caps(uint8_t * local_caps);
     void update_caps(const uint8_t * remote_caps);
-    const char * transport_name() const;
 
     static socket_ptr create_server(const char * host, int port);
     static socket_ptr connect(const char * host, int port);

ggml/src/ggml-sycl/convert.cpp

@@ -537,18 +537,6 @@ static void dequantize_row_mxfp4_sycl(const void * vx, dst_t * y, const int64_t
         });
 }
 
-template <typename dst_t>
-static void dequantize_row_mxfp4_sycl_reorder(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
-    GGML_ASSERT(k % QK_MXFP4 == 0);
-    const int n_warp = (k / QK_MXFP4 + WARP_SIZE - 1) / WARP_SIZE;
-    stream->parallel_for(
-        sycl::nd_range<3>(sycl::range<3>(1, 1, n_warp) * sycl::range<3>(1, 1, WARP_SIZE),
-                          sycl::range<3>(1, 1, WARP_SIZE)),
-        [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-            dequantize_block_mxfp4_reorder(vx, y, k, item_ct1);
-        });
-}
-
 template <typename dst_t>
 static void dequantize_row_nvfp4_sycl(const void * vx, dst_t * y, const int64_t k, dpct::queue_ptr stream) {
     GGML_ASSERT(k % QK_NVFP4 == 0);
@@ -740,9 +728,6 @@ to_fp16_sycl_t ggml_get_to_fp16_sycl(ggml_type type, ggml_tensor * dst) {
         case GGML_TYPE_IQ4_NL:
             return dequantize_row_iq4_nl_sycl;
         case GGML_TYPE_MXFP4:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_mxfp4_sycl_reorder;
-            }
             return dequantize_row_mxfp4_sycl;
         case GGML_TYPE_NVFP4:
             return dequantize_row_nvfp4_sycl;
@@ -834,9 +819,6 @@ to_fp32_sycl_t ggml_get_to_fp32_sycl(ggml_type type, ggml_tensor *dst) {
         case GGML_TYPE_IQ4_NL:
             return dequantize_row_iq4_nl_sycl;
         case GGML_TYPE_MXFP4:
-            if (dst->src[0]->extra && ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                return dequantize_row_mxfp4_sycl_reorder;
-            }
             return dequantize_row_mxfp4_sycl;
         case GGML_TYPE_NVFP4:
             return dequantize_row_nvfp4_sycl;

ggml/src/ggml-sycl/dequantize.hpp

@@ -1646,26 +1646,6 @@ static void dequantize_block_mxfp4(const void * __restrict__ vx, dst_t * __restr
     }
 }
 
-// Reordered MXFP4 ([qs...][e...], see ggml_sycl_reordered::block_q_t<MXFP4>): one work-item per block.
-template <typename dst_t>
-static void dequantize_block_mxfp4_reorder(const void * __restrict__ vx, dst_t * __restrict__ yy, int64_t k,
-                                           const sycl::nd_item<3> & item_ct1) {
-    const int64_t ib = (int64_t) item_ct1.get_group(2) * WARP_SIZE + item_ct1.get_local_id(2);
-    if (ib >= k / QK_MXFP4) {
-        return;
-    }
-
-    const uint8_t * qs = (const uint8_t *) vx + ib * (QK_MXFP4 / 2);
-    const float     d  = ggml_sycl_e8m0_to_fp32(((const uint8_t *) vx)[k / 2 + ib]) * 0.5f;
-    dst_t *         y  = yy + ib * QK_MXFP4;
-
-#pragma unroll
-    for (int j = 0; j < QK_MXFP4 / 2; ++j) {
-        y[j]                = d * kvalues_mxfp4[qs[j] & 0xf];
-        y[j + QK_MXFP4 / 2] = d * kvalues_mxfp4[qs[j] >> 4];
-    }
-}
-
 
 template <typename dst_t>
 static void dequantize_block_nvfp4(

ggml/src/ggml-sycl/dmmv.cpp

@@ -66,9 +66,6 @@ static void dequantize_mul_mat_vec(const void * __restrict__ vx, const dfloat *
 
     for (int i = 0; i < ncols; i += iter_stride) {
         const int col = i + vals_per_iter*tid;
-        if (col >= ncols) {
-            continue;
-        }
         const int ib = (row*ncols + col)/qk; // x block index
         const int iqs = (col%qk)/qr; // x quant index
         const int iybs = col - col%qk; // y block start index

ggml/src/ggml-sycl/ggml-sycl.cpp

@@ -773,8 +773,7 @@ ggml_backend_sycl_buffer_init_tensor(ggml_backend_buffer_t buffer,
             case GGML_TYPE_Q3_K:
             case GGML_TYPE_Q4_K:
             case GGML_TYPE_Q5_K:
-            case GGML_TYPE_Q6_K:
-            case GGML_TYPE_MXFP4:{
+            case GGML_TYPE_Q6_K:{
                 ggml_tensor_extra_gpu * extra = new ggml_tensor_extra_gpu{};
                 tensor->extra                 = extra;
                 ctx->tensor_extras.push_back(extra);
@@ -4624,58 +4623,6 @@ static bool reorder_qw_q6_k_moe(uint8_t * data_device, size_t expert_bytes, int6
     return true;
 }
 
-// Reorder each MXFP4 expert slice into [qs][e]: 16-byte nibble blocks, then one E8M0 byte per block.
-// Experts are self-contained, so the tensor is reordered a few experts at a time through a small
-// temporary: a whole-tensor temporary (hundreds of MB) can exceed the VRAM left on a nearly full card,
-// and on Windows the driver then pages device memory out to host RAM instead of failing.
-static bool reorder_qw_mxfp4_moe(uint8_t * data_device, size_t expert_bytes, int64_t n_expert, dpct::queue_ptr stream) {
-    GGML_ASSERT(expert_bytes % sizeof(block_mxfp4) == 0);
-    const int     blocks_per_expert = (int) (expert_bytes / sizeof(block_mxfp4));
-    const size_t  max_chunk_bytes   = 32u << 20;
-    const int64_t chunk_experts     = std::max<int64_t>(1, std::min<int64_t>(n_expert, (int64_t) (max_chunk_bytes / expert_bytes)));
-
-    sycl_reorder_temp_buffer tmp(stream, (size_t) chunk_experts * expert_bytes);
-    if (!tmp) {
-        GGML_LOG_WARN("%s: failed to allocate %zu bytes for reorder temp buffer, skipping reorder\n", __func__,
-                      (size_t) chunk_experts * expert_bytes);
-        return false;
-    }
-    uint8_t * tmp_buf = static_cast<uint8_t *>(tmp.ptr);
-
-    // the queue is in-order: each chunk's copy into tmp_buf waits for the previous chunk's kernel
-    for (int64_t e0 = 0; e0 < n_expert; e0 += chunk_experts) {
-        const int64_t n_chunk = std::min(chunk_experts, n_expert - e0);
-        uint8_t *     chunk   = data_device + (size_t) e0 * expert_bytes;
-
-        sycl::event copy_event;
-        SYCL_CHECK(CHECK_TRY_ERROR(copy_event = stream->memcpy(tmp_buf, chunk, (size_t) n_chunk * expert_bytes)));
-        if (!g_ggml_sycl_use_async_mem_op) {
-            copy_event.wait();
-        }
-
-        const int total_blocks = blocks_per_expert * (int) n_chunk;
-        auto reorder_event = stream->parallel_for(total_blocks, [=](auto gb_) {
-            const int           gb   = gb_;
-            const int           e    = gb / blocks_per_expert;
-            const int           ib   = gb % blocks_per_expert;
-            const block_mxfp4 * x    = (const block_mxfp4 *) (tmp_buf + (size_t) e * expert_bytes);
-            uint8_t *           base = chunk + (size_t) e * expert_bytes;
-
-            uint8_t * qs_ptr = base;
-            uint8_t * e_ptr  = qs_ptr + (QK_MXFP4 / 2) * (size_t) blocks_per_expert;
-
-            for (int j = 0; j < QK_MXFP4 / 2; ++j) {
-                qs_ptr[(size_t) ib * (QK_MXFP4 / 2) + j] = x[ib].qs[j];
-            }
-            e_ptr[ib] = x[ib].e;
-        });
-        if (!g_ggml_sycl_use_async_mem_op) {
-            reorder_event.wait_and_throw();
-        }
-    }
-    return true;
-}
-
 static bool reorder_qw_q2_k(uint8_t * data_device, size_t size, size_t offset, dpct::queue_ptr stream) {
     GGML_ASSERT(size % sizeof(block_q2_K) == 0);
     GGML_ASSERT(offset % sizeof(block_q2_K) == 0);
@@ -4885,8 +4832,6 @@ static bool reorder_qw(const ggml_tensor * src0, dpct::queue_ptr stream) {
                 return reorder_qw_q5_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
             case GGML_TYPE_Q6_K:
                 return reorder_qw_q6_k_moe(data_device, src0->nb[2], src0->ne[2], stream);
-            case GGML_TYPE_MXFP4:
-                return reorder_qw_mxfp4_moe(data_device, src0->nb[2], src0->ne[2], stream);
             default:
                 return false;
         }
@@ -4960,12 +4905,7 @@ static void opt_for_reorder_id(ggml_backend_sycl_context * ctx, const ggml_tenso
     if (!g_ggml_sycl_enable_optimize || !ctx->opt_feature.reorder) {
         return;
     }
-    if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K &&
-        src0->type != GGML_TYPE_MXFP4) {
-        return;
-    }
-    // The MXFP4 reorder kernels use 8-byte vector loads, so every expert slice must stay aligned.
-    if (src0->type == GGML_TYPE_MXFP4 && (src0->nb[2] % 16 != 0 || (uintptr_t) src0->data % 16 != 0)) {
+    if (src0->type != GGML_TYPE_Q4_K && src0->type != GGML_TYPE_Q5_K && src0->type != GGML_TYPE_Q6_K) {
         return;
     }
     ggml_tensor_extra_gpu * extra = static_cast<ggml_tensor_extra_gpu *>(src0->extra);
@@ -5448,11 +5388,6 @@ static void ggml_sycl_mul_mat_id(ggml_backend_sycl_context & ctx,
         }
     }
 
-    // The per-expert loop below reads the experts in whatever layout they have: reorder MXFP4 here as well, so prompt processing does not depend on a single-token decode having run first.
-    if (src0->type == GGML_TYPE_MXFP4) {
-        opt_for_reorder_id(&ctx, src0);
-    }
-
     std::vector<char> ids_host(ggml_nbytes(ids));
     const char * ids_dev = (const char *) ids->data;
 

ggml/src/ggml-sycl/mmvq.cpp

@@ -1285,65 +1285,6 @@ static void reorder_mul_mat_vec_q8_0_q8_1_sycl_switch_ncols(
     }
 }
 
-// MXFP4 reorder GEMV. Only MoE expert slices are reordered (opt_for_reorder_id); these dense entry
-// points serve per-expert ggml_sycl_mul_mat calls from multi-token MUL_MAT_ID after that reorder.
-static void reorder_mul_mat_vec_mxfp4_q8_1_sycl(const void * vx, const void * vy, float * dst, const int ncols,
-                                                const int nrows, dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK_MXFP4 == 0);
-    constexpr size_t     num_subgroups = WARP_SIZE;
-    const int            block_num_y   = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
-    const sycl::range<3> block_nums(1, 1, block_num_y);
-    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
-
-    stream->submit([&](sycl::handler & cgh) {
-        cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
-                         [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                             mul_mat_vec_q_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4>>(vx, vy, dst, ncols, nrows,
-                                                                                            nd_item);
-                         });
-    });
-}
-
-template <int ncols_dst>
-static void reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols(
-        const void * vx, const void * vy, float * dst,
-        const int ncols, const int nrows,
-        const int stride_col_y_bytes, const int stride_col_dst,
-        dpct::queue_ptr stream) {
-    GGML_ASSERT(ncols % QK_MXFP4 == 0);
-    constexpr size_t     num_subgroups = WARP_SIZE;
-    const int            block_num_y   = ceil_div(nrows, GGML_SYCL_MMV_Y * (int) num_subgroups);
-    const sycl::range<3> block_nums(1, 1, block_num_y);
-    const sycl::range<3> block_dims(1, GGML_SYCL_MMV_Y, num_subgroups * WARP_SIZE);
-
-    stream->submit([&](sycl::handler & cgh) {
-        cgh.parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
-                         [=](sycl::nd_item<3> nd_item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
-                             mul_mat_vec_q_reorder_ncols<reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4>, ncols_dst>(
-                                 vx, /*vgate=*/ nullptr, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst,
-                                 /*glu_op=*/ GGML_GLU_OP_SWIGLU, nd_item);
-                         });
-    });
-}
-
-static void reorder_mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(
-        const void * vx, const void * vy, float * dst,
-        const int ncols, const int nrows, const int ncols_dst,
-        const int stride_col_y_bytes, const int stride_col_dst,
-        dpct::queue_ptr stream) {
-    switch (ncols_dst) {
-        case 1: reorder_mul_mat_vec_mxfp4_q8_1_sycl(vx, vy, dst, ncols, nrows, stream); break;
-        case 2: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<2>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 3: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<3>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 4: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<4>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 5: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<5>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 6: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<6>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 7: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<7>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        case 8: reorder_mul_mat_vec_mxfp4_q8_1_sycl_ncols<8>(vx, vy, dst, ncols, nrows, stride_col_y_bytes, stride_col_dst, stream); break;
-        default: GGML_ABORT("unsupported ncols_dst=%d for MXFP4 reorder multi-col MMVQ", ncols_dst);
-    }
-}
-
 static void mul_mat_vec_q8_0_q8_1_sycl(const void *vx, const void *vy,
                                        float *dst, const int ncols,
                                        const int nrows,
@@ -2824,21 +2765,7 @@ void ggml_sycl_op_mul_mat_vec_q(ggml_backend_sycl_context & ctx, const ggml_tens
                 }
                 break;
             case GGML_TYPE_MXFP4:
-                if ((ggml_tensor_extra_gpu *) dst->src[0]->extra &&
-                    ((ggml_tensor_extra_gpu *) dst->src[0]->extra)->optimized_feature.reorder) {
-                    if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
-                        const int stride_col_y_bytes = src1_padded_col_size * q8_1_ts / q8_1_bs;
-                        const int stride_col_dst     = dst->ne[0];
-                        GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
-                        reorder_mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols(
-                            src0_dd_i, src1_ddq_i, dst_dd_i, ne00, row_diff,
-                            src1_ncols, stride_col_y_bytes, stride_col_dst, stream);
-                        return;
-                    } else {
-                        GGML_SYCL_DEBUG("Calling reorder_mul_mat_vec_mxfp4_q8_1_sycl\n");
-                        reorder_mul_mat_vec_mxfp4_q8_1_sycl(src0_dd_i, src1_ddq_i_bs, dst_dd_i_bs, ne00, row_diff, stream);
-                    }
-                } else if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
+                if (i == 0 && src1_ncols > 1 && src1_ncols <= 8) {
                     const int stride_col_y   = src1_padded_col_size / QK8_1;
                     const int stride_col_dst = dst->ne[0];
                     GGML_SYCL_DEBUG("Calling mul_mat_vec_mxfp4_q8_1_sycl_switch_ncols ncols=%d\n", (int)src1_ncols);
@@ -3184,11 +3111,6 @@ bool ggml_sycl_mul_mat_vec_q_id_reorder(
                 vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
                 expert_weight_stride, dst_row_stride, src1_row_stride, stream);
             return true;
-        case GGML_TYPE_MXFP4:
-            launch_mul_mat_vec_q_moe_reorder<reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4>>(
-                vx_base, vy, ids_dev, dst_base, ncols, nrows, n_experts_used,
-                expert_weight_stride, dst_row_stride, src1_row_stride, stream);
-            return true;
         default:
             return false;
     }

ggml/src/ggml-sycl/quants.hpp

@@ -199,27 +199,6 @@ template <> struct block_q_t<GGML_TYPE_Q8_0> {
     static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }  // 1
 };
 
-template <> struct block_q_t<GGML_TYPE_MXFP4> {
-    struct traits {
-        static constexpr uint32_t qk       = QK_MXFP4;  // 32
-        static constexpr uint32_t qi       = QI_MXFP4;  // 4
-        static constexpr uint32_t qr       = QR_MXFP4;  // 2
-        static constexpr uint32_t vdr_mmvq = 2;
-    };
-
-    // MXFP4 reorder layout: [qs0|qs1|...|qsN][e0|e1|...|eN]
-    // The 17-byte AoS block leaves qs unaligned; split out, every 16-byte nibble block is aligned.
-    static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
-        return { block_index * (QK_MXFP4 / 2), 0 };
-    }
-
-    static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
-        return { (ncols / 2 * nrows) + block_index, 0 };
-    }
-
-    static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }  // 1
-};
-
 }  // namespace ggml_sycl_reordered
 
 #endif  // GGML_SYCL_QUANTS_HPP

ggml/src/ggml-sycl/vecdotq.hpp

@@ -148,28 +148,6 @@ static __dpct_inline__ sycl::int2 get_int_from_table_16(
       dpct::byte_level_permute(tmp[0], tmp[1], 0x7531));
 }
 
-// Four E2M1 codes (one per byte, bits 0..3) to their kvalues_mxfp4 int8 values. SWAR arithmetic
-// replaces get_int_from_table_16 for MXFP4: dpct::byte_level_permute is emulated with 64-bit shifts,
-// eight per int, which made the MXFP4 GEMV compute-bound on Intel GPUs.
-// Magnitudes 0,1,2,3,4,6,8,12 = m + [m>=5] + [m>=6] + 3*[m>=7]; each byte stays below 256, so the
-// byte-wise adds never carry. -0 (code 8) is left as 0 so the two's-complement +1 cannot carry either.
-static __dpct_inline__ int mxfp4_codes_to_int8(const uint32_t x) {
-    const uint32_t m   = x & 0x07070707u;
-    const uint32_t ge5 = ((m + 0x03030303u) >> 3) & 0x01010101u;
-    const uint32_t ge6 = ((m + 0x02020202u) >> 3) & 0x01010101u;
-    const uint32_t ge7 = ((m + 0x01010101u) >> 3) & 0x01010101u;
-    const uint32_t mag = m + ge5 + ge6 + 3u * ge7;
-    const uint32_t nz  = ((mag + 0x7f7f7f7fu) >> 7) & 0x01010101u;
-    const uint32_t neg = (x >> 3) & nz & 0x01010101u;
-    return (int) ((mag ^ (neg * 0xffu)) + neg);
-}
-
-// Same result as get_int_from_table_16(q4, kvalues_mxfp4): x = low nibbles, y = high nibbles.
-static __dpct_inline__ sycl::int2 get_int_from_mxfp4(const int q4) {
-    return sycl::int2(mxfp4_codes_to_int8((uint32_t) q4 & 0x0f0f0f0fu),
-                      mxfp4_codes_to_int8(((uint32_t) q4 >> 4) & 0x0f0f0f0fu));
-}
-
 #define VDR_Q2_K_Q8_1_MMVQ 1
 
 // contiguous v/x values
@@ -817,41 +795,6 @@ template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_Q6_K> {
             vl, vh, u0, u1, scs[0], scs[4], *d, d80, d81);
     }
 };
-
-template <> struct reorder_vec_dot_q_sycl<GGML_TYPE_MXFP4> {
-    static constexpr ggml_type gtype = GGML_TYPE_MXFP4;
-
-    using mxfp4_block  = ggml_sycl_reordered::block_q_t<GGML_TYPE_MXFP4>;
-    using mxfp4_traits = typename mxfp4_block::traits;
-
-    __dpct_inline__ float operator()(const void * __restrict__ vbq, const std::pair<int, int> ibx_offset,
-                                     const std::pair<int, int> d_offset, const int8_t * q8_1_quant_ptr,
-                                     const sycl::half2 * q8_1_ds, const int & iqs) {
-        static_assert(mxfp4_traits::vdr_mmvq == 2, "vector load assumes vdr_mmvq == 2");
-        const uint8_t * base = static_cast<const uint8_t *>(vbq);
-
-        // Reordered nibble blocks are 16 contiguous bytes and iqs is 0 or 2, so each lane's two
-        // weight ints are one aligned 8-byte load (the AoS layout needed eight byte loads).
-        const sycl::int2 q4 = *reinterpret_cast<const sycl::int2 *>(base + ibx_offset.first + sizeof(int) * iqs);
-        const uint8_t    e  = base[d_offset.first];
-
-        // Low nibbles pair with q8_1 ints iqs..iqs+1, high nibbles with iqs+4..iqs+5.
-        const sycl::int2 u_lo = *reinterpret_cast<const sycl::int2 *>(q8_1_quant_ptr + sizeof(int) * iqs);
-        const sycl::int2 u_hi = *reinterpret_cast<const sycl::int2 *>(q8_1_quant_ptr + sizeof(int) * (iqs + 4));
-
-        const sycl::int2 v0 = get_int_from_mxfp4(q4.x());
-        const sycl::int2 v1 = get_int_from_mxfp4(q4.y());
-
-        int sumi = 0;
-        sumi = ggml_sycl_dp4a(v0.x(), u_lo.x(), sumi);
-        sumi = ggml_sycl_dp4a(v0.y(), u_hi.x(), sumi);
-        sumi = ggml_sycl_dp4a(v1.x(), u_lo.y(), sumi);
-        sumi = ggml_sycl_dp4a(v1.y(), u_hi.y(), sumi);
-
-        const float d = ggml_sycl_e8m0_to_fp32(e) * 0.5f * static_cast<float>((*q8_1_ds)[0]);
-        return d * sumi;
-    }
-};
 #define VDR_Q4_0_Q8_1_MMVQ 2
 #define VDR_Q4_0_Q8_1_MMQ  4
 
@@ -1181,7 +1124,7 @@ static __dpct_inline__ float vec_dot_mxfp4_q8_1(const void * __restrict__ vbq,
 #pragma unroll
     for (int l = 0; l < VDR_MXFP4_Q8_1_MMVQ; ++l) {
         const int aux_q4 = get_int_b1(bq4->qs, iqs + l);
-        const sycl::int2 v      = get_int_from_mxfp4(aux_q4);
+        const sycl::int2 v      = get_int_from_table_16(aux_q4, kvalues_mxfp4);
         sumi = ggml_sycl_dp4a(v.x(), q8[l + 0], sumi);
         sumi = ggml_sycl_dp4a(v.y(), q8[l + 4], sumi);
     }

ggml/src/ggml-vulkan/ggml-vulkan-common.h

@@ -1,10 +1,6 @@
 #pragma once
 #include "ggml-vulkan-push-constants.h"
 
-inline std::ostream & operator<<(std::ostream & os, vk::Buffer buffer) {
-    return os << static_cast<VkBuffer>(buffer);
-}
-
 // shared globals
 extern ggml_backend_buffer_type_i ggml_backend_vk_buffer_type_interface;
 extern bool vk_memory_logger_enabled;

ggml/src/ggml-vulkan/ggml-vulkan-push-constants.h

@@ -63,7 +63,7 @@ struct vk_mat_mat_id_push_constants {
     uint32_t batch_stride_a; uint32_t batch_stride_b; uint32_t batch_stride_d;
     uint32_t nei0; uint32_t nei1; uint32_t nbi1; uint32_t ne11;
     uint32_t n_experts;
-    uint32_t row_ids_offset;
+    uint32_t hoist_row_ids;
 };
 
 struct vk_mat_vec_id_push_constants {
@@ -218,10 +218,9 @@ struct vk_op_count_experts_push_constants {
     uint32_t nb01;
     uint32_t a_offset;
     uint32_t n_experts;
+    uint32_t hoist_row_ids;
     uint32_t ne00mp;
     uint32_t ne00L;
-    uint32_t row_tile_size;
-    uint32_t row_ids_offset;
 };
 
 struct vk_op_glu_push_constants {

ggml/src/ggml-vulkan/ggml-vulkan.cpp

@@ -1,5 +1,10 @@
 #include "ggml-vulkan-common.h"
 
+namespace {
+inline std::ostream & operator<<(std::ostream & os, vk::Buffer buffer) {
+    return os << static_cast<VkBuffer>(buffer);
+}
+}
 static vk_device_architecture get_device_architecture(const vk::PhysicalDevice& device) {
     vk::PhysicalDeviceProperties props = device.getProperties();
 
@@ -1468,8 +1473,9 @@ static bool ggml_vk_matmul_shmem_support(const vk_device& device, const std::vec
     const uint32_t load_bufs = (warptile[1] + warptile[2]) * (warptile[3] + bank_conflict_offset) * type_size;
     const uint32_t mmid_row_ids = mul_mat_id ? (warptile[2] * 2 * sizeof(uint16_t)) : 0;
     const uint32_t coopmat_stage = device->coopmat_support ? warptile[7] * warptile[8] / warps * sizeof(float) : 0;
+    const uint32_t ballots_sh = mul_mat_id ? (warps * 4 * sizeof(uint32_t)) : 0;
 
-    const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size;
+    const uint32_t total_size = load_bufs + mmid_row_ids + coopmat_stage + lut_size + ballots_sh;
     const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;
 
     VK_LOG_DEBUG("ggml_vk_matmul_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "
@@ -1534,7 +1540,10 @@ static bool ggml_vk_matmul_int_shmem_support(const vk_device& device, const std:
     const uint32_t buf_b_size = BN * BK_STEP * block_b_size;
     const uint32_t mmid_row_ids = mul_mat_id ? (BN * 2u * (uint32_t)sizeof(uint16_t)) : 0u;
 
-    const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + lut_size;
+    const uint32_t warps = warptile[0] / warptile[10];
+    const uint32_t ballots_sh = mul_mat_id ? (warps * 4u * (uint32_t)sizeof(uint32_t)) : 0u;
+
+    const uint32_t total_size = buf_a_size + buf_b_size + mmid_row_ids + ballots_sh + lut_size;
     const bool supported = total_size <= device->properties.limits.maxComputeSharedMemorySize;
 
     VK_LOG_DEBUG("ggml_vk_matmul_int_shmem_support(warptile=(" << warptile[0] << "," << warptile[1] << "," << warptile[2] << "), "
@@ -1564,8 +1573,10 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const
             return false;
     }
 
+    const uint32_t BLOCK_SIZE = warptile[0];
     const uint32_t BM         = warptile[1];
     const uint32_t BN         = warptile[2];
+    const uint32_t WARP       = warptile[10];
 
     const uint32_t BK      = 32;
     const uint32_t BK_STEP = mul_mat_id ? 2u : 4u;
@@ -1589,6 +1600,8 @@ static bool ggml_vk_matmul_cm1_int_shmem_support(const vk_device& device, const
     }
     if (mul_mat_id) {
         total += BN * 2u * (uint32_t)sizeof(uint16_t);   // row_ids[BN] (u16vec2)
+        const uint32_t num_warps = BLOCK_SIZE / std::max(WARP, 1u);
+        total += num_warps * 4u * (uint32_t)sizeof(uint32_t); // ballots_sh[NUM_WARPS] (uvec4)
     }
 
     const bool supported = total <= device->properties.limits.maxComputeSharedMemorySize;
@@ -2879,14 +2892,8 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
     // RDNA3/4: above four columns, static 4 rows for all types bench faster than the default
     const bool is_rdna3_or_4 = device->vendor_id == VK_VENDOR_ID_AMD && (device->architecture == AMD_RDNA3 || device->architecture == AMD_RDNA4);
     auto const &rm_int_n = [&](uint32_t rows, uint32_t i) { return (is_rdna3_or_4 && i >= 4) ? 4u : rows; };
-    // RDNA3/4 and NVIDIA except pre-Turing: use 4 rows for MUL_MAT_ID MMVQ.
-    auto const &rm_id = [&](uint32_t rows) {
-        if (device->vendor_id == VK_VENDOR_ID_NVIDIA &&
-            device->architecture != vk_device_architecture::NVIDIA_PRE_TURING) {
-            return 4u;
-        }
-        return is_rdna3_or_4 ? 4u : rows;
-    };
+    // RDNA3/4: Static 4 rows for all types bench faster than the default
+    auto const &rm_id = [&](uint32_t rows) { return is_rdna3_or_4 ? 4u : rows; };
     uint32_t rm_iq = 2 * rm_kq;
 
     const bool use_subgroups = device->subgroup_arithmetic;
@@ -3582,7 +3589,7 @@ void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) {
 
     ggml_vk_create_pipeline(device, device->pipeline_count_equal_i32, "count_equal_i32", count_equal_i32_len, count_equal_i32_data, "main", 3, sizeof(vk_op_push_constants), {512, 1, 1}, { device->subgroup_size }, 1);
 
-    if (device->subgroup_arithmetic && device->subgroup_vote && device->subgroup_require_full_support) {
+    if (device->subgroup_arithmetic && device->subgroup_require_full_support) {
         ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_subgroup_len, count_experts_subgroup_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true, true);
     } else {
         ggml_vk_create_pipeline(device, device->pipeline_count_experts, "count_experts", count_experts_len, count_experts_data, "main", 2, sizeof(vk_op_count_experts_push_constants), {1, 1, 1}, {}, 1, true);
@@ -6001,27 +6008,20 @@ static uint32_t ggml_vk_guess_matmul_pipeline_align_map(ggml_backend_vk_context
     return configs[idx].align;
 }
 
-static uint64_t ggml_vk_mul_mat_id_max_tiles(uint64_t n_experts, uint64_t n_rows, uint32_t bn) {
-    // Each nonempty expert can add at most one partial tile to the combined grid.
-    return std::min(n_rows, CEIL_DIV(n_rows, bn) + std::min(n_experts, n_rows) - 1);
-}
-
 static void ggml_vk_matmul_id(
         ggml_backend_vk_context * ctx, vk_context& subctx, vk_pipeline& pipeline,
         vk_subbuffer&& a, vk_subbuffer&& b, vk_subbuffer&& d, vk_subbuffer&& ids, const vk_subbuffer & expert_count_buf,
         uint32_t m, uint32_t n, uint32_t k, uint32_t stride_a, uint32_t stride_b, uint32_t stride_d,
         uint32_t batch_stride_a, uint32_t batch_stride_b, uint32_t batch_stride_d,
         uint32_t n_as, uint32_t nei0, uint32_t nei1, uint32_t nbi1, uint32_t ne11,
-        uint32_t row_ids_offset, uint32_t max_tiles) {
+        bool hoist_row_ids) {
     VK_LOG_DEBUG("ggml_vk_matmul_id(a: (" << a.buffer->buffer << ", " << a.offset << ", " << a.size << "), b: (" << b.buffer->buffer << ", " << b.offset << ", " << b.size << "), d: (" << d.buffer->buffer << ", " << d.offset << ", " << d.size << "), ids: (" << ids.buffer->buffer << ", " << ids.offset << ", " << ids.size << "), expert_count: (" << expert_count_buf.buffer->buffer << ", " << expert_count_buf.offset << ", " << expert_count_buf.size << "), " <<
         "m: " << m << ", n: " << n << ", k: " << k << ", stride_a: " << stride_a << ", stride_b: " << stride_b << ", stride_d: " << stride_d << ", " <<
         "batch_stride_a: " << batch_stride_a << ", batch_stride_b: " << batch_stride_b << ", batch_stride_d: " << batch_stride_d << ", " <<
         "n_as: " << n_as << ", nei0: " << nei0 << ", nei1: " << nei1 << ", nbi1: " << nbi1 << ", ne11: " << ne11 << ")");
     const vk_mat_mat_id_push_constants pc = { m, n, k, stride_a, stride_b, stride_d, batch_stride_a, batch_stride_b, batch_stride_d,
-                                              nei0, nei1, nbi1, ne11, n_as, row_ids_offset };
-    const uint32_t tiles_y = std::min(max_tiles, ctx->device->properties.limits.maxComputeWorkGroupCount[1]);
-    const std::array<uint32_t, 3> elements = { m, tiles_y * pipeline->wg_denoms[1], uint32_t(CEIL_DIV(max_tiles, tiles_y)) };
-    ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, elements);
+                                              nei0, nei1, nbi1, ne11, n_as, uint32_t(hoist_row_ids) };
+    ggml_vk_dispatch_pipeline(ctx, subctx, pipeline, { a, b, d, ids, expert_count_buf }, pc, { m, nei1, n_as });
 }
 
 bool ggml_vk_dim01_contiguous(const ggml_tensor * tensor) {
@@ -7330,6 +7330,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
 
     const uint32_t nbi0 = ids->nb[0];
     const uint32_t nbi1 = ids->nb[1];
+    const uint32_t nbi2 = ids->nb[2];
 
     const uint64_t ne20 = dst->ne[0];
     const uint64_t ne21 = dst->ne[1];
@@ -7337,24 +7338,38 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
     // const uint64_t ne23 = dst->ne[3];
 
     const uint64_t n_as = ne02;
+    // n_as counts, n_as offsets, one total, then one packed row id per (expert, token).
+    // Hoisting requires 16-bit indices for the packing and a table that fits one binding.
+    const uint64_t hoisted_row_id_words = 2 * n_as + 1 + nei0 * nei1;
+    // 1024 matches MAX_EXPERTS in count_experts.comp and LLAMA_MAX_EXPERTS. It costs
+    // 3 * 1024 * 4 = 12 KiB of shared memory, within the 16 KiB Vulkan guarantees.
+    const bool hoist_row_ids = n_as <= 1024 && nei0 <= 0xffff && nei1 <= 0xffff &&
+                                hoisted_row_id_words * sizeof(uint32_t) <=
+                                    ctx->device->properties.limits.maxStorageBufferRange;
 
     ggml_backend_vk_buffer_context * dst_buf_ctx = (ggml_backend_vk_buffer_context *)dst->buffer->context;
     ggml_backend_vk_buffer_context * src0_buf_ctx = (ggml_backend_vk_buffer_context *)src0->buffer->context;
     ggml_backend_vk_buffer_context * src1_buf_ctx = (ggml_backend_vk_buffer_context *)src1->buffer->context;
+    ggml_backend_vk_buffer_context * ids_buf_ctx = (ggml_backend_vk_buffer_context *)ids->buffer->context;
 
     vk_buffer d_Qx = nullptr;
     size_t qx_buf_offset = 0;
     vk_buffer d_Qy = nullptr;
     size_t qy_buf_offset = 0;
+    vk_buffer d_ids = nullptr;
+    size_t ids_buf_offset = 0;
 
     bool src0_uma = false;
     bool src1_uma = false;
+    bool ids_uma = false;
 
     if (ctx->device->uma) {
         ggml_vk_host_get(ctx->device, src0->data, d_Qx, qx_buf_offset);
         ggml_vk_host_get(ctx->device, src1->data, d_Qy, qy_buf_offset);
+        ggml_vk_host_get(ctx->device, ids->data, d_ids, ids_buf_offset);
         src0_uma = d_Qx != nullptr;
         src1_uma = d_Qy != nullptr;
+        ids_uma = d_ids != nullptr;
     }
 
     // Reformat and convert to fp16 if non-contiguous, or for coopmat2 for better perf
@@ -7424,11 +7439,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
 
     vk_pipeline pipeline = ggml_vk_guess_matmul_pipeline_map(ctx, *mmp_map, ne01, nei1, aligned, true);
 
-    const uint32_t bn = pipeline->wg_denoms[1];
-    const uint64_t max_tiles = ggml_vk_mul_mat_id_max_tiles(n_as, nei0 * nei1, bn);
-    const uint64_t row_ids_offset = 1 + 2 * max_tiles;
-    const uint64_t row_map_words = row_ids_offset + max_tiles * bn;
-
     if (ggml_nbytes(src0) > ctx->device->properties.limits.maxStorageBufferRange) {
         pipeline = ggml_vk_get_64b_indexing_pipeline(ctx, pipeline);
     }
@@ -7440,6 +7450,7 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
     const uint64_t qy_sz = ggml_type_size(src1->type) * ggml_nelements(src1) / ggml_blck_size(src1->type);
     const uint64_t x_sz = !qx_needs_dequant ? qx_sz : sizeof(ggml_fp16_t) * x_ne;
     const uint64_t y_sz = quantize_y ? (ggml_vk_align_size(y_ne, 128) * ggml_type_size(GGML_TYPE_Q8_1) / ggml_blck_size(GGML_TYPE_Q8_1)) : (y_f32_kernel ? sizeof(float) * y_ne : sizeof(ggml_fp16_t) * y_ne);
+    const uint64_t ids_sz = nbi2;
     const uint64_t d_sz = sizeof(float) * d_ne;
 
     vk_pipeline to_fp16_vk_0 = nullptr;
@@ -7481,7 +7492,8 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
     }
     vk_pipeline count_experts = ctx->device->pipeline_count_experts;
 
-    const size_t expert_data_size = sizeof(uint32_t) * row_map_words;
+    const size_t expert_data_size = sizeof(uint32_t) *
+        (hoist_row_ids ? hoisted_row_id_words : n_as);
 
     {
         if (
@@ -7533,6 +7545,11 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
         qy_buf_offset = vk_tensor_offset(src1) + src1->view_offs;
         GGML_ASSERT(d_Qy != nullptr);
     }
+    if (!ids_uma) {
+        d_ids = ids_buf_ctx->dev_buffer;
+        ids_buf_offset = vk_tensor_offset(ids) + ids->view_offs;
+        GGML_ASSERT(d_ids != nullptr);
+    }
     if (qx_needs_dequant) {
         d_X = ctx->prealloc_x;
         GGML_ASSERT(d_X->size >= x_sz);
@@ -7558,8 +7575,6 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
             ggml_vk_sync_buffers(ctx, subctx);
         }
     }
-    vk_subbuffer d_ids = ggml_vk_tensor_subbuffer(ctx, ids, true);
-
     // Count how many times each expert is used
     vk_subbuffer expert_count_buf = { ctx->prealloc_split_k, 0, expert_data_size };
     if (ctx->prealloc_split_k_need_sync) {
@@ -7572,11 +7587,12 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
                                            (uint32_t)(nbi1 / ggml_type_size(ids->type)),
                                            (uint32_t)(get_misalign_bytes(ctx, ids) / ggml_type_size(ids->type)),
                                            (uint32_t)n_as,
-                                           0, 0, bn, (uint32_t)row_ids_offset };
+                                           uint32_t(hoist_row_ids),
+                                           0, 0 };
         init_pushconst_fastdiv(pc);
         ggml_vk_dispatch_pipeline(ctx, subctx, count_experts,
-            { d_ids, expert_count_buf }, pc,
-            { 1, 1, 1 });
+            { vk_subbuffer{ d_ids, ids_buf_offset, ids_sz }, expert_count_buf }, pc,
+            { hoist_row_ids ? 1u : (uint32_t)n_as, 1, 1});
     }
 
     if (x_non_contig) {
@@ -7649,10 +7665,10 @@ static void ggml_vk_mul_mat_id_q_f16(ggml_backend_vk_context * ctx, vk_context&
     ggml_vk_matmul_id(
         ctx, subctx, pipeline,
         { d_X, x_buf_offset, x_range }, { d_Y, y_buf_offset, y_range },
-        { d_D, d_buf_offset, d_sz }, std::move(d_ids), expert_count_buf,
+        { d_D, d_buf_offset, d_sz }, { d_ids, ids_buf_offset, ids_sz }, expert_count_buf,
         ne01, ne21, ne10, ne10, stride_b_y, ne01,
         stride_batch_x, stride_batch_y, ne20*ne21,
-        n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, row_ids_offset, max_tiles
+        n_as, nei0, nei1, nbi1 / ggml_type_size(ids->type), ne11, hoist_row_ids
     );  // NOLINT
 
     if (x_non_contig || qx_needs_dequant) {
@@ -7901,14 +7917,11 @@ static void ggml_vk_mul_mat_vec_id_q_f16(ggml_backend_vk_context * ctx, vk_conte
     }
 }
 
-static bool ggml_vk_use_mul_mat_vec_id(const ggml_tensor * dst) {
-    const ggml_tensor * src0 = dst->src[0];
-    const ggml_tensor * ids = dst->src[2];
-    return ids->ne[1] <= 8 && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type));
-}
-
 bool ggml_vk_use_mul_mat_vec_id(const struct ggml_cgraph * cgraph, int node_idx) {
-    return ggml_vk_use_mul_mat_vec_id(cgraph->nodes[node_idx]);
+    ggml_tensor * dst = cgraph->nodes[node_idx];
+    ggml_tensor * src0 = dst->src[0];
+    ggml_tensor * src2 = dst->src[2];
+    return (src2->ne[1] <= 8) && (src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16 || ggml_is_quantized(src0->type));
 }
 
 void ggml_vk_mul_mat_id(ggml_backend_vk_context * ctx, vk_context& subctx, const struct ggml_cgraph * cgraph, int node_idx) {
@@ -15412,13 +15425,6 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
             {
                 ggml_type src0_type = op->src[0]->type;
                 if (op->op == GGML_OP_MUL_MAT_ID) {
-                    if (!ggml_vk_use_mul_mat_vec_id(op)) {
-                        const ggml_tensor * ids = op->src[2];
-                        // Shaders store row IDs as uint16_t slot and token indices.
-                        if (ids->ne[0] > int64_t(UINT16_MAX) + 1 || ids->ne[1] > int64_t(UINT16_MAX) + 1) {
-                            return false;
-                        }
-                    }
                     if (!device->mul_mat_id_s[src0_type] && !device->mul_mat_id_m[src0_type] && !device->mul_mat_id_l[src0_type]) {
                         // If there's not enough shared memory for row_ids and the result tile, fallback to CPU
                         return false;

ggml/src/ggml-vulkan/vulkan-shaders/count_experts.comp

@@ -5,7 +5,6 @@
 #ifdef USE_SUBGROUPS
 #extension GL_KHR_shader_subgroup_basic : enable
 #extension GL_KHR_shader_subgroup_arithmetic : enable
-#extension GL_KHR_shader_subgroup_vote : enable
 #endif
 
 #include "types.glsl"
@@ -19,47 +18,40 @@ layout (push_constant) uniform parameter
     uint32_t nb01;
     uint32_t a_offset;
     uint32_t n_experts;
+    uint32_t hoist_row_ids;
     uint32_t ne00mp;
     uint32_t ne00L;
-    uint32_t row_tile_size;
-    uint32_t row_ids_offset;
 } p;
 
 #define BLOCK_SIZE 256
-#define EXPERTS_PER_CHUNK 1024
 
 layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
 
 layout (binding = 0) readonly buffer A {uint data_a[];};
 layout (binding = 1) writeonly buffer D {uint data_d[];};
 
-shared uint vals[EXPERTS_PER_CHUNK];
-shared uint offsets[EXPERTS_PER_CHUNK];
-shared uint cursors[EXPERTS_PER_CHUNK];
+// Upper bound on n_experts for the hoisted row-id path. Must match the limit in
+// ggml_vk_mul_mat_id_q_f16 (hoist_row_ids). The non-hoisted reduction below only
+// needs BLOCK_SIZE entries.
+#define MAX_EXPERTS 1024
 
-// data_d[0] is the tile count. Each tile has (expert, valid rows).
-// Packed (token << 16) | slot row ids start at p.row_ids_offset, with each expert on a tile boundary.
-void write_tiles(uint expert, uint count, uint tile_begin, uint n_tiles) {
-    for (uint t = 0; t < n_tiles; ++t) {
-        const uint descriptor = 1 + 2 * (tile_begin + t);
-        data_d[descriptor] = expert;
-        data_d[descriptor + 1] = min(p.row_tile_size, count - t * p.row_tile_size);
-    }
-}
+shared uint vals[MAX_EXPERTS];
+shared uint offsets[MAX_EXPERTS];
+shared uint cursors[MAX_EXPERTS];
 
+// data_d layout when p.hoist_row_ids is set:
+//   [0,              n_experts)   per-expert row count
+//   [n_experts,    2*n_experts)   per-expert start offset into the row id region
+//   [2*n_experts]                 total row count
+//   [2*n_experts + 1,         )   row ids grouped by expert, packed as (i01 << 16) | (i00 & 0xffff)
+// Otherwise only data_d[expert_id] is written, holding that expert's row count.
 void main() {
+    const uint expert_id = gl_WorkGroupID.x;
     const uint num_elements = p.ne00 * p.ne01;
     const uint tid = gl_LocalInvocationID.x;
-    const uint tile_shift = findLSB(p.row_tile_size);
-    uint total_tiles = 0;
-#ifdef USE_SUBGROUPS
-    // Use the subgroup that contains invocation 0.
-    const bool prefix_subgroup = subgroupAny(tid == 0);
-#endif
 
-    for (uint expert_base = 0; expert_base < p.n_experts;) {
-        const uint n_experts = min(EXPERTS_PER_CHUNK, p.n_experts - expert_base);
-        for (uint e = tid; e < n_experts; e += BLOCK_SIZE) {
+    if (p.hoist_row_ids != 0) {
+        for (uint e = tid; e < p.n_experts; e += BLOCK_SIZE) {
             vals[e] = 0;
         }
         barrier();
@@ -67,43 +59,46 @@ void main() {
         for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
             const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
             const uint i00 = idx - i01 * p.ne00;
-            const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00] - expert_base;
-            if (expert < n_experts) {
+            const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
+            if (expert < p.n_experts) {
                 atomicAdd(vals[expert], 1);
             }
         }
         barrier();
 
 #ifdef USE_SUBGROUPS
-        if (prefix_subgroup) {
-            const uint n_experts_padded = (n_experts + gl_SubgroupSize - 1) & ~(gl_SubgroupSize - 1);
-            uint base = subgroupAdd(tid == 0 ? total_tiles : 0u);
-            for (uint e = gl_SubgroupInvocationID; e < n_experts_padded; e += gl_SubgroupSize) {
-                const bool in_range = e < n_experts;
-                const uint count = in_range ? vals[e] : 0;
-                const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift;
-                const uint tile_begin = base + subgroupExclusiveAdd(n_tiles);
+        if (gl_SubgroupID == 0) {
+            // pad the trip count so the subgroup ops stay in uniform control flow
+            const uint n_experts_padded = (p.n_experts + gl_SubgroupSize - 1) & ~(gl_SubgroupSize - 1);
+            uint base = 0;
+            for (uint expert = gl_SubgroupInvocationID; expert < n_experts_padded; expert += gl_SubgroupSize) {
+                const bool in_range = expert < p.n_experts;
+                const uint count = in_range ? vals[expert] : 0;
+                const uint offset = base + subgroupExclusiveAdd(count);
                 if (in_range) {
-                    offsets[e] = tile_begin * p.row_tile_size;
-                    cursors[e] = 0;
-                    write_tiles(expert_base + e, count, tile_begin, n_tiles);
+                    data_d[expert] = count;
+                    data_d[p.n_experts + expert] = offset;
+                    offsets[expert] = offset;
+                    cursors[expert] = 0;
                 }
-                base += subgroupAdd(n_tiles);
+                base += subgroupAdd(count);
             }
-            if (tid == 0) {
-                total_tiles = base;
+            if (subgroupElect()) {
+                data_d[2 * p.n_experts] = base;
             }
         }
 #else
         if (tid == 0) {
-            for (uint e = 0; e < n_experts; ++e) {
-                const uint count = vals[e];
-                const uint n_tiles = (count + p.row_tile_size - 1) >> tile_shift;
-                offsets[e] = total_tiles * p.row_tile_size;
-                cursors[e] = 0;
-                write_tiles(expert_base + e, count, total_tiles, n_tiles);
-                total_tiles += n_tiles;
+            uint offset = 0;
+            for (uint expert = 0; expert < p.n_experts; ++expert) {
+                const uint count = vals[expert];
+                data_d[expert] = count;
+                data_d[p.n_experts + expert] = offset;
+                offsets[expert] = offset;
+                cursors[expert] = 0;
+                offset += count;
             }
+            data_d[2 * p.n_experts] = offset;
         }
 #endif
         barrier();
@@ -111,16 +106,35 @@ void main() {
         for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
             const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
             const uint i00 = idx - i01 * p.ne00;
-            const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00] - expert_base;
-            if (expert < n_experts) {
+            const uint expert = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
+            if (expert < p.n_experts) {
                 const uint row = atomicAdd(cursors[expert], 1);
-                data_d[p.row_ids_offset + offsets[expert] + row] = (i01 << 16) | i00;
+                const uint packed_row_id = (i01 << 16) | (i00 & 0xffffu);
+                data_d[2 * p.n_experts + 1 + offsets[expert] + row] = packed_row_id;
             }
         }
-        // The next chunk clears vals. Scatter uses only offsets and cursors.
-        expert_base += n_experts;
+        return;
     }
+
+    uint count = 0;
+    for (uint idx = tid; idx < num_elements; idx += BLOCK_SIZE) {
+        const uint i01 = fastdiv(idx, p.ne00mp, p.ne00L);
+        const uint i00 = idx - i01 * p.ne00;
+        const uint a = data_a[p.a_offset + i01 * p.nb01 + i00 * p.nb00];
+
+        count += uint(a == expert_id);
+    }
+
+    vals[tid] = count;
+    barrier();
+    [[unroll]] for (uint s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
+        if (tid < s) {
+            vals[tid] += vals[tid + s];
+        }
+        barrier();
+    }
+
     if (tid == 0) {
-        data_d[0] = total_tiles;
+        data_d[expert_id] = vals[0];
     }
 }

ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp

@@ -124,7 +124,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
 
 #ifdef MUL_MAT_ID
 layout (binding = 3) readonly buffer IDS {int data_ids[];};
-layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};
+layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
 #endif
 
 layout (push_constant) uniform parameter
@@ -146,7 +146,7 @@ layout (push_constant) uniform parameter
     uint nbi1;
     uint ne11;
     uint n_experts;
-    uint row_ids_offset;
+    uint hoist_row_ids;
 #else
     uint base_work_group_z;
     uint num_batches;
@@ -208,17 +208,13 @@ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS];
 #endif
 
 void main() {
+    const uint ic = gl_WorkGroupID.y;
+
 #ifdef MUL_MAT_ID
-    const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;
-    if (tile_idx >= data_row_map[0]) {
+    const uint expert_idx = gl_WorkGroupID.z;
+    if (ic * BN >= data_expert_count[expert_idx]) {
         return;
     }
-    const uint expert_idx = data_row_map[1 + 2 * tile_idx];
-    const uint row_begin = tile_idx * BN;
-    _ne1 = data_row_map[2 + 2 * tile_idx];
-    const uint ic = 0;
-#else
-    const uint ic = gl_WorkGroupID.y;
 #endif
 #if defined(NEEDS_INIT_IQ_SHMEM) || defined(MULMAT_QUANT)
     init_iq_shmem(gl_WorkGroupSize);
@@ -289,9 +285,37 @@ void main() {
     const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;
 
 #ifdef MUL_MAT_ID
-    load_row_ids(row_begin);
-    uint required_work_items = _ne1 * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B;
-    uint required_warp_c = (_ne1 + WN - 1) / WN;
+    if (p.hoist_row_ids != 0) {
+        load_row_ids_hoisted(expert_idx, ic);
+    } else {
+#ifdef MUL_MAT_ID_USE_SUBGROUPS
+        if (bitCount(p.nei0) == 1) {
+            load_row_ids(expert_idx, true, ic);
+        } else {
+            load_row_ids(expert_idx, false, ic);
+        }
+#else
+        _ne1 = 0;
+        for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
+            for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
+                if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
+                    if (_ne1 >= ic * BN) {
+                        row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
+                    }
+                    _ne1++;
+                }
+            }
+        }
+
+        barrier();
+#endif
+    }
+
+    // Workgroup has no work
+    if (ic * BN >= _ne1) return;
+
+    uint required_work_items = (_ne1 - ic * BN) * BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B;
+    uint required_warp_c = (_ne1 - ic * BN + WN - 1) / WN;
 #endif
 
 #ifdef MUL_MAT_ID

ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp

@@ -49,12 +49,17 @@ layout (constant_id = 6) const uint subgroup_size = 32;
 #ifdef MULMAT_QUANT
 
 uint mm_quant_k() {
-    // Use ternary expressions instead of a switch to keep qk as a specialization-constant expression.
-    return (MmTypeA == GGML_TYPE_Q1_0) ? 128u :
-           (MmTypeA == GGML_TYPE_Q2_0) ?  64u :
-          ((MmTypeA == GGML_TYPE_Q4_0 || MmTypeA == GGML_TYPE_Q4_1 ||
-            MmTypeA == GGML_TYPE_Q5_0 || MmTypeA == GGML_TYPE_Q5_1 ||
-            MmTypeA == GGML_TYPE_Q8_0) ? 32u : 256u);
+    switch (MmTypeA) {
+    case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1:
+    case GGML_TYPE_Q8_0:
+        return 32u;
+    case GGML_TYPE_Q1_0:
+        return 128u;
+    case GGML_TYPE_Q2_0:
+        return 64u;
+    default:
+        return 256u;
+    }
 }
 #endif
 
@@ -77,7 +82,7 @@ layout (push_constant) uniform parameter
     uint nbi1;
     uint ne11;
     uint n_experts;
-    uint row_ids_offset;
+    uint hoist_row_ids;
 #else
     uint base_work_group_z;
     uint num_batches;
@@ -177,7 +182,7 @@ f16vec4 mmDecodeA_v(const in decodeBufA bl_in, const in uint blockCoords[2], con
 
 #ifdef MUL_MAT_ID
 layout (binding = 3) readonly buffer IDS {int data_ids[];};
-layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};
+layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
 
 shared u16vec4 row_ids[BN];
 
@@ -186,6 +191,7 @@ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB {
 };
 
 uint _ne1;
+shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size];
 
 B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2])
 {
@@ -232,9 +238,82 @@ D_TYPE perElemOpD(const in uint32_t r, const in uint32_t c, const in D_TYPE elem
     return elem;
 }
 
-void load_row_ids(uint row_begin) {
-    for (uint i = gl_LocalInvocationIndex; i < _ne1; i += BLOCK_SIZE) {
-        const uint packed_row_id = data_row_map[p.row_ids_offset + row_begin + i];
+void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
+    _ne1 = 0;
+    uint num_elements = p.nei1 * p.nei0;
+    uint nei0shift = findLSB(p.nei0);
+
+    uint ids[16];
+    uint iter = 0;
+
+    uint expert_count = data_expert_count[expert_idx];
+
+    for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {
+        // prefetch up to 16 elements
+        if (iter == 0) {
+            [[unroll]] for (uint k = 0; k < 16; ++k) {
+                uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;
+                bool in_range = i < num_elements;
+                uint ii1;
+                if (nei0_is_pow2) {
+                    ii1 = i >> nei0shift;
+                } else {
+                    ii1 = i / p.nei0;
+                }
+                uint ii0 = i - ii1 * p.nei0;
+                ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;
+            }
+        }
+        uint i = j + gl_LocalInvocationIndex;
+        bool in_range = i < num_elements;
+        uint ii1;
+        if (nei0_is_pow2) {
+            ii1 = i >> nei0shift;
+        } else {
+            ii1 = i / p.nei0;
+        }
+        uint ii0 = i - ii1 * p.nei0;
+        uint id = ids[iter++];
+        uvec4 ballot = subgroupBallot(in_range && id == expert_idx);
+
+        if (gl_SubgroupInvocationID == 0) {
+            ballots_sh[gl_SubgroupID] = ballot;
+        }
+        barrier();
+
+        uint subgroup_base = 0;
+        uint total = 0;
+        for (uint k = 0; k < gl_NumSubgroups; ++k) {
+            if (k == gl_SubgroupID) {
+                subgroup_base = total;
+            }
+            total += subgroupBallotBitCount(ballots_sh[k]);
+        }
+        barrier();
+
+        uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);
+        if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {
+            row_ids[_ne1 + idx - ic * BN] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0);
+        }
+        _ne1 += total;
+        iter &= 15;
+        if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {
+            break;
+        }
+    }
+    barrier();
+}
+
+void load_row_ids_hoisted(uint expert_idx, uint ic) {
+    _ne1 = uint(data_expert_count[expert_idx]);
+
+    const uint tile_begin = ic * BN;
+    const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0;
+    const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]);
+    const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin;
+
+    for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) {
+        const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]);
         const uint ii0 = packed_row_id & 0xffffu;
         const uint ii1 = packed_row_id >> 16;
         row_ids[i] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0);
@@ -249,20 +328,13 @@ void load_row_ids(uint row_begin) {
 
 void main() {
     const uint tid = gl_LocalInvocationIndex;
-#ifdef MUL_MAT_ID
-    const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;
-    if (tile_idx >= data_row_map[0]) {
-        return;
-    }
-    const uint expert_idx = data_row_map[1 + 2 * tile_idx];
-    const uint row_begin = tile_idx * BN;
-    _ne1 = data_row_map[2 + 2 * tile_idx];
-    const uint ic = 0;
-#else
     const uint ic = gl_WorkGroupID.y;
-#endif
 
 #ifdef MUL_MAT_ID
+    const uint expert_idx = gl_WorkGroupID.z;
+    if (ic * BN >= data_expert_count[expert_idx]) {
+        return;
+    }
     // initialize to row 0 so we don't need to bounds check
     if (tid < BN) {
         row_ids[tid] = u16vec4(0);
@@ -293,7 +365,16 @@ void main() {
     const uint ik = gl_WorkGroupID.x / blocks_m;
 
 #ifdef MUL_MAT_ID
-    load_row_ids(row_begin);
+    if (p.hoist_row_ids != 0) {
+        load_row_ids_hoisted(expert_idx, ic);
+    } else if (bitCount(p.nei0) == 1) {
+        load_row_ids(expert_idx, true, ic);
+    } else {
+        load_row_ids(expert_idx, false, ic);
+    }
+
+    // Workgroup has no work
+    if (ic * BN >= _ne1) return;
 #endif
 
 #ifdef MUL_MAT_ID

ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl

@@ -2,9 +2,86 @@
 shared u16vec2 row_ids[BN];
 uint _ne1;
 
-void load_row_ids(uint row_begin) {
-    for (uint i = gl_LocalInvocationIndex; i < _ne1; i += BLOCK_SIZE) {
-        const uint packed_row_id = data_row_map[p.row_ids_offset + row_begin + i];
+#ifdef MUL_MAT_ID_USE_SUBGROUPS
+shared uvec4 ballots_sh[NUM_WARPS];
+
+void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
+    _ne1 = 0;
+    uint num_elements = p.nei1 * p.nei0;
+    uint nei0shift = findLSB(p.nei0);
+
+    uint ids[16];
+    uint iter = 0;
+
+    uint expert_count = data_expert_count[expert_idx];
+
+    for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {
+        // prefetch up to 16 elements
+        if (iter == 0) {
+            [[unroll]] for (uint k = 0; k < 16; ++k) {
+                uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;
+                bool in_range = i < num_elements;
+                uint ii1;
+                if (nei0_is_pow2) {
+                    ii1 = i >> nei0shift;
+                } else {
+                    ii1 = i / p.nei0;
+                }
+                uint ii0 = i - ii1 * p.nei0;
+                ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;
+            }
+        }
+        uint i = j + gl_LocalInvocationIndex;
+        bool in_range = i < num_elements;
+        uint ii1;
+        if (nei0_is_pow2) {
+            ii1 = i >> nei0shift;
+        } else {
+            ii1 = i / p.nei0;
+        }
+        uint ii0 = i - ii1 * p.nei0;
+        uint id = ids[iter++];
+        uvec4 ballot = subgroupBallot(in_range && id == expert_idx);
+
+        if (gl_SubgroupInvocationID == 0) {
+            ballots_sh[gl_SubgroupID] = ballot;
+        }
+        barrier();
+
+        uint subgroup_base = 0;
+        uint total = 0;
+        for (uint k = 0; k < gl_NumSubgroups; ++k) {
+            if (k == gl_SubgroupID) {
+                subgroup_base = total;
+            }
+            total += subgroupBallotBitCount(ballots_sh[k]);
+        }
+        barrier();
+
+        uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);
+        if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {
+            row_ids[_ne1 + idx - ic * BN] = u16vec2(ii0, ii1);
+        }
+        _ne1 += total;
+        iter &= 15;
+        if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {
+            break;
+        }
+    }
+    barrier();
+}
+#endif // MUL_MAT_ID_USE_SUBGROUPS
+
+void load_row_ids_hoisted(uint expert_idx, uint ic) {
+    _ne1 = uint(data_expert_count[expert_idx]);
+
+    const uint tile_begin = ic * BN;
+    const uint tile_count = tile_begin < _ne1 ? min(BN, _ne1 - tile_begin) : 0;
+    const uint expert_offset = uint(data_expert_count[p.n_experts + expert_idx]);
+    const uint row_ids_offset = 2 * p.n_experts + 1 + expert_offset + tile_begin;
+
+    for (uint i = gl_LocalInvocationIndex; i < tile_count; i += BLOCK_SIZE) {
+        const uint packed_row_id = uint(data_expert_count[row_ids_offset + i]);
         row_ids[i] = u16vec2(packed_row_id & 0xffffu, packed_row_id >> 16);
     }
     barrier();

ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp

@@ -37,7 +37,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
 
 #ifdef MUL_MAT_ID
 layout (binding = 3) readonly buffer IDS {int data_ids[];};
-layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};
+layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
 #endif
 
 layout (push_constant) uniform parameter
@@ -59,7 +59,7 @@ layout (push_constant) uniform parameter
     uint nbi1;
     uint ne11;
     uint n_experts;
-    uint row_ids_offset;
+    uint hoist_row_ids;
 #else
     uint base_work_group_z;
     uint num_batches;
@@ -105,22 +105,19 @@ block_b_cache cache_b;
 #define LOAD_VEC_A (4 * QUANT_R_MMQ)
 #define LOAD_VEC_B 16
 
+#define NUM_WARPS (BLOCK_SIZE / WARP)
 
 #include "mul_mm_id_funcs.glsl"
 #include "mul_mmq_funcs.glsl"
 
 void main() {
+    const uint ic = gl_WorkGroupID.y;
+
 #ifdef MUL_MAT_ID
-    const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;
-    if (tile_idx >= data_row_map[0]) {
+    const uint expert_idx = gl_WorkGroupID.z;
+    if (ic * BN >= data_expert_count[expert_idx]) {
         return;
     }
-    const uint expert_idx = data_row_map[1 + 2 * tile_idx];
-    const uint row_begin = tile_idx * BN;
-    _ne1 = data_row_map[2 + 2 * tile_idx];
-    const uint ic = 0;
-#else
-    const uint ic = gl_WorkGroupID.y;
 #endif
 #ifdef NEEDS_INIT_IQ_SHMEM
     init_iq_shmem(gl_WorkGroupSize);
@@ -164,7 +161,34 @@ void main() {
     const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK;
 
 #ifdef MUL_MAT_ID
-    load_row_ids(row_begin);
+    if (p.hoist_row_ids != 0) {
+        load_row_ids_hoisted(expert_idx, ic);
+    } else {
+#ifdef MUL_MAT_ID_USE_SUBGROUPS
+        if (bitCount(p.nei0) == 1) {
+            load_row_ids(expert_idx, true, ic);
+        } else {
+            load_row_ids(expert_idx, false, ic);
+        }
+#else
+        _ne1 = 0;
+        for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
+            for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
+                if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
+                    if (_ne1 >= ic * BN) {
+                        row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
+                    }
+                    _ne1++;
+                }
+            }
+        }
+
+        barrier();
+#endif
+    }
+
+    // Workgroup has no work
+    if (ic * BN >= _ne1) return;
 #endif
 
 #ifdef MUL_MAT_ID

ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_cm1.comp

@@ -39,7 +39,7 @@ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
 
 #ifdef MUL_MAT_ID
 layout (binding = 3) readonly buffer IDS {int data_ids[];};
-layout (binding = 4) readonly buffer RowMap {uint data_row_map[];};
+layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
 #endif
 
 layout (push_constant) uniform parameter
@@ -61,7 +61,7 @@ layout (push_constant) uniform parameter
     uint nbi1;
     uint ne11;
     uint n_experts;
-    uint row_ids_offset;
+    uint hoist_row_ids;
 #else
     uint base_work_group_z;
     uint num_batches;
@@ -142,22 +142,13 @@ ACC_TYPE cm1_accumulate(ACC_TYPE prev, int acc_e, float scale_a, float nbias_a,
 }
 
 #ifdef MUL_MAT_ID
+#define NUM_WARPS (BLOCK_SIZE / WARP)
 #include "mul_mm_id_funcs.glsl"
 #endif
 
 #include "mul_mmq_cm1_funcs.glsl"
 
 void main() {
-#ifdef MUL_MAT_ID
-    const uint tile_idx = gl_WorkGroupID.y + gl_WorkGroupID.z * gl_NumWorkGroups.y;
-    if (tile_idx >= data_row_map[0]) {
-        return;
-    }
-    const uint expert_idx = data_row_map[1 + 2 * tile_idx];
-    const uint row_begin = tile_idx * BN;
-    _ne1 = data_row_map[2 + 2 * tile_idx];
-    const uint ic = 0;
-#endif
 #if defined(DATA_A_IQ4_NL) || defined(DATA_A_IQ4_XS)
     if (gl_LocalInvocationIndex < 16u) {
         cm1_kvalues[gl_LocalInvocationIndex] = kvalues_iq4nl_const[gl_LocalInvocationIndex];
@@ -184,7 +175,12 @@ void main() {
     const uint ik = gl_WorkGroupID.x / blocks_m;
 
 #ifdef MUL_MAT_ID
+    const uint ic = gl_WorkGroupID.y;
     const uint ir = gl_WorkGroupID.x % blocks_m;
+    const uint expert_idx = gl_WorkGroupID.z;
+    if (ic * BN >= data_expert_count[expert_idx]) {
+        return;
+    }
 #else
     // L2-friendly workgroup scheduling
     const uint blocks_n = (p.N + BN - 1) / BN;
@@ -235,7 +231,33 @@ void main() {
     const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK;
 
 #ifdef MUL_MAT_ID
-    load_row_ids(row_begin);
+    if (p.hoist_row_ids != 0) {
+        load_row_ids_hoisted(expert_idx, ic);
+    } else {
+#ifdef MUL_MAT_ID_USE_SUBGROUPS
+        if (bitCount(p.nei0) == 1) {
+            load_row_ids(expert_idx, true, ic);
+        } else {
+            load_row_ids(expert_idx, false, ic);
+        }
+#else
+        _ne1 = 0;
+        for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
+            for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
+                if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
+                    if (_ne1 >= ic * BN) {
+                        row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
+                    }
+                    _ne1++;
+                }
+            }
+        }
+
+        barrier();
+#endif
+    }
+
+    if (ic * BN >= _ne1) return;
 #endif
 
 #ifdef MUL_MAT_ID

gguf-py/gguf/constants.py

@@ -259,7 +259,6 @@ class Keys:
         DIMENSION_COUNT           = "{arch}.rope.dimension_count"
         DIMENSION_COUNT_SWA       = "{arch}.rope.dimension_count_swa"
         DIMENSION_SECTIONS        = "{arch}.rope.dimension_sections"
-        SECTION_ORDER             = "{arch}.rope.section_order"
         FREQ_BASE                 = "{arch}.rope.freq_base"
         FREQ_BASE_SWA             = "{arch}.rope.freq_base_swa"
         SCALING_TYPE              = "{arch}.rope.scaling.type"
@@ -332,7 +331,6 @@ class Keys:
         HEAD_COUNT         = "{arch}.decision.head_count"
         MAX_HEAD_TOKENS    = "{arch}.decision.max_head_tokens"
         TEMPERATURE        = "{arch}.decision.temperature.{name}"  # name: "<type>" or "<type>.<n_opt bucket>"
-        SCORE_BIAS         = "{arch}.decision.score_bias.{n}.{i}"  # added to the logit of level i of a score question with n levels
 
     class Tokenizer:
         MODEL                = "tokenizer.ggml.model"
@@ -411,7 +409,6 @@ class Keys:
         BLOCK_COUNT           = "clip.vision.block_count"
         IMAGE_MEAN            = "clip.vision.image_mean"
         IMAGE_STD             = "clip.vision.image_std"
-        MAX_SLICE_NUMS        = "clip.vision.max_slice_nums"
         IMAGE_RESIZE_ALGO     = "clip.vision.image_resize_algo"
         SPATIAL_MERGE_SIZE    = "clip.vision.spatial_merge_size"
         SWIGLU_CLAMP          = "clip.vision.swiglu_clamp"
@@ -496,19 +493,6 @@ class Keys:
         BETA                = "xielu.beta"
         EPS                 = "xielu.eps"
 
-    class PrismHadamard:
-        VERSION              = "prism.hadamard.version"
-        TIED_OUTPUT          = "prism.hadamard.tied_output"
-        BLOCK_SIZE           = "prism.hadamard.block_size"
-        TRANSFORM            = "prism.hadamard.transform"
-        AXIS                 = "prism.hadamard.axis"
-        SIGN_MODE            = "prism.hadamard.sign_mode"
-        SIGN_WIDTHS          = "prism.hadamard.sign_widths"
-        SIGN_VALUES          = "prism.hadamard.sign_values"
-        WEIGHT_NAMES         = "prism.hadamard.weight_names"
-        INVERSE_WEIGHT_NAMES = "prism.hadamard.inverse_weight_names"
-        GDN_V_GROUPED        = "prism.hadamard.gdn_v_grouped"
-
 
 #
 # recommended mapping of model tensor names for storage in gguf
@@ -560,7 +544,6 @@ class MODEL_ARCH(IntEnum):
     QWEN35           = auto()
     QWEN35MOE        = auto()
     CLEF             = auto()
-    VLLM_DECISION2   = auto()
     QWEN4EXP         = auto()
     PHI2             = auto()
     PHI3             = auto()
@@ -925,11 +908,6 @@ class MODEL_TENSOR(IntEnum):
     DECISION_SCALES               = auto()
     DECISION_SCORER               = auto()
     DECISION_SCORER_OUT           = auto()
-    DECISION_QUERY_NORM           = auto()
-    DECISION_KEY                  = auto()
-    DECISION_QUERY                = auto()
-    DECISION_OPTION_MLP           = auto()
-    DECISION_QUERY_MLP            = auto()
     CONV1D               = auto()
     CONVNEXT_DW          = auto()
     CONVNEXT_NORM        = auto()
@@ -1073,7 +1051,6 @@ class MODEL_TENSOR(IntEnum):
     V_SAM_NET_3          = auto() # Deepseek-OCR
     V_ENC_EMBD_IMGNL     = auto() # Deepseek-OCR
     V_ENC_EMBD_VSEP      = auto() # Deepseek-OCR
-    V_TOK_EMBD_SEP       = auto() # MiniCPM-V 4.7
     V_RESMPL_QUERY_768   = auto() # Deepseek-OCR-2
     V_RESMPL_QUERY_1024  = auto() # Deepseek-OCR-2
 
@@ -1350,7 +1327,6 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.QWEN35:           "qwen35",
     MODEL_ARCH.QWEN35MOE:        "qwen35moe",
     MODEL_ARCH.CLEF:             "clef",
-    MODEL_ARCH.VLLM_DECISION2:   "vllm-decision2",
     MODEL_ARCH.QWEN4EXP:         "qwen4exp",
     MODEL_ARCH.PHI2:             "phi2",
     MODEL_ARCH.PHI3:             "phi3",
@@ -1714,11 +1690,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.DECISION_SCALES:               "decision.scales",
     MODEL_TENSOR.DECISION_SCORER:               "decision.scorer",
     MODEL_TENSOR.DECISION_SCORER_OUT:           "decision.scorer_out",
-    MODEL_TENSOR.DECISION_QUERY_NORM:           "decision.query_norm",
-    MODEL_TENSOR.DECISION_KEY:                  "decision.key",
-    MODEL_TENSOR.DECISION_QUERY:                "decision.query",
-    MODEL_TENSOR.DECISION_OPTION_MLP:           "decision.option_mlp",
-    MODEL_TENSOR.DECISION_QUERY_MLP:            "decision.query_mlp",
     MODEL_TENSOR.CONV1D:                    "conv1d",
     MODEL_TENSOR.CONVNEXT_DW:               "convnext.{bid}.dw",
     MODEL_TENSOR.CONVNEXT_NORM:             "convnext.{bid}.norm",
@@ -1861,7 +1832,6 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.V_SAM_NET_3:               "v.sam.net_3",
     MODEL_TENSOR.V_ENC_EMBD_IMGNL:          "v.image_newline", # Deepseek-OCR, Granite4Vision
     MODEL_TENSOR.V_ENC_EMBD_VSEP:           "v.view_seperator", # Deepseek-OCR
-    MODEL_TENSOR.V_TOK_EMBD_SEP:            "v.tok_embd_sep", # MiniCPM-V 4.7
     MODEL_TENSOR.V_RESMPL_QUERY_768:        "v.resample_query_768", # Deepseek-OCR-2 qwen2
     MODEL_TENSOR.V_RESMPL_QUERY_1024:       "v.resample_query_1024", # Deepseek-OCR-2 qwen2
     # Granite4 Vision
@@ -2109,7 +2079,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.V_ENC_EMBD_POS,
         MODEL_TENSOR.V_ENC_EMBD_IMGNL,
         MODEL_TENSOR.V_ENC_EMBD_VSEP,
-        MODEL_TENSOR.V_TOK_EMBD_SEP,
         MODEL_TENSOR.V_ENC_INPUT_NORM,
         MODEL_TENSOR.V_ENC_ATTN_QKV,
         MODEL_TENSOR.V_ENC_ATTN_Q,
@@ -3047,29 +3016,6 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.DECISION_SCORER,
         MODEL_TENSOR.DECISION_SCORER_OUT,
     ],
-    MODEL_ARCH.VLLM_DECISION2: [
-        MODEL_TENSOR.TOKEN_EMBD,
-        MODEL_TENSOR.OUTPUT_NORM,
-        MODEL_TENSOR.ATTN_NORM,
-        MODEL_TENSOR.ATTN_Q,
-        MODEL_TENSOR.ATTN_Q_NORM,
-        MODEL_TENSOR.ATTN_K,
-        MODEL_TENSOR.ATTN_K_NORM,
-        MODEL_TENSOR.ATTN_V,
-        MODEL_TENSOR.ATTN_OUT,
-        MODEL_TENSOR.FFN_NORM,
-        MODEL_TENSOR.FFN_GATE,
-        MODEL_TENSOR.FFN_DOWN,
-        MODEL_TENSOR.FFN_UP,
-        # decision head
-        MODEL_TENSOR.DECISION_OPTION_NORM,
-        MODEL_TENSOR.DECISION_QUERY_NORM,
-        MODEL_TENSOR.DECISION_KEY,
-        MODEL_TENSOR.DECISION_QUERY,
-        MODEL_TENSOR.DECISION_OPTION_MLP,
-        MODEL_TENSOR.DECISION_QUERY_MLP,
-        MODEL_TENSOR.DECISION_SCORER_OUT,
-    ],
     MODEL_ARCH.QWEN35MOE: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -5981,12 +5927,6 @@ class RopeScalingType(Enum):
     LONGROPE = 'longrope'
 
 
-# M-RoPE: input position slot (t, y, x, z) that feeds each RoPE section, in section order
-class RopeSectionOrder(Enum):
-    TYXZ = 'tyxz' # default
-    ZYXT = 'zyxt'
-
-
 class PoolingType(IntEnum):
     NONE = 0
     MEAN = 1
@@ -6136,7 +6076,6 @@ class DecisionType:
     NIMBLE  = "nimble"   # same as openjev, the prompt lists all the questions of the request
     CLEF    = "clef"     # joint head over all questions, one score per option
     PPLX_DECIDER = "pplx-decider"  # same as openjev, label codes of 1 or 2 letters
-    VLLM_DECISION2 = "vllm-decision2"  # head over the end token of each option and the last token, one score per option
     LFM2_D1 = "lfm2-d1"  # same as openjev, the labels depend on the question type
     LFM2_D1_OMNI = "lfm2-d1-omni"  # same head as laya on a bidirectional LFM2 trunk, other prompt layout
 
@@ -6194,7 +6133,6 @@ class VisionProjectorType:
     PARAKEET       = "parakeet"  # audio
     MINIMAXM3      = "minimax_m3"
     MINICPMV4_6    = "minicpmv4_6"
-    MINICPMV4_7    = "minicpmv4_7"
     GRANITE_SPEECH = "granite_speech"  # audio
     MIMOVL         = "mimovl"
     MIMO_AUDIO     = "mimo_audio"

gguf-py/gguf/gguf_writer.py

@@ -24,7 +24,6 @@ from .constants import (
     GGUFEndian,
     GGUFValueType,
     Keys,
-    RopeSectionOrder,
     RopeScalingType,
     PoolingType,
     TokenType,
@@ -1155,9 +1154,6 @@ class GGUFWriter:
     def add_rope_dimension_sections(self, dims: Sequence[int]) -> None:
         self.add_array(Keys.Rope.DIMENSION_SECTIONS.format(arch=self.arch), dims)
 
-    def add_rope_section_order(self, value: RopeSectionOrder) -> None:
-        self.add_string(Keys.Rope.SECTION_ORDER.format(arch=self.arch), value.value)
-
     def add_rope_freq_base(self, value: float) -> None:
         self.add_float32(Keys.Rope.FREQ_BASE.format(arch=self.arch), value)
 
@@ -1365,10 +1361,6 @@ class GGUFWriter:
     def add_decision_temperature(self, name: str, value: float) -> None:
         self.add_float32(Keys.Decision.TEMPERATURE.format(arch=self.arch, name=name), value)
 
-    def add_decision_score_bias(self, values: Sequence[float]) -> None:
-        for i, value in enumerate(values):
-            self.add_float32(Keys.Decision.SCORE_BIAS.format(arch=self.arch, n=len(values), i=i), value)
-
     # for vision models
 
     def add_clip_has_vision_encoder(self, value: bool) -> None:
@@ -1470,9 +1462,6 @@ class GGUFWriter:
     def add_vision_projector_scale_factor(self, value: int) -> None:
         self.add_uint32(Keys.ClipVision.Projector.SCALE_FACTOR, value)
 
-    def add_vision_max_slice_nums(self, value: int) -> None:
-        self.add_uint32(Keys.ClipVision.MAX_SLICE_NUMS, value)
-
     def add_vision_n_wa_pattern(self, value: int) -> None:
         """Add window attention pattern interval for vision models.
 
@@ -1638,39 +1627,6 @@ class GGUFWriter:
     def add_xielu_eps(self, values: Sequence[float]):
         self.add_array(Keys.xIELU.EPS, values)
 
-    def add_prism_hadamard_version(self, value: int) -> None:
-        self.add_uint32(Keys.PrismHadamard.VERSION, value)
-
-    def add_prism_hadamard_tied_output(self, value: bool) -> None:
-        self.add_bool(Keys.PrismHadamard.TIED_OUTPUT, value)
-
-    def add_prism_hadamard_block_size(self, value: int) -> None:
-        self.add_uint32(Keys.PrismHadamard.BLOCK_SIZE, value)
-
-    def add_prism_hadamard_transform(self, value: str) -> None:
-        self.add_string(Keys.PrismHadamard.TRANSFORM, value)
-
-    def add_prism_hadamard_axis(self, value: str) -> None:
-        self.add_string(Keys.PrismHadamard.AXIS, value)
-
-    def add_prism_hadamard_sign_mode(self, value: str) -> None:
-        self.add_string(Keys.PrismHadamard.SIGN_MODE, value)
-
-    def add_prism_hadamard_sign_widths(self, values: Sequence[int]) -> None:
-        self.add_array(Keys.PrismHadamard.SIGN_WIDTHS, values)
-
-    def add_prism_hadamard_sign_values(self, values: Sequence[int]) -> None:
-        self.add_array(Keys.PrismHadamard.SIGN_VALUES, values)
-
-    def add_prism_hadamard_weight_names(self, names: Sequence[str]) -> None:
-        self.add_array(Keys.PrismHadamard.WEIGHT_NAMES, names)
-
-    def add_prism_hadamard_inverse_weight_names(self, names: Sequence[str]) -> None:
-        self.add_array(Keys.PrismHadamard.INVERSE_WEIGHT_NAMES, names)
-
-    def add_prism_hadamard_gdn_v_grouped(self, value: bool) -> None:
-        self.add_bool(Keys.PrismHadamard.GDN_V_GROUPED, value)
-
     def add_attention_value_expert_count(self, count: int):
         self.add_uint32(Keys.Attention.VALUE_EXPERT_COUNT.format(arch=self.arch), count)
 

gguf-py/gguf/tensor_mapping.py

@@ -1328,8 +1328,7 @@ class TensorNameMap:
         ),
 
         MODEL_TENSOR.DECISION_OPTION_NORM: (
-            "joint_head.option_norm",        # clef
-            "decision_head.candidate_norm",  # vllm-decision2
+            "joint_head.option_norm",  # clef
         ),
 
         MODEL_TENSOR.DECISION_SCORER: (
@@ -1338,27 +1337,6 @@ class TensorNameMap:
 
         MODEL_TENSOR.DECISION_SCORER_OUT: (
             "joint_head.residual_scorer.3",  # clef
-            "decision_head.scalar",          # vllm-decision2
-        ),
-
-        MODEL_TENSOR.DECISION_QUERY_NORM: (
-            "decision_head.query_norm",  # vllm-decision2
-        ),
-
-        MODEL_TENSOR.DECISION_KEY: (
-            "decision_head.key",  # vllm-decision2
-        ),
-
-        MODEL_TENSOR.DECISION_QUERY: (
-            "decision_head.query",  # vllm-decision2
-        ),
-
-        MODEL_TENSOR.DECISION_OPTION_MLP: (
-            "decision_head.candidate_mlp",  # vllm-decision2
-        ),
-
-        MODEL_TENSOR.DECISION_QUERY_MLP: (
-            "decision_head.query_mlp",  # vllm-decision2
         ),
 
         MODEL_TENSOR.ENC_ATTN_NORM: (

include/llama.h

@@ -1056,7 +1056,6 @@ extern "C" {
     // "state" here means extra hidden state carried over from a previous stage, e.g.:
     //   - MTP: state from N layers of the target model
     //   - Qwen3 VL (deepstack): state from N layers of the vision encoder
-    // Returns false if the context does not take a state embedding (currently only MTP contexts do)
     LLAMA_API bool llama_batch_ext_set_embd_state(
                                 struct llama_batch_ext * batch,
                                                int32_t   idx,
@@ -1078,7 +1077,7 @@ extern "C" {
 
     // Set custom position for the token at index idx in the batch
     // For M-RoPE models:
-    //     - Embedding tokens must have n_pos_per_embd positions per token, in order [t, y, x, z]; t is also the KV cache position
+    //     - Embedding tokens must have multiple positions per token
     //     - Text token only requires one single position per token
     LLAMA_API bool llama_batch_ext_set_pos(
                                 struct llama_batch_ext * batch,
@@ -1686,8 +1685,6 @@ extern "C" {
         int32_t n_p_eval;   // number of prompt tokens
         int32_t n_eval;     // number of generated tokens
         int32_t n_reused;   // number of times a ggml compute graph had been reused
-        int32_t n_reallocs_same; // number of graph reallocations during inference with an unchanged graph size
-        int32_t n_reallocs_diff; // number of graph reallocations during inference with a changed graph size
     };
 
     struct llama_perf_sampler_data {

skills/code-review/SKILL.md

@@ -132,7 +132,6 @@ Public API changes carry a higher bar than internal ones (`CONTRIBUTING.md`). Re
 - Only add tests that bring meaningful results. Too-trivial tests just bloat the suite and CI.
 - No time-sensitive tests (timing thresholds, sleeps, races against wall-clock); they are flaky on CI.
 - Think twice about tests that significantly increase CI run time (expensive computation, large inputs, or long sleep/wait delays) or download large amounts of data from the internet (big models, datasets). Flag them and ask whether a smaller model/input or an existing fixture would do.
-- Server: small tests of internal components (helpers, `server_tokens`, queues, format converters, ...) belong in `tests/test-server-component.cpp`. The end-to-end pytest suite in `tools/server/tests` is costly in CI time; only add to it when the test requires end-to-end behavior with a real model.
 
 ## General (always)
 

src/llama-adapter.cpp

@@ -334,10 +334,6 @@ static void llama_adapter_lora_init_impl(llama_model & model, FILE * file, llama
         if (!model_tensor) {
             throw std::runtime_error("LoRA tensor '" + name + "' does not exist in base model (hint: maybe wrong base model?)");
         }
-        // the LoRA delta of a Hadamard-folded weight reads the transformed input, so the result is wrong
-        if (model.hdmd.weight_blocks.count(name) || model.hdmd.inverse_blocks.count(name)) {
-            throw std::runtime_error("LoRA tensor '" + name + "' targets a prism.hadamard folded weight, which is not supported");
-        }
 
         auto * buft = ggml_backend_buffer_get_type(model_tensor->buffer);
 

src/llama-arch.cpp

@@ -41,7 +41,6 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_QWEN35,           "qwen35"           },
     { LLM_ARCH_QWEN35MOE,        "qwen35moe"        },
     { LLM_ARCH_CLEF,             "clef"             },
-    { LLM_ARCH_VLLM_DECISION2,   "vllm-decision2"   },
     { LLM_ARCH_QWEN4EXP,         "qwen4exp"         },
     { LLM_ARCH_PHI2,             "phi2"             },
     { LLM_ARCH_PHI3,             "phi3"             },
@@ -331,7 +330,6 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_ROPE_DIMENSION_COUNT,           "%s.rope.dimension_count"                 },
     { LLM_KV_ROPE_DIMENSION_COUNT_SWA,       "%s.rope.dimension_count_swa"             },
     { LLM_KV_ROPE_DIMENSION_SECTIONS,        "%s.rope.dimension_sections"              },
-    { LLM_KV_ROPE_SECTION_ORDER,             "%s.rope.section_order"                   },
     { LLM_KV_ROPE_FREQ_BASE,                 "%s.rope.freq_base"                       },
     { LLM_KV_ROPE_FREQ_BASE_SWA,             "%s.rope.freq_base_swa"                   },
     { LLM_KV_ROPE_SCALE_LINEAR,              "%s.rope.scale_linear"                    },
@@ -442,18 +440,6 @@ static const std::map<llm_kv, const char *> LLM_KV_NAMES = {
     { LLM_KV_XIELU_BETA,            "xielu.beta"            },
     { LLM_KV_XIELU_EPS,             "xielu.eps"             },
 
-    { LLM_KV_PRISM_HADAMARD_VERSION,              "prism.hadamard.version"              },
-    { LLM_KV_PRISM_HADAMARD_TIED_OUTPUT,          "prism.hadamard.tied_output"          },
-    { LLM_KV_PRISM_HADAMARD_BLOCK_SIZE,           "prism.hadamard.block_size"           },
-    { LLM_KV_PRISM_HADAMARD_TRANSFORM,            "prism.hadamard.transform"            },
-    { LLM_KV_PRISM_HADAMARD_AXIS,                 "prism.hadamard.axis"                 },
-    { LLM_KV_PRISM_HADAMARD_SIGN_MODE,            "prism.hadamard.sign_mode"            },
-    { LLM_KV_PRISM_HADAMARD_SIGN_WIDTHS,          "prism.hadamard.sign_widths"          },
-    { LLM_KV_PRISM_HADAMARD_SIGN_VALUES,          "prism.hadamard.sign_values"          },
-    { LLM_KV_PRISM_HADAMARD_WEIGHT_NAMES,         "prism.hadamard.weight_names"         },
-    { LLM_KV_PRISM_HADAMARD_INVERSE_WEIGHT_NAMES, "prism.hadamard.inverse_weight_names" },
-    { LLM_KV_PRISM_HADAMARD_GDN_V_GROUPED,        "prism.hadamard.gdn_v_grouped"        },
-
     // deprecated
     { LLM_KV_TOKENIZER_PREFIX_ID, "tokenizer.ggml.prefix_token_id" },
     { LLM_KV_TOKENIZER_SUFFIX_ID, "tokenizer.ggml.suffix_token_id" },
@@ -525,11 +511,6 @@ static const std::map<llm_tensor, const char *> LLM_TENSOR_NAMES = {
     { LLM_TENSOR_DECISION_SCALES,                        "decision.scales" },
     { LLM_TENSOR_DECISION_SCORER,                        "decision.scorer" },
     { LLM_TENSOR_DECISION_SCORER_OUT,                    "decision.scorer_out" },
-    { LLM_TENSOR_DECISION_QUERY_NORM,                    "decision.query_norm" },
-    { LLM_TENSOR_DECISION_KEY,                           "decision.key" },
-    { LLM_TENSOR_DECISION_QUERY,                         "decision.query" },
-    { LLM_TENSOR_DECISION_OPTION_MLP,                    "decision.option_mlp" },
-    { LLM_TENSOR_DECISION_QUERY_MLP,                     "decision.query_mlp" },
     { LLM_TENSOR_ENC_OUTPUT_NORM,                        "enc.output_norm" },
     { LLM_TENSOR_FFN_GATE_INP_SHEXP,                     "blk.%d.ffn_gate_inp_shexp" },
     { LLM_TENSOR_SSM_A_NOSCAN,                           "blk.%d.ssm_a" },
@@ -795,11 +776,6 @@ static const std::map<llm_tensor, llm_tensor_info> LLM_TENSOR_INFOS = {
     {LLM_TENSOR_DECISION_SCALES,            {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL}},
     {LLM_TENSOR_DECISION_SCORER,            {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
     {LLM_TENSOR_DECISION_SCORER_OUT,        {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_DECISION_QUERY_NORM,        {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL}},
-    {LLM_TENSOR_DECISION_KEY,               {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_DECISION_QUERY,             {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_DECISION_OPTION_MLP,        {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
-    {LLM_TENSOR_DECISION_QUERY_MLP,         {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}},
     {LLM_TENSOR_DENSE_2_OUT,                {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}}, // Dense layer output
     {LLM_TENSOR_DENSE_3_OUT,                {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL_MAT}}, // Dense layer output
     {LLM_TENSOR_OUTPUT_NORM,                {LLM_TENSOR_LAYER_OUTPUT,    GGML_OP_MUL}},

src/llama-arch.h

@@ -46,7 +46,6 @@ enum llm_arch {
     LLM_ARCH_QWEN35,
     LLM_ARCH_QWEN35MOE,
     LLM_ARCH_CLEF,
-    LLM_ARCH_VLLM_DECISION2,
     LLM_ARCH_QWEN4EXP,
     LLM_ARCH_PHI2,
     LLM_ARCH_PHI3,
@@ -336,7 +335,6 @@ enum llm_kv {
     LLM_KV_ROPE_DIMENSION_COUNT,
     LLM_KV_ROPE_DIMENSION_COUNT_SWA,
     LLM_KV_ROPE_DIMENSION_SECTIONS,
-    LLM_KV_ROPE_SECTION_ORDER,
     LLM_KV_ROPE_FREQ_BASE,
     LLM_KV_ROPE_FREQ_BASE_SWA,
     LLM_KV_ROPE_SCALE_LINEAR,
@@ -441,18 +439,6 @@ enum llm_kv {
     LLM_KV_XIELU_BETA,
     LLM_KV_XIELU_EPS,
 
-    LLM_KV_PRISM_HADAMARD_VERSION,
-    LLM_KV_PRISM_HADAMARD_TIED_OUTPUT,
-    LLM_KV_PRISM_HADAMARD_BLOCK_SIZE,
-    LLM_KV_PRISM_HADAMARD_TRANSFORM,
-    LLM_KV_PRISM_HADAMARD_AXIS,
-    LLM_KV_PRISM_HADAMARD_SIGN_MODE,
-    LLM_KV_PRISM_HADAMARD_SIGN_WIDTHS,
-    LLM_KV_PRISM_HADAMARD_SIGN_VALUES,
-    LLM_KV_PRISM_HADAMARD_WEIGHT_NAMES,
-    LLM_KV_PRISM_HADAMARD_INVERSE_WEIGHT_NAMES,
-    LLM_KV_PRISM_HADAMARD_GDN_V_GROUPED,
-
     // deprecated:
     LLM_KV_TOKENIZER_PREFIX_ID,
     LLM_KV_TOKENIZER_SUFFIX_ID,
@@ -696,11 +682,6 @@ enum llm_tensor {
     LLM_TENSOR_DECISION_SCALES,
     LLM_TENSOR_DECISION_SCORER,
     LLM_TENSOR_DECISION_SCORER_OUT,
-    LLM_TENSOR_DECISION_QUERY_NORM,
-    LLM_TENSOR_DECISION_KEY,
-    LLM_TENSOR_DECISION_QUERY,
-    LLM_TENSOR_DECISION_OPTION_MLP,
-    LLM_TENSOR_DECISION_QUERY_MLP,
     LLM_TENSOR_CONV1D,
     LLM_TENSOR_CONVNEXT_DW,
     LLM_TENSOR_CONVNEXT_NORM,

src/llama-batch.cpp

@@ -31,10 +31,9 @@ bool llama_batch_allocr::init(
         bool output_all) {
     clear();
 
-    this->vocab        = &vocab;
-    this->n_embd       = batch_inp.n_embd > 0 ? batch_inp.n_embd : batch_inp.n_embd_inp;
-    this->n_embd_state = batch_inp.n_embd_state;
-    this->n_seq_max    = batch_inp.n_seq_max;
+    this->vocab     = &vocab;
+    this->n_embd    = batch_inp.n_embd > 0 ? batch_inp.n_embd : batch_inp.n_embd_inp;
+    this->n_seq_max = batch_inp.n_seq_max;
 
     const int32_t n_tok = (int32_t) batch_inp.tokens.size();
 
@@ -49,17 +48,14 @@ bool llama_batch_allocr::init(
 
     //
     // determine the content types of the batch
-    // an entry can carry a token id, a token embedding, or both
+    // an entry can carry a token id, a token embedding, or both (e.g. MTP hook batches)
     // all entries must carry the same combination, or be a mix of token and embd entries
-    // a state embedding (e.g. MTP hook batches) is set on all entries or on none
     //
 
     int32_t n_tok_only  = 0;
     int32_t n_embd_only = 0;
     int32_t n_both      = 0;
 
-    const bool has_state = batch_inp.tokens[0].has_state;
-
     for (int32_t i = 0; i < n_tok; ++i) {
         const bool is_tok = batch_inp.tokens[i].id != LLAMA_TOKEN_NULL;
         const bool is_emb = batch_inp.tokens[i].has_embd;
@@ -69,11 +65,6 @@ bool llama_batch_allocr::init(
             return false;
         }
 
-        if (batch_inp.tokens[i].has_state != has_state) {
-            LLAMA_LOG_ERROR("%s: all entries in the batch must have the same state embedding presence\n", __func__);
-            return false;
-        }
-
         n_tok_only  += is_tok && !is_emb;
         n_embd_only += is_emb && !is_tok;
         n_both      += is_tok &&  is_emb;
@@ -133,10 +124,6 @@ bool llama_batch_allocr::init(
         embd_vec = batch_inp.embd;
     }
 
-    if (has_state) {
-        state_vec = batch_inp.state;
-    }
-
     //
     // build flat pos array, section-major: pos[j*n_tok + i] = section j of entry i
     // token entry: [p, p, p, 0] (M-RoPE text position)
@@ -305,7 +292,6 @@ bool llama_batch_allocr::init(
             /*.n_pos        =*/ n_pos_per_embd,
             /*.token        =*/ batch.token,
             /*.embd         =*/ batch.embd,
-            /*.embd_state   =*/ state_vec.empty() ? nullptr : state_vec.data(),
             /*.pos          =*/ batch.pos,
             /*.n_seq_id     =*/ batch.n_seq_id,
             /*.seq_id       =*/ batch.seq_id,
@@ -507,7 +493,6 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
 
     udata->token     .resize(n_tokens);
     udata->embd      .clear();
-    udata->embd_state.clear();
     udata->pos       .resize(n_pos_all);
     udata->n_seq_id  .resize(n_tokens);
     udata->seq_id    .resize(n_tokens);
@@ -530,7 +515,6 @@ llama_ubatch llama_batch_allocr::ubatch_reserve(uint32_t n_seq_tokens, uint32_t
 
         /*.token        =*/ udata->token.data(),
         /*.embd         =*/ nullptr,
-        /*.embd_state   =*/ nullptr,
         /*.pos          =*/ udata->pos.data(),
         /*.n_seq_id     =*/ udata->n_seq_id.data(),
         /*.seq_id       =*/ udata->seq_id.data(),
@@ -837,7 +821,6 @@ void llama_batch_allocr::clear() {
     token_vec   .clear();
     embd_vec    .clear();
     is_embd_vec .clear();
-    state_vec   .clear();
     seq_id_data .clear();
     pos         .clear();
     n_seq_id    .clear();
@@ -880,15 +863,12 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
     const bool mixed     = mixed_batch && n_embd_rows > 0 && n_embd_rows < n_tokens;
     const bool use_token = batch.token && !(mixed_batch && n_embd_rows == n_tokens);
     const bool use_embd  = batch.embd  && !(mixed_batch && n_embd_rows == 0);
-    const bool has_state = !state_vec.empty();
 
-    const int64_t n_embd_all  = use_embd  ? (int64_t) n_tokens*n_embd       : 0;
-    const int64_t n_state_all = has_state ? (int64_t) n_tokens*n_embd_state : 0;
-    const int64_t n_pos_all   =             (int64_t) n_tokens*n_pos_per_embd;
+    const int64_t n_embd_all = use_embd ? (int64_t) n_tokens*n_embd : 0;
+    const int64_t n_pos_all  =              (int64_t) n_tokens*n_pos_per_embd;
 
     udata->token     .resize(n_tokens);
     udata->embd      .resize(n_embd_all);
-    udata->embd_state.resize(n_state_all);
     udata->pos       .resize(n_pos_all);
     udata->n_seq_id  .resize(n_tokens);
     udata->seq_id    .resize(n_tokens);
@@ -916,10 +896,6 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
             udata->type[i] = is_embd_vec[idxs[i]];
         }
 
-        if (has_state) {
-            memcpy(udata->embd_state.data() + i*n_embd_state, state_vec.data() + (int64_t) idxs[i]*n_embd_state, n_embd_state*sizeof(float));
-        }
-
         for (size_t j = 0; j < (size_t)n_pos_per_embd; ++j) {
             udata->pos[j*n_tokens + i] = batch.pos[j*batch.n_tokens + idxs[i]];
         }
@@ -966,7 +942,6 @@ llama_ubatch llama_batch_allocr::ubatch_add(const std::vector<int32_t> & idxs, u
 
         /*.token        =*/ use_token ? udata->token.data() : nullptr,
         /*.embd         =*/ use_embd  ? udata->embd.data()  : nullptr,
-        /*.embd_state   =*/ has_state ? udata->embd_state.data() : nullptr,
         /*.pos          =*/ udata->pos.data(),
         /*.n_seq_id     =*/ udata->n_seq_id.data(),
         /*.seq_id       =*/ udata->seq_id.data(),
@@ -1018,7 +993,6 @@ void llama_batch_allocr::ubatch_print(const llama_ubatch & ubatch, int debug) {
 
         LLAMA_LOG_DEBUG("%s:   token      = %p\n", __func__, (void *) ubatch.token);
         LLAMA_LOG_DEBUG("%s:   embd       = %p\n", __func__, (void *) ubatch.embd);
-        LLAMA_LOG_DEBUG("%s:   embd_state = %p\n", __func__, (void *) ubatch.embd_state);
         LLAMA_LOG_DEBUG("%s:   pos        = %p\n", __func__, (void *) ubatch.pos);
         LLAMA_LOG_DEBUG("%s:   n_seq_id   = %p\n", __func__, (void *) ubatch.n_seq_id);
         LLAMA_LOG_DEBUG("%s:   seq_id     = %p\n", __func__, (void *) ubatch.seq_id);
@@ -1136,25 +1110,19 @@ void llama_batch_free(struct llama_batch batch) {
 // llama_batch_ext
 
 size_t llama_batch_ext_select_n_embd_inp(llama_context_type ctx_type, llm_arch arch, const llama_hparams & hparams) {
-    GGML_UNUSED(ctx_type);
+    if (ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
+        return hparams.n_embd_out();
+    }
     if (arch == LLM_ARCH_DFLASH) {
         return hparams.n_embd_inp_enc();
     }
     return hparams.n_embd_inp();
 }
 
-size_t llama_batch_ext_select_n_embd_state(llama_context_type ctx_type, const llama_hparams & hparams) {
-    if (ctx_type == LLAMA_CONTEXT_TYPE_MTP) {
-        return hparams.n_embd_out();
-    }
-    return 0;
-}
-
 llama_batch_ext::llama_batch_ext(llama_context * ctx) :
         n_tokens_max(llama_n_batch(ctx)),
         n_embd_inp(llama_batch_ext_select_n_embd_inp(ctx->get_cparams().ctx_type, llama_get_model(ctx)->arch, llama_get_model(ctx)->hparams)),
         n_embd_inp_enc(llama_get_model(ctx)->hparams.n_embd_inp_enc()),
-        n_embd_state(llama_batch_ext_select_n_embd_state(ctx->get_cparams().ctx_type, llama_get_model(ctx)->hparams)),
         n_seq_max(llama_n_seq_max(ctx)),
         mem(llama_get_memory(ctx)),
         n_vocab(llama_vocab_n_tokens(llama_model_get_vocab(llama_get_model(ctx)))),
@@ -1173,7 +1141,6 @@ llama_batch_ext::llama_batch_ext(
         n_tokens_max(n_tokens_max),
         n_embd_inp(n_embd_inp),
         n_embd_inp_enc(n_embd_inp_enc),
-        n_embd_state(0),
         n_seq_max(n_seq_max),
         mem(mem),
         n_vocab(n_vocab),
@@ -1184,7 +1151,6 @@ llama_batch_ext::llama_batch_ext(
 void llama_batch_ext::clear() {
     tokens.clear();
     embd  .clear();
-    state .clear();
     n_embd = 0;
 }
 
@@ -1267,38 +1233,6 @@ bool llama_batch_ext::set_token_embd(int32_t idx, llama_embd embd_in) {
     return true;
 }
 
-bool llama_batch_ext::set_token_state(int32_t idx, llama_embd state_in) {
-    if (idx < 0 || idx >= (int32_t) tokens.size()) {
-        return false;
-    }
-    if (!state_in.data) {
-        return false;
-    }
-    if (n_embd_state == 0) {
-        return false; // this context does not take state embeddings
-    }
-
-    const size_t n_total = state_in.n_rows * state_in.n_embd;
-    if (n_total != n_embd_state) {
-        LLAMA_LOG_ERROR("%s: state size mismatch, got %zu rows x %zu = %zu, expected %zu\n",
-                __func__, state_in.n_rows, state_in.n_embd, n_total, n_embd_state);
-        return false;
-    }
-
-    token & t = tokens[idx];
-
-    if (t.has_state) {
-        LLAMA_LOG_ERROR("%s: state for token %d is already set\n", __func__, idx);
-        return false;
-    }
-
-    t.has_state = true;
-    t.state_off = state.size();
-    state.insert(state.end(), state_in.data, state_in.data + n_total);
-
-    return true;
-}
-
 bool llama_batch_ext::set_token_pos(int32_t idx, const llama_pos * pos_in) {
     if (idx < 0 || idx >= (int32_t) tokens.size()) {
         return false;
@@ -1386,7 +1320,11 @@ bool llama_batch_ext_set_embd_token(llama_batch_ext * batch, int32_t idx, llama_
 }
 
 bool llama_batch_ext_set_embd_state(llama_batch_ext * batch, int32_t idx, llama_embd embd) {
-    return batch->set_token_state(idx, embd);
+    // TODO
+    GGML_UNUSED(batch);
+    GGML_UNUSED(idx);
+    GGML_UNUSED(embd);
+    return false;
 }
 
 bool llama_batch_ext_set_output_embd(llama_batch_ext * batch, int32_t idx, bool value) {
@@ -1455,13 +1393,7 @@ void llama_batch_compat::init(llama_batch_ext & dst, const llama_batch & batch_i
             t.id = batch_inp.token[i];
         }
 
-        // legacy MTP hook batches carry the hidden state next to the token ids
-        if (has_embd && has_token && batch_ext->n_embd_state > 0) {
-            t.has_state = true;
-            t.state_off = batch_ext->state.size();
-            const float * src = batch_inp.embd + (size_t) i * batch_ext->n_embd_state;
-            batch_ext->state.insert(batch_ext->state.end(), src, src + batch_ext->n_embd_state);
-        } else if (has_embd) {
+        if (has_embd) {
             t.has_embd = true;
             t.embd_off = batch_ext->embd.size();
             const float * src = batch_inp.embd + (size_t) i * n_embd_row;

src/llama-batch.h

@@ -48,11 +48,10 @@ struct llama_ubatch {
     // seq_idx:    indices of the unique sequence ids in the ubatch in [0, n_seqs_unq)
     //             used for extracting sequence pooled embeddings
 
-    //                          // size                     | idx | val
-    llama_token  *  token;      // [n_tokens]               | i   | id, token
-    float        *  embd;       // [n_embd, n_tokens]       | i   | embd
-    float        *  embd_state; // [n_embd_state, n_tokens] | i   | hidden state carried over from a previous stage (e.g. MTP)
-    llama_pos    *  pos;        // [n_tokens*n_pos]         | i   | pos
+    //                          // size               | idx | val
+    llama_token  *  token;      // [n_tokens]         | i   | id, token
+    float        *  embd;       // [n_embd, n_tokens] | i   | embd
+    llama_pos    *  pos;        // [n_tokens*n_pos]   | i   | pos
     int32_t      *  n_seq_id;   // [n_tokens]         | i   | -
     llama_seq_id ** seq_id;     // [n_tokens]         | s   | s0, s1, seq_id
     llama_seq_id *  seq_id_unq; // [n_seqs_unq]       | s   | seq_id
@@ -64,7 +63,6 @@ struct llama_ubatch {
     struct data_t {
         std::vector<llama_token>    token;
         std::vector<float>          embd;
-        std::vector<float>          embd_state;
         std::vector<llama_pos>      pos;
         std::vector<int32_t>        n_seq_id;
         std::vector<llama_seq_id *> seq_id;      // these point into the seq_id_data below
@@ -87,18 +85,15 @@ struct llama_ubatch {
 
 struct llama_hparams;
 
+// MTP hook batches carry the target model's hidden state (n_embd_out size).
 // DFlash batches carry the fused target features at the encoder input width (n_embd_inp_enc size).
-// Other batches carry token embeddings (n_embd_inp size).
+// Normal batches carry token embeddings (n_embd_inp size).
 size_t llama_batch_ext_select_n_embd_inp(llama_context_type ctx_type, llm_arch arch, const llama_hparams & hparams);
 
-// MTP contexts also take the target model's hidden state (n_embd_out size), 0 = no state input
-size_t llama_batch_ext_select_n_embd_state(llama_context_type ctx_type, const llama_hparams & hparams);
-
 struct llama_batch_ext {
     const size_t n_tokens_max;     // max number of tokens that can be stored in the batch
     const size_t n_embd_inp;       // decoder embd row width
     const size_t n_embd_inp_enc;   // encoder embd row width (e.g. eagle3/dflash extracted features)
-    const size_t n_embd_state;     // state embd row width, 0 if the context takes no state
     const llama_seq_id n_seq_max;  // max number of sequences
     llama_memory_i * mem;          // memory for position inference
     const llama_token n_vocab;     // max token ID that we accept
@@ -112,8 +107,6 @@ struct llama_batch_ext {
         llama_token  id = LLAMA_TOKEN_NULL;
         bool         has_embd = false; // whether embd_off is set
         size_t       embd_off = 0; // index offset in the embd array
-        bool         has_state = false; // whether state_off is set
-        size_t       state_off = 0; // index offset in the state array
         bool         output = false; // TODO: have dedicated output flags
         int32_t      decision_order = 0; // see llama_batch_ext_set_decision_order()
         std::unordered_set<llama_seq_id> seq_ids;
@@ -121,7 +114,6 @@ struct llama_batch_ext {
     };
     std::vector<token> tokens;
     std::vector<float> embd;
-    std::vector<float> state;
 
     llama_batch_ext(llama_context * ctx);
 
@@ -144,7 +136,6 @@ struct llama_batch_ext {
     bool add_seq(int32_t idx, llama_seq_id seq_id);
     bool set_token_id(int32_t idx, llama_token id);
     bool set_token_embd(int32_t idx, llama_embd embd_in);
-    bool set_token_state(int32_t idx, llama_embd state_in);
     bool set_token_pos(int32_t idx, const llama_pos * pos_in);
     bool set_output(int32_t idx, bool output_last);
     bool set_decision_order(int32_t idx, int32_t order);
@@ -214,14 +205,12 @@ private:
     const bool allow_mixed;
 
     uint32_t n_embd;
-    uint32_t n_embd_state;
     uint32_t n_seq_max;
     uint32_t n_outputs;
 
     std::vector<llama_token>    token_vec;    // owned token IDs built from llama_batch_ext
     std::vector<float>          embd_vec;     // owned embeddings built from llama_batch_ext
     std::vector<int8_t>         is_embd_vec;  // mixed batch only (= 1 if embd, 0 if text token)
-    std::vector<float>          state_vec;    // owned state embeddings built from llama_batch_ext, llama_batch has no slot for them
     std::vector<llama_seq_id>   seq_id_data;  // flat storage for seq_id pointers below
 
     std::vector<llama_pos>      pos;

src/llama-context.cpp

@@ -26,89 +26,6 @@
 // llama_context
 //
 
-// check that each folded weight in the graph gets its Hadamard transform, and each latent lookup gets the inverse
-// without this check, an arch that skips the transform helpers loads and computes wrong results
-static void llama_verify_hadamard_graph(
-        ggml_cgraph * gf,
-        const llama_hadamard_rotations & rotations,
-        const llama_hadamard_rotations & inverses,
-        const llama_moe_cache * moe_cache) {
-    // the MoE cache gives the matmul a copy of the expert weights, so the copy also needs the transform of its source
-    llama_hadamard_rotations forward = rotations;
-    for (const auto & [w, t] : rotations) {
-        if (const ggml_tensor * cached = moe_cache ? moe_cache->get_experts(w) : nullptr) {
-            forward.emplace(cached, t);
-        }
-    }
-
-    auto unwrap = [](const ggml_tensor * t) {
-        while (t && (t->op == GGML_OP_RESHAPE || t->op == GGML_OP_VIEW)) {
-            t = t->src[0];
-        }
-        return t;
-    };
-
-    std::map<const ggml_tensor *, bool> lookups; // get_rows results of latent tables
-
-    for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) {
-        const ggml_tensor * node = ggml_graph_node(gf, i);
-
-        if (node->op == GGML_OP_GET_ROWS && inverses.count(node->src[0])) {
-            lookups.emplace(node, false);
-            continue;
-        }
-
-        if (node->op != GGML_OP_MUL_MAT && node->op != GGML_OP_MUL_MAT_ID) {
-            continue;
-        }
-
-        if (node->op == GGML_OP_MUL_MAT && ((const int32_t *) node->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD) {
-            const auto lk = lookups.find(unwrap(node->src[1]));
-            if (lk != lookups.end()) {
-                lk->second = true;
-            }
-            continue;
-        }
-
-        const auto it = forward.find(node->src[0]);
-        if (it == forward.end()) {
-            if (inverses.count(node->src[0])) {
-                throw std::runtime_error(format(
-                    "Hadamard-latent table '%s' is used as a head without a forward transform", node->src[0]->name));
-            }
-            continue;
-        }
-        const ggml_tensor * src = unwrap(node->src[1]);
-        const bool transformed = src && src->op == GGML_OP_MUL_MAT &&
-            ((const int32_t *) src->op_params)[1] == GGML_HINT_SRC0_IS_HADAMARD &&
-            src->src[0] == it->second.rot;
-        if (!transformed) {
-            throw std::runtime_error(format(
-                "Hadamard-folded weight '%s' is consumed without its activation transform; "
-                "this graph's matmul path does not support prism.hadamard folding",
-                node->src[0]->name));
-        }
-    }
-
-    for (const auto & [node, ok] : lookups) {
-        if (!ok) {
-            for (int i = 0; i < ggml_graph_n_nodes(gf); ++i) {
-                const ggml_tensor * n2 = ggml_graph_node(gf, i);
-                for (int s = 0; s < GGML_MAX_SRC && n2->src[s]; ++s) {
-                    if (unwrap(n2->src[s]) == node) {
-                        LLAMA_LOG_WARN("%s: latent lookup '%s' consumed by op=%s name='%s' src%d hint=%d\n",
-                                __func__, node->name, ggml_op_name(n2->op), n2->name, s,
-                                ((const int32_t *) n2->op_params)[1]);
-                    }
-                }
-            }
-            throw std::runtime_error(format(
-                "Hadamard-latent table '%s' is read without the inverse transform",
-                node->src[0]->name));
-        }
-    }
-}
-
 static llm_graph_type ctx_type_to_graph_type(llama_context_type ctx_type) {
     switch (ctx_type) {
         case LLAMA_CONTEXT_TYPE_DEFAULT: return LLM_GRAPH_TYPE_DEFAULT;
@@ -704,14 +621,6 @@ static int llama_graph_n_input_tensors(ggml_cgraph * gf) {
     return (int) users.size();
 }
 
-void llama_context::sched_new(size_t max_nodes, bool parallel) {
-    sched.reset(ggml_backend_sched_new(
-            backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, parallel, cparams.op_offload));
-
-    ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
-    ggml_backend_sched_set_realloc_callback(sched.get(), sched_graph_realloc, this);
-}
-
 void llama_context::sched_reserve() {
     if (!sched_need_reserve) {
         return;
@@ -738,7 +647,8 @@ void llama_context::sched_reserve() {
     gf_res_reserve.reset(new llm_graph_result(max_nodes));
     gf_res_prev_active = nullptr;
 
-    sched_new(max_nodes, cparams.pipeline_parallel);
+    sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, cparams.pipeline_parallel, cparams.op_offload));
+    ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
 
     llama_memory_context_ptr mctx;
     if (memory) {
@@ -777,7 +687,8 @@ void llama_context::sched_reserve() {
             if (cparams.pipeline_parallel) {
                 LLAMA_LOG_WARN("%s: compute buffer allocation failed, retrying without pipeline parallelism\n", __func__);
                 cparams.pipeline_parallel = false;
-                sched_new(max_nodes, false);
+                sched.reset(ggml_backend_sched_new(backend_ptrs.data(), backend_buft.data(), backend_ptrs.size(), max_nodes, false, cparams.op_offload));
+                ggml_backend_sched_set_copy_callback(sched.get(), sched_copy_experts, this);
                 gf = graph_reserve(n_tokens, n_seqs, n_outputs_pp, mctx.get());
             }
             if (!gf) {
@@ -2651,12 +2562,6 @@ ggml_cgraph * llama_context::graph_reserve(
 
     auto * gf = model.build_graph(gparams);
 
-    // check the graph before scheduling: cross-backend copies break the producer chain that the check follows
-    if (!hadamard_verified && gf && (!model.hdmd.rot.empty() || !model.hdmd.inv.empty())) {
-        llama_verify_hadamard_graph(gf, model.hdmd.rot, model.hdmd.inv, moe_cache.get());
-        hadamard_verified = true;
-    }
-
     this->n_input_tensors = llama_graph_n_input_tensors(gf);
     this->n_outputs = save_n_outputs;
 
@@ -2695,7 +2600,6 @@ llm_graph_params llama_context::graph_params(
         /*.cross       =*/ &cross,
         /*.moe_cache   =*/ moe_cache.get(),
         /*.prec_policy =*/ &model.prec_policy,
-        /*.hdmd        =*/ model.hdmd.rot.empty() ? nullptr : &model.hdmd,
         /*.samplers    =*/ sampling.samplers,
         /*.n_outputs   =*/ n_outputs,
         /*.cb          =*/ graph_get_cb(),
@@ -2703,25 +2607,6 @@ llm_graph_params llama_context::graph_params(
     };
 }
 
-void llama_context::sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * gf, bool backend_ids_changed, bool graph_size_changed, void * user_data) {
-    // reallocating after the backend assignments changed is expected, trace only the unexpected ones
-    if (backend_ids_changed) {
-        return;
-    }
-
-    auto * lctx = static_cast<llama_context *>(user_data);
-
-    if (graph_size_changed) {
-        lctx->n_reallocs_diff++;
-    } else {
-        lctx->n_reallocs_same++;
-    }
-
-    LLAMA_LOG_INFO("%s: graph reallocation: nodes = %d, splits = %d, n_same = %d, n_diff = %d\n",
-            __func__, ggml_graph_n_nodes(gf), ggml_backend_sched_get_n_splits(sched),
-            lctx->n_reallocs_same, lctx->n_reallocs_diff);
-}
-
 ggml_status llama_context::graph_compute(
             ggml_cgraph * gf,
                    bool   batched) {
@@ -3676,9 +3561,6 @@ llama_perf_context_data llama_context::perf_get_data() const {
     data.n_eval      = std::max(1, n_eval);
     data.n_reused    = std::max(0, n_reused);
 
-    data.n_reallocs_same = n_reallocs_same;
-    data.n_reallocs_diff = n_reallocs_diff;
-
     return data;
 }
 
@@ -3687,8 +3569,6 @@ void llama_context::perf_reset() {
     t_eval_us   = n_eval = 0;
     t_p_eval_us = n_p_eval = 0;
     n_reused    = 0;
-
-    // note: the reallocation counters are not reset - they track the lifetime totals for this context
 }
 
 llama_memory_breakdown llama_context::memory_breakdown() const {
@@ -4654,7 +4534,6 @@ void llama_perf_context_print(const llama_context * ctx) {
             __func__, data.t_eval_ms, data.n_eval, data.t_eval_ms / data.n_eval, 1e3 / data.t_eval_ms * data.n_eval);
     LLAMA_LOG_INFO("%s:       total time = %10.2f ms / %5d tokens\n", __func__, (t_end_ms - data.t_start_ms), (data.n_p_eval + data.n_eval));
     LLAMA_LOG_INFO("%s:    graphs reused = %10d\n", __func__, data.n_reused);
-    LLAMA_LOG_INFO("%s:   graph reallocs = %10d    / %5d        (same size) / (changed size)\n", __func__, data.n_reallocs_same, data.n_reallocs_diff);
 }
 
 void llama_perf_context_reset(llama_context * ctx) {

src/llama-context.h

@@ -273,15 +273,9 @@ private:
 
     llm_graph_cb graph_get_cb() const;
 
-    // create a new backend scheduler for a graph of the specified size and register the callbacks below
-    void sched_new(size_t max_nodes, bool parallel);
-
     // ggml_backend_sched copy callback, copies only the experts used by MUL_MAT_ID and updates the MoE cache
     static bool sched_copy_experts(ggml_backend_t backend, const ggml_tensor * src, ggml_tensor * dst, ggml_cgraph * graph, void * user_data);
 
-    // ggml_backend_sched realloc callback, counts and traces the graph reallocations of this context
-    static void sched_graph_realloc(ggml_backend_sched_t sched, ggml_cgraph * graph, bool backend_ids_changed, bool graph_size_changed, void * user_data);
-
     // disable auto fused ops (Flash Attention, Gated Delta Net) whose op lands on a device
     // that differs from the layer it belongs to (usually due to missing backend support)
     void resolve_fused_ops(const llama_memory_context_i * mctx, uint32_t n_seqs);
@@ -383,11 +377,6 @@ private:
 
     copy_experts_info copy_experts;
 
-    // graph reallocations during inference where the backend assignments did not change, split by whether the
-    // graph size is the same as the previously scheduled one (unexpected) or different
-    int32_t n_reallocs_same = 0;
-    int32_t n_reallocs_diff = 0;
-
     ggml_backend_t backend_cpu = nullptr;
     std::vector<ggml_backend_ptr> backends;
 
@@ -424,9 +413,6 @@ private:
     // env: LLAMA_GRAPH_REUSE_DISABLE
     bool graph_reuse_disable = false;
 
-    // true after the prism.hadamard coverage check passes on a reserved graph
-    bool hadamard_verified = false;
-
     // perf
     mutable int64_t t_start_us  = 0;
     mutable int64_t t_load_us   = 0;

src/llama-ext.h

@@ -100,7 +100,7 @@ LLAMA_API void llama_set_embeddings_nextn(struct llama_context * ctx, bool value
 // chain multiple trained NextN heads. Default 0 (first head).
 LLAMA_API void llama_set_nextn_layer_offset(struct llama_context * ctx, int32_t offset);
 
-// Marks the entries that a decision head (clef, vllm-decision2) reads, the default is 0
+// Marks the entries that a joint decision head (clef) reads, the default is 0
 // See https://github.com/ggml-org/llama.cpp/pull/29831 for details
 // A run of entries with the same value is one span, spans must be separated by entries with value 0
 // An option belongs to the last question before it

src/llama-graph.cpp

@@ -149,21 +149,25 @@ void llm_graph_input_embd_h::set_input(const llama_ubatch * ubatch) {
         GGML_ASSERT(ubatch->embd);
         GGML_ASSERT(n_embd == embd->ne[0]);
 
-        ggml_backend_tensor_set(embd, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(embd));
+        ggml_backend_tensor_set(embd, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(h));
     }
 
-    GGML_ASSERT(ubatch->embd_state && "this graph requires a state embedding, see llama_batch_ext_set_embd_state()");
-    GGML_ASSERT(n_embd_state == h->ne[0]);
+    // TODO: extend llama_ubatch to differentiate between token embeddings and hidden states
+    //       for now, we assume that the hidden state is always provided as an embedding
+    //       ref: https://github.com/ggml-org/llama.cpp/pull/23643
+    if (ubatch->embd) {
+        GGML_ASSERT(n_embd == h->ne[0]);
 
-    ggml_backend_tensor_set(h, ubatch->embd_state, 0, n_tokens*n_embd_state*ggml_element_size(h));
+        ggml_backend_tensor_set(h, ubatch->embd, 0, n_tokens*n_embd*ggml_element_size(h));
+    }
 }
 
 bool llm_graph_input_embd_h::can_reuse(const llm_graph_params & params) {
     bool res = true;
 
-    res &= (!params.ubatch.token)      || (tokens && tokens->ne[0] == params.ubatch.n_tokens);
-    res &= (!params.ubatch.embd)       || (embd   && embd->ne[1]   == params.ubatch.n_tokens);
-    res &= (!params.ubatch.embd_state) || (h      && h->ne[1]      == params.ubatch.n_tokens);
+    res &= (!params.ubatch.token) || (tokens && tokens->ne[0] == params.ubatch.n_tokens);
+    res &= (!params.ubatch.embd)  || (embd   && embd->ne[1]   == params.ubatch.n_tokens);
+    res &= (!params.ubatch.embd)  || (h      && h->ne[1]      == params.ubatch.n_tokens);
 
     return res;
 }
@@ -172,33 +176,7 @@ void llm_graph_input_pos::set_input(const llama_ubatch * ubatch) {
     if (ubatch->pos && pos) {
         const int64_t n_tokens = ubatch->n_tokens;
 
-        const bool has_embd = ubatch->is_mixed() || ubatch->token == nullptr;
-        if (rope_section_order == LLAMA_ROPE_SECTION_ORDER_TYXZ || !has_embd) {
-            ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
-            return;
-        }
-
-        // input is always [t, y, x, z]
-        // token entries are expanded by the batch to [p, p, p, 0]
-        GGML_ASSERT(n_pos_per_embd == 4);
-
-        // slot index per section, slots are t = 0, y = 1, x = 2, z = 3
-        std::array<int64_t, 4> slot_of_section = { 0, 1, 2, 3 };
-        switch (rope_section_order) {
-            case LLAMA_ROPE_SECTION_ORDER_TYXZ: break;
-            case LLAMA_ROPE_SECTION_ORDER_ZYXT: slot_of_section = { 3, 1, 2, 0 }; break;
-            default: GGML_ABORT("unsupported rope section order");
-        }
-
-        std::vector<llama_pos> pos_data(n_tokens*n_pos_per_embd);
-        for (int64_t i = 0; i < n_tokens; ++i) {
-            const bool is_embd = ubatch->is_mixed() ? ubatch->type[i] != 0 : true;
-            for (int64_t s = 0; s < 4; ++s) {
-                const int64_t slot = is_embd ? slot_of_section[s] : s;
-                pos_data[s*n_tokens + i] = ubatch->pos[slot*n_tokens + i];
-            }
-        }
-        ggml_backend_tensor_set(pos, pos_data.data(), 0, pos_data.size()*ggml_element_size(pos));
+        ggml_backend_tensor_set(pos, ubatch->pos, 0, n_tokens*n_pos_per_embd*ggml_element_size(pos));
     }
 }
 
@@ -1396,7 +1374,6 @@ void llm_graph_result::reset() {
 
     inputs.clear();
     fused_nodes.clear();
-    hdmd_inputs.clear();
 
     buf_compute_meta.resize(ggml_tensor_overhead()*max_nodes + ggml_graph_overhead_custom(max_nodes, false));
 
@@ -1502,15 +1479,6 @@ void llm_graph_result::add_fused_node(llm_graph_fused_node result) {
     fused_nodes.push_back(result);
 }
 
-ggml_tensor * llm_graph_result::get_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot) const {
-    const auto it = hdmd_inputs.find({ cur, rot });
-    return it == hdmd_inputs.end() ? nullptr : it->second;
-}
-
-void llm_graph_result::set_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot, ggml_tensor * res) {
-    hdmd_inputs[{ cur, rot }] = res;
-}
-
 void llm_graph_result::set_params(const llm_graph_params & params) {
     this->params = params;
 }
@@ -1558,7 +1526,6 @@ llm_graph_context::llm_graph_context(const llm_graph_params & params) :
     cross            (params.cross),
     moe_cache        (params.moe_cache),
     prec_policy      (params.prec_policy),
-    hdmd             (params.hdmd),
     samplers         (params.samplers),
     cb_func          (params.cb),
     res              (params.res),
@@ -1581,42 +1548,10 @@ ggml_tensor * llm_graph_context::build_cvec(
     return cvec->apply_to(ctx0, cur, il);
 }
 
-ggml_tensor * llm_graph_context::build_hadamard_input(
-          ggml_tensor * w,
-          ggml_tensor * cur) const {
-    if (!hdmd) {
-        return cur;
-    }
-    const auto it = hdmd->rot.find(w);
-    if (it == hdmd->rot.end()) {
-        return cur;
-    }
-    const auto & t = it->second;
-    if (ggml_tensor * x = res->get_hdmd_input(cur, t.rot)) {
-        return x;
-    }
-    ggml_tensor * x = cur;
-    if (t.perm_rep > 1) {
-        // tiled [hd, nk, rep] -> grouped [hd, rep, nk] feature order
-        x = ggml_is_contiguous(x) ? x : ggml_cont(ctx0, x);
-        const int64_t ne1 = x->ne[1], ne2 = x->ne[2], ne3 = x->ne[3];
-        x = ggml_reshape_4d(ctx0, x, t.perm_hd, t.perm_nk, t.perm_rep, ne1*ne2*ne3);
-        x = ggml_cont(ctx0, ggml_permute(ctx0, x, 0, 2, 1, 3));
-        x = ggml_reshape_4d(ctx0, x, t.perm_hd*t.perm_nk*t.perm_rep, ne1, ne2, ne3);
-    }
-    if (t.signs) {
-        x = ggml_mul(ctx0, x, t.signs);
-    }
-    x = llama_mul_mat_hadamard(ctx0, x, t.rot);
-    res->set_hdmd_input(cur, t.rot, x);
-    return x;
-}
-
 ggml_tensor * llm_graph_context::build_lora_mm(
           ggml_tensor * w,
           ggml_tensor * cur,
           ggml_tensor * w_s) const {
-    cur = build_hadamard_input(w, cur);
     ggml_tensor * res = ggml_mul_mat(ctx0, w, cur);
 
     if (prec_policy) {
@@ -1658,7 +1593,6 @@ ggml_tensor * llm_graph_context::build_lora_mm_id(
           ggml_tensor * ids,
           ggml_tensor * w_s,
           ggml_tensor * slots) const {
-    cur = build_hadamard_input(w, cur);
     // the experts in the MoE cache are selected by their slots
     ggml_tensor * res = slots == nullptr ?
         ggml_mul_mat_id(ctx0, w, cur, ids) :
@@ -2525,9 +2459,6 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots(
     ggml_backend_sched_set_tensor_backend(sched, slots, moe_cache->backend(il));
     cb(slots, "ffn_moe_slots", il);
 
-    // add the lookup now: the transform of a Hadamard-folded expert reads host weights and would start its split first
-    ggml_build_forward_expand(gf, slots);
-
     return ggml_reshape_2d(ctx0, slots, selected_experts->ne[0], selected_experts->ne[1]); // [n_expert_used, n_tokens]
 }
 
@@ -2591,16 +2522,6 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
     // TODO: when lora is active, this is likely going to cause issues similar to https://github.com/ggml-org/llama.cpp/pull/30160
     //       need to add lora tests and refactor the logic to make the lora GET_ROWS go at the front of the graph
     auto build_tok = [&](ggml_tensor * cur, ggml_tensor * ids) {
-        // a Hadamard-latent table stores rotated rows: restore the primal basis, h = s * (H z)
-        if (hdmd) {
-            if (const auto it = hdmd->inv.find(tok_embd); it != hdmd->inv.end()) {
-                cur = llama_mul_mat_hadamard(ctx0, cur, it->second.rot);
-                if (it->second.signs) {
-                    cur = ggml_mul(ctx0, cur, it->second.signs);
-                }
-            }
-        }
-
         // apply lora for embedding tokens if needed
         for (const auto & lora : *loras) {
             llama_adapter_lora_weight * lw = lora.first->get_weight(tok_embd);
@@ -2703,7 +2624,7 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to
 }
 
 ggml_tensor * llm_graph_context::build_inp_pos() const {
-    auto inp = std::make_unique<llm_graph_input_pos>(hparams.n_pos_per_embd(), hparams.rope_section_order);
+    auto inp = std::make_unique<llm_graph_input_pos>(hparams.n_pos_per_embd());
 
     auto & cur = inp->pos;
 

src/llama-graph.h

@@ -20,7 +20,6 @@ struct ggml_tensor;
 struct llama_cparams;
 struct llama_layer;
 struct llama_prec_policy;
-struct llama_hadamard;
 
 class llama_moe_cache;
 
@@ -150,10 +149,10 @@ public:
     const int64_t n_embd = 0;
 };
 
-// similar to llm_graph_input_embd but with an additional hidden state input, fed from ubatch.embd_state
+// similar to llm_graph_input_embd but with an additional hidden state input
 class llm_graph_input_embd_h : public llm_graph_input_i {
 public:
-    llm_graph_input_embd_h(int64_t n_embd, int64_t n_embd_state) : n_embd(n_embd), n_embd_state(n_embd_state) {}
+    llm_graph_input_embd_h(int64_t n_embd) : n_embd(n_embd) {}
     virtual ~llm_graph_input_embd_h() = default;
 
     void set_input(const llama_ubatch * ubatch) override;
@@ -162,16 +161,14 @@ public:
 
     ggml_tensor * tokens = nullptr; // I32 [n_batch]
     ggml_tensor * embd   = nullptr; // F32 [n_embd, n_batch]
-    ggml_tensor * h      = nullptr; // F32 [n_embd_state, n_batch]
+    ggml_tensor * h      = nullptr; // F32 [n_embd, n_batch]
 
-    const int64_t n_embd       = 0;
-    const int64_t n_embd_state = 0;
+    const int64_t n_embd = 0;
 };
 
 class llm_graph_input_pos : public llm_graph_input_i {
 public:
-    llm_graph_input_pos(uint32_t n_pos_per_embd, llama_rope_section_order rope_section_order = LLAMA_ROPE_SECTION_ORDER_TYXZ)
-        : n_pos_per_embd(n_pos_per_embd), rope_section_order(rope_section_order) {}
+    llm_graph_input_pos(uint32_t n_pos_per_embd) : n_pos_per_embd(n_pos_per_embd) {}
     virtual ~llm_graph_input_pos() = default;
 
     void set_input(const llama_ubatch * ubatch) override;
@@ -181,7 +178,6 @@ public:
     ggml_tensor * pos = nullptr; // I32 [n_batch]
 
     const uint32_t n_pos_per_embd = 1;
-    const llama_rope_section_order rope_section_order = LLAMA_ROPE_SECTION_ORDER_TYXZ;
 };
 
 // temperature tuning, used by llama4
@@ -804,8 +800,6 @@ struct llm_graph_params {
 
     const llama_prec_policy * prec_policy = nullptr;
 
-    const llama_hadamard * hdmd = nullptr;
-
     std::map<llama_seq_id, llama_sampler *> samplers;
 
     static bool samplers_equal(
@@ -844,8 +838,7 @@ struct llm_graph_params {
                 (!ubatch.token && !other.ubatch.token) ||
                 (!ubatch.embd  && !other.ubatch.embd)  ||
                 (ubatch.token && other.ubatch.token && ubatch.embd && other.ubatch.embd)
-            ) &&
-            (!ubatch.embd_state == !other.ubatch.embd_state);
+            );
 
         // when we split the batch using "equal_seqs" we have to verify that the participating sequences are the same
         //   the reason is because the set of attention streams would be different for different sequences
@@ -946,10 +939,6 @@ public:
 
     void add_fused_node(llm_graph_fused_node result);
 
-    // Hadamard-transformed activations, keyed by (input, rotation): folded weights that read the same activation share one transform
-    ggml_tensor * get_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot) const;
-    void          set_hdmd_input(const ggml_tensor * cur, const ggml_tensor * rot, ggml_tensor * res);
-
     const std::vector<llm_graph_fused_node> & get_fused_nodes() const { return fused_nodes; }
 
     void set_params(const llm_graph_params & params);
@@ -972,8 +961,6 @@ public:
     std::vector<llm_graph_input_ptr> inputs;
     std::vector<llm_graph_fused_node> fused_nodes;
 
-    std::map<std::pair<const ggml_tensor *, const ggml_tensor *>, ggml_tensor *> hdmd_inputs;
-
     ggml_context_ptr ctx_compute;
 
     // memory buffers used to evaluate the model
@@ -1057,8 +1044,6 @@ struct llm_graph_context {
 
     const llama_prec_policy * prec_policy;
 
-    const llama_hadamard * hdmd;
-
     std::map<llama_seq_id, llama_sampler *> samplers;
 
     const llm_graph_cb & cb_func;
@@ -1091,11 +1076,6 @@ struct llm_graph_context {
              ggml_tensor * cur,
                      int   il) const;
 
-    // apply the activation-side transform of a Hadamard-folded weight, if any
-    ggml_tensor * build_hadamard_input(
-              ggml_tensor * w,
-              ggml_tensor * cur) const;
-
     // do mat_mul, while optionally apply lora and per-tensor scale
     ggml_tensor * build_lora_mm(
               ggml_tensor * w,

src/llama-hparams.h

@@ -36,13 +36,6 @@ enum llama_non_causal_type {
     LLAMA_NON_CAUSAL_TYPE_SWA_FULL = 2, // all layers non-causal, SWA not applied between tokens of the current ubatch (deepseek 4)
 };
 
-// M-RoPE: which input position slot feeds each RoPE section
-enum llama_rope_section_order {
-    LLAMA_ROPE_SECTION_ORDER_UNSPECIFIED = -1,
-    LLAMA_ROPE_SECTION_ORDER_TYXZ        = 0, // default, slot i feeds section i
-    LLAMA_ROPE_SECTION_ORDER_ZYXT        = 1, // MiniCPM-V 4.7: time last
-};
-
 // forward declaration; full definition in llama-graph.h
 enum llm_ffn_op_type : int;
 
@@ -173,8 +166,6 @@ struct llama_hparams {
 
     std::array<int, 4> rope_sections;
 
-    enum llama_rope_section_order rope_section_order = LLAMA_ROPE_SECTION_ORDER_TYXZ;
-
     // Per-layer RoPE enable flags (1 = use RoPE, 0 = NoPE)
     // by default, all layers use RoPE (controlled by rope_finetuned)
     std::array<uint32_t, LLAMA_MAX_LAYERS> rope_pattern;

src/llama-kv-cache-dsv4.cpp

@@ -159,7 +159,6 @@ static llama_ubatch dsv4_build_raw_write_ubatch(const llama_ubatch & ubatch) {
         /*.n_pos        =*/ ubatch.n_pos,
         /*.token        =*/ data->token.empty() ? nullptr : data->token.data(),
         /*.embd         =*/ nullptr,
-        /*.embd_state   =*/ nullptr,
         /*.pos          =*/ data->pos.data(),
         /*.n_seq_id     =*/ data->n_seq_id.data(),
         /*.seq_id       =*/ data->seq_id.data(),

src/llama-model-saver.cpp

@@ -21,7 +21,6 @@ bool llama_model_saver_supports_arch(llm_arch arch) {
         case LLM_ARCH_APERTUS:
         case LLM_ARCH_STEP35:
         case LLM_ARCH_CLEF: // the head tensors are not saved
-        case LLM_ARCH_VLLM_DECISION2:
             return false;
         default:
             return true;
@@ -366,7 +365,6 @@ void llama_model_saver::add_kv_from_model() {
     add_kv(LLM_KV_ROPE_DIMENSION_COUNT,              hparams.n_rot_full);
     add_kv(LLM_KV_ROPE_DIMENSION_COUNT_SWA,          hparams.n_rot_swa);
     add_kv(LLM_KV_ROPE_DIMENSION_SECTIONS,           hparams.rope_sections);
-    add_kv(LLM_KV_ROPE_SECTION_ORDER,                llama_rope_section_order_name(hparams.rope_section_order));
     add_kv(LLM_KV_ROPE_FREQ_BASE,                    hparams.rope_freq_base_train);
     add_kv(LLM_KV_ROPE_FREQ_BASE_SWA,                hparams.rope_freq_base_train_swa);
     // add_kv(LLM_KV_ROPE_SCALE_LINEAR,                 rope_scaling_factor); // old name

src/llama-model.cpp

@@ -330,8 +330,6 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_qwen35moe(params);
         case LLM_ARCH_CLEF:
             return new llama_model_clef(params);
-        case LLM_ARCH_VLLM_DECISION2:
-            return new llama_model_vllm_decision2(params);
         case LLM_ARCH_QWEN4EXP:
             return new llama_model_qwen4exp(params);
         case LLM_ARCH_MISTRAL3:
@@ -1066,25 +1064,6 @@ static llama_rope_scaling_type llama_rope_scaling_type_from_string(const std::st
     return LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED;
 }
 
-static const std::map<llama_rope_section_order, const char *> LLAMA_ROPE_SECTION_ORDERS = {
-    { LLAMA_ROPE_SECTION_ORDER_TYXZ, "tyxz" },
-    { LLAMA_ROPE_SECTION_ORDER_ZYXT, "zyxt" },
-};
-
-std::string llama_rope_section_order_name(llama_rope_section_order rope_section_order) {
-    return LLAMA_ROPE_SECTION_ORDERS.at(rope_section_order);
-}
-
-static llama_rope_section_order llama_rope_section_order_from_string(const std::string & name) {
-    for (const auto & kv : LLAMA_ROPE_SECTION_ORDERS) {
-        if (kv.second == name) {
-            return kv.first;
-        }
-    }
-
-    return LLAMA_ROPE_SECTION_ORDER_UNSPECIFIED;
-}
-
 // Maps GGUF activation names to the FFN op type used by the graph builders.
 static const std::map<std::string, llm_ffn_op_type> LLM_FFN_OP_TYPES_FROM_STRING = {
     { "gelu",              LLM_FFN_GEGLU_ERF },
@@ -1352,8 +1331,6 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
         gguf_kv.emplace(name, value);
     }
 
-    load_hparams_hadamard(ml);
-
     // get general kv
     ml.get_key(LLM_KV_GENERAL_NAME, name, false);
 
@@ -1470,13 +1447,6 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
     hparams.rope_scaling_type_train = llama_rope_scaling_type_from_string(rope_scaling);
     GGML_ASSERT(hparams.rope_scaling_type_train != LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED);
 
-    std::string rope_section_order("tyxz");
-    ml.get_key(LLM_KV_ROPE_SECTION_ORDER, rope_section_order, false);
-    hparams.rope_section_order = llama_rope_section_order_from_string(rope_section_order);
-    if (hparams.rope_section_order == LLAMA_ROPE_SECTION_ORDER_UNSPECIFIED) {
-        throw std::runtime_error("unknown rope section order: " + rope_section_order);
-    }
-
     // TODO: Handle SWA metadata similarly when models start implementing it
     // rope_freq_scale (inverse of the kv) is optional
     float ropescale = 0.0f;
@@ -1547,10 +1517,6 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
     }
 
     hparams.rope_type = llama_model_rope_type(this);
-
-    if (hparams.rope_section_order != LLAMA_ROPE_SECTION_ORDER_TYXZ && hparams.n_pos_per_embd() != 4) {
-        throw std::runtime_error("rope section order " + llama_rope_section_order_name(hparams.rope_section_order) + " requires M-RoPE");
-    }
 }
 
 void llama_model_base::load_vocab(llama_model_loader & ml) {
@@ -2019,342 +1985,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
         }
     }
 
-    load_tensors_hadamard();
-
     return true;
 }
 
-// read and check the prism.hadamard metadata, and record the folded weights (load_tensors_hadamard makes the tensors)
-void llama_model_base::load_hparams_hadamard(llama_model_loader & ml) {
-    uint32_t hadamard_version = 0;
-    ml.get_key(LLM_KV_PRISM_HADAMARD_TIED_OUTPUT, hdmd.tied_output, false);
-    if (ml.get_key(LLM_KV_PRISM_HADAMARD_VERSION, hadamard_version, false)) {
-        if (hadamard_version != 1 && hadamard_version != 2) {
-            throw std::runtime_error(format("unsupported prism.hadamard.version: %u", hadamard_version));
-        }
-
-        if ((hadamard_version == 2) != hdmd.tied_output) {
-            throw std::runtime_error("prism.hadamard version 2 requires tied_output=true; version 1 forbids it");
-        }
-        if (hdmd.tied_output && ml.get_weight("output.weight")) {
-            throw std::runtime_error("prism.hadamard.tied_output requires output.weight to be absent");
-        }
-
-        uint32_t block_size = 0;
-        std::string transform;
-        std::string axis;
-        std::string sign_mode;
-        std::vector<std::string> weight_names;
-
-        ml.get_key(LLM_KV_PRISM_HADAMARD_BLOCK_SIZE, block_size);
-        ml.get_key(LLM_KV_PRISM_HADAMARD_TRANSFORM, transform);
-        ml.get_key(LLM_KV_PRISM_HADAMARD_AXIS, axis);
-        ml.get_key(LLM_KV_PRISM_HADAMARD_SIGN_MODE, sign_mode);
-        ml.get_arr(LLM_KV_PRISM_HADAMARD_WEIGHT_NAMES, weight_names);
-
-        if (block_size == 0 || (block_size & (block_size - 1)) != 0) {
-            throw std::runtime_error(format("invalid prism.hadamard.block_size: %u", block_size));
-        }
-        if (transform != "normalized-sylvester-walsh-hadamard") {
-            throw std::runtime_error(format("unsupported prism.hadamard.transform: %s", transform.c_str()));
-        }
-        if (axis != "input-last-dimension") {
-            throw std::runtime_error(format("unsupported prism.hadamard.axis: %s", axis.c_str()));
-        }
-        if (sign_mode != "identity" && sign_mode != "explicit") {
-            throw std::runtime_error(format("unsupported prism.hadamard.sign_mode: %s", sign_mode.c_str()));
-        }
-        if (weight_names.empty()) {
-            throw std::runtime_error("prism.hadamard.weight_names is empty");
-        }
-
-        if (sign_mode == "explicit") {
-            std::vector<int32_t> sign_widths;
-            std::vector<int32_t> sign_values;
-            ml.get_arr(LLM_KV_PRISM_HADAMARD_SIGN_WIDTHS, sign_widths);
-            ml.get_arr(LLM_KV_PRISM_HADAMARD_SIGN_VALUES, sign_values);
-            // explicit mode with no widths gives an empty sign table, which acts as identity and changes the model
-            if (sign_widths.empty()) {
-                throw std::runtime_error("prism.hadamard.sign_mode is explicit but sign_widths is empty");
-            }
-            size_t off = 0;
-            for (const int32_t width : sign_widths) {
-                if (width <= 0 || (uint32_t) width % block_size != 0 || off + width > sign_values.size()) {
-                    throw std::runtime_error(format("invalid prism.hadamard sign width: %d", width));
-                }
-                if (hdmd.sign_data.count(width)) {
-                    throw std::runtime_error(format("duplicate prism.hadamard sign width: %d", width));
-                }
-                auto & vec = hdmd.sign_data[width];
-                vec.assign(sign_values.begin() + off, sign_values.begin() + off + width);
-                for (const int32_t v : vec) {
-                    if (v != 1 && v != -1) {
-                        throw std::runtime_error("prism.hadamard sign values must be +/-1");
-                    }
-                }
-                off += width;
-            }
-            if (off != sign_values.size()) {
-                throw std::runtime_error("prism.hadamard.sign_values length mismatch");
-            }
-        }
-
-        ml.get_key(LLM_KV_PRISM_HADAMARD_GDN_V_GROUPED, hdmd.gdn_v_grouped, false);
-
-        // only build_lora_mm/build_lora_mm_id apply the transform: refuse archs and tensor kinds that can skip them
-        switch (arch) {
-            case LLM_ARCH_LLAMA:
-            case LLM_ARCH_QWEN3:
-            case LLM_ARCH_QWEN3MOE:
-            case LLM_ARCH_QWEN35:
-            case LLM_ARCH_QWEN35MOE:
-            case LLM_ARCH_QWEN3NEXT:
-                break;
-            default:
-                throw std::runtime_error(format(
-                    "prism.hadamard: arch '%s' is not verified to apply the activation transform to all folded weights",
-                    llm_arch_name(arch)));
-        }
-
-        // a folded weight W_f = W*D*H is the weight W with the +1/-1 signs D and the normalized block Hadamard H folded in
-        // H*H = I and D*D = I, so W*x = W_f*(H*(D*x)): the graph applies D, then H, to the matmul input
-        const auto is_foldable_weight = [](const std::string & name) {
-            static const char * kinds[] = {
-                "attn_q", "attn_k", "attn_v", "attn_qkv", "attn_gate", "attn_output",
-                "ffn_gate", "ffn_up", "ffn_down",
-                "ffn_gate_exps", "ffn_up_exps", "ffn_down_exps", "ffn_gate_up_exps",
-                "ffn_gate_shexp", "ffn_up_shexp", "ffn_down_shexp",
-                "ssm_out",
-            };
-            if (name == "output.weight") {
-                return true; // the output head is built through build_lora_mm in every arch
-            }
-            if (name.compare(0, 4, "blk.") != 0) {
-                return false;
-            }
-            size_t pos = 4;
-            while (pos < name.size() && isdigit((unsigned char) name[pos])) {
-                pos++;
-            }
-            if (pos == 4 || pos >= name.size() || name[pos] != '.') {
-                return false;
-            }
-            pos++;
-            for (const char * kind : kinds) {
-                const std::string suffix = std::string(kind) + ".weight";
-                if (name.compare(pos, std::string::npos, suffix) == 0) {
-                    return true;
-                }
-            }
-            return false;
-        };
-
-        for (const auto & weight_name : weight_names) {
-            if (!is_foldable_weight(weight_name)) {
-                throw std::runtime_error(format(
-                    "prism.hadamard: weight '%s' is not on a verified Hadamard-aware matmul path", weight_name.c_str()));
-            }
-            if (!hdmd.weight_blocks.emplace(weight_name, block_size).second) {
-                throw std::runtime_error(format("duplicate prism.hadamard weight: %s", weight_name.c_str()));
-            }
-        }
-
-        // tables read by row lookup store latent rows: the inverse transform goes on the lookup result
-        std::vector<std::string> inverse_names;
-        ml.get_arr(LLM_KV_PRISM_HADAMARD_INVERSE_WEIGHT_NAMES, inverse_names, false);
-        for (const auto & name : inverse_names) {
-            // the graph applies the inverse only after the token-embedding lookup, other latent tables stay rotated
-            if (name != "token_embd.weight") {
-                throw std::runtime_error(format(
-                    "prism.hadamard: weight '%s' is not a verified inverse-after-lookup table", name.c_str()));
-            }
-            if (hdmd.weight_blocks.count(name) || !hdmd.inverse_blocks.emplace(name, block_size).second) {
-                throw std::runtime_error(format("duplicate prism.hadamard inverse weight: %s", name.c_str()));
-            }
-        }
-    }
-
-    if (hdmd.tied_output) {
-        if (hadamard_version != 2) {
-            throw std::runtime_error("prism.hadamard.tied_output requires version 2");
-        }
-        const auto it = hdmd.inverse_blocks.find("token_embd.weight");
-        if (it == hdmd.inverse_blocks.end()) {
-            throw std::runtime_error("prism.hadamard.tied_output requires a latent token embedding");
-        }
-        hdmd.weight_blocks.emplace("token_embd.weight", it->second);
-    } else if (hdmd.inverse_blocks.count("token_embd.weight") && !ml.get_weight("output.weight")) {
-        throw std::runtime_error("a tied Hadamard output requires version 2 and tied_output=true");
-    }
-}
-
-// make one Hadamard matrix per block size and one sign vector per width (the GGUF does not contain them)
-// each tensor goes on the buffer type of its weights (never a CPU extra type), and each transform goes in hdmd.rot or hdmd.inv
-void llama_model_base::load_tensors_hadamard() {
-    if (hdmd.weight_blocks.empty() && hdmd.inverse_blocks.empty()) {
-        return;
-    }
-
-    struct hadamard_rotation {
-        uint32_t block_size;
-        ggml_backend_buffer_type_t buft;
-        ggml_tensor * tensor;
-    };
-
-    std::vector<hadamard_rotation> rotations;
-    std::map<std::pair<uint32_t, ggml_backend_buffer_type_t>, ggml_tensor *> sign_tensors;
-
-    const std::pair<const std::unordered_map<std::string, uint32_t> *, llama_hadamard_rotations *> groups[] = {
-        { &hdmd.weight_blocks,  &hdmd.rot },
-        { &hdmd.inverse_blocks, &hdmd.inv  },
-    };
-    // inverse transforms use the buffer type of the forward rotations, not the host type of a CPU-mapped table (no PCIe round trip per token)
-    ggml_backend_buffer_type_t preferred_buft = nullptr;
-
-    for (const auto & [blocks, target] : groups) {
-        for (const auto & entry : *blocks) {
-            const std::string & weight_name = entry.first;
-            const uint32_t block_size = entry.second;
-            const ggml_tensor * weight = get_tensor(weight_name.c_str());
-            if (hdmd.tied_output && weight_name == "token_embd.weight") {
-                weight = target == &hdmd.rot ? output : tok_embd;
-                if (!weight || strcmp(weight->name, "token_embd.weight") != 0) {
-                    throw std::runtime_error("prism.hadamard.tied_output is not bound to the token embedding");
-                }
-            }
-            if (weight == nullptr) {
-                throw std::runtime_error(format("prism.hadamard weight not found: %s", weight_name.c_str()));
-            }
-            if (weight->ne[0] % block_size != 0) {
-                throw std::runtime_error(format(
-                    "prism.hadamard block size %u does not divide input dimension %lld for %s",
-                    block_size, (long long) weight->ne[0], weight_name.c_str()));
-            }
-            if (weight->buffer == nullptr) {
-                throw std::runtime_error(format("prism.hadamard weight has no buffer: %s", weight_name.c_str()));
-            }
-
-            ggml_backend_buffer_type_t buft = ggml_backend_buffer_get_type(weight->buffer);
-            // CPU extra buffer types (e.g. CPU_REPACK) only accept tensors they can repack
-            if (ggml_backend_dev_t dev = ggml_backend_buft_get_device(buft)) {
-                if (ggml_backend_dev_type(dev) == GGML_BACKEND_DEVICE_TYPE_CPU) {
-                    buft = ggml_backend_dev_buffer_type(dev);
-                }
-            }
-            if (target == &hdmd.rot) {
-                preferred_buft = buft;
-            } else if (preferred_buft) {
-                buft = preferred_buft;
-            }
-            auto it = std::find_if(rotations.begin(), rotations.end(),
-                    [block_size, buft](const hadamard_rotation & rotation) {
-                        return rotation.block_size == block_size && rotation.buft == buft;
-                    });
-
-            if (it == rotations.end()) {
-                ggml_init_params params = {
-                    /*.mem_size   =*/ ggml_tensor_overhead(),
-                    /*.mem_buffer =*/ NULL,
-                    /*.no_alloc   =*/ true,
-                };
-                ggml_context_ptr ctx { ggml_init(params) };
-                if (!ctx) {
-                    throw std::runtime_error("failed to create Hadamard rotation context");
-                }
-
-                ggml_tensor * rotation = ggml_new_tensor_2d(ctx.get(), GGML_TYPE_F32, block_size, block_size);
-                char rotation_name[GGML_MAX_NAME];
-                snprintf(rotation_name, sizeof(rotation_name), "prism.hadamard.%u", block_size);
-                ggml_set_name(rotation, rotation_name);
-
-                ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) };
-                if (!buffer) {
-                    throw std::runtime_error(format("unable to allocate %s Hadamard rotation buffer", ggml_backend_buft_name(buft)));
-                }
-                ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
-
-                std::vector<float> data((size_t) block_size * block_size);
-                const float scale = 1.0f / sqrtf((float) block_size);
-                for (uint32_t row = 0; row < block_size; ++row) {
-                    for (uint32_t col = 0; col < block_size; ++col) {
-                        uint32_t parity = row & col;
-                        parity ^= parity >> 16;
-                        parity ^= parity >> 8;
-                        parity ^= parity >> 4;
-                        parity ^= parity >> 2;
-                        parity ^= parity >> 1;
-                        data[(size_t) row * block_size + col] = (parity & 1) ? -scale : scale;
-                    }
-                }
-                ggml_backend_tensor_set(rotation, data.data(), 0, data.size() * sizeof(float));
-
-                std::vector<ggml_backend_buffer_ptr> buffers;
-                buffers.emplace_back(std::move(buffer));
-                pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
-                rotations.push_back({ block_size, buft, rotation });
-                it = std::prev(rotations.end());
-            }
-
-            ggml_tensor * sign_tensor = nullptr;
-            if (!hdmd.sign_data.empty()) {
-                const uint32_t width = (uint32_t) weight->ne[0];
-                const auto sd = hdmd.sign_data.find(width);
-                if (sd == hdmd.sign_data.end()) {
-                    throw std::runtime_error(format(
-                        "prism.hadamard has no sign vector for width %u (%s)", width, weight_name.c_str()));
-                }
-                const auto key = std::make_pair(width, buft);
-                auto st = sign_tensors.find(key);
-                if (st == sign_tensors.end()) {
-                    ggml_init_params params = {
-                        /*.mem_size   =*/ ggml_tensor_overhead(),
-                        /*.mem_buffer =*/ NULL,
-                        /*.no_alloc   =*/ true,
-                    };
-                    ggml_context_ptr ctx { ggml_init(params) };
-                    if (!ctx) {
-                        throw std::runtime_error("failed to create Hadamard sign context");
-                    }
-
-                    ggml_tensor * signs = ggml_new_tensor_1d(ctx.get(), GGML_TYPE_F32, width);
-                    char sign_name[GGML_MAX_NAME];
-                    snprintf(sign_name, sizeof(sign_name), "prism.hadamard.signs.%u", width);
-                    ggml_set_name(signs, sign_name);
-
-                    ggml_backend_buffer_ptr buffer { ggml_backend_alloc_ctx_tensors_from_buft(ctx.get(), buft) };
-                    if (!buffer) {
-                        throw std::runtime_error(format("unable to allocate %s Hadamard sign buffer", ggml_backend_buft_name(buft)));
-                    }
-                    ggml_backend_buffer_set_usage(buffer.get(), GGML_BACKEND_BUFFER_USAGE_WEIGHTS);
-
-                    std::vector<float> data(width);
-                    for (uint32_t i = 0; i < width; ++i) {
-                        data[i] = (float) sd->second[i];
-                    }
-                    ggml_backend_tensor_set(signs, data.data(), 0, data.size() * sizeof(float));
-
-                    std::vector<ggml_backend_buffer_ptr> buffers;
-                    buffers.emplace_back(std::move(buffer));
-                    pimpl->ctxs_bufs.emplace_back(std::move(ctx), std::move(buffers));
-                    st = sign_tensors.emplace(key, signs).first;
-                }
-                sign_tensor = st->second;
-            }
-
-            llama_hadamard_transform transform { it->tensor, sign_tensor };
-            // the GDN output projection reads its value heads in tiled order, see set_gdn_v_perm
-            if (hdmd.gdn_v_grouped && weight_name.find(".ssm_out.") != std::string::npos &&
-                !transform.set_gdn_v_perm(weight->ne[0], hparams.ssm_dt_rank, hparams.ssm_n_group)) {
-                throw std::runtime_error(format("prism.hadamard: bad GDN head geometry for %s", weight_name.c_str()));
-            }
-            target->emplace(weight, transform);
-        }
-    }
-
-    LLAMA_LOG_INFO("%s: loaded %zu Hadamard-folded weight(s) (%zu inverse-lookup) using %zu rotation(s) and %zu sign vector(s)\n",
-            __func__, hdmd.rot.size() + hdmd.inv.size(), hdmd.inv.size(), rotations.size(), sign_tensors.size());
-}
-
 ggml_tensor * llama_model_base::create_tensor(llama_model_loader & ml, const LLM_TN_IMPL & tn, const std::initializer_list<int64_t> & ne, int flags) {
     const buft_list_t * buft_list_layer = nullptr;
     if (tn.bid != -1) {
@@ -2525,9 +2158,6 @@ void llama_model::print_info() const {
         if (const auto & s = hparams.rope_sections; s[0] || s[1] || s[2] || s[3]) {
             LLAMA_LOG_INFO("%s: mrope sections        = [%d, %d, %d, %d]\n", __func__, s[0], s[1], s[2], s[3]);
         }
-        if (hparams.rope_section_order != LLAMA_ROPE_SECTION_ORDER_TYXZ) {
-            LLAMA_LOG_INFO("%s: rope section order    = %s\n", __func__, llama_rope_section_order_name(hparams.rope_section_order).c_str());
-        }
         if (!classifier_labels.empty()) {
             LLAMA_LOG_INFO("%s: n_cls_out             = %u\n", __func__, hparams.n_cls_out);
 
@@ -3536,7 +3166,6 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_DREAM:
         case LLM_ARCH_QWEN2MOE:
         case LLM_ARCH_QWEN3:
-        case LLM_ARCH_VLLM_DECISION2:
         case LLM_ARCH_QWEN3MOE:
         case LLM_ARCH_LLADA_MOE:
         case LLM_ARCH_RND1:

src/llama-model.h

@@ -160,7 +160,6 @@ enum llm_type {
 };
 
 std::string llama_rope_scaling_type_name(llama_rope_scaling_type rope_scaling_type);
-std::string llama_rope_section_order_name(llama_rope_section_order rope_section_order);
 
 // Map a GGUF activation-name string to llm_ffn_op_type. Returns `fallback` if
 // the string is empty or not recognized.
@@ -638,44 +637,6 @@ struct llama_prec_policy {
     void load(llama_model_loader & ml, const llama_model & model);
 };
 
-// transform of a folded weight, applied to the matmul input: optional sign flip, then the normalized block Hadamard rotation
-struct llama_hadamard_transform {
-    ggml_tensor * rot;
-    ggml_tensor * signs; // nullptr for identity sign mode
-
-    // if perm_rep > 1, permute the input from tiled head order [hd, nk, rep] to grouped order [hd, rep, nk] before signs and rotation
-    int64_t perm_hd  = 0;
-    int64_t perm_nk  = 0;
-    int64_t perm_rep = 0;
-
-    // ssm_out of a gated delta net gets its value heads in tiled order, but the fold used grouped order
-    // record the head geometry for that permutation, return false if it does not match the input width
-    bool set_gdn_v_perm(int64_t n_in, int64_t n_v, int64_t n_k) {
-        if (n_k <= 0 || n_v <= 0 || n_v % n_k != 0 || n_in % n_v != 0) {
-            return false;
-        }
-        perm_hd  = n_in / n_v;
-        perm_nk  = n_k;
-        perm_rep = n_v / n_k;
-        return true;
-    }
-};
-using llama_hadamard_rotations = std::unordered_map<const ggml_tensor *, llama_hadamard_transform>;
-
-struct llama_hadamard {
-    // names and sign data come from the GGUF metadata in load_hparams, the transforms are made in load_tensors
-    std::unordered_map<std::string, uint32_t> weight_blocks;
-    std::unordered_map<std::string, uint32_t> inverse_blocks;
-
-    std::map<uint32_t, std::vector<int32_t>> sign_data;
-
-    bool gdn_v_grouped = false;
-    bool tied_output = false;
-
-    llama_hadamard_rotations rot; // folded weight -> activation transform
-    llama_hadamard_rotations inv; // latent lookup table -> inverse transform
-};
-
 struct llama_model {
     llm_type type = LLM_TYPE_UNKNOWN;
     llm_arch arch = LLM_ARCH_UNKNOWN;
@@ -688,8 +649,6 @@ struct llama_model {
     // per-tensor activation precision policy
     llama_prec_policy prec_policy;
 
-    llama_hadamard hdmd;
-
     // for classifier models
     std::vector<std::string> classifier_labels;
 
@@ -897,12 +856,6 @@ struct llama_model_base : public llama_model {
     };
     nextn_flags_t nextn_flags(llama_model_loader & ml, llm_tensor trunk_probe = LLM_TENSOR_ATTN_NORM) const;
 
-    // helper: read the prism.hadamard metadata and record which weights are folded
-    void load_hparams_hadamard(llama_model_loader & ml);
-
-    // helper: make the prism.hadamard rotation and sign tensors for the folded weights
-    void load_tensors_hadamard();
-
     // helper: read the SWA pattern as one flag per layer, or as a period expanded by set_swa_pattern
     void load_swa_pattern(llama_model_loader & ml, uint32_t n_pattern, bool dense_first = false);
 

src/llama-quant.cpp

@@ -346,11 +346,6 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
         quantize &= name.find("attn_v_b.weight")         == std::string::npos;
     }
 
-    // the decision head of vllm-decision2 is small and computed in F32
-    if (arch == LLM_ARCH_VLLM_DECISION2) {
-        quantize &= name.compare(0, 9, "decision.") != 0;
-    }
-
     // do not quantize RWKV's small yet 2D weights
     quantize &= name.find("time_mix_first.weight") == std::string::npos;
     quantize &= name.find("time_mix_w0.weight") == std::string::npos;

src/models/bailingmoe3.cpp

@@ -438,17 +438,15 @@ llama_model_bailingmoe3::graph_mtp::graph_mtp(const llama_model & model, const l
     const int64_t kv_lora_rank = hparams.n_lora_kv;
     const float kq_scale = 1.0f / sqrtf((float) qk_head_dim);
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
     ggml_set_input(inp->embd);
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
+    ggml_set_name(inp->embd, "mtp_h_input");
 
-    ggml_tensor * tok_embd = ubatch.token ? ggml_get_rows(ctx0, model.tok_embd, inp->tokens) : inp->embd;
-    ggml_tensor * h_norm = build_norm(inp->h, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
+    ggml_tensor * tok_embd = ggml_get_rows(ctx0, model.tok_embd, inp->tokens);
+    ggml_tensor * h_norm = build_norm(inp->embd, layer.nextn.hnorm, nullptr, LLM_NORM_RMS, il);
     ggml_tensor * e_norm = build_norm(tok_embd, layer.nextn.enorm, nullptr, LLM_NORM_RMS, il);
     ggml_tensor * cur = ggml_mul_mat(ctx0, layer.nextn.eh_proj, ggml_concat(ctx0, e_norm, h_norm, 0));
     cb(cur, "mtp_eh_proj", il);

src/models/cohere2moe.cpp

@@ -297,7 +297,7 @@ llama_model_cohere2moe::graph_mtp::graph_mtp(const llama_model & model, const ll
     const llm_norm_type cohere2moe_norm_type = hparams.f_norm_rms_eps == 0.0f ? LLM_NORM : LLM_NORM_RMS;
 
     // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/deepseek2.cpp

@@ -206,7 +206,7 @@ llama_model_deepseek2::graph_mtp::graph_mtp(const llama_model & model, const llm
     GGML_ASSERT(layer.ffn_down_shexp);
     GGML_ASSERT(layer.ffn_up_shexp);
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/deepseek32.cpp

@@ -520,7 +520,7 @@ llama_model_deepseek32::graph_mtp::graph_mtp(const llama_model & model, const ll
     const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k));
 
     // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/deepseek4.cpp

@@ -1378,26 +1378,20 @@ llama_model_deepseek4::graph_mtp::graph_mtp(const llama_model & model, const llm
     GGML_ASSERT(layer.nextn.enorm   && "MTP block missing nextn.enorm");
     GGML_ASSERT(layer.nextn.hnorm   && "MTP block missing nextn.hnorm");
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd_out());
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_out());
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
 
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens);
     ggml_set_input(inp->embd);
 
     inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens);
     ggml_set_input(inp->h);
     ggml_set_name(inp->h, "mtp_h_input");
 
-    ggml_tensor * tok_embd;
-    if (ubatch.token) {
-        ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
-
-        tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
-    } else {
-        tok_embd = inp->embd;
-    }
+    ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
+    ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
     cb(tok_embd, "mtp_tok_embd", il);
 
     ggml_tensor * h_state = ggml_reshape_3d(ctx0, inp->h, n_embd, hc, n_tokens);

src/models/gemma4-assistant.cpp

@@ -86,10 +86,9 @@ llama_model_gemma4_assistant::graph::graph(const llama_model & model, const llm_
     const int64_t n_embd_backbone = hparams.n_embd_inp();
 
     ggml_tensor * inp_tokens;
-    ggml_tensor * inp_embd;
     ggml_tensor * inp_h;
     {
-        auto inp = std::make_unique<llm_graph_input_embd_h>(n_embd_backbone, n_embd_backbone);
+        auto inp = std::make_unique<llm_graph_input_embd>(n_embd_backbone);
 
         inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, ubatch.n_tokens);
         cb(inp->tokens, "inp_tokens", -1);
@@ -98,23 +97,18 @@ llama_model_gemma4_assistant::graph::graph(const llama_model & model, const llm_
         res->t_inp_tokens = inp->tokens;
 
         inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_backbone, ubatch.n_tokens);
-        cb(inp->embd, "inp_embd", -1);
+        cb(inp->embd, "inp_h", -1);
         ggml_set_input(inp->embd);
-        inp_embd = inp->embd;
+        inp_h = inp->embd;
         res->t_inp_embd = inp->embd;
 
-        inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd_backbone, ubatch.n_tokens);
-        cb(inp->h, "inp_h", -1);
-        ggml_set_input(inp->h);
-        inp_h = inp->h;
-
         res->add_input(std::move(inp));
     }
 
     GGML_ASSERT(cparams.ctx_other != nullptr);
     const auto * model_other = llama_get_model(cparams.ctx_other);
 
-    ggml_tensor * x = ubatch.token ? ggml_get_rows(ctx0, model_other->tok_embd, inp_tokens) : inp_embd;
+    ggml_tensor * x = ggml_get_rows(ctx0, model_other->tok_embd, inp_tokens);
     x = ggml_scale(ctx0, x, sqrtf((float) n_embd_backbone));
     cb(x, "inp_embd_target", -1);
 

src/models/glm-dsa.cpp

@@ -560,7 +560,7 @@ llama_model_glm_dsa::graph_mtp::graph_mtp(const llama_model & model, const llm_g
     const float kq_scale = 1.0f * mscale * mscale / sqrtf(float(n_embd_head_k));
 
     // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/glm4-moe.cpp

@@ -143,7 +143,7 @@ llama_model_glm4_moe::graph_mtp::graph_mtp(const llama_model & model, const llm_
     GGML_ASSERT(layer.nextn.hnorm   && "MTP block missing nextn.hnorm");
     GGML_ASSERT(layer.ffn_gate_inp  && "MTP block missing ffn_gate_inp");
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/glm5-next.cpp

@@ -568,25 +568,20 @@ llama_model_glm5_next::graph_mtp::graph_mtp(const llama_model & model, const llm
 
     ggml_tensor * inp_out_ids = build_inp_out_ids();
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
 
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);
     ggml_set_input(inp->embd);
 
     inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, n_embd, n_tokens);
     ggml_set_input(inp->h);
     ggml_set_name(inp->h, "mtp_h_input");
 
-    ggml_tensor * tok_embd;
-    if (ubatch.token) {
-        tok_embd = ggml_get_rows(ctx0,
-                layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd, inp->tokens);
-    } else {
-        tok_embd = inp->embd;
-    }
+    ggml_tensor * tok_embd = ggml_get_rows(ctx0,
+            layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd, inp->tokens);
     cb(tok_embd, "mtp_tok_embd", il);
 
     ggml_tensor * h = inp->h;

src/models/hy-v3.cpp

@@ -245,22 +245,19 @@ llama_model_hy_v3::graph_mtp::graph_mtp(const llama_model & model, const llm_gra
     GGML_ASSERT(layer.nextn.enorm   && "MTP block missing nextn.enorm");
     GGML_ASSERT(layer.nextn.hnorm   && "MTP block missing nextn.hnorm");
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
 
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
     ggml_set_input(inp->embd);
-
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
+    ggml_set_name(inp->embd, "mtp_h_input");
 
     ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
 
-    ggml_tensor * h_input  = inp->h;
-    ggml_tensor * tok_embd = ubatch.token ? ggml_get_rows(ctx0, tok_embd_w, inp->tokens) : inp->embd;
+    ggml_tensor * h_input  = inp->embd;
+    ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
     cb(tok_embd, "mtp_tok_embd", il);
 
     res->add_input(std::move(inp));

src/models/mimo2.cpp

@@ -282,21 +282,18 @@ llama_model_mimo2::graph_mtp::graph_mtp(const llama_model & model, const llm_gra
     const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
     const float v_scale      = hparams.f_attn_value_scale;
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
 
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
     ggml_set_input(inp->embd);
-
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
+    ggml_set_name(inp->embd, "mtp_h_input");
 
     ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
-    ggml_tensor * h_input    = inp->h;
-    ggml_tensor * tok_embd   = ubatch.token ? ggml_get_rows(ctx0, tok_embd_w, inp->tokens) : inp->embd;
+    ggml_tensor * h_input    = inp->embd;
+    ggml_tensor * tok_embd   = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
     cb(tok_embd, "mtp_tok_embd", il);
 
     res->add_input(std::move(inp));

src/models/models.h

@@ -583,44 +583,6 @@ struct llama_model_qwen3 : public llama_model_base {
 
     struct graph : public llm_graph_context {
         graph(const llama_model & model, const llm_graph_params & params);
-    protected:
-        // the helpers alone, a derived graph builds its own body
-        struct no_build {};
-        graph(const llm_graph_params & params, no_build) : llm_graph_context(params) {}
-
-        // the layers and the final norm: [n_embd, n_outputs]
-        ggml_tensor * build_backbone(const llama_model & model);
-    };
-
-    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
-};
-
-// Qwen3 with the decision head of vllm-sr Decision 2.0: one score per option
-struct llama_model_vllm_decision2 : public llama_model_qwen3 {
-    llama_model_vllm_decision2(const struct llama_model_params & params) : llama_model_qwen3(params) {}
-    void load_arch_hparams(llama_model_loader & ml) override;
-    void load_arch_tensors(llama_model_loader & ml) override;
-
-    ggml_tensor * option_norm     = nullptr;
-    ggml_tensor * option_norm_b   = nullptr;
-    ggml_tensor * query_norm      = nullptr;
-    ggml_tensor * query_norm_b    = nullptr;
-    ggml_tensor * key             = nullptr;
-    ggml_tensor * query           = nullptr;
-    ggml_tensor * option_mlp      = nullptr;
-    ggml_tensor * option_mlp_b    = nullptr;
-    ggml_tensor * query_mlp       = nullptr;
-    ggml_tensor * scorer_out      = nullptr;
-
-    class input_decision;
-
-    struct graph : public llama_model_qwen3::graph {
-        graph(const llama_model & model, const llm_graph_params & params);
-    private:
-        // hidden: [n_embd, n_outputs], returns one score per option: [1, n_options]
-        ggml_tensor * build_head(ggml_tensor * hidden, input_decision * inp);
-
-        const llama_model_vllm_decision2 & model;
     };
 
     std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;

src/models/nemotron-h-moe.cpp

@@ -25,7 +25,7 @@ llama_model_nemotron_h_moe::graph_mtp::graph_mtp(const llama_model & model, cons
     ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
     GGML_ASSERT(tok_embd_w != nullptr && "NEMOTRON_H_MOE MTP requires token embeddings");
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/qwen3.cpp

@@ -51,18 +51,6 @@ std::unique_ptr<llm_graph_context> llama_model_qwen3::build_arch_graph(const llm
 }
 
 llama_model_qwen3::graph::graph(const llama_model & model, const llm_graph_params & params) : llm_graph_context(params) {
-    ggml_tensor * cur = build_backbone(model);
-
-    // lm_head
-    cur = build_lora_mm(model.output, cur, model.output_s);
-
-    cb(cur, "result_output", -1);
-    res->t_logits = cur;
-
-    ggml_build_forward_expand(gf, cur);
-}
-
-ggml_tensor * llama_model_qwen3::graph::build_backbone(const llama_model & model) {
     const int64_t n_embd_head = hparams.n_embd_head_v();
 
     GGML_ASSERT(n_embd_head == hparams.n_embd_head_k());
@@ -161,5 +149,11 @@ ggml_tensor * llama_model_qwen3::graph::build_backbone(const llama_model & model
     cb(cur, "result_norm", -1);
     res->t_embd = cur;
 
-    return cur;
+    // lm_head
+    cur = build_lora_mm(model.output, cur, model.output_s);
+
+    cb(cur, "result_output", -1);
+    res->t_logits = cur;
+
+    ggml_build_forward_expand(gf, cur);
 }

src/models/qwen35.cpp

@@ -518,7 +518,7 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr
     std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
 
     // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
@@ -534,16 +534,6 @@ llama_model_qwen35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr
         ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
 
         tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
-
-        // a Hadamard-latent table stores rotated rows; restore the primal basis
-        if (hdmd) {
-            if (const auto it = hdmd->inv.find(tok_embd_w); it != hdmd->inv.end()) {
-                tok_embd = llama_mul_mat_hadamard(ctx0, tok_embd, it->second.rot);
-                if (it->second.signs) {
-                    tok_embd = ggml_mul(ctx0, tok_embd, it->second.signs);
-                }
-            }
-        }
     } else {
         tok_embd = inp->embd;
     }

src/models/qwen35moe.cpp

@@ -568,7 +568,7 @@ llama_model_qwen35moe::graph_mtp::graph_mtp(const llama_model & model, const llm
     std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
 
     // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/qwen3next.cpp

@@ -642,7 +642,7 @@ llama_model_qwen3next::graph_mtp::graph_mtp(const llama_model & model, const llm
     GGML_ASSERT(layer.ffn_gate_inp     && "MTP block missing ffn_gate_inp");
 
     // TODO: extract in a common llm_graph_context::build_inp_embd_h()
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);

src/models/qwen4exp.cpp

@@ -540,24 +540,19 @@ llama_model_qwen4exp::graph_mtp::graph_mtp(const llama_model & model, const llm_
     int sections[4];
     std::copy(std::begin(hparams.rope_sections), std::begin(hparams.rope_sections) + 4, sections);
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd_out());
+    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_out());
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
 
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens);
     ggml_set_input(inp->embd);
 
     inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_out(), n_tokens);
     ggml_set_input(inp->h);
     ggml_set_name(inp->h, "mtp_h_input");
 
-    ggml_tensor * tok_embd;
-    if (ubatch.token) {
-        tok_embd = ggml_get_rows(ctx0, model.tok_embd, inp->tokens);
-    } else {
-        tok_embd = inp->embd;
-    }
+    ggml_tensor * tok_embd = ggml_get_rows(ctx0, model.tok_embd, inp->tokens);
     cb(tok_embd, "mtp_tok_embd", il);
 
     ggml_tensor * h = inp->h;

src/models/step35.cpp

@@ -380,22 +380,19 @@ llama_model_step35::graph_mtp::graph_mtp(const llama_model & model, const llm_gr
     const float freq_base_l  = model.get_rope_freq_base(cparams, il);
     const float freq_scale_l = model.get_rope_freq_scale(cparams, il);
 
-    auto inp = std::make_unique<llm_graph_input_embd_h>(hparams.n_embd_inp(), hparams.n_embd);
+    auto inp = std::make_unique<llm_graph_input_embd>(hparams.n_embd);
 
     inp->tokens = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_tokens);
     ggml_set_input(inp->tokens);
 
-    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd_inp(), n_tokens);
+    inp->embd = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
     ggml_set_input(inp->embd);
-
-    inp->h = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, hparams.n_embd, n_tokens);
-    ggml_set_input(inp->h);
-    ggml_set_name(inp->h, "mtp_h_input");
+    ggml_set_name(inp->embd, "mtp_h_input");
 
     ggml_tensor * tok_embd_w = layer.nextn.embed_tokens ? layer.nextn.embed_tokens : model.tok_embd;
 
-    ggml_tensor * h_input  = inp->h;
-    ggml_tensor * tok_embd = ubatch.token ? ggml_get_rows(ctx0, tok_embd_w, inp->tokens) : inp->embd;
+    ggml_tensor * h_input  = inp->embd;
+    ggml_tensor * tok_embd = ggml_get_rows(ctx0, tok_embd_w, inp->tokens);
     cb(tok_embd, "mtp_tok_embd", il);
 
     res->add_input(std::move(inp));