cont : move the loader change to #31321

* server : add realtime conversation and transcription on /v1/realtime

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>
Assisted-by: Claude Opus 5.5

* examples : default the realtime client to the server port 9931

Assisted-by: Claude Opus 5.5

* server : sample realtime responses right after their own decode

Slots respond in turns on one context: sampling at the start of the next step read the logits of whichever slot decoded last, so concurrent sessions mixed their outputs.

Assisted-by: Claude Opus 5.5

* move more stuff to server-realtime, making it self-contained

* examples : fix the type check of the realtime client

Assisted-by: Claude Opus 5.5

* examples : use the model card prompts in the realtime client

Assisted-by: Claude Opus 5.5

---------

Co-authored-by: Xuan Son Nguyen <son@huggingface.co>

common/arg.cpp

@@ -4512,6 +4512,14 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
             params.tts_lang = value;
         }
     ).set_examples({LLAMA_EXAMPLE_TTS}));
+    add_opt(common_arg(
+        {"--tts-voice"}, "VOICE",
+        "voice for audio generation (model-dependent)\n"
+        "see tts/README.md for per-model usage notes",
+        [](common_params & params, const std::string & value) {
+            params.tts_voice = value;
+        }
+    ).set_examples({LLAMA_EXAMPLE_TTS}));
     add_opt(common_arg(
         {"--tts-speaker-file"}, "FNAME",
         "speaker file path for audio generation",

common/chat.cpp

@@ -190,10 +190,17 @@ json common_chat_msg::to_json_oaicompat(bool concat_typed_text) const {
         } else {
             auto & parts = jmsg["content"] = json::array();
             for (const auto & part : content_parts) {
-                parts.push_back({
-                    {"type", part.type},
-                    {"text", part.text},
-                });
+                if (part.type == "tool_reference") {
+                    parts.push_back({
+                        {"type", part.type},
+                        {"name", part.name},
+                    });
+                } else {
+                    parts.push_back({
+                        {"type", part.type},
+                        {"text", part.text},
+                    });
+                }
             }
         }
     } else {
@@ -309,6 +316,7 @@ struct common_chat_templates {
     bool has_explicit_template;  // Model had builtin template or template overridden was specified.
     std::unique_ptr<common_chat_template> template_default;  // always set (defaults to chatml)
     std::unique_ptr<common_chat_template> template_tool_use;
+    std::unique_ptr<common_chat_template> template_tts;  // the prompt of a text to speak
 };
 
 common_chat_tool_choice common_chat_tool_choice_parse_oaicompat(const std::string & tool_choice) {
@@ -370,12 +378,16 @@ std::vector<common_chat_msg> common_chat_msgs_parse_oaicompat(const json & messa
                             throw std::invalid_argument("Missing content part type: " + part.dump());
                         }
                         const auto & type = part.at("type");
-                        if (type != "text" && type != "media_marker") {
-                            throw std::invalid_argument("Unsupported content part type: " + type.dump());
-                        }
                         common_chat_msg_content_part msg_part;
                         msg_part.type = type;
-                        msg_part.text = part.at("text");
+                        if (type == "text" || type == "media_marker") {
+                            msg_part.text = part.at("text");
+                        } else if (type == "tool_reference") {
+                            // names a request tool, the template renders its definition
+                            msg_part.name = part.at("name");
+                        } else {
+                            throw std::invalid_argument("Unsupported content part type: " + type.dump());
+                        }
                         msg.content_parts.push_back(msg_part);
                     }
                 } else if (!content.is_null()) {
@@ -610,13 +622,17 @@ json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_tool> & t
 
     auto result = json::array();
     for (const auto & tool : tools) {
+        json function {
+            { "name",        tool.name },
+            { "description", tool.description },
+            { "parameters",  json::parse(tool.parameters) },
+        };
+        if (tool.defer_loading) {
+            function["defer_loading"] = true;
+        }
         result.push_back({
             { "type",     "function" },
-            { "function", {
-                { "name", tool.name },
-                { "description", tool.description },
-                { "parameters", json::parse(tool.parameters) },
-            }},
+            { "function", function },
         });
     }
     return result;
@@ -653,10 +669,19 @@ std::vector<common_chat_tool> common_chat_tools_parse_oaicompat(const json & too
                 }
 
                 const auto & function = tool.at("function");
+
+                // "defer_loading" sits on the function, or on the tool next to "type"; the function wins
+                const json & defer = function.contains("defer_loading") ? function.at("defer_loading")
+                                                                        : tool.value("defer_loading", json(false));
+                if (!defer.is_boolean()) {
+                    throw std::invalid_argument("Expected 'defer_loading' to be a boolean, got " + defer.dump());
+                }
+
                 result.push_back({
                     /* .name = */ function.at("name"),
                     /* .description = */ function.value("description", ""),
                     /* .parameters = */ function.value("parameters", json::object()).dump(),
+                    /* .defer_loading = */ defer.get<bool>(),
                 });
             }
         }
@@ -779,19 +804,70 @@ bool common_chat_templates_was_explicit(const struct common_chat_templates * tmp
     return tmpls->has_explicit_template;
 }
 
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates) {
+// renders a task template (e.g. "tts") with the given variables
+static std::string common_chat_template_render(const common_chat_template & tmpl, const json & vars) {
+    jinja::context ctx(tmpl.source());
+    jinja::global_from_json(ctx, vars, false);
+    ctx.set_val("bos_token", jinja::mk_val<jinja::value_string>(tmpl.bos_token()));
+    ctx.set_val("eos_token", jinja::mk_val<jinja::value_string>(tmpl.eos_token()));
+    jinja::runtime runtime(ctx);
+    return jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
+}
+
+// LFM3 format detection: template renders deferred tools inside <additional_tools>...</additional_tools>
+static bool is_lfm3_template(const std::string & src) {
+    return src.find("<additional_tools>") != std::string::npos;
+}
+
+// LFM3-Audio: a known language code maps to its name, no language lets the model detect it
+static std::string lfm3_asr_system_prompt(const std::string & language) {
+    static const std::map<std::string, std::string> names = {
+        { "ar", "Arabic" }, { "da", "Danish" }, { "de", "German" }, { "el", "Greek" }, { "en", "English" },
+        { "es", "Spanish" }, { "fi", "Finnish" }, { "fr", "French" }, { "he", "Hebrew" }, { "it", "Italian" },
+        { "ja", "Japanese" }, { "ko", "Korean" }, { "nl", "Dutch" }, { "pl", "Polish" }, { "pt", "Portuguese" },
+        { "sv", "Swedish" }, { "tr", "Turkish" }, { "zh", "Chinese" },
+    };
+    if (language.empty()) {
+        return "Perform ASR.";
+    }
+    std::string code = language;
+    std::transform(code.begin(), code.end(), code.begin(), [](unsigned char c) { return (char) std::tolower(c); });
+    const auto it = names.find(code);
+    return "Perform ASR in " + (it != names.end() ? it->second : language) + ".";
+}
+
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language) {
     common_chat_prompt_preset asr_preset;
     asr_preset.system = "";
     asr_preset.user   = "Transcribe audio to text";
 
-    if (chat_templates && chat_templates->template_default && is_lfm2_template(chat_templates->template_default->source())) {
-        asr_preset.system = "Perform ASR.";
-        asr_preset.user   = "";
+    if (chat_templates && chat_templates->template_default) {
+        const std::string src = chat_templates->template_default->source();
+        if (is_lfm2_template(src)) {
+            asr_preset.system = "Perform ASR.";
+            asr_preset.user   = "";
+        } else if (is_lfm3_template(src)) {
+            asr_preset.system = lfm3_asr_system_prompt(language);
+            asr_preset.user   = "";
+            asr_preset.has_language = true;
+        }
     }
 
     return asr_preset;
 }
 
+std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
+                                       const std::string & language, const std::string & voice) {
+    if (!chat_templates || !chat_templates->template_tts) {
+        return "";
+    }
+    return common_chat_template_render(*chat_templates->template_tts, {
+        {"messages", json::array({{{"role", "user"}, {"content", text}}})},
+        {"language", language},
+        {"voice",    voice},
+    });
+}
+
 std::string common_chat_templates_source(const struct common_chat_templates * tmpls, const std::string & variant) {
     if (!variant.empty()) {
         if (variant == "tool_use") {
@@ -902,6 +978,16 @@ common_chat_templates_ptr common_chat_templates_init(const struct llama_model *
             LOG_ERR("%s: failed to parse tool use chat template (ignoring it): %s\n", __func__, e.what());
         }
     }
+    // task templates of the model, kept with a chat template override
+    if (model) {
+        if (const char * src = llama_model_chat_template(model, "tts")) {
+            try {
+                tmpls->template_tts = std::make_unique<common_chat_template>(src, token_bos, token_eos);
+            } catch (const std::exception & e) {
+                LOG_ERR("%s: failed to parse the tts chat template (ignoring it): %s\n", __func__, e.what());
+            }
+        }
+    }
     return tmpls;
 }
 
@@ -1854,12 +1940,13 @@ const common_chat_msg & common_chat_session::finish(const common_chat_input & ch
     return result;
 }
 
-std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
+common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates) {
     GGML_ASSERT(chat_templates != nullptr);
     GGML_ASSERT(chat_templates->template_default != nullptr);
-    if (chat_templates->template_tool_use != nullptr) {
-        // take the more expressive template when available
-        return chat_templates->template_tool_use->caps.to_map();
-    }
-    return chat_templates->template_default->caps.to_map();
+    // take the more expressive template when available
+    const auto & caps = chat_templates->template_tool_use != nullptr ? chat_templates->template_tool_use->caps
+                                                                      : chat_templates->template_default->caps;
+    common_json result = caps.to_map();
+    result["reasoning_effort_levels"] = caps.reasoning_effort_levels;
+    return result;
 }

common/chat.h

@@ -45,6 +45,7 @@ struct common_chat_tool_call {
 struct common_chat_msg_content_part {
     std::string type;
     std::string text;
+    std::string name; // tool_reference: name of a tool from the request tools
 
     // TODO @ngxson : no known chat templates support reasoning_content in content parts yet
     //                this can be useful for models with interleaved thinking (like Kimi-K2)
@@ -52,7 +53,7 @@ struct common_chat_msg_content_part {
     // std::string reasoning_content;
 
     bool operator==(const common_chat_msg_content_part & other) const {
-        return type == other.type && text == other.text;
+        return type == other.type && text == other.text && name == other.name;
     }
 };
 
@@ -226,6 +227,9 @@ struct common_chat_tool {
     std::string name;
     std::string description;
     std::string parameters;
+
+    // withheld from the system prompt, a tool_reference reveals it later in the conversation
+    bool defer_loading = false;
 };
 
 enum common_chat_tool_choice {
@@ -468,7 +472,7 @@ common_json common_chat_tools_to_json_oaicompat(const std::vector<common_chat_to
 common_json common_chat_tool_parameters(const common_json & function);
 
 // get template caps, useful for reporting to server /props endpoint
-std::map<std::string, bool> common_chat_templates_get_caps(const common_chat_templates * chat_templates);
+common_json common_chat_templates_get_caps(const common_chat_templates * chat_templates);
 
 common_chat_prompt common_chat_template_direct_apply(
     const common_chat_template & tmpl,
@@ -487,6 +491,12 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
 struct common_chat_prompt_preset {
     std::string system;
     std::string user;
+    bool has_language = false; // the preset already names the requested language
 };
 
-common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates);
+common_chat_prompt_preset common_chat_get_asr_prompt(const common_chat_templates * chat_templates, const std::string & language = "");
+
+// prompt of the model's "tts" chat template for the text to speak, empty if the model has none
+// language and voice are optional, the template defines their defaults
+std::string common_chat_get_tts_prompt(const common_chat_templates * chat_templates, const std::string & text,
+                                       const std::string & language = "", const std::string & voice = "");

common/common.cpp

@@ -1188,6 +1188,8 @@ static const std::map<common_decision_type, std::string> COMMON_DECISION_TYPE_NA
     { COMMON_DECISION_TYPE_GRANITE_SWITCH, "granite-switch"},
     { COMMON_DECISION_TYPE_H2O_LIGHTNING,  "h2o-lightning" },
     { COMMON_DECISION_TYPE_MINICPM_OCR_JEV, "openbmb-minicpm-ocr-jev" },
+    { COMMON_DECISION_TYPE_LFM3_D1,        "lfm3-d1"       },
+    { COMMON_DECISION_TYPE_LFM3_D1_OMNI,   "lfm3-d1-omni"  },
     // for future models, ALWAYS add org name prefix to avoid name collisions, e.g. "orgname-modelname"
 };
 

common/common.h

@@ -772,6 +772,7 @@ struct common_params {
 
     // TTS params
     std::string tts_lang = "";
+    std::string tts_voice = "";
     std::string tts_speaker_file = "";
 
     bool is_gen_docs = false; // whether we are running inside llama-gen-docs
@@ -980,6 +981,8 @@ enum common_decision_type {
     COMMON_DECISION_TYPE_GRANITE_SWITCH, // same as openjev, run by the decision aLoRA of granite switch
     COMMON_DECISION_TYPE_H2O_LIGHTNING, // same as openjev, the labels are read after an answer prefill
     COMMON_DECISION_TYPE_MINICPM_OCR_JEV, // one score per class, the pooled embeddings output of the last token
+    COMMON_DECISION_TYPE_LFM3_D1,       // same as lfm2-d1, non-causal, label codes A..Z then AA, AB..
+    COMMON_DECISION_TYPE_LFM3_D1_OMNI,  // same as lfm3-d1
     COMMON_DECISION_TYPE_UNKNOWN, // a decision model of a type that is not supported
 };
 

common/jinja/caps.cpp

@@ -6,7 +6,9 @@
 // we can remove it in the future when we figure out a better way to define inputs using jinja::value
 #include "json.h"
 
+#include <algorithm>
 #include <functional>
+#include <optional>
 #include <sstream>
 
 #define FILENAME "jinja-caps"
@@ -184,6 +186,10 @@ caps caps_get(jinja::program & prog) {
                     {
                         {"role", "user"},
                         {"content", json::array({
+                            {
+                                {"type", "text"},
+                                {"text", content_marker}
+                            }
                         })}
                     }
                 });
@@ -584,6 +590,58 @@ caps caps_get(jinja::program & prog) {
         }
     );
 
+    if (result.supports_reasoning_effort) {
+        JJ_DEBUG("%s\n", ">>> Running capability check: reasoning effort levels");
+
+        static const std::vector<std::string> candidates = { "minimal", "low", "medium", "high", "xhigh", "max" };
+
+        // renders a single user turn with the given effort, nothing when the template raises
+        const auto render = [&](const std::string & effort) {
+            std::optional<std::string> rendered;
+            caps_try_execute(
+                prog,
+                [&]() {
+                    // messages
+                    return json::array({
+                        {
+                            {"role", "user"},
+                            {"content", "User message"}
+                        },
+                    });
+                },
+                [&](context & ctx) {
+                    ctx.set_val("enable_thinking", mk_val<value_bool>(true));
+                    caps_apply_reasoning_effort(ctx, effort);
+                },
+                nullptr, // tools_fn
+                [&](context &, bool success, value &, value &, const std::string & output) {
+                    if (success) {
+                        rendered = output;
+                    }
+                }
+            );
+            return rendered;
+        };
+
+        // an unknown level shows the render of the template fallback, a level that renders the same is not honored
+        // levels sharing a render are aliases and the highest one is kept
+        const auto unknown = render("REASONING_EFFORT_UNKNOWN");
+        std::vector<std::string> renders;
+        for (const auto & level : candidates) {
+            const auto rendered = render(level);
+            if (!rendered || rendered == unknown) {
+                continue;
+            }
+            const auto it = std::find(renders.begin(), renders.end(), *rendered);
+            if (it != renders.end()) {
+                result.reasoning_effort_levels[it - renders.begin()] = level;
+            } else {
+                renders.push_back(*rendered);
+                result.reasoning_effort_levels.push_back(level);
+            }
+        }
+    }
+
     JJ_DEBUG("%s\n", result.to_string().c_str());
 
     return result;

common/jinja/caps.h

@@ -4,6 +4,7 @@
 
 #include <string>
 #include <map>
+#include <vector>
 
 namespace jinja {
 
@@ -19,6 +20,9 @@ struct caps {
     // supports reasoning effort levels
     bool supports_reasoning_effort = false;
 
+    // reasoning effort levels the template renders distinctly, from lowest to highest
+    std::vector<std::string> reasoning_effort_levels;
+
     // one of the 2 content capabilities must be true
     bool supports_string_content = true;
     bool supports_typed_content = false;

conversion/__init__.py

@@ -172,6 +172,10 @@ TEXT_MODEL_MAP: dict[str, str] = {
     "D1OmniModel": "lfm2",
     "Lfm2Model": "lfm2",
     "Lfm2MoeForCausalLM": "lfm2",
+    "Lfm3D1Model": "lfm3",
+    "Lfm3ForCausalLM": "lfm3",
+    "Lfm3OmniForConditionalGeneration": "lfm3",
+    "Lfm3VlForConditionalGeneration": "lfm3",
     "Llama4ForCausalLM": "llama",
     "Llama4ForConditionalGeneration": "llama",
     "LlamaBidirectionalModel": "llama",
@@ -358,6 +362,9 @@ MMPROJ_MODEL_MAP: dict[str, str] = {
     "Lfm2AudioForConditionalGeneration": "lfm2",
     "D1OmniModel": "lfm2",
     "Lfm2VlForConditionalGeneration": "lfm2",
+    "Lfm3AudioForConditionalGeneration": "lfm3",
+    "Lfm3OmniForConditionalGeneration": "lfm3",
+    "Lfm3VlForConditionalGeneration": "lfm3",
     "LightOnOCRForConditionalGeneration": "lighton_ocr",
     "Llama4ForConditionalGeneration": "llama4",
     "LlavaForConditionalGeneration": "llava",

conversion/lfm2.py

@@ -15,25 +15,30 @@ from .base import MmprojModel, ModelBase, TextModel, gguf, jinja_str_or_json, lo
 from .gemma import ConformerAudioModel
 
 
+def lfm2_feed_forward_length(hparams: dict[str, Any]) -> int:
+    ff_dim = hparams.get("block_ff_dim", hparams.get("intermediate_size"))
+    assert ff_dim is not None
+    auto_adjust_ff_dim = hparams["block_auto_adjust_ff_dim"]
+    ffn_dim_multiplier = hparams["block_ffn_dim_multiplier"]
+    multiple_of = hparams["block_multiple_of"]
+
+    if auto_adjust_ff_dim:
+        ff_dim = int(2 * ff_dim / 3)
+        # custom dim factor multiplier
+        if ffn_dim_multiplier is not None:
+            ff_dim = int(ffn_dim_multiplier * ff_dim)
+        ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
+
+    return ff_dim
+
+
 @ModelBase.register("Lfm2ForCausalLM", "LFM2ForCausalLM")
 @ModelBase.example("LiquidAI/LFM2-1.2B", "LiquidAI/LFM2.5-350M")
 class LFM2Model(TextModel):
     model_arch = gguf.MODEL_ARCH.LFM2
 
     def _add_feed_forward_length(self):
-        ff_dim = self.find_hparam(["block_ff_dim", "intermediate_size"])
-        auto_adjust_ff_dim = self.hparams["block_auto_adjust_ff_dim"]
-        ffn_dim_multiplier = self.hparams["block_ffn_dim_multiplier"]
-        multiple_of = self.hparams["block_multiple_of"]
-
-        if auto_adjust_ff_dim:
-            ff_dim = int(2 * ff_dim / 3)
-            # custom dim factor multiplier
-            if ffn_dim_multiplier is not None:
-                ff_dim = int(ffn_dim_multiplier * ff_dim)
-            ff_dim = multiple_of * ((ff_dim + multiple_of - 1) // multiple_of)
-
-        self.gguf_writer.add_feed_forward_length(ff_dim)
+        self.gguf_writer.add_feed_forward_length(lfm2_feed_forward_length(self.hparams))
 
     def set_gguf_parameters(self):
         # set num_key_value_heads only for attention layers
@@ -72,7 +77,9 @@ def _is_d1_checkpoint(dir_model: Path) -> bool:
     if not (dir_model / "config.json").is_file():
         return False
     with open(dir_model / "config.json", encoding="utf-8") as f:
-        return json.load(f).get("auto_map", {}).get("AutoModel", "").endswith(".D1Model")
+        config = json.load(f)
+    # d1 on LFM3 is converted by conversion/lfm3.py
+    return config.get("auto_map", {}).get("AutoModel", "").endswith(".D1Model") and not config.get("model_type", "").startswith("lfm3")
 
 
 @ModelBase.register_hparams_loader(_is_d1_checkpoint)

conversion/lfm3.py

@@ -0,0 +1,613 @@
+from __future__ import annotations
+
+import json
+import math
+
+from pathlib import Path
+from typing import Any, Callable, Iterable, TYPE_CHECKING
+
+import torch
+
+if TYPE_CHECKING:
+    from torch import Tensor
+
+from .base import MmprojModel, ModelBase, TextModel, gguf, logger
+from .lfm2 import D1Model, lfm2_feed_forward_length
+
+
+@ModelBase.register("Lfm3ForCausalLM", "Lfm3VlForConditionalGeneration", "Lfm3OmniForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-1.7B", "LiquidAI/LFM3-1.7B-Base")
+class LFM3Model(TextModel):
+    model_arch = gguf.MODEL_ARCH.LFM3
+
+    # LFM3-Audio/Omni: encoder, projector and audio-output heads live next to the text model
+    _media_prefixes = ("encoder.", "mlp.", "codebook_embedding.", "depth_embeddings.", "depth_linear.", "depthformer.",
+                       "mask_embedding", "model.codebook_embedding.", "model.depth_decoder.")
+
+    @classmethod
+    def filter_tensors(cls, item: tuple[str, Callable[[], Tensor]]) -> tuple[str, Callable[[], Tensor]] | None:
+        name, gen = item
+
+        if name.startswith(cls._media_prefixes):
+            return None
+
+        name = name.removeprefix("lfm.")
+
+        return super().filter_tensors((name, gen))
+
+    # LFM3-Audio live ASR prompt of the model card, the audio follows in the assistant turn
+    _asr_template = (
+        "{{- bos_token -}}"
+        "{{- '<|im_start|>system\\nPerform live ASR.<|im_end|>\\n' -}}"
+        "{%- if add_generation_prompt -%}{{- '<|im_start|>assistant\\n' -}}{%- endif -%}"
+    )
+
+    # TTS prompt of the model card: a language code is looked up, anything else is taken as the language name
+    _tts_langs = {
+        "ar": "Arabic", "da": "Danish", "de": "German", "el": "Greek", "en": "American English", "en-us": "American English",
+        "en-gb": "British English", "fi": "Finnish", "fr": "French", "he": "Hebrew", "it": "Italian", "ja": "Japanese",
+        "ko": "Korean", "nl": "Dutch", "pl": "Polish", "pt": "Portuguese", "es": "Spanish", "sv": "Swedish", "tr": "Turkish",
+        "zh": "Chinese",
+    }
+
+    # "language" and "voice" are optional, the text to speak is the first message
+    def _tts_template(self, think: bool) -> str:
+        return (
+            "{%- set names = " + json.dumps(self._tts_langs) + " -%}"
+            "{%- set code = (language | lower) if language is defined and language else 'en' -%}"
+            "{%- set lang = names[code] if code in names else language -%}"
+            "{%- set v = (voice | lower) if voice is defined and voice and (voice | lower) in ['female', 'male'] else 'female' -%}"
+            "{{- bos_token -}}"
+            "{{- '<|im_start|>system\\nPerform TTS in ' + lang + ' using a ' + v + ' voice.<|im_end|>\\n' -}}"
+            "{{- '<|im_start|>user\\n' + messages[0]['content'] + '<|im_end|>\\n' -}}"
+            "{{- '<|im_start|>assistant\\n" + ("<think></think>" if think else "") + "<|audio_start|>' -}}"
+        )
+
+    def set_vocab(self):
+        super().set_vocab()
+        # hparams hold the merged text config, the checkpoint's own model_type is in config.json
+        with open(self.dir_model / "config.json", encoding="utf-8") as f:
+            model_type = json.load(f).get("model_type")
+        is_audio = model_type == "lfm3_audio"
+        is_omni  = model_type == "lfm3_omni"
+        templates = []
+        if is_audio:
+            templates.append({"name": "asr", "template": self._asr_template})
+        if is_audio or is_omni:
+            # LFM3-Omni opens every assistant turn with an empty think block
+            templates.append({"name": "tts", "template": self._tts_template(think=is_omni)})
+        if templates:
+            self.gguf_writer.add_chat_template(templates)
+
+    def set_gguf_parameters(self):
+        layer_types = self.hparams["layer_types"]
+        n_kv_heads = self.hparams["num_key_value_heads"]
+
+        # set num_key_value_heads only for attention layers
+        self.hparams["num_key_value_heads"] = [
+            n_kv_heads if lt != "conv" else 0
+            for lt in layer_types
+        ]
+
+        super().set_gguf_parameters()
+
+        self.gguf_writer.add_rope_dimension_count(self.hparams["head_dim"])
+
+        self.gguf_writer.add_sliding_window_pattern([lt == "sliding_attention" for lt in layer_types])
+        self.gguf_writer.add_sliding_window(self.hparams["sliding_window"])
+
+        self.gguf_writer.add_shortconv_l_cache(self.hparams["conv_L_cache"])
+
+    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+        # Partial RoPE is represented by masking trailing frequency pairs.
+        # rope_parameters is keyed by layer type; only full_attention layers use partial RoPE.
+        rope_parameters = self.hparams["rope_parameters"]["full_attention"]
+        partial = rope_parameters.get("partial_rotary_factor", 1.0)
+        head_dim = self.hparams["head_dim"]
+        rope_angles = int(partial * head_dim) // 2
+        n_pairs = head_dim // 2
+        assert 0 < rope_angles <= n_pairs
+        values = [1.0] * rope_angles + [1e30] * (n_pairs - rope_angles)
+        rope_freqs = torch.tensor(values, dtype=torch.float32)
+        yield (self.format_tensor_name(gguf.MODEL_TENSOR.ROPE_FREQS), rope_freqs)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        # conv op requires 2d tensor
+        if "conv.conv" in name:
+            data_torch = data_torch.squeeze(1)
+
+        yield from super().modify_tensors(data_torch, name, bid)
+
+
+# d1 on LFM3-VL / LFM3-Omni: the stock weights, the d1 prompt read at its last token
+_D1_TYPES = {"lfm3_vl": gguf.DecisionType.LFM3_D1, "lfm3_omni": gguf.DecisionType.LFM3_D1_OMNI}
+
+
+def _is_lfm3_d1_checkpoint(dir_model: Path) -> bool:
+    if not (dir_model / "config.json").is_file():
+        return False
+    with open(dir_model / "config.json", encoding="utf-8") as f:
+        config = json.load(f)
+    return config.get("architectures") == ["D1Model"] and config.get("model_type") in _D1_TYPES
+
+
+@ModelBase.register_hparams_loader(_is_lfm3_d1_checkpoint)
+def _load_lfm3_d1_hparams(dir_model: Path) -> dict[str, Any]:
+    logger.info("gguf: detected LFM3 d1 checkpoint")
+    hparams = ModelBase.load_hparams(dir_model, False, guess=False)
+    hparams["d1_family"] = hparams["model_type"]
+    # the mmproj is the base model's, the text model writes the decision
+    hparams["architectures"] = ["Lfm3VlForConditionalGeneration" if hparams["model_type"] == "lfm3_vl" else "Lfm3OmniForConditionalGeneration"]
+    hparams["text_config"]["architectures"] = ["Lfm3D1Model"]
+    return hparams
+
+
+@ModelBase.register("Lfm3D1Model")
+@ModelBase.example("LiquidAI/d1-2B", "LiquidAI/d1-omni-2B")
+class LFM3D1Model(LFM3Model):
+    model_arch = gguf.MODEL_ARCH.LFM3
+
+    # how the checkpoint serves, hard-coded in the server: assert the checkpoint records the same
+    _served = {"d1_codes": 2, "d1_attention": "set"}
+
+    def set_vocab(self):
+        # the vocab without the speech task templates of LFM3-Audio/Omni, d1 only decides
+        TextModel.set_vocab(self)
+        # the d1 prompt of LFM2, the LFM3 template writes an empty thought before the answer
+        self.gguf_writer.add_chat_template([{"name": "systemone", "template": D1Model._systemone_template() + "<think></think>"}])
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+
+        for key, value in self._served.items():
+            if self.hparams.get(key) != value:
+                raise ValueError(f"unsupported d1 checkpoint: {key} = {self.hparams.get(key)!r}, expected {value!r}")
+
+        # a lone question reads the whole prompt both ways on the full attention layers
+        self.gguf_writer.add_causal_attention(False)
+        self.gguf_writer.add_decision_type(_D1_TYPES[self.hparams["d1_family"]])
+
+        # the option answers share one temperature
+        with open(self.dir_model / "calibration.json", encoding="utf-8") as f:
+            calibration = json.load(f)
+        if calibration.get("temperatures"):
+            raise ValueError("unsupported d1 calibration: per-bucket temperatures")
+        for name in ("noul", "choice", "score"):
+            self.gguf_writer.add_decision_temperature(name, calibration["default"])
+
+
+class LFM3VLVisionMixin(MmprojModel):
+    """LFM3-VL vision tower and projector, shared with LFM3-Omni."""
+
+    def _set_lfm3vl_gguf_parameters(self):
+        assert self.hparams_vision is not None
+        # clip.cpp hardcodes the GELU projector and its input norm eps
+        if (self.global_config["projector_hidden_act"] != "gelu" or not self.global_config["projector_bias"]
+                or self.global_config["projector_layer_norm_eps"] != 1e-6):
+            raise ValueError("unsupported projector_hidden_act / projector_bias / projector_layer_norm_eps")
+        # clip.cpp hardcodes the vision rope base, the RGB patchifier and the 3x3 spatial conv
+        if (self.hparams_vision["rope_parameters"]["rope_theta"] != 1e6 or self.hparams_vision["in_channels"] != 3
+                or self.hparams_vision["conv_kernel_size"] != 3):
+            raise ValueError("unsupported vision rope_theta / in_channels / conv_kernel_size")
+        self.gguf_writer.add_clip_vision_projector_type(gguf.VisionProjectorType.LFM3VL)
+        self.gguf_writer.add_vision_head_count_kv(self.hparams_vision["num_key_value_heads"])
+        self.gguf_writer.add_vision_spatial_merge_size(self.global_config["downsample_factor"])
+        self.gguf_writer.add_vision_attention_layernorm_eps(self.hparams_vision["norm_eps"])
+        self.gguf_writer.add_vision_use_silu(True)
+        size = self.preprocessor_config["size"]
+        self.gguf_writer.add_vision_min_pixels(size["shortest_edge"])
+        self.gguf_writer.add_vision_max_pixels(size["longest_edge"])
+        # PIL resample enum, the torchvision backend has no lanczos
+        resample = self.preprocessor_config["resample"]
+        resize_algo = {2: "bilinear", 3: "bicubic"}.get(resample)
+        if resize_algo is None:
+            raise ValueError(f"unsupported resample: {resample}")
+        self.gguf_writer.add_vision_image_resize_algo(resize_algo)
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        if "patch_embd" in new_name:
+            return gguf.GGMLQuantizationType.F16
+        if "conv2d_spatial" in new_name:
+            # CUDA CONV_2D_DW requires an f32 kernel; the tensor is tiny
+            return gguf.GGMLQuantizationType.F32
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def _lfm3vl_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name == "model.vision_tower.patchifier.weight":
+            # qwen2vl-style layout [channel, temporal, ph, pw], one conv kernel per temporal slice
+            assert self.hparams_vision is not None
+            t = self.hparams_vision["temporal_patch_size"]
+            p = self.hparams_vision["patch_size"]
+            n_embd = data_torch.shape[0]
+            w = data_torch.view(n_embd, 3, t, p, p)
+            yield ("v.patch_embd.weight", w[:, :, 0].contiguous())
+            if t == 2:
+                yield ("v.patch_embd.weight.1", w[:, :, 1].contiguous())
+            else:
+                assert t == 1, f"unsupported vision_temporal_patch_size {t}"
+            return
+        if name == "model.vision_tower.embedding_norm.weight":
+            yield ("v.embedding_norm.weight", data_torch)
+            return
+        if "conv.in_proj.weight" in name:
+            yield (f"v.blk.{bid}.conv2d_in_proj.weight", data_torch)
+            return
+        if "conv.spatial_conv.weight" in name:
+            yield (f"v.blk.{bid}.conv2d_spatial.weight", data_torch)
+            return
+        if "conv.out_proj.weight" in name:
+            yield (f"v.blk.{bid}.conv2d_out_proj.weight", data_torch)
+            return
+
+        # projector: input LayerNorm -> linear_1 -> GELU -> linear_2
+        if name.startswith("model.multi_modal_projector."):
+            layer, suffix = name.removeprefix("model.multi_modal_projector.").split(".")
+            yield ({"layer_norm": "mm.input_norm", "linear_1": "mm.0", "linear_2": "mm.2"}[layer] + "." + suffix, data_torch)
+            return
+
+        yield (self.map_tensor_name(name), data_torch)
+
+
+@ModelBase.register("Lfm3VlForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-VL-2B")
+class LFM3VLMmprojModel(LFM3VLVisionMixin):
+    has_vision_encoder = True
+    has_audio_encoder = False
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        assert self.hparams_vision is not None
+        # dynamic resolution, image_size is only set for MmprojModel compatibility
+        self.hparams_vision["image_size"] = 512
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        self._set_lfm3vl_gguf_parameters()
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
+            yield from self._lfm3vl_tensors(data_torch, name, bid)
+
+
+def _is_lfm3_audio_checkpoint(dir_model: Path) -> bool:
+    if not (dir_model / "config.json").is_file():
+        return False
+    with open(dir_model / "config.json", encoding="utf-8") as f:
+        config = json.load(f)
+    return config.get("model_type") == "lfm3_audio" and "lfm" in config
+
+
+# read the release config.json as is: a transformers that knows lfm3_audio rewrites it into another layout
+@ModelBase.register_hparams_loader(_is_lfm3_audio_checkpoint)
+def _load_lfm3_audio_hparams(dir_model: Path) -> dict[str, Any]:
+    with open(dir_model / "config.json", encoding="utf-8") as f:
+        config = json.load(f)
+    config["text_config"] = config["lfm"]
+    return config
+
+
+@ModelBase.register("Lfm3AudioForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-Audio-2B")
+class LFM3AudioMmprojModel(MmprojModel):
+    has_vision_encoder = False
+    has_audio_encoder = True
+
+    # the downsampler has the LFM2-Audio pre-encoder layout: conv, dw conv, pw conv, dw conv, pw conv
+    _downsample_idx = {"conv_in": 0, "conv_mid": 2, "lin_mid": 3, "conv_end": 5, "lin_end": 6}
+
+    # audio output: MaskGIT depthformer (a.gen.code.*) and the codec decoder (a.gen.wav.*)
+    _depth_layer_map = {
+        "operator_norm":      gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_NORM,
+        "self_attn.q_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q,
+        "self_attn.q_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_Q_NORM,
+        "self_attn.k_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K,
+        "self_attn.k_norm":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_K_NORM,
+        "self_attn.v_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_V,
+        "self_attn.o_proj":   gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_OUT,
+        "self_attn.gate_proj": gguf.MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
+        "ffn_norm":           gguf.MODEL_TENSOR.A_GEN_CODE_FFN_NORM,
+        "feed_forward.w1":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_GATE,
+        "feed_forward.w3":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_UP,
+        "feed_forward.w2":    gguf.MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
+    }
+    _codec_layer_map = {
+        "operator_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_NORM,
+        "post_operator_norm":    gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM,
+        "self_attn.q_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q,
+        "self_attn.q_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM,
+        "self_attn.k_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K,
+        "self_attn.k_norm":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM,
+        "self_attn.v_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_V,
+        "self_attn.o_proj":      gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_OUT,
+        "self_attn.gate_proj":   gguf.MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE,
+        "conv.in_proj":          gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1,
+        "conv.conv":             gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW,
+        "conv.out_proj":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2,
+        "ffn_norm":              gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_NORM,
+        "post_ffn_norm":         gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM,
+        "feed_forward.w1":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_GATE,
+        "feed_forward.w3":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
+        "feed_forward.w2":       gguf.MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
+    }
+    # the codec graph in clip.cpp hardcodes the liquid-codec FSQ front end and decoder shape, conversion fails on a mismatch
+    _codec = {
+        "model_type": "liquid_codec", "fsq_levels": [8, 7, 6, 6], "n_q": 12, "quantizer_dim": 256,
+        "encoder_dim": 512, "stft_n_fft": 1280, "stft_hop_length": 320, "stft_win_length": 1280, "stft_frame_stack": 6,
+        "decoder_window": 30, "sampling_rate": 24000,
+    }
+    # the lfm3a pipeline in mtmd-helper-gen.cpp hardcodes these
+    _generation = {"codebooks": 12, "maskgit_schedule": [4, 4, 4], "interleaved_n_text": 6, "interleaved_n_audio": 12}
+
+    # speech-code sampling of the model card, generation_config.json overrides it
+    _audio_sampling = {
+        "audio_temperature": 1.0, "audio_top_k": 128, "audio_repetition_penalty": 1.1, "audio_repetition_window": 4096,
+    }
+
+    # mtmd_audio_preprocessor_lfm3a hardcodes this front end, conversion fails on a mismatch
+    _feature_extractor = {
+        "sampling_rate": 16000, "feature_size": 128, "n_fft": 400, "win_length": 400, "hop_length": 160,
+        "log_floor": -12.0, "normalization_offset": 4.0, "normalization_divisor": 4.0, "encoder_stride": 8,
+    }
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # audio output is exported when the checkpoint ships the codec
+        self._audio_output = (self.dir_model / "audio_tokenizer").is_dir()
+        gen = self.get_generation_config()
+        self._n_codebooks: int = gen["codebooks"]
+        self._audio_vocab: int = gen["audio_vocab_size"]
+        self._depth_embd: dict[int, Tensor] = {}
+        self._depth_norm: dict[int, Tensor] = {}
+
+    def get_audio_config(self) -> dict[str, Any] | None:
+        enc = self.global_config["encoder"]
+        cfg = dict(enc["lfm"])
+        cfg["num_mel_bins"] = enc["dim_in"]
+        cfg["sliding_window"] = enc["sliding_window"]
+        return cfg
+
+    def get_generation_config(self) -> dict[str, Any]:
+        c = self.global_config
+        steps = c["maskgit_steps"]
+        return {
+            "codebooks": c["codebooks"], "audio_vocab_size": c["audio_vocab_size"],
+            "maskgit_schedule": [c["codebooks"] // steps] * steps,
+            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
+        }
+
+    def get_depth_config(self) -> dict[str, Any]:
+        return self.global_config["depthformer"]
+
+    def set_gguf_parameters(self):
+        assert self.hparams_audio is not None
+        fe = self.preprocessor_config["feature_extractor"]
+        if fe["feature_extractor_type"] not in ("Lfm3AudioFeatureExtractor", "Lfm3OmniFeatureExtractor"):
+            raise ValueError(f"unsupported feature_extractor_type {fe['feature_extractor_type']!r}")
+        for k, v in self._feature_extractor.items():
+            if fe[k] != v:
+                raise ValueError(f"unsupported feature_extractor.{k}={fe[k]!r}, expected {v!r}")
+
+        # clip.cpp hardcodes the encoder rope base
+        if self.hparams_audio["rope_parameters"]["rope_theta"] != 1e6:
+            raise ValueError("unsupported audio encoder rope_theta")
+
+        self.hparams_audio["intermediate_size"] = lfm2_feed_forward_length(self.hparams_audio)
+        super().set_gguf_parameters()
+        self.gguf_writer.add_clip_audio_projector_type(gguf.VisionProjectorType.LFM3A)
+        self.gguf_writer.add_audio_head_count_kv(self.hparams_audio["num_key_value_heads"])
+        self.gguf_writer.add_audio_attention_layernorm_eps(self.hparams_audio["norm_eps"])
+        self.gguf_writer.add_audio_num_mel_bins(self.hparams_audio["num_mel_bins"])
+        self.gguf_writer.add_audio_window_size(self.hparams_audio["sliding_window"])
+
+        if not self._audio_output:
+            return
+
+        # audio generation: depthformer hparams, the codec ones are constant and live in clip.cpp
+        gen = self.get_generation_config()
+        for k, v in self._generation.items():
+            if gen[k] != v:
+                raise ValueError(f"unsupported {k}={gen[k]!r}, expected {v!r}")
+        depth = self.get_depth_config()
+        if depth.get("is_causal", False) or depth["use_sandwich_norm"] or set(depth["layer_types"]) != {"full_attention"}:
+            raise ValueError("unsupported depthformer, expected a bidirectional all-attention stack")
+        if depth["rope_parameters"]["full_attention"]["rope_theta"] != 1e6:
+            raise ValueError("unsupported depthformer rope_theta")
+        self.gguf_writer.add_clip_has_gen_audio_encoder(True)
+        self.gguf_writer.add_clip_gen_audio_projector_type(gguf.VisionProjectorType.LFM3A_GEN)
+        self.gguf_writer.add_gen_audio_projection_dim(self.n_embd_text)
+        self.gguf_writer.add_gen_audio_embedding_length(depth["hidden_size"])
+        self.gguf_writer.add_gen_audio_feed_forward_length(depth["intermediate_size"])
+        self.gguf_writer.add_gen_audio_block_count(depth["num_hidden_layers"])
+        self.gguf_writer.add_gen_audio_head_count(depth["num_attention_heads"])
+        self.gguf_writer.add_gen_audio_head_count_kv(depth["num_key_value_heads"])
+        self.gguf_writer.add_gen_audio_attention_layernorm_eps(depth["norm_eps"])
+
+        # sampling defaults of the speech codes, next to the text ones that become general.sampling.*
+        gen = gguf.Metadata.load_generation_config(self.dir_model)
+        for key, add in (("audio_temperature",        self.gguf_writer.add_gen_audio_sampling_temp),
+                         ("audio_top_k",              self.gguf_writer.add_gen_audio_sampling_top_k),
+                         ("audio_repetition_penalty", self.gguf_writer.add_gen_audio_sampling_penalty_repeat),
+                         ("audio_repetition_window",  self.gguf_writer.add_gen_audio_sampling_penalty_last_n)):
+            value = gen.get(key, self._audio_sampling.get(key))
+            if value is not None:
+                add(value)
+
+    def tensor_force_quant(self, name, new_name, bid, n_dims):
+        if new_name.startswith("a.conv1d.") or "conv_dw" in new_name or new_name.startswith("a.gen.wav.quant."):
+            return gguf.GGMLQuantizationType.F32
+        # the codec decoder is small and its log-magnitude/phase output does not survive quantization
+        if new_name.startswith("a.gen.wav."):
+            return gguf.GGMLQuantizationType.F16
+        return super().tensor_force_quant(name, new_name, bid, n_dims)
+
+    def generate_extra_tensors(self) -> Iterable[tuple[str, Tensor]]:
+        if self._audio_output:
+            yield from self._codec_tensors()
+
+    def _codec_tensors(self) -> Iterable[tuple[str, Tensor]]:
+        import json
+        from safetensors.torch import load_file
+
+        T = gguf.MODEL_TENSOR
+        codec_dir = self.dir_model / "audio_tokenizer"
+        with open(codec_dir / "config.json", encoding="utf-8") as f:
+            cfg = json.load(f)
+        for k, v in self._codec.items():
+            if cfg.get(k) != v:
+                raise ValueError(f"unsupported audio_tokenizer {k}={cfg.get(k)!r}, expected {v!r}")
+        if math.prod(cfg["fsq_levels"]) != self._audio_vocab - 1:
+            raise ValueError(f"audio_vocab_size {self._audio_vocab} does not match the codec, expected codebook size + 1")
+
+        state_dict = load_file(codec_dir / "model.safetensors")
+
+        # FSQ code table: index -> the len(levels) quantized values, see FiniteScalarQuantization._indices_to_codes
+        levels = torch.tensor(cfg["fsq_levels"], dtype=torch.int64)
+        basis = torch.cumprod(torch.cat([torch.ones(1, dtype=torch.int64), levels[:-1]]), dim=0)
+        half = levels // 2
+        idx = torch.arange(int(levels.prod()), dtype=torch.int64)[:, None]
+        codes = ((idx // basis) % levels - half) / half
+        yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_CB), codes.to(torch.float32))
+
+        # one projection over the concatenated code vectors of all codebooks, under quantizer.fsq. in newer exports
+        quantizer = "quantizer.fsq." if "quantizer.fsq.project_out.weight" in state_dict else "quantizer."
+        for suffix in ("weight", "bias"):
+            yield (self.format_tensor_name(T.A_GEN_WAV_QUANT_FIRST_OUT, suffix=f".{suffix}"), state_dict[f"{quantizer}project_out.{suffix}"])
+            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_IN_PROJ, suffix=f".{suffix}"), state_dict[f"from_quantizer.{suffix}"])
+            yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUT_PROJ, suffix=f".{suffix}"), state_dict[f"decoder.lin.{suffix}"])
+        yield (self.format_tensor_name(T.A_GEN_WAV_TFM_OUTPUT_NORM), state_dict["decoder.lfm.embedding_norm.weight"])
+
+        for name, data_torch in state_dict.items():
+            if not name.startswith("decoder.lfm.layers."):
+                continue
+            bid_str, key = name.removeprefix("decoder.lfm.layers.").rsplit(".weight", 1)[0].split(".", 1)
+            if key == "conv.conv":
+                data_torch = data_torch.squeeze(1)
+            yield (self.format_tensor_name(self._codec_layer_map[key], int(bid_str)), data_torch)
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.startswith("a.gen.wav."):
+            yield (name, data_torch)
+            return
+
+        if name.startswith(("depthformer.", "depth_", "codebook_embedding.", "mask_embedding")):
+            if self._audio_output:
+                yield from self._depthformer_tensor(data_torch, name, bid)
+            return
+
+        if not name.startswith(("encoder.", "mlp.")):
+            return
+
+        if name.startswith("encoder.downsample."):
+            layer, suffix = name.removeprefix("encoder.downsample.").split(".")
+            if layer == "lin_out":
+                if suffix == "weight":
+                    data_torch = data_torch.squeeze(-1)
+                yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_OUT, suffix=f".{suffix}"), data_torch)
+                return
+            if suffix == "bias":
+                data_torch = data_torch[:, None, None]
+            yield (self.format_tensor_name(gguf.MODEL_TENSOR.A_ENC_CONV1D, self._downsample_idx[layer], f".{suffix}"), data_torch)
+            return
+
+        # conv op requires 2d tensor
+        if name.endswith("conv.conv.weight"):
+            data_torch = data_torch.squeeze(1)
+
+        yield (self.map_tensor_name(name), data_torch)
+
+    def _depthformer_tensor(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        T = gguf.MODEL_TENSOR
+
+        if name.startswith("depthformer.model.layers."):
+            key = name.removeprefix("depthformer.model.layers.").split(".", 1)[1].rsplit(".weight", 1)[0]
+            yield (self.format_tensor_name(self._depth_layer_map[key], bid), data_torch)
+            return
+        if name == "depthformer.model.embedding_norm.weight":
+            yield (self.format_tensor_name(T.A_GEN_CODE_OUTPUT_NORM), data_torch)
+            return
+        if name.startswith("depth_linear."):
+            yield (self.format_tensor_name(T.A_GEN_CODE_PROJ_IN, suffix="." + name.rsplit(".", 1)[1]), data_torch)
+            return
+        if name == "mask_embedding":
+            yield (self.format_tensor_name(T.A_GEN_CODE_MASK_EMBD, suffix=""), data_torch)
+            return
+        if name == "codebook_embedding.weight":
+            assert data_torch.shape[0] == self._n_codebooks * self._audio_vocab
+            yield (self.format_tensor_name(T.A_GEN_CODE_OUT_EMBD), data_torch)
+            return
+
+        # per-codebook heads: the embedding table is tied to to_logits, merge the 12 tables into one tensor
+        assert name.startswith("depth_embeddings.")
+        idx = int(name.split(".")[1])
+        key = name.split(".", 2)[2]
+        if key == "embedding.weight":
+            self._depth_embd[idx] = data_torch
+            if len(self._depth_embd) == self._n_codebooks:
+                yield (self.format_tensor_name(T.A_GEN_CODE_EMBD), torch.stack([self._depth_embd[i] for i in range(self._n_codebooks)], dim=0))
+        elif key == "embedding_norm.weight":
+            self._depth_norm[idx] = data_torch
+            if len(self._depth_norm) == self._n_codebooks:
+                yield (self.format_tensor_name(T.A_GEN_CODE_HEAD_NORM), torch.stack([self._depth_norm[i] for i in range(self._n_codebooks)], dim=0))
+        elif key == "to_logits.weight":
+            if not torch.equal(data_torch, self._depth_embd[idx]):
+                raise ValueError(f"depth_embeddings.{idx} is not tied")
+        else:
+            raise ValueError(f"unexpected tensor {name}")
+
+
+@ModelBase.register("Lfm3OmniForConditionalGeneration")
+@ModelBase.example("LiquidAI/LFM3-Omni-2B")
+class LFM3OmniMmprojModel(LFM3VLVisionMixin, LFM3AudioMmprojModel):
+    has_vision_encoder = True
+    has_audio_encoder = True
+
+    _audio_sampling = {"audio_temperature": 0.4, "audio_repetition_penalty": 1.2, "audio_repetition_window": 64}
+
+    # the audio side is the LFM3-Audio model under the HF-native names
+    _audio_renames = (
+        ("model.audio_encoder.lfm.", "encoder.lfm.lfm."),
+        ("model.audio_encoder.", "encoder."),
+        ("model.audio_projector.layer_norm.", "mlp.model.0."),
+        ("model.audio_projector.linear_1.", "mlp.model.1."),
+        ("model.audio_projector.linear_2.", "mlp.model.3."),
+        ("model.depth_decoder.model.", "depthformer.model."),
+        ("model.depth_decoder.", ""),
+        ("model.codebook_embedding.", "codebook_embedding."),
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        assert self.hparams_vision is not None
+        # dynamic resolution, image_size is only set for MmprojModel compatibility
+        self.hparams_vision["image_size"] = 512
+
+    def get_audio_config(self) -> dict[str, Any] | None:
+        enc = self.global_config["audio_encoder_config"]
+        cfg = dict(enc["text_config"])
+        cfg["num_mel_bins"] = enc["num_mel_bins"]
+        cfg["sliding_window"] = enc["sliding_window"]
+        return cfg
+
+    def get_generation_config(self) -> dict[str, Any]:
+        c = self.global_config
+        return {
+            "codebooks": c["num_codebooks"], "audio_vocab_size": c["codebook_size"], "maskgit_schedule": c["maskgit_schedule"],
+            "interleaved_n_text": c["interleaved_n_text"], "interleaved_n_audio": c["interleaved_n_audio"],
+        }
+
+    def get_depth_config(self) -> dict[str, Any]:
+        return self.global_config["depth_config"]
+
+    def set_gguf_parameters(self):
+        super().set_gguf_parameters()
+        self._set_lfm3vl_gguf_parameters()
+
+    def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iterable[tuple[str, Tensor]]:
+        if name.startswith(("model.vision_tower.", "model.multi_modal_projector.")):
+            yield from self._lfm3vl_tensors(data_torch, name, bid)
+            return
+        for old, new in self._audio_renames:
+            if name.startswith(old):
+                yield from super().modify_tensors(data_torch, new + name.removeprefix(old), bid)
+                return
+        yield from super().modify_tensors(data_torch, name, bid)

examples/realtime_client.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""Example client for the llama-server /v1/realtime websocket, with the openai SDK.
+
+Start a server with an audio model, e.g. LFM3-Audio:
+
+    llama-server -hf LiquidAI/LFM3-Audio-2B-GGUF
+
+Spoken conversation, the audio of each wav file is one user turn. It is sent in chunks as it would come from a
+microphone, the server prefills it as it arrives so the response starts right after the last chunk:
+
+    python examples/realtime_client.py --instructions "Respond with interleaved text and audio." question.wav
+
+Output modalities: left out, the model picks; "text", "audio" or "text,audio" (interleaved) force the response.
+Without --instructions, a forced mode comes with the system prompt of the LFM3-Audio or LFM3-Omni model card:
+
+    python examples/realtime_client.py --output-modalities text,audio --out answer.wav question.wav
+
+Text turns go in with --text, transcription sessions transcribe each file:
+
+    python examples/realtime_client.py --text "What is the capital of France?"
+    python examples/realtime_client.py --type transcription --language en speech.wav
+
+Live transcription models (Voxtral Realtime, LFM3-Audio live ASR) transcribe while the audio is sent, a transcription
+session opens with ?intent=transcription:
+
+    python examples/realtime_client.py --type transcription speech.wav
+
+Requirements: pip install "openai[realtime]" numpy
+"""
+
+import argparse
+import asyncio
+import base64
+import time
+import wave
+
+import numpy as np
+from openai import AsyncOpenAI
+
+
+# system prompts of the LiquidAI/LFM3-Audio-2B and LiquidAI/LFM3-Omni-2B model cards for a forced output mode
+MODE_INSTRUCTIONS = {
+    "audio": {"text,audio": "Respond with interleaved text and audio."},
+    "omni":  {"text": "Respond in text.", "text,audio": "Respond in interleaved text and audio."},
+}
+
+
+def read_wav(path, rate):
+    # PCM16 wav -> mono int16 at the given rate
+    with wave.open(path, "rb") as f:
+        assert f.getsampwidth() == 2, "the example reads PCM16 wav files"
+        x = np.frombuffer(f.readframes(f.getnframes()), dtype=np.int16).astype(np.float32)
+        x = x.reshape(-1, f.getnchannels()).mean(axis=1)
+        sr = f.getframerate()
+    if sr != rate:
+        n = int(round(len(x) * rate / sr))
+        x = np.interp(np.arange(n) * sr / rate, np.arange(len(x)), x)
+    return np.clip(x, -32768, 32767).astype(np.int16)
+
+
+def write_wav(path, pcm, rate):
+    with wave.open(path, "wb") as f:
+        f.setnchannels(1)
+        f.setsampwidth(2)
+        f.setframerate(rate)
+        f.writeframes(pcm.tobytes())
+
+
+async def main(args):
+    # the SDK derives a wss:// URL from base_url, a plain http server needs ws://
+    client = AsyncOpenAI(base_url=args.url, api_key=args.api_key, websocket_base_url=args.url.replace("http", "ws", 1))
+    query = {"intent": "transcription"} if args.type == "transcription" else {}
+    async with client.realtime.connect(model=args.model, extra_query=query) as conn:
+        ev = await conn.recv()
+        if ev.type == "error":
+            print("error:", ev.error.message)
+            return
+        # the audio goes in at the rate of the model, PCM16 mono
+        session_info = ev.model_dump(warnings=False)["session"]
+        audio_fmt = session_info["audio"]
+        card = "omni" if "omni" in str(session_info.get("model", "")).lower() else "audio"
+        in_rate = audio_fmt["input"]["format"]["rate"]
+        out_rate = (audio_fmt.get("output") or {}).get("format", {}).get("rate", 24000)
+
+        session = {"type": args.type}
+        if args.type == "realtime":
+            instructions = args.instructions
+            if instructions is None:
+                instructions = MODE_INSTRUCTIONS[card].get(args.output_modalities)
+            if instructions is not None:
+                session["instructions"] = instructions
+            if args.output_modalities:
+                session["output_modalities"] = args.output_modalities.split(",")
+        elif args.language:
+            session["audio"] = {"input": {"transcription": {"language": args.language}}}
+        await conn.session.update(session=session)  # ty: ignore[invalid-argument-type]
+        ev = await conn.recv()
+        if ev.type == "error":
+            print("error:", ev.error.message)
+            return
+
+        turns = [("audio", path) for path in args.wav] + [("text", text) for text in args.text]
+        for i, (kind, src) in enumerate(turns):
+            t_last = 0.0
+
+            # the input is sent while the events are read: live transcription arrives while the audio is sent
+            async def send_turn():
+                nonlocal t_last
+                if kind == "audio":
+                    pcm = read_wav(src, in_rate)
+                    step = in_rate * args.chunk_ms // 1000
+                    for j in range(0, len(pcm), step):
+                        chunk = pcm[j:j + step]
+                        await conn.input_audio_buffer.append(audio=base64.b64encode(chunk.tobytes()).decode())
+                        if args.realtime:
+                            await asyncio.sleep(len(chunk) / in_rate)
+                    if args.type == "transcription":
+                        # "final" ends the input, models that transcribe with a delay then complete the last item
+                        await conn.send({"type": "input_audio_buffer.commit", "final": i == len(args.wav) - 1})  # ty: ignore[invalid-argument-type]
+                else:
+                    await conn.conversation.item.create(item={
+                        "type": "message", "role": "user", "content": [{"type": "input_text", "text": src}]})
+                if args.type == "realtime":
+                    # the audio not committed yet ends the user turn
+                    await conn.response.create()
+                t_last = time.perf_counter()
+
+            print(f"--- turn {i} ({kind}: {src})")
+            sender = asyncio.create_task(send_turn())
+            t_first = None
+            audio = []
+            async for ev in conn:
+                if ev.type == "response.output_text.delta" or ev.type == "conversation.item.input_audio_transcription.delta":
+                    t_first = t_first or time.perf_counter()
+                    print(ev.delta, end="", flush=True)
+                elif ev.type == "response.output_audio.delta":
+                    t_first = t_first or time.perf_counter()
+                    audio.append(np.frombuffer(base64.b64decode(ev.delta), dtype=np.int16))
+                elif ev.type in ("response.done", "conversation.item.input_audio_transcription.completed"):
+                    break
+                elif ev.type == "error":
+                    print("\nerror:", ev.error.message)
+                    return
+            await sender
+            print()
+            if t_first is not None and t_first > t_last:
+                print(f"first output {1000 * (t_first - t_last):.0f} ms after the end of the input")
+            elif t_first is not None:
+                print(f"output started {1000 * (t_last - t_first):.0f} ms before the end of the input, "
+                      f"done {1000 * (time.perf_counter() - t_last):.0f} ms after it")
+            if audio:
+                pcm_out = np.concatenate(audio)
+                path = f"{args.out.removesuffix('.wav')}_{i}.wav" if len(turns) > 1 else args.out
+                write_wav(path, pcm_out, out_rate)
+                print(f"audio: {len(pcm_out) / out_rate:.2f} s -> {path}")
+
+
+if __name__ == "__main__":
+    p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    p.add_argument("wav", nargs="*", help="PCM16 wav files, one user turn each")
+    p.add_argument("--text", action="append", default=[], help="a user turn as text, after the audio turns, repeatable")
+    p.add_argument("--type", choices=["realtime", "transcription"], default="realtime", help="session type")
+    p.add_argument("--instructions", help="system prompt of the conversation")
+    p.add_argument("--output-modalities", choices=["text", "audio", "text,audio"], help="default: the model picks")
+    p.add_argument("--language", help="transcription language, e.g. en")
+    p.add_argument("--chunk-ms", type=int, default=100, help="audio chunk size")
+    p.add_argument("--realtime", action=argparse.BooleanOptionalAction, default=True, help="send the audio in real time")
+    p.add_argument("--out", default="response.wav", help="where the audio of the responses goes")
+    p.add_argument("--url", default="http://127.0.0.1:9931/v1")
+    p.add_argument("--api-key", default="none")
+    p.add_argument("--model", default="default", help="model name, used by the router")
+    asyncio.run(main(p.parse_args()))

ggml/src/ggml-cuda/conv2d.cu

@@ -180,7 +180,6 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
     const int                  end            = int(int64_t(tiles) * (split + 1) / split_k) * BK;
     for (int k0 = begin; k0 < end; k0 += BK) {
         if (k_total % 8 == 0 && uintptr_t(weight) % 16 == 0) {
-#pragma unroll
             for (int i = tid; i < BM * BK / 8; i += nthreads) {
                 const int  row = i / (BK / 8), col = 8 * (i % (BK / 8));
                 const int4 v                 = m0 + row < oc && k0 + col < k_total ?
@@ -189,7 +188,6 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 *(int4 *) &a_s[row][col / 2] = v;
             }
         } else {
-#pragma unroll
             for (int i = tid; i < BM * BK / 2; i += nthreads) {
                 const int row = i / (BK / 2), col = 2 * (i % (BK / 2));
                 half      lo = __float2half(0.0f), hi = lo;
@@ -202,7 +200,6 @@ static __global__ void conv2d_implicit_gemm_f16(const half * __restrict__ input,
                 a_s[row][col / 2] = __halves2half2(lo, hi);
             }
         }
-#pragma unroll
         for (int k = load_row; k < BK; k += nthreads / (BN / 2)) {
             const int ki = k0 + k;
             const int ci = ki / (kw * kh), ky = ki / kw % kh, kx = ki % kw;

ggml/src/ggml-rpc/ggml-rpc.cpp

@@ -125,11 +125,16 @@ struct rpc_msg_hello_req {
     uint8_t conn_caps[RPC_CONN_CAPS_SIZE];
 };
 
+// server feature flags in rpc_msg_hello_rsp::flags; old servers send 0
+enum rpc_server_flags : uint8_t {
+    RPC_SERVER_FLAG_CACHE = 1 << 0, // server has a local tensor cache (rpc-server -c)
+};
+
 struct rpc_msg_hello_rsp {
     uint8_t major;
     uint8_t minor;
     uint8_t patch;
-    uint8_t padding;
+    uint8_t flags;
     uint8_t conn_caps[RPC_CONN_CAPS_SIZE];
 };
 
@@ -373,21 +378,30 @@ static bool parse_endpoint(const std::string & endpoint, std::string & host, int
 }
 
 // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |
+// request_data is input followed by an optional payload, so large data does not need to be copied into input
 // No response
-static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) {
+static bool send_rpc_cmd_payload(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, const void * payload, size_t payload_size) {
     uint8_t cmd_byte = cmd;
     if (!sock->send_data(&cmd_byte, sizeof(cmd_byte))) {
         return false;
     }
-    if (!sock->send_data(&input_size, sizeof(input_size))) {
+    const size_t request_size = input_size + payload_size;
+    if (!sock->send_data(&request_size, sizeof(request_size))) {
         return false;
     }
     if (!sock->send_data(input, input_size)) {
         return false;
     }
+    if (payload_size > 0 && !sock->send_data(payload, payload_size)) {
+        return false;
+    }
     return sock->flush();
 }
 
+static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size) {
+    return send_rpc_cmd_payload(sock, cmd, input, input_size, nullptr, 0);
+}
+
 // RPC request : | rpc_cmd (1 byte) | request_size (8 bytes) | request_data (request_size bytes) |
 // RPC response: | response_size (8 bytes) | response_data (response_size bytes) |
 static bool send_rpc_cmd(socket_ptr sock, enum rpc_cmd cmd, const void * input, size_t input_size, void * output, size_t output_size) {
@@ -423,7 +437,7 @@ static inline void rpc_cpu_relax() {
 // Performs HELLO handshake with transport auto-negotiation.
 // Advertises local capabilities via conn_caps; if the server responds with
 // matching capabilities, the socket is upgraded transparently.
-static bool negotiate_hello(const std::shared_ptr<socket_t> & sock) {
+static bool negotiate_hello(const std::shared_ptr<socket_t> & sock, uint8_t & server_flags) {
     rpc_msg_hello_req request = {};
     rpc_msg_hello_rsp response = {};
 
@@ -439,6 +453,7 @@ static bool negotiate_hello(const std::shared_ptr<socket_t> & sock) {
     }
 
     sock->update_caps(response.conn_caps);
+    server_flags = response.flags;
     LOG_DBG("[%s] handshake ok: server version %d.%d.%d, transport: %s\n",
             __func__, response.major, response.minor, response.patch, sock->transport_name());
     return true;
@@ -503,6 +518,8 @@ public:
     void send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size);
     void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size);
     void send_async(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size);
+    // payload is sent after input without a copy, the call returns once it is sent
+    void send_payload(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, const void * payload, size_t payload_size);
 
     ggml_backend_event_t event_new(ggml_backend_dev_t dev);
     void event_free(ggml_backend_event_t event);
@@ -516,6 +533,8 @@ public:
     void start(const std::string & endpoint);
     void work();
 
+    bool server_has_cache() const { return server_flags & RPC_SERVER_FLAG_CACHE; }
+
     ~rpc_dispatcher();
 
 private:
@@ -525,6 +544,8 @@ private:
         size_t                        input_size;
         void                        * output;
         size_t                        output_size;
+        const void                  * payload      = nullptr;
+        size_t                        payload_size = 0;
         std::promise<void>            completion;
     };
     using rpc_msg_ptr   = std::shared_ptr<rpc_msg>;
@@ -537,6 +558,7 @@ private:
     std::unordered_map<uint32_t, std::unordered_set<uint64_t>> graph_uids;
     rpc_msg_queue    queue;
     socket_ptr       sock;
+    uint8_t          server_flags = 0;
     std::atomic_uint busy_spin_users = 0;
     std::atomic_bool running;
     std::thread      thread;
@@ -569,6 +591,20 @@ void rpc_dispatcher::send_async(enum rpc_cmd cmd, std::shared_ptr<const void> in
     GGML_ASSERT(queue.push(msg));
 }
 
+void rpc_dispatcher::send_payload(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, const void * payload, size_t payload_size) {
+    auto msg = std::make_shared<rpc_msg>();
+    msg->cmd = cmd;
+    msg->input = input;
+    msg->input_size = input_size;
+    msg->output = nullptr;
+    msg->output_size = 0;
+    msg->payload = payload;
+    msg->payload_size = payload_size;
+    GGML_ASSERT(queue.push(msg));
+    auto future = msg->completion.get_future();
+    future.wait();
+}
+
 void rpc_dispatcher::send(enum rpc_cmd cmd, std::shared_ptr<const void> input, size_t input_size, void * output, size_t output_size) {
     auto msg = std::make_shared<rpc_msg>();
     msg->cmd = cmd;
@@ -652,7 +688,7 @@ void rpc_dispatcher::start(const std::string & endpoint) {
     if (sock == nullptr) {
         GGML_ABORT("Failed to connect to %s\n", endpoint.c_str());
     }
-    if (!negotiate_hello(sock)) {
+    if (!negotiate_hello(sock, server_flags)) {
         GGML_ABORT("RPC handshake failed for %s\n", endpoint.c_str());
     }
     LOG_DBG("[%s] connected to %s\n", __func__, endpoint.c_str());
@@ -684,7 +720,7 @@ void rpc_dispatcher::work() {
                 bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->output, msg_ptr->output_size);
                 RPC_STATUS_ASSERT(status);
             } else {
-                bool status = send_rpc_cmd(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size);
+                bool status = send_rpc_cmd_payload(sock, msg_ptr->cmd, msg_ptr->input.get(), msg_ptr->input_size, msg_ptr->payload, msg_ptr->payload_size);
                 RPC_STATUS_ASSERT(status);
             }
             if (trace) {
@@ -829,14 +865,17 @@ static void ggml_backend_rpc_buffer_memset_tensor(
 }
 
 // input serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes)
+// with size == 0 only the header is serialized and data is sent separately as payload
 static std::shared_ptr<uint8_t> serialize_set_tensor(const rpc_tensor & rpc_tensor, uint8_t cache_flag, uint64_t offset, const void * data, size_t size, size_t & input_size) {
     input_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset) + size;
-    uint8_t * input = new uint8_t[input_size]();
+    uint8_t * input = new uint8_t[input_size];
     uint8_t * p = input;
     memcpy(p, &rpc_tensor, sizeof(rpc_tensor)); p += sizeof(rpc_tensor);
     memcpy(p, &cache_flag, sizeof(cache_flag)); p += sizeof(cache_flag);
     memcpy(p, &offset,     sizeof(offset));     p += sizeof(offset);
-    memcpy(p, data, size);
+    if (size > 0) {
+        memcpy(p, data, size);
+    }
     return std::shared_ptr<uint8_t>(input, std::default_delete<uint8_t[]>());
 }
 
@@ -844,8 +883,9 @@ static std::shared_ptr<uint8_t> serialize_set_tensor(const rpc_tensor & rpc_tens
 // compute-buffer inputs (the activations ggml_backend_sched copies between backends) must not
 // take this path, otherwise with `rpc-server -c` every ubatch above the threshold is written
 // to the cache directory and later served from there.
-static bool rpc_use_hash_cache(const ggml_tensor * tensor, size_t size) {
-    return size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
+// hashing is slow (~1 GB/s), so skip it when the server has no cache to look up.
+static bool rpc_use_hash_cache(const rpc_dispatcher & dispatcher, const ggml_tensor * tensor, size_t size) {
+    return dispatcher.server_has_cache() && size > HASH_THRESHOLD && tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_WEIGHTS;
 }
 
 static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data, size_t offset, size_t size) {
@@ -854,7 +894,7 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm
     uint8_t cache_flag = 0;
     LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset,
              rpc_log_format_size(size).c_str());
-    if (rpc_use_hash_cache(tensor, size)) {
+    if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) {
         auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
         request->tensor = rpc_tensor;
         request->offset = offset;
@@ -870,9 +910,9 @@ static void ggml_backend_rpc_buffer_set_tensor(ggml_backend_buffer_t buffer, ggm
         // the server has no cache entry for this tensor - ask it to save one
         cache_flag = 1;
     }
-    size_t input_size;
-    auto input = serialize_set_tensor(rpc_tensor, cache_flag, offset, data, size, input_size);
-    ctx->dispatcher->send(RPC_CMD_SET_TENSOR, input, input_size);
+    size_t header_size;
+    auto header = serialize_set_tensor(rpc_tensor, cache_flag, offset, nullptr, 0, header_size);
+    ctx->dispatcher->send_payload(RPC_CMD_SET_TENSOR, header, header_size, data, size);
 }
 
 static void ggml_backend_rpc_buffer_set_tensor_2d(ggml_backend_buffer_t buffer, ggml_tensor * tensor, const void * data,
@@ -1154,7 +1194,7 @@ static void ggml_backend_rpc_set_tensor_async(ggml_backend_t backend, ggml_tenso
     uint8_t cache_flag = 0;
     LOG_DBG2("[%s] tensor: %s, offset: %zu, size: %s\n", __func__, rpc_log_format_tensor(tensor).c_str(), offset,
              rpc_log_format_size(size).c_str());
-    if (rpc_use_hash_cache(tensor, size)) {
+    if (rpc_use_hash_cache(*ctx->dispatcher, tensor, size)) {
         auto request = std::make_shared<rpc_msg_set_tensor_hash_req>();
         request->tensor = rpc_tensor;
         request->offset = offset;
@@ -1405,7 +1445,7 @@ public:
     bool free_buffer(const rpc_msg_free_buffer_req & request);
     bool buffer_clear(const rpc_msg_buffer_clear_req & request);
     bool memset_tensor(const rpc_msg_memset_tensor_req & request);
-    bool set_tensor(const std::vector<uint8_t> & input);
+    bool set_tensor(socket_ptr sock, uint64_t input_size);
     bool set_tensor_2d(const std::vector<uint8_t> & input);
     bool set_tensor_hash(const rpc_msg_set_tensor_hash_req & request, rpc_msg_set_tensor_hash_rsp & response);
     bool get_tensor(const rpc_msg_get_tensor_req & request, std::vector<uint8_t> & response);
@@ -1452,12 +1492,15 @@ private:
     // computed graphs cached per backend, keyed by uid
     std::vector<std::unordered_map<uint64_t, stored_graph>> stored_graphs;
     std::vector<comm_state> comm_states;
+    // SET_TENSOR data is received in chunks through this buffer
+    std::vector<uint8_t> set_tensor_buf;
 };
 
 void rpc_server::hello(rpc_msg_hello_rsp & response) {
     response.major = RPC_PROTO_MAJOR_VERSION;
     response.minor = RPC_PROTO_MINOR_VERSION;
     response.patch = RPC_PROTO_PATCH_VERSION;
+    response.flags = cache_dir ? RPC_SERVER_FLAG_CACHE : 0;
     LOG_DBG("[%s] version: %d.%d.%d\n", __func__, response.major, response.minor, response.patch);
 }
 
@@ -1698,19 +1741,21 @@ ggml_tensor * rpc_server::deserialize_tensor(struct ggml_context * ctx, const rp
 }
 
 
-bool rpc_server::set_tensor(const std::vector<uint8_t> & input) {
+bool rpc_server::set_tensor(socket_ptr sock, uint64_t input_size) {
     sync_all_backends();
     // serialization format: | rpc_tensor | cache_flag (1 byte) | offset (8 bytes) | data (size bytes) |
+    // only the header is read here, the data is streamed from the socket below
     uint8_t  cache_flag;
     uint64_t offset;
-    const size_t header_size = sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset);
-    if (input.size() < header_size) {
+    uint8_t  header[sizeof(rpc_tensor) + sizeof(cache_flag) + sizeof(offset)];
+    const size_t header_size = sizeof(header);
+    if (input_size < header_size || !sock->recv_data(header, header_size)) {
         return false;
     }
-    const rpc_tensor * in_tensor = (const rpc_tensor *)input.data();
-    memcpy(&cache_flag, input.data() + sizeof(rpc_tensor), sizeof(cache_flag));
-    memcpy(&offset,     input.data() + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset));
-    const size_t size = input.size() - header_size;
+    const rpc_tensor * in_tensor = (const rpc_tensor *)header;
+    memcpy(&cache_flag, header + sizeof(rpc_tensor), sizeof(cache_flag));
+    memcpy(&offset,     header + sizeof(rpc_tensor) + sizeof(cache_flag), sizeof(offset));
+    const size_t size = input_size - header_size;
 
     struct ggml_init_params params {
         /*.mem_size   =*/ ggml_tensor_overhead(),
@@ -1741,18 +1786,37 @@ bool rpc_server::set_tensor(const std::vector<uint8_t> & input) {
         }
     }
 
-    const void * data = input.data() + header_size;
     if (cache_dir && cache_flag) {
-        uint64_t hash = fnv_hash((const uint8_t*)data, size);
+        // the whole data is needed to compute the hash
+        std::vector<uint8_t> data(size);
+        if (!sock->recv_data(data.data(), size)) {
+            return false;
+        }
+        uint64_t hash = fnv_hash(data.data(), size);
         char hash_str[17];
         snprintf(hash_str, sizeof(hash_str), "%016" PRIx64, hash);
         // save to cache_dir/hash_str
         fs::path cache_file = fs::path(cache_dir) / hash_str;
         std::ofstream ofs(cache_file, std::ios::binary);
-        ofs.write((const char *)data, size);
+        ofs.write((const char *)data.data(), size);
         LOG_INFO("[%s] saved to '%s'\n", __func__, cache_file.string().c_str());
+        ggml_backend_tensor_set(tensor, data.data(), offset, size);
+        return true;
+    }
+
+    // a buffer of the tensor size would be allocated and page faulted for every tensor, which is slow
+    const size_t chunk_size = 16*1024*1024;
+    if (set_tensor_buf.size() < std::min(size, chunk_size)) {
+        set_tensor_buf.resize(std::min(size, chunk_size));
+    }
+    for (size_t done = 0; done < size; ) {
+        const size_t n = std::min(size - done, chunk_size);
+        if (!sock->recv_data(set_tensor_buf.data(), n)) {
+            return false;
+        }
+        ggml_backend_tensor_set(tensor, set_tensor_buf.data(), offset + done, n);
+        done += n;
     }
-    ggml_backend_tensor_set(tensor, data, offset, size);
     return true;
 }
 
@@ -2601,11 +2665,11 @@ static void rpc_serve_client(const std::vector<ggml_backend_t> & backends, const
                 break;
             }
             case RPC_CMD_SET_TENSOR: {
-                std::vector<uint8_t> input;
-                if (!recv_msg(sock, input)) {
+                uint64_t input_size;
+                if (!sock->recv_data(&input_size, sizeof(input_size))) {
                     return;
                 }
-                if (!server.set_tensor(input)) {
+                if (!server.set_tensor(sock, input_size)) {
                     return;
                 }
                 break;

ggml/src/ggml-rpc/transport.cpp

@@ -71,6 +71,11 @@ struct rdma_conn {
     struct ibv_mr * rx_mr  = nullptr;
     int             rx_head = 0;
 
+    // received slot that is only partly consumed, so recv sizes need not match send sizes
+    int             rx_cur  = -1;
+    size_t          rx_off  = 0;
+    size_t          rx_len  = 0;
+
     uint32_t        max_inline = 0;
 
     uint8_t * rx_slot(int i) const {
@@ -512,19 +517,24 @@ bool socket_t::impl::rdma_recv(void * data, size_t size) {
     uint8_t * dst = (uint8_t *)data;
     size_t rem = size;
     while (rem > 0) {
-        struct ibv_wc wc;
-        if (!rdma_poll(c->rcq, &wc)) return false;
+        if (c->rx_cur < 0) {
+            struct ibv_wc wc;
+            if (!rdma_poll(c->rcq, &wc)) return false;
 
-        int slot = (int)wc.wr_id;
-        size_t got = wc.byte_len;
-        if (got > rem) {
-            LOG_ERROR("RDMA recv overrun: got=%zu rem=%zu\n", got, rem);
-            c->post_rx(slot);
-            return false;
+            c->rx_cur = (int)wc.wr_id;
+            c->rx_off = 0;
+            c->rx_len = wc.byte_len;
         }
-        memcpy(dst, c->rx_slot(slot), got);
 
-        if (!c->post_rx(slot)) return false;
+        size_t got = std::min(rem, c->rx_len - c->rx_off);
+        memcpy(dst, c->rx_slot(c->rx_cur) + c->rx_off, got);
+        c->rx_off += got;
+
+        // repost the slot only once it is fully consumed
+        if (c->rx_off == c->rx_len) {
+            if (!c->post_rx(c->rx_cur)) return false;
+            c->rx_cur = -1;
+        }
 
         dst += got;
         rem -= got;

gguf-py/gguf/constants.py

@@ -501,6 +501,7 @@ class Keys:
 
         class Attention:
             HEAD_COUNT      = "clip.audio.attention.head_count"
+            HEAD_COUNT_KV   = "clip.audio.attention.head_count_kv"
             HEAD_DIM        = "clip.audio.attention.head_dim" # set when qkv width != n_embd
             LAYERNORM_EPS   = "clip.audio.attention.layer_norm_epsilon"
 
@@ -518,6 +519,11 @@ class Keys:
         FEED_FORWARD_LENGTH = "clip.gen.audio.feed_forward_length"
         BLOCK_COUNT         = "clip.gen.audio.block_count"
         PROJECTION_DIM      = "clip.gen.audio.projection_dim"
+        # sampling defaults of the generated audio, like general.sampling.* for text
+        SAMPLING_TEMP           = "clip.gen.audio.sampling.temp"
+        SAMPLING_TOP_K          = "clip.gen.audio.sampling.top_k"
+        SAMPLING_PENALTY_REPEAT = "clip.gen.audio.sampling.penalty_repeat"
+        SAMPLING_PENALTY_LAST_N = "clip.gen.audio.sampling.penalty_last_n"
 
         class Attention:
             HEAD_COUNT      = "clip.gen.audio.attention.head_count"
@@ -691,6 +697,7 @@ class MODEL_ARCH(IntEnum):
     GPT_OSS          = auto()
     LFM2             = auto()
     LFM2MOE          = auto()
+    LFM3             = auto()
     DREAM            = auto()
     SMALLTHINKER     = auto()
     LLADA            = auto()
@@ -1175,6 +1182,8 @@ class MODEL_TENSOR(IntEnum):
     A_ENC_ATTN_Q          = auto()
     A_ENC_ATTN_K          = auto()
     A_ENC_ATTN_V          = auto()
+    A_ENC_ATTN_Q_NORM     = auto() # lfm3a
+    A_ENC_ATTN_K_NORM     = auto() # lfm3a
     A_ENC_ATTN_POST_NORM  = auto()
     A_ENC_ATTN_PRE_NORM   = auto()
     A_ENC_ATTN_K_REL      = auto() # gemma4
@@ -1219,6 +1228,9 @@ class MODEL_TENSOR(IntEnum):
     A_GEN_CODE_FFN_UP      = auto()
     A_GEN_CODE_FFN_DOWN    = auto()
     A_GEN_CODE_OUTPUT_NORM = auto()
+    A_GEN_CODE_ATTN_GATE   = auto() # lfm3a: per-head sigmoid gate on the attention output
+    A_GEN_CODE_HEAD_NORM   = auto() # lfm3a: per-codebook norm before the output head, merged 2D [n_codebooks, dim]
+    A_GEN_CODE_MASK_EMBD   = auto() # lfm3a: MaskGIT mask embedding
     # qwen3tts code2wav: RVQ codes -> raw PCM
     A_GEN_WAV_QUANT_FIRST_IN       = auto() # semantic RVQ, in_proj (1x1 conv, loaded as 2D)
     A_GEN_WAV_QUANT_FIRST_OUT      = auto() # semantic RVQ, out_proj
@@ -1241,6 +1253,14 @@ class MODEL_TENSOR(IntEnum):
     A_GEN_WAV_TFM_FFN_UP           = auto()
     A_GEN_WAV_TFM_FFN_DOWN         = auto()
     A_GEN_WAV_TFM_FFN_SCALE        = auto() # layer scale (gamma) on the FFN output
+    A_GEN_WAV_TFM_ATTN_Q_NORM      = auto() # lfm3a
+    A_GEN_WAV_TFM_ATTN_K_NORM      = auto() # lfm3a
+    A_GEN_WAV_TFM_CONV_PW1         = auto() # lfm3a: short conv in_proj
+    A_GEN_WAV_TFM_CONV_DW          = auto() # lfm3a: short conv depthwise kernel
+    A_GEN_WAV_TFM_CONV_PW2         = auto() # lfm3a: short conv out_proj
+    A_GEN_WAV_TFM_ATTN_GATE        = auto() # lfm3a: per-head sigmoid gate on the attention output
+    A_GEN_WAV_TFM_ATTN_POST_NORM   = auto() # lfm3a: sandwich norm after the operator
+    A_GEN_WAV_TFM_FFN_POST_NORM    = auto() # lfm3a: sandwich norm after the FFN
     A_GEN_WAV_UP_CONV              = auto() # causal ConvTranspose1d, 2x upsample
     A_GEN_WAV_UP_DWCONV            = auto() # ConvNeXt depthwise conv
     A_GEN_WAV_UP_NORM              = auto() # ConvNeXt LayerNorm
@@ -1523,6 +1543,7 @@ MODEL_ARCH_NAMES: dict[MODEL_ARCH, str] = {
     MODEL_ARCH.GPT_OSS:          "gpt-oss",
     MODEL_ARCH.LFM2:             "lfm2",
     MODEL_ARCH.LFM2MOE:          "lfm2moe",
+    MODEL_ARCH.LFM3:             "lfm3",
     MODEL_ARCH.DREAM:            "dream",
     MODEL_ARCH.SMALLTHINKER:     "smallthinker",
     MODEL_ARCH.LLADA:            "llada",
@@ -2002,6 +2023,8 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_ENC_ATTN_Q:              "a.blk.{bid}.attn_q",
     MODEL_TENSOR.A_ENC_ATTN_K:              "a.blk.{bid}.attn_k",
     MODEL_TENSOR.A_ENC_ATTN_V:              "a.blk.{bid}.attn_v",
+    MODEL_TENSOR.A_ENC_ATTN_Q_NORM:         "a.blk.{bid}.attn_q_norm",
+    MODEL_TENSOR.A_ENC_ATTN_K_NORM:         "a.blk.{bid}.attn_k_norm",
     MODEL_TENSOR.A_ENC_ATTN_POST_NORM:      "a.blk.{bid}.attn_post_norm",
     MODEL_TENSOR.A_ENC_ATTN_PRE_NORM:       "a.blk.{bid}.attn_pre_norm",
     MODEL_TENSOR.A_ENC_ATTN_K_REL:          "a.blk.{bid}.attn_k_rel",
@@ -2045,6 +2068,9 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_GEN_CODE_FFN_UP:         "a.gen.code.blk.{bid}.ffn_up",
     MODEL_TENSOR.A_GEN_CODE_FFN_DOWN:       "a.gen.code.blk.{bid}.ffn_down",
     MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM:    "a.gen.code.output_norm",
+    MODEL_TENSOR.A_GEN_CODE_ATTN_GATE:      "a.gen.code.blk.{bid}.attn_gate",
+    MODEL_TENSOR.A_GEN_CODE_HEAD_NORM:      "a.gen.code.head_norm",
+    MODEL_TENSOR.A_GEN_CODE_MASK_EMBD:      "a.gen.code.mask_embd",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN:  "a.gen.wav.quant.first.in_proj",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT: "a.gen.wav.quant.first.out_proj",
     MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB:  "a.gen.wav.quant.first.codebook",
@@ -2066,6 +2092,14 @@ TENSOR_NAMES: dict[MODEL_TENSOR, str] = {
     MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP:      "a.gen.wav.tfm.blk.{bid}.ffn_up",
     MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN:    "a.gen.wav.tfm.blk.{bid}.ffn_down",
     MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE:   "a.gen.wav.tfm.blk.{bid}.ls2",
+    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM: "a.gen.wav.tfm.blk.{bid}.attn_q_norm",
+    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM: "a.gen.wav.tfm.blk.{bid}.attn_k_norm",
+    MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1:    "a.gen.wav.tfm.blk.{bid}.conv_pw1",
+    MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW:     "a.gen.wav.tfm.blk.{bid}.conv_dw",
+    MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2:    "a.gen.wav.tfm.blk.{bid}.conv_pw2",
+    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE:   "a.gen.wav.tfm.blk.{bid}.attn_gate",
+    MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM: "a.gen.wav.tfm.blk.{bid}.attn_post_norm",
+    MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM:  "a.gen.wav.tfm.blk.{bid}.ffn_post_norm",
     MODEL_TENSOR.A_GEN_WAV_UP_CONV:         "a.gen.wav.up.blk.{bid}.conv",
     MODEL_TENSOR.A_GEN_WAV_UP_DWCONV:       "a.gen.wav.up.blk.{bid}.dwconv",
     MODEL_TENSOR.A_GEN_WAV_UP_NORM:         "a.gen.wav.up.blk.{bid}.norm",
@@ -2362,6 +2396,8 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_ENC_ATTN_Q,
         MODEL_TENSOR.A_ENC_ATTN_K,
         MODEL_TENSOR.A_ENC_ATTN_V,
+        MODEL_TENSOR.A_ENC_ATTN_Q_NORM,
+        MODEL_TENSOR.A_ENC_ATTN_K_NORM,
         MODEL_TENSOR.A_ENC_ATTN_POST_NORM,
         MODEL_TENSOR.A_ENC_ATTN_PRE_NORM,
         MODEL_TENSOR.A_ENC_ATTN_K_REL,
@@ -2429,6 +2465,9 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_GEN_CODE_FFN_UP,
         MODEL_TENSOR.A_GEN_CODE_FFN_DOWN,
         MODEL_TENSOR.A_GEN_CODE_OUTPUT_NORM,
+        MODEL_TENSOR.A_GEN_CODE_ATTN_GATE,
+        MODEL_TENSOR.A_GEN_CODE_HEAD_NORM,
+        MODEL_TENSOR.A_GEN_CODE_MASK_EMBD,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_IN,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_OUT,
         MODEL_TENSOR.A_GEN_WAV_QUANT_FIRST_CB,
@@ -2450,6 +2489,14 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.A_GEN_WAV_TFM_FFN_UP,
         MODEL_TENSOR.A_GEN_WAV_TFM_FFN_DOWN,
         MODEL_TENSOR.A_GEN_WAV_TFM_FFN_SCALE,
+        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_Q_NORM,
+        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_K_NORM,
+        MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW1,
+        MODEL_TENSOR.A_GEN_WAV_TFM_CONV_DW,
+        MODEL_TENSOR.A_GEN_WAV_TFM_CONV_PW2,
+        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_GATE,
+        MODEL_TENSOR.A_GEN_WAV_TFM_ATTN_POST_NORM,
+        MODEL_TENSOR.A_GEN_WAV_TFM_FFN_POST_NORM,
         MODEL_TENSOR.A_GEN_WAV_UP_CONV,
         MODEL_TENSOR.A_GEN_WAV_UP_DWCONV,
         MODEL_TENSOR.A_GEN_WAV_UP_NORM,
@@ -5522,6 +5569,29 @@ MODEL_TENSORS: dict[MODEL_ARCH, list[MODEL_TENSOR]] = {
         MODEL_TENSOR.FFN_UP_EXP,
         MODEL_TENSOR.FFN_EXP_PROBS_B,
     ],
+    MODEL_ARCH.LFM3: [
+        MODEL_TENSOR.TOKEN_EMBD,
+        MODEL_TENSOR.OUTPUT_NORM,
+        MODEL_TENSOR.OUTPUT,
+        MODEL_TENSOR.ROPE_FREQS,
+        MODEL_TENSOR.SHORTCONV_CONV,
+        MODEL_TENSOR.SHORTCONV_INPROJ,
+        MODEL_TENSOR.SHORTCONV_OUTPROJ,
+        MODEL_TENSOR.FFN_GATE,
+        MODEL_TENSOR.FFN_DOWN,
+        MODEL_TENSOR.FFN_UP,
+        MODEL_TENSOR.FFN_NORM,
+        MODEL_TENSOR.FFN_POST_NORM,
+        MODEL_TENSOR.ATTN_NORM, # operator_norm
+        MODEL_TENSOR.ATTN_POST_NORM,
+        MODEL_TENSOR.ATTN_Q_NORM,
+        MODEL_TENSOR.ATTN_K_NORM,
+        MODEL_TENSOR.ATTN_Q,
+        MODEL_TENSOR.ATTN_K,
+        MODEL_TENSOR.ATTN_V,
+        MODEL_TENSOR.ATTN_OUT,
+        MODEL_TENSOR.ATTN_GATE,
+    ],
     MODEL_ARCH.SMALLTHINKER: [
         MODEL_TENSOR.TOKEN_EMBD,
         MODEL_TENSOR.OUTPUT_NORM,
@@ -6474,6 +6544,8 @@ class DecisionType:
     VLLM_D3 = "vllm-d3"  # same as pplx-decider, non-causal attention
     LFM2_D1 = "lfm2-d1"  # same as openjev, the labels depend on the question type
     LFM2_D1_OMNI = "lfm2-d1-omni"  # same head as laya on a bidirectional LFM2 trunk, other prompt layout
+    LFM3_D1 = "lfm3-d1"  # same as lfm2-d1 on LFM3-VL, full attention layers read the prompt both ways
+    LFM3_D1_OMNI = "lfm3-d1-omni"  # same as lfm3-d1 on LFM3-Omni
     GRANITE_SWITCH = "granite-switch"  # same as openjev, run by the decision aLoRA of granite switch
     H2O_LIGHTNING = "h2o-lightning"  # same as openjev, labels read after an answer prefill
     MINICPM_OCR_JEV = "openbmb-minicpm-ocr-jev"  # class scores of a value head read at the last token, one fixed task
@@ -6507,6 +6579,7 @@ class VisionProjectorType:
     VOXTRAL_RT = "voxtral_rt"
     MERALION = "meralion"  # audio: Whisper + gated MLP adaptor
     LFM2 = "lfm2"
+    LFM3VL = "lfm3vl"
     KIMIVL = "kimivl"
     PADDLEOCR = "paddleocr"
     KIMIK25 = "kimik25"
@@ -6521,6 +6594,8 @@ class VisionProjectorType:
     DEEPSEEK4V = "deepseek4v"
     DEEPSEEK41V = "deepseek41v"
     LFM2A = "lfm2a" # audio
+    LFM3A = "lfm3a" # audio
+    LFM3A_GEN = "lfm3a_gen" # audio generation: MaskGIT depthformer + liquid-codec decoder
     MUSIC_FLAMINGO = "musicflamingo" # audio
     GLM4V = "glm4v"
     GLM5V = "glm5v"

gguf-py/gguf/gguf_writer.py

@@ -1628,6 +1628,9 @@ class GGUFWriter:
     def add_audio_head_count(self, value: int) -> None:
         self.add_uint32(Keys.ClipAudio.Attention.HEAD_COUNT, value)
 
+    def add_audio_head_count_kv(self, value: int) -> None:
+        self.add_uint32(Keys.ClipAudio.Attention.HEAD_COUNT_KV, value)
+
     def add_audio_head_dim(self, value: int) -> None:
         self.add_uint32(Keys.ClipAudio.Attention.HEAD_DIM, value)
 
@@ -1708,6 +1711,18 @@ class GGUFWriter:
     def add_gen_audio_block_count(self, value: int) -> None:
         self.add_uint32(Keys.ClipGenAudio.BLOCK_COUNT, value)
 
+    def add_gen_audio_sampling_temp(self, value: float) -> None:
+        self.add_float32(Keys.ClipGenAudio.SAMPLING_TEMP, value)
+
+    def add_gen_audio_sampling_top_k(self, value: int) -> None:
+        self.add_int32(Keys.ClipGenAudio.SAMPLING_TOP_K, value)
+
+    def add_gen_audio_sampling_penalty_repeat(self, value: float) -> None:
+        self.add_float32(Keys.ClipGenAudio.SAMPLING_PENALTY_REPEAT, value)
+
+    def add_gen_audio_sampling_penalty_last_n(self, value: int) -> None:
+        self.add_int32(Keys.ClipGenAudio.SAMPLING_PENALTY_LAST_N, value)
+
     def add_gen_audio_head_count(self, value: int) -> None:
         self.add_uint32(Keys.ClipGenAudio.Attention.HEAD_COUNT, value)
 

gguf-py/gguf/tensor_mapping.py

@@ -119,6 +119,7 @@ class TensorNameMap:
             "model.transformer.ln_f",                  # llada
             "final_norm",                              # modern-bert
             "model.norm",                              # cogvlm
+            "model.embedding_norm",                    # lfm3
         ),
 
         # Rope frequencies
@@ -418,6 +419,7 @@ class TensorNameMap:
             "model.layers.{bid}.post_self_attn_layernorm",       # glm-4-0414
             "model.layers.{bid}.post_attn_norm",                 # kolibri1
             "model.layers.layers.{bid}.post_mixer_norm.weight",  # plamo2
+            "model.layers.{bid}.post_operator_norm",             # lfm3
         ),
 
         # Rotary embeddings
@@ -504,6 +506,7 @@ class TensorNameMap:
             "model.layers.layers.{bid}.post_mlp_norm.weight", # plamo2
             "model.layers.{bid}.feed_forward.up_proj",
             "model.layers.{bid}.post_moe_norm",               # grok-2
+            "model.layers.{bid}.post_ffn_norm",               # lfm3
         ),
 
         MODEL_TENSOR.FFN_POST_NORM_1: (
@@ -1855,6 +1858,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.q_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.q_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wq", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.q_proj", # lfm3vl
             "visual.blocks.{bid}.attn.q", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wq", # kimi-vl, generated
             "siglip2.vision_model.encoder.layers.{bid}.self_attn.q_proj", # youtuvl
@@ -1870,6 +1874,7 @@ class TensorNameMap:
             "model.vision_tower.encoder.layer.{bid}.attention.q_norm", # Intern-S1
             "visual.blocks.{bid}.attn.q_norm", # GLM-OCR
             "vision_model.model.layers.{bid}.self_attn.q_norm", # gemma4
+            "model.vision_tower.layers.{bid}.self_attn.q_layernorm", # lfm3vl
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_K: (
@@ -1884,6 +1889,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.k_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.k_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wk", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.k_proj", # lfm3vl
             "visual.blocks.{bid}.attn.k", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wk", # kimi-vl, generated
             "model.vision_model.transformer.layers.{bid}.self_attn.k_proj", # Deepseek-OCR CLIP, generated
@@ -1899,6 +1905,7 @@ class TensorNameMap:
             "model.vision_tower.encoder.layer.{bid}.attention.k_norm", # Intern-S1
             "visual.blocks.{bid}.attn.k_norm", # GLM-OCR
             "vision_model.model.layers.{bid}.self_attn.k_norm", # gemma4
+            "model.vision_tower.layers.{bid}.self_attn.k_layernorm", # lfm3vl
         ),
 
         MODEL_TENSOR.V_ENC_ATTN_V: (
@@ -1913,6 +1920,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.v_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.v_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wv", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.v_proj", # lfm3vl
             "visual.blocks.{bid}.attn.v", # qwen2vl, generated
             "vision_tower.encoder.blocks.{bid}.wv", # kimi-vl, generated
             "siglip2.vision_model.encoder.layers.{bid}.self_attn.v_proj",
@@ -1935,6 +1943,7 @@ class TensorNameMap:
             "vit.layers.{bid}.input_layernorm", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.attention_norm", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention_norm", # pixtral
+            "model.vision_tower.layers.{bid}.operator_norm", # lfm3vl
             "vision_model.model.layers.{bid}.input_layernorm", # llama4, gemma4
             "visual.blocks.{bid}.norm1", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm0", # kimi-vl (norm0/norm1)
@@ -1964,6 +1973,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.self_attn.o_proj", # llama4
             "vision_tower.transformer.layers.{bid}.attention.o_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.attention.wo", # pixtral
+            "model.vision_tower.layers.{bid}.self_attn.out_proj", # lfm3vl
             "visual.blocks.{bid}.attn.proj", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.wo", # kimi-vl
             "model.vision.transformer.layers.{bid}.attention.dense", # cogvlm
@@ -1996,6 +2006,7 @@ class TensorNameMap:
             "vision_model.model.layers.{bid}.post_attention_layernorm", # llama4
             "vision_tower.transformer.layers.{bid}.ffn_norm", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.ffn_norm", # pixtral
+            "model.vision_tower.layers.{bid}.ffn_norm", # lfm3vl
             "visual.blocks.{bid}.norm2", # qwen2vl
             "vision_tower.encoder.blocks.{bid}.norm1", # kimi-vl (norm0/norm1)
             "model.vision.transformer.layers.{bid}.post_attention_layernorm", # cogvlm
@@ -2023,6 +2034,7 @@ class TensorNameMap:
             "vit.layers.{bid}.mlp.dense_h_to_4h", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.feed_forward.up_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w3", # pixtral
+            "model.vision_tower.layers.{bid}.feed_forward.w3", # lfm3vl
             "vision_model.model.layers.{bid}.mlp.fc1", # llama4
             "visual.blocks.{bid}.mlp.fc1", # qwen2vl
             "visual.blocks.{bid}.mlp.up_proj", # qwen2.5vl
@@ -2043,6 +2055,7 @@ class TensorNameMap:
             "vision_encoder.blocks.{bid}.mlp.fc1", # dots3note
             "vision_tower.transformer.layers.{bid}.feed_forward.gate_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w1", # pixtral
+            "model.vision_tower.layers.{bid}.feed_forward.w1", # lfm3vl
             "visual.blocks.{bid}.mlp.gate_proj", # qwen2.5vl
             "vision_model.model.layers.{bid}.mlp.gate_proj", # gemma4
             "model.qwen2_model.model.model.layers.{bid}.mlp.gate_proj", # Deepseek-OCR-2 qwen2
@@ -2061,6 +2074,7 @@ class TensorNameMap:
             "vit.layers.{bid}.mlp.dense_4h_to_h", # HunyuanVL
             "vision_tower.transformer.layers.{bid}.feed_forward.down_proj", # pixtral-hf
             "vision_encoder.transformer.layers.{bid}.feed_forward.w2", # pixtral
+            "model.vision_tower.layers.{bid}.feed_forward.w2", # lfm3vl
             "vision_model.model.layers.{bid}.mlp.fc2", # llama4
             "visual.blocks.{bid}.mlp.fc2", # qwen2vl
             "visual.blocks.{bid}.mlp.down_proj", # qwen2.5vl
@@ -2540,12 +2554,14 @@ class TensorNameMap:
             "audio_encoder.dots_encoder.speech_encoder.layer_norm", # dots3note
             "audio_tower.ln_post", # qwen2omni
             "encoder.layer_norm", # mimo-audio-tokenizer
+            "encoder.lfm.lfm.embedding_norm", # lfm3a
         ),
 
         MODEL_TENSOR.A_ENC_ATTN_Q: (
             "audio_tower.layers.{bid}.self_attn.q_proj", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.q_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_q", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.self_attn.q_proj", # lfm3a
             "conformer.layers.{bid}.attention.attn.q_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.q_proj", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.q_proj", # parakeet
@@ -2557,6 +2573,7 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn.k_proj", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.k_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_k", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.self_attn.k_proj", # lfm3a
             "conformer.layers.{bid}.attention.attn.k_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.k_proj", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.k_proj", # parakeet
@@ -2568,6 +2585,7 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn.v_proj", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.v_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_v", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.self_attn.v_proj", # lfm3a
             "conformer.layers.{bid}.attention.attn.v_proj", # gemma3n
             "conformer.layers.{bid}.self_attn.v_proj", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.v_proj", # parakeet
@@ -2575,6 +2593,14 @@ class TensorNameMap:
             "encoder.layers.{bid}.self_attn.v_proj", # mimo-audio-tokenizer
         ),
 
+        MODEL_TENSOR.A_ENC_ATTN_Q_NORM: (
+            "encoder.lfm.lfm.layers.{bid}.self_attn.q_layernorm", # lfm3a
+        ),
+
+        MODEL_TENSOR.A_ENC_ATTN_K_NORM: (
+            "encoder.lfm.lfm.layers.{bid}.self_attn.k_layernorm", # lfm3a
+        ),
+
         MODEL_TENSOR.A_ENC_ATTN_K_REL: (
             "conformer.layers.{bid}.self_attn.relative_k_proj", # gemma4
         ),
@@ -2600,6 +2626,7 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn_layer_norm", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn_layer_norm", # dots3note
             "conformer.layers.{bid}.norm_self_att", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.operator_norm", # lfm3a
             "conformer.layers.{bid}.attention.pre_attn_norm", # gemma3n
             "sound_encoder.encoder.layers.{bid}.norm_self_att", # parakeet
             "encoder.layers.{bid}.attn.pre_norm", # granite_speech
@@ -2611,6 +2638,7 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.self_attn.o_proj", # voxtral realtime
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.self_attn.out_proj", # dots3note
             "conformer.layers.{bid}.self_attn.linear_out", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.self_attn.out_proj", # lfm3a
             "conformer.layers.{bid}.attention.post", # gemma3n
             "conformer.layers.{bid}.self_attn.post", # gemma4
             "sound_encoder.encoder.layers.{bid}.self_attn.o_proj", # parakeet
@@ -2622,6 +2650,7 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.final_layer_norm", # ultravox
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.final_layer_norm", # dots3note
             "conformer.layers.{bid}.norm_out", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.ffn_norm", # lfm3a
             "conformer.layers.{bid}.attention.post_norm", # gemma3n
             "sound_encoder.encoder.layers.{bid}.norm_out", # parakeet
             "encoder.layers.{bid}.post_norm", # granite_speech
@@ -2650,6 +2679,7 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.fc1", # ultravox
             "audio_tower.layers.{bid}.mlp.up_proj", # voxtral realtime
             "conformer.layers.{bid}.feed_forward1.linear1", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.feed_forward.w3", # lfm3a
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_1", # gemma3n
             "conformer.layers.{bid}.feed_forward1.ffw_layer_1", # gemma4
             "sound_encoder.encoder.layers.{bid}.feed_forward1.linear1", # parakeet
@@ -2659,6 +2689,7 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_FFN_GATE: (
             "audio_encoder.dots_encoder.speech_encoder.layers.{bid}.fc1_gate", # dots3note (split from fc1 in conversion code)
+            "encoder.lfm.lfm.layers.{bid}.feed_forward.w1", # lfm3a
             "audio_tower.layers.{bid}.mlp.gate_proj", # voxtral realtime
         ),
 
@@ -2667,6 +2698,7 @@ class TensorNameMap:
             "audio_tower.layers.{bid}.fc2", # ultravox
             "audio_tower.layers.{bid}.mlp.down_proj", # voxtral realtime
             "conformer.layers.{bid}.feed_forward1.linear2", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.feed_forward.w2", # lfm3a
             "conformer.layers.{bid}.ffw_layer_start.ffw_layer_2", # gemma3n
             "conformer.layers.{bid}.feed_forward1.ffw_layer_2", # gemma4
             "sound_encoder.encoder.layers.{bid}.feed_forward1.linear2", # parakeet
@@ -2750,6 +2782,7 @@ class TensorNameMap:
             "audio.multi_modal_projector.linear_{bid}", # ultravox, meralion
             "audio_encoder.audio_adapter.proj.{bid}", # dots3note (proj.1, proj.3)
             "audio_adapter.model.{bid}", # lfm2
+            "mlp.model.{bid}", # lfm3a
             "audio_tower.proj{bid}", # qwen3omni
             "sound_projection.linear{bid}", # parakeet (linear1, linear2)
         ),
@@ -2809,6 +2842,7 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_CONV_DW: (
             "conformer.layers.{bid}.conv.depthwise_conv", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.conv.conv", # lfm3a
             "conformer.layers.{bid}.lconv1d.depthwise_conv1d", # gemma3n
             "sound_encoder.encoder.layers.{bid}.conv.depthwise_conv", # parakeet
             "encoder.layers.{bid}.conv.depth_conv.conv", # granite_speech
@@ -2831,6 +2865,7 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_CONV_PW1: (
             "conformer.layers.{bid}.conv.pointwise_conv1", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.conv.in_proj", # lfm3a
             "conformer.layers.{bid}.lconv1d.linear_start", # gemma3n
             "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv1", # parakeet
             "encoder.layers.{bid}.conv.up_conv", # granite_speech
@@ -2839,6 +2874,7 @@ class TensorNameMap:
 
         MODEL_TENSOR.A_ENC_CONV_PW2: (
             "conformer.layers.{bid}.conv.pointwise_conv2", # lfm2
+            "encoder.lfm.lfm.layers.{bid}.conv.out_proj", # lfm3a
             "conformer.layers.{bid}.lconv1d.linear_end", # gemma3n
             "sound_encoder.encoder.layers.{bid}.conv.pointwise_conv2", # parakeet
             "encoder.layers.{bid}.conv.down_conv", # granite_speech

models/templates/LFM3.jinja

@@ -0,0 +1,280 @@
+{{- bos_token -}}
+{%- set preserve_thinking = preserve_thinking | default(false) -%}
+{%- set enable_thinking = enable_thinking | default(true) -%}
+
+{%- macro render_value(v) -%}
+    {%- if v is string -%}
+        {{- v -}}
+    {%- elif v is boolean -%}
+        {{- "true" if v else "false" -}}
+    {%- elif v is none -%}
+        {{- "null" -}}
+    {%- elif v is mapping or (v is sequence and v is not string) -%}
+        {{- v | tojson -}}
+    {%- else -%}
+        {{- v -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro parse_content(content) -%}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping and content.get("type") == "image" -%}
+        {{- "<|vision_start|><|image|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "video" -%}
+        {{- "<|vision_start|><|video|><|vision_end|>" -}}
+    {%- elif content is mapping and content.get("type") == "text" -%}
+        {{- (content.get("text") or "") | string -}}
+    {%- elif content is mapping -%}
+        {{- content | tojson -}}
+    {%- elif content is iterable -%}
+        {%- set _ns = namespace(result="") -%}
+        {%- for item in content -%}
+            {%- if item is string -%}
+                {%- set _ns.result = _ns.result + item -%}
+            {%- elif item is mapping and item.get("type") == "image" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|image|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "video" -%}
+                {%- set _ns.result = _ns.result + "<|vision_start|><|video|><|vision_end|>" -%}
+            {%- elif item is mapping and item.get("type") == "text" -%}
+                {%- set _ns.result = _ns.result + ((item.get("text") or "") | string) -%}
+            {%- else -%}
+                {%- set _ns.result = _ns.result + (item | tojson) -%}
+            {%- endif -%}
+        {%- endfor -%}
+        {{- _ns.result -}}
+    {%- elif content is not none -%}
+        {{- content | tojson -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- macro render_tool_calls(tool_calls) -%}
+    {%- for tool_call in tool_calls -%}
+        {%- if not loop.first -%}{{- "\n\n" -}}{%- endif -%}
+        {%- set func = tool_call["function"] if "function" in tool_call else tool_call -%}
+        {%- set func_args = func.get("arguments") -%}
+        {%- if func_args is string and (func_args | trim) not in ["", "{}", "null"] -%}
+            {{- raise_exception("Tool call arguments must be a mapping, got a JSON-encoded string: parse arguments with json.loads() before applying the chat template") -}}
+        {%- endif -%}
+        {{- "<tool_call>\n<function=" + func["name"] + ">\n" -}}
+        {%- if func_args is mapping -%}
+            {%- for arg_name, arg_value in func_args.items() -%}
+                {{- "<parameter=" + arg_name + ">\n" + render_value(arg_value) + "\n</parameter>\n" -}}
+            {%- endfor -%}
+        {%- endif -%}
+        {{- "</function>\n</tool_call>" -}}
+    {%- endfor -%}
+{%- endmacro -%}
+
+{%- macro tool_to_json(tool) -%}
+    {%- set fn = tool.function if tool.function is defined else tool -%}
+    {%- set ns_t = namespace(first=true) -%}
+    {{- "{" -}}
+    {%- for k, v in fn.items() -%}
+        {%- if k != "defer_loading" -%}
+            {%- if not ns_t.first -%}{{- ", " -}}{%- endif -%}
+            {%- set ns_t.first = false -%}
+            {{- k | tojson -}}{{- ": " -}}{{- v | tojson -}}
+        {%- endif -%}
+    {%- endfor -%}
+    {{- "}" -}}
+{%- endmacro -%}
+
+{%- macro is_tool_reference(item) -%}
+    {{- "1" if item is mapping and item.get("type") == "tool_reference" else "" -}}
+{%- endmacro -%}
+
+{%- macro referenced_tool_to_json(name) -%}
+    {%- set _ns = namespace(found=none) -%}
+    {%- for tool in tools or [] -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- if fn.name == name -%}
+            {%- set _ns.found = fn -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if _ns.found is none -%}
+        {{- raise_exception("Tool reference '" + name | string + "' not found in available tools. Every tool referenced by a tool_reference must be in `tools` on every request.") -}}
+    {%- endif -%}
+    {{- tool_to_json(_ns.found) -}}
+{%- endmacro -%}
+
+{%- macro render_tool_response(content) -%}
+    {%- if content is string and "<tool_response>" in content -%}
+        {{- content -}}
+    {%- else -%}
+    {{- "<tool_response>\n" -}}
+    {%- if content is string -%}
+        {{- content -}}
+    {%- elif content is mapping -%}
+        {{- parse_content(content) -}}
+    {%- elif content is iterable -%}
+        {%- for item in content -%}
+            {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+            {%- if is_tool_reference(item) -%}
+                {%- if loop.first or not is_tool_reference(loop.previtem) -%}
+                    {{- "<additional_tools>\n" -}}
+                {%- endif -%}
+                {{- referenced_tool_to_json(item.name) -}}
+                {%- if loop.last or not is_tool_reference(loop.nextitem) -%}
+                    {{- "\n</additional_tools>" -}}
+                {%- endif -%}
+            {%- elif item is mapping and item.result is defined and item.result is mapping and item.result.tools is defined -%}
+                {{- "<additional_tools>\n" -}}
+                {%- set _tl = item.result.tools -%}
+                {%- if _tl is string -%}
+                    {{- _tl -}}
+                {%- elif _tl is mapping -%}
+                    {{- tool_to_json(_tl) -}}
+                {%- else -%}
+                    {%- for t in _tl -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>" -}}
+            {%- else -%}
+                {{- parse_content(item) -}}
+            {%- endif -%}
+        {%- endfor -%}
+    {%- elif content is not none -%}
+        {{- content | tojson -}}
+    {%- endif -%}
+    {{- "\n</tool_response>" -}}
+    {%- endif -%}
+{%- endmacro -%}
+
+{%- set ns = namespace(system_prompt="", last_user_index=-1) -%}
+{%- if messages and messages[0]["role"] == "system" -%}
+    {%- if messages[0].get("content") -%}
+        {%- set ns.system_prompt = parse_content(messages[0]["content"]) -%}
+    {%- endif -%}
+    {%- set messages = messages[1:] -%}
+{%- endif -%}
+{%- if tools -%}
+    {%- set tools_ns = namespace(lines=[]) -%}
+    {%- for tool in tools -%}
+        {%- set fn = tool.function if tool.function is defined else tool -%}
+        {%- set _dyn = (fn.defer_loading is defined and fn.defer_loading) or (tool.defer_loading is defined and tool.defer_loading) -%}
+        {%- if not _dyn -%}
+            {%- set tools_ns.lines = tools_ns.lines + [tool_to_json(fn)] -%}
+        {%- endif -%}
+    {%- endfor -%}
+    {%- if tools_ns.lines -%}
+        {%- set ns.system_prompt = ns.system_prompt + ("\n\n" if ns.system_prompt else "") + "<tools>\n" + (tools_ns.lines | join("\n")) + "\n</tools>" -%}
+    {%- endif -%}
+{%- endif -%}
+{%- if ns.system_prompt -%}
+    {{- "<|im_start|>system\n" + ns.system_prompt + "<|im_end|>\n" -}}
+{%- endif -%}
+{%- for message in messages -%}
+    {%- if message["role"] == "user" -%}
+        {%- set ns.last_user_index = loop.index0 -%}
+    {%- endif -%}
+{%- endfor -%}
+{%- set carry = namespace(pending=[]) -%}
+{%- for message in messages -%}
+    {%- set _next = messages[loop.index0 + 1] if not loop.last else none -%}
+    {%- set _merge = message.role == "tool" and not message.get("additional_tools")
+        and _next is not none and _next.role == "tool" and not _next.get("additional_tools")
+        and message.get("tool_call_id") and _next.get("tool_call_id") == message.get("tool_call_id") -%}
+    {%- set _items = [] -%}
+    {%- if message.role == "tool" and not message.get("additional_tools") -%}
+        {%- set _c = message.get("content") -%}
+        {%- if _c is string -%}
+            {%- set _items = [{"type": "text", "text": _c}] if _c else [] -%}
+        {%- elif _c is mapping -%}
+            {%- set _items = [_c] -%}
+        {%- elif _c is iterable -%}
+            {%- set _items = _c | list -%}
+        {%- elif _c is not none -%}
+            {%- set _items = [_c] -%}
+        {%- endif -%}
+    {%- endif -%}
+    {%- if _merge -%}
+        {%- set carry.pending = carry.pending + _items -%}
+    {%- else -%}
+    {{- "<|im_start|>" + message.role + "\n" -}}
+    {%- if message.role == "assistant" -%}
+        {%- generation -%}
+        {%- set thinking = message.thinking if message.thinking is defined else message.reasoning if message.reasoning is defined else message.reasoning_content -%}
+        {%- set keep_thinking = preserve_thinking or loop.index0 > ns.last_user_index -%}
+        {%- if thinking is defined and keep_thinking -%}
+            {{- "<think>" + (thinking or "") + "</think>" -}}
+        {%- elif keep_thinking and "</think>" not in (message.get("content") if message.get("content") is string else "") -%}
+            {{- "<think></think>" -}}
+        {%- endif -%}
+        {%- set _cfm_tag = "CONTINUE_FINAL_MESSAGE_TAG " -%}
+        {%- set _has_cfm = false -%}
+        {%- set content = "" -%}
+        {%- if message.get("content") -%}
+            {%- set content = parse_content(message.content) -%}
+        {%- endif -%}
+        {%- if not keep_thinking -%}
+            {%- if "</think>" in content -%}
+                {%- set content = content.split("</think>")[-1] | trim -%}
+            {%- endif -%}
+            {%- set content = "<think></think>" + content -%}
+        {%- endif -%}
+        {%- if content.endswith(_cfm_tag) -%}
+            {%- set _has_cfm = true -%}
+            {%- set _trunc_len = (content | length) - (_cfm_tag | length) -%}
+            {%- set content = content[:_trunc_len] -%}
+        {%- endif -%}
+        {{- content -}}
+        {%- if message.tool_calls -%}
+            {{- render_tool_calls(message.tool_calls) -}}
+        {%- endif -%}
+        {%- if _has_cfm -%}
+            {{- _cfm_tag -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+        {%- endgeneration -%}
+    {%- elif message.role == "tool" -%}
+        {%- if message.get("additional_tools") -%}
+            {%- set _at = message["additional_tools"] -%}
+            {%- if _at is mapping or (_at is sequence and _at is not string) -%}
+                {%- set _src = _at -%}
+            {%- else -%}
+                {%- set _src = message.get("content") -%}
+            {%- endif -%}
+            {%- if _src is string and "<tool_response>" in _src -%}
+                {{- _src -}}
+            {%- elif _src is string and "<additional_tools>" in _src -%}
+                {{- "<tool_response>\n" + _src + "\n</tool_response>" -}}
+            {%- elif _src -%}
+                {{- "<tool_response>\n<additional_tools>\n" -}}
+                {%- if _src is string -%}
+                    {{- _src -}}
+                {%- elif _src is mapping -%}
+                    {{- tool_to_json(_src) -}}
+                {%- else -%}
+                    {%- for t in _src -%}
+                        {%- if not loop.first -%}{{- "\n" -}}{%- endif -%}
+                        {{- tool_to_json(t) -}}
+                    {%- endfor -%}
+                {%- endif -%}
+                {{- "\n</additional_tools>\n</tool_response>" -}}
+            {%- endif -%}
+        {%- elif carry.pending -%}
+            {{- render_tool_response(carry.pending + _items) -}}
+            {%- set carry.pending = [] -%}
+        {%- else -%}
+            {{- render_tool_response("" if message.get("content") is none else message.get("content")) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- else %}
+        {%- if message.get("content") -%}
+            {{- parse_content(message["content"]) -}}
+        {%- endif -%}
+        {{- "<|im_end|>\n" -}}
+    {%- endif %}
+    {%- endif -%}
+{%- endfor -%}
+{%- if add_generation_prompt -%}
+    {{- "<|im_start|>assistant\n" -}}
+    {%- if enable_thinking -%}
+        {{- "<think>" -}}
+    {%- else -%}
+        {{- "<think></think>" -}}
+    {%- endif -%}
+{%- endif -%}

scripts/snapdragon/qdc/run_qdc_jobs.py

@@ -36,6 +36,7 @@ import sys
 import tempfile
 import time
 import xml.etree.ElementTree as ET
+import zipfile
 from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Callable
@@ -406,11 +407,19 @@ def fetch_logs_and_parse_tests(
 
     with tempfile.TemporaryDirectory() as tmpdir:
         for lf in log_files:
-            zip_path = os.path.join(tmpdir, "log.zip")
+            dl_path = os.path.join(tmpdir, "log.bin")
             log.info("Downloading log file: %s", lf.filename)
-            qdc_api.download_job_log_files(client, lf.filename, zip_path)
             try:
-                shutil.unpack_archive(zip_path, tmpdir, "zip")
+                qdc_api.download_job_log_files_v2(client, lf.filename, dl_path)
+            except Exception as e:
+                log.warning("Could not download %s: %s", lf.filename, e)
+                continue
+            # v2 returns some files raw instead of zipped
+            if not zipfile.is_zipfile(dl_path):
+                os.replace(dl_path, os.path.join(tmpdir, os.path.basename(lf.filename)))
+                continue
+            try:
+                shutil.unpack_archive(dl_path, tmpdir, "zip")
             except Exception as e:
                 log.warning("Could not unpack %s as zip: %s", lf.filename, e)
 
@@ -617,7 +626,7 @@ def main() -> int:
             args.pkg_dir, Path(tmpdir), args.test, args.model_url
         )
         log.info("Uploading artifact (%d MB) ...", zip_path.stat().st_size // 1_000_000)
-        artifact_id = qdc_api.upload_file(client, str(zip_path), ArtifactType.TESTSCRIPT)
+        artifact_id = qdc_api.upload_file_v2(client, str(zip_path), ArtifactType.TESTSCRIPT)
 
     if artifact_id is None:
         log.error("Artifact upload failed")

src/llama-arch.cpp

@@ -135,6 +135,7 @@ static const std::map<llm_arch, const char *> LLM_ARCH_NAMES = {
     { LLM_ARCH_OPENAI_MOE,       "gpt-oss"          },
     { LLM_ARCH_LFM2,             "lfm2"             },
     { LLM_ARCH_LFM2MOE,          "lfm2moe"          },
+    { LLM_ARCH_LFM3,             "lfm3"             },
     { LLM_ARCH_DREAM,            "dream"            },
     { LLM_ARCH_SMALLTHINKER,     "smallthinker"     },
     { LLM_ARCH_LLADA,            "llada"            },
@@ -1251,6 +1252,7 @@ bool llm_arch_is_hybrid(const llm_arch & arch) {
         case LLM_ARCH_GRANITE_HYBRID:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_QWEN3NEXT:
@@ -1295,6 +1297,7 @@ bool llm_arch_supports_rs_rollback(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_BAILINGMOE3:
             return true;
         default:
@@ -1341,6 +1344,7 @@ bool llm_arch_supports_sm_tensor(const llm_arch & arch) {
         case LLM_ARCH_NEMOTRON_H:
         case LLM_ARCH_NEMOTRON_H_MOE:
         case LLM_ARCH_GRANITE_HYBRID:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_MINIMAX_01:
         case LLM_ARCH_MINIMAX_M2:
         case LLM_ARCH_MINIMAX_M3:

src/llama-arch.h

@@ -139,6 +139,7 @@ enum llm_arch {
     LLM_ARCH_OPENAI_MOE,
     LLM_ARCH_LFM2,
     LLM_ARCH_LFM2MOE,
+    LLM_ARCH_LFM3,
     LLM_ARCH_DREAM,
     LLM_ARCH_SMALLTHINKER,
     LLM_ARCH_LLADA,

src/llama-context.cpp

@@ -1614,8 +1614,9 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
     // eagle3/DFlash: features as encoder input, and non-draft paths fall back to model's input dim
     const int64_t n_vocab = model.vocab.n_tokens();
 
-    // note: during encode, we always output all tokens and skip position continuity checks (output_all=true)
-    if (!balloc->init(batch_inp, model.vocab, true)) {
+    // note: during encode, we skip position continuity checks and output all tokens
+    //       a decoder without memory (see decode()) outputs the marked tokens, unless it extracts embeddings
+    if (!balloc->init(batch_inp, model.vocab, memory || cparams.embeddings)) {
         LLAMA_LOG_ERROR("%s: failed to initialize batch\n", __func__);
         return -1;
     }
@@ -1644,17 +1645,20 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
 
     n_queued_tokens += n_tokens;
 
+    const uint32_t n_outputs_all = balloc->get_n_outputs();
+
     // reserve output buffer
-    if (output_reserve(n_tokens) < n_tokens) {
-        LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_tokens);
+    if (output_reserve(n_outputs_all) < n_outputs_all) {
+        LLAMA_LOG_ERROR("%s: could not reserve space for batch with %u outputs\n", __func__, n_outputs_all);
         return -2;
     };
 
-    for (uint32_t i = 0; i < n_tokens; ++i) {
-        output_ids[i] = i;
+    const auto & out_ids = balloc->get_out_ids();
+    for (uint32_t i = 0; i < n_outputs_all; ++i) {
+        output_ids[out_ids[i]] = i;
     }
 
-    n_outputs = n_tokens;
+    n_outputs = n_outputs_all;
 
     const auto causal_attn_org = cparams.causal_attn;
 
@@ -1687,7 +1691,7 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
         GGML_ASSERT(backend_res != nullptr);
         GGML_ASSERT(logits.data != nullptr);
 
-        ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_tokens*n_vocab*sizeof(float));
+        ggml_backend_tensor_get_async(backend_res, t_logits, logits.data, 0, n_outputs*n_vocab*sizeof(float));
     }
 
     // extract embeddings
@@ -1702,8 +1706,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
                     GGML_ASSERT(embd.data != nullptr);
                     const uint32_t n_embd_out = hparams.n_embd_out();
 
-                    GGML_ASSERT(n_tokens*n_embd_out <= (int64_t) embd.size);
-                    ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_tokens*n_embd_out*sizeof(float));
+                    GGML_ASSERT(n_outputs*n_embd_out <= (int64_t) embd.size);
+                    ggml_backend_tensor_get_async(backend_embd, t_embd, embd.data, 0, n_outputs*n_embd_out*sizeof(float));
                 } break;
             case LLAMA_POOLING_TYPE_MEAN:
             case LLAMA_POOLING_TYPE_CLS:
@@ -1751,8 +1755,8 @@ int llama_context::encode(const llama_batch_ext & batch_inp) {
         GGML_ASSERT(backend_h != nullptr);
 
         const uint32_t n_embd = hparams.n_embd_out();
-        GGML_ASSERT(n_tokens*n_embd <= (int64_t) embd_nextn.size);
-        ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_tokens*n_embd*sizeof(float));
+        GGML_ASSERT(n_outputs*n_embd <= (int64_t) embd_nextn.size);
+        ggml_backend_tensor_get_async(backend_h, t_h_nextn, embd_nextn.data, 0, n_outputs*n_embd*sizeof(float));
     }
 
     // TODO: hacky solution

src/llama-graph.cpp

@@ -479,7 +479,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
     const int64_t n_kv     = self_kq_mask->ne[0];
     const int64_t n_tokens = ubatch->n_tokens;
 
-    const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type) {
+    const auto fill_mask = [&](auto * data, int64_t ne, int n_swa, llama_swa_type swa_type, bool causal) {
         using T = std::remove_reference_t<decltype(*data)>;
         std::fill(data, data + ne, llama_cast<T>(-INFINITY));
 
@@ -499,7 +499,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
                 }
 
                 // mask future tokens
-                if (cparams.causal_attn && p0 > p1) {
+                if (causal && p0 > p1) {
                     continue;
                 }
 
@@ -517,21 +517,25 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
         }
     };
 
+    // see llama_non_causal_type
+    const bool causal     = cparams.causal_attn;
+    const bool causal_swa = cparams.causal_attn || hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY;
+
     GGML_ASSERT(self_kq_mask);
     GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask->buffer));
     if (self_kq_mask->type == GGML_TYPE_F16) {
-        fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE);
+        fill_mask((ggml_fp16_t *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal);
     } else {
-        fill_mask((float       *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE);
+        fill_mask((float       *) self_kq_mask->data, ggml_nelements(self_kq_mask), 0, LLAMA_SWA_TYPE_NONE, causal);
     }
 
     if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
         GGML_ASSERT(self_kq_mask_swa);
         GGML_ASSERT(ggml_backend_buffer_is_host(self_kq_mask_swa->buffer));
         if (self_kq_mask_swa->type == GGML_TYPE_F16) {
-            fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type);
+            fill_mask((ggml_fp16_t *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa);
         } else {
-            fill_mask((float       *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type);
+            fill_mask((float       *) self_kq_mask_swa->data, ggml_nelements(self_kq_mask_swa), hparams.n_swa, hparams.swa_type, causal_swa);
         }
     }
 }

src/llama-hparams.h

@@ -34,6 +34,7 @@ enum llama_non_causal_type {
     LLAMA_NON_CAUSAL_TYPE_ALL      = 0, // all layers non-causal, SWA still applied (gemma 3, qwen-vl, ...)
     LLAMA_NON_CAUSAL_TYPE_SWA_ONLY = 1, // SWA layers non-causal, dense layers stay causal (gemma 4)
     LLAMA_NON_CAUSAL_TYPE_SWA_FULL = 2, // all layers non-causal, SWA not applied between tokens of the current ubatch (deepseek 4)
+    LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY = 3, // dense layers non-causal, SWA layers stay causal (lfm3 d1)
 };
 
 // M-RoPE: which input position slot feeds each RoPE section

src/llama-kv-cache.cpp

@@ -1749,6 +1749,9 @@ void llama_kv_cache::set_input_kq_mask(ggml_tensor * dst, const llama_ubatch * u
     if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_SWA_ONLY) {
         causal_attn = swa_type == LLAMA_SWA_TYPE_NONE;
     }
+    if (!causal_attn && hparams.non_causal_type == LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY) {
+        causal_attn = swa_type != LLAMA_SWA_TYPE_NONE;
+    }
 
     //const int64_t t_start = ggml_time_us();
 

src/llama-model.cpp

@@ -314,6 +314,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
             return new llama_model_lfm2(params);
         case LLM_ARCH_LFM2MOE:
             return new llama_model_lfm2moe(params);
+        case LLM_ARCH_LFM3:
+            return new llama_model_lfm3(params);
         case LLM_ARCH_SMALLTHINKER:
             return new llama_model_smallthinker(params);
         case LLM_ARCH_GROVEMOE:
@@ -2799,10 +2801,13 @@ ggml_tensor * llama_model::get_rope_factors(const llama_cparams & cparams, int i
 llama_memory_i * llama_model::create_memory(const llama_memory_params & params, const llama_cparams & cparams) const {
     llama_memory_i * res;
 
-    // the non-causal LFM2 decision graph reads the whole prompt in one batch, nothing is kept
+    // the non-causal LFM2 and LFM3 decision graphs read the whole prompt in one batch, nothing is kept
     if (arch == LLM_ARCH_LFM2 && !hparams.causal_attn && hparams.n_layer_decision > 0) {
         return nullptr;
     }
+    if (arch == LLM_ARCH_LFM3 && !hparams.causal_attn) {
+        return nullptr;
+    }
 
     switch (arch) {
         // Models that need specific instantiation should be handled in the
@@ -3637,6 +3642,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
         case LLM_ARCH_HY_V3:
         case LLM_ARCH_LFM2:
         case LLM_ARCH_LFM2MOE:
+        case LLM_ARCH_LFM3:
         case LLM_ARCH_SMALLTHINKER:
         case LLM_ARCH_SEED_OSS:
         case LLM_ARCH_GROVEMOE:

src/llama-quant.cpp

@@ -342,6 +342,14 @@ static bool tensor_allows_quantization(const llama_model_quantize_params * param
     quantize &= name.find("ssm_conv1d") == std::string::npos;
     quantize &= name.find("shortconv.conv.weight") == std::string::npos;
 
+    // do not quantize LFM3-VL's depthwise conv kernels, CONV_2D_DW takes F32 kernels
+    quantize &= name.find("conv2d_spatial.weight") == std::string::npos;
+
+    // do not quantize LFM3-Audio's encoder convs and codec, the codec output does not survive quantization
+    quantize &= name.rfind("a.conv1d.", 0) != 0;
+    quantize &= name.find("conv_dw") == std::string::npos;
+    quantize &= name.rfind("a.gen.wav.", 0) != 0;
+
     // do not quantize MiniMax's indexer projection weights, they are tiny
     quantize &= name.find("indexer.k_proj.weight") == std::string::npos;
     quantize &= name.find("indexer.q_proj.weight") == std::string::npos;

src/models/lfm-base.cpp

@@ -84,6 +84,33 @@ ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model,
     return y;
 }
 
+ggml_tensor * llm_build_lfm_base::build_lfm_shortconv(const llama_model & model,
+                                                      ggml_tensor * cur,
+                                                      int il) {
+    const int64_t d_conv = hparams.n_shortconv_l_cache - 1;
+
+    auto * bcx = build_lora_mm(model.layers[il].shortconv.in_proj, cur);
+    cb(bcx, "model.layers.{}.conv.in_proj", il);
+
+    const int64_t chunk_size = bcx->ne[0] / 3;
+    auto * b = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 0 * chunk_size * ggml_element_size(bcx));
+    auto * c = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 1 * chunk_size * ggml_element_size(bcx));
+    auto * x = ggml_view_2d(ctx0, bcx, chunk_size, bcx->ne[1], bcx->nb[1], 2 * chunk_size * ggml_element_size(bcx));
+
+    // causal from a zero state
+    auto * bx = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mul(ctx0, b, x)));
+    bx = ggml_pad_ext(ctx0, bx, d_conv, 0, 0, 0, 0, 0, 0, 0);
+
+    auto * conv_out = ggml_ssm_conv(ctx0, bx, model.layers[il].shortconv.conv);
+    cb(conv_out, "model.layers.{}.conv.conv", il);
+
+    auto * y = ggml_mul(ctx0, c, ggml_reshape_2d(ctx0, conv_out, chunk_size, n_tokens));
+    y = build_lora_mm(model.layers[il].shortconv.out_proj, y);
+    cb(y, "model.layers.{}.conv.out_proj", il);
+
+    return y;
+}
+
 ggml_tensor * llm_build_lfm_base::build_lfm_dense_ffn(const llama_model & model,
                                                       ggml_tensor * cur,
                                                       int il) {

src/models/lfm3.cpp

@@ -0,0 +1,217 @@
+#include "models.h"
+#include "../llama-memory-hybrid-iswa.h"
+
+void llama_model_lfm3::load_arch_hparams(llama_model_loader & ml) {
+    ml.get_key(LLM_KV_SHORTCONV_L_CACHE,           hparams.n_shortconv_l_cache);
+    ml.get_key(LLM_KV_ATTENTION_LAYERNORM_RMS_EPS, hparams.f_norm_rms_eps);
+
+    for (uint32_t il = 0; il < hparams.n_layer(); ++il) {
+        hparams.is_recr_impl[il] = hparams.n_head_kv(il) == 0;
+    }
+
+    // LFM3 always mixes full and sliding window attention layers
+    ml.get_arr(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, hparams.is_swa_impl);
+    if (!hparams.is_swa_any()) {
+        throw std::runtime_error("LFM3 requires sliding window attention layers");
+    }
+    hparams.swa_type = LLAMA_SWA_TYPE_STANDARD;
+    ml.get_key(LLM_KV_ATTENTION_SLIDING_WINDOW, hparams.n_swa);
+
+    ml.get_key(LLM_KV_ROPE_FREQ_BASE_SWA, hparams.rope_freq_base_train_swa);
+
+    // d1: full attention layers read the prompt both ways, sliding window layers stay causal
+    if (!hparams.causal_attn) {
+        hparams.non_causal_type = LLAMA_NON_CAUSAL_TYPE_DENSE_ONLY;
+    }
+
+    switch (hparams.n_ff()) {
+        case 7680: type = LLM_TYPE_1_7B; break;
+        default:   type = LLM_TYPE_UNKNOWN;
+    }
+}
+
+void llama_model_lfm3::load_arch_tensors(llama_model_loader &) {
+    LLAMA_LOAD_LOCALS;
+
+    tok_embd    = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD,  "weight"), {n_embd, n_vocab}, 0);
+    output_norm = create_tensor(tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd}, 0);
+    output      = create_tensor(tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab}, TENSOR_NOT_REQUIRED);
+    if (output == NULL) {
+        output = create_tensor(tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, TENSOR_DUPLICATED);
+    }
+
+    int rope_freqs_flag = 0;
+
+    for (int i = 0; i < n_layer; ++i) {
+        auto & layer = layers[i];
+
+        layer.attn_norm      = create_tensor(tn(LLM_TENSOR_ATTN_NORM,      "weight", i), {n_embd}, 0);
+        layer.ffn_norm       = create_tensor(tn(LLM_TENSOR_FFN_NORM,       "weight", i), {n_embd}, 0);
+        // absent when use_sandwich_norm is false
+        layer.attn_post_norm = create_tensor(tn(LLM_TENSOR_ATTN_POST_NORM, "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
+        layer.ffn_post_norm  = create_tensor(tn(LLM_TENSOR_FFN_POST_NORM,  "weight", i), {n_embd}, TENSOR_NOT_REQUIRED);
+        if ((layer.attn_post_norm == nullptr) != (layer.ffn_post_norm == nullptr)) {
+            throw std::runtime_error("LFM3 requires both post norms or neither");
+        }
+
+        layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd,   n_ff}, 0);
+        layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {  n_ff, n_embd}, 0);
+        layer.ffn_up   = create_tensor(tn(LLM_TENSOR_FFN_UP,   "weight", i), {n_embd,   n_ff}, 0);
+
+        if (!hparams.is_recr(i)) {
+            // note: n_embd_head_k * n_head may differ from n_embd
+            const int64_t n_embd_q = n_embd_head_k * n_head;
+            const int64_t n_embd_k = hparams.n_embd_k_gqa(i);
+            const int64_t n_embd_v = hparams.n_embd_v_gqa(i);
+            if (n_embd_k != n_embd_v) {
+                throw std::runtime_error("LFM3 requires equal key and value sizes");
+            }
+
+            layer.attn_q_norm = create_tensor(tn(LLM_TENSOR_ATTN_Q_NORM, "weight", i), {n_embd_head_k}, 0);
+            layer.attn_k_norm = create_tensor(tn(LLM_TENSOR_ATTN_K_NORM, "weight", i), {n_embd_head_k}, 0);
+
+            create_tensor_qkv(layer, i, n_embd, n_embd_q, n_embd_k, n_embd_v, 0);
+
+            layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd_q, n_embd}, 0);
+
+            layer.wqkv_gate = create_tensor(tn(LLM_TENSOR_ATTN_GATE, "weight", i), {n_embd, n_head}, 0);
+
+            if (!hparams.is_swa(i)) {
+                layer.rope_freqs = create_tensor(tn(LLM_TENSOR_ROPE_FREQS, "weight", i), {n_embd_head_k / 2}, rope_freqs_flag);
+                rope_freqs_flag = TENSOR_DUPLICATED;
+            }
+        } else {
+            layer.shortconv.conv     = create_tensor(tn(LLM_TENSOR_SHORTCONV_CONV,    "weight", i), {hparams.n_shortconv_l_cache, n_embd}, 0);
+            layer.shortconv.in_proj  = create_tensor(tn(LLM_TENSOR_SHORTCONV_INPROJ,  "weight", i), {n_embd, 3 * n_embd}, 0);
+            layer.shortconv.out_proj = create_tensor(tn(LLM_TENSOR_SHORTCONV_OUTPROJ, "weight", i), {n_embd, n_embd}, 0);
+        }
+    }
+}
+
+std::unique_ptr<llm_graph_context> llama_model_lfm3::build_arch_graph(const llm_graph_params & params) const {
+    if (!hparams.causal_attn) {
+        return std::make_unique<graph<true>>(*this, params);
+    }
+    return std::make_unique<graph<false>>(*this, params);
+}
+
+template <bool decision>
+llama_model_lfm3::graph<decision>::graph(const llama_model & model, const llm_graph_params & params) :
+    llm_build_lfm_base(params) {
+    auto build_attn_block = [&model, this](ggml_tensor * cur_in,
+                                           ggml_tensor * inp_pos,
+                                           auto *        inp_attn,
+                                           int           il) -> ggml_tensor * {
+        const int64_t n_embd_head = hparams.n_embd_head_k(il);
+        const int64_t n_head_il   = hparams.n_head(il);
+        const int64_t n_head_kv   = hparams.n_head_kv(il);
+
+        const int     n_rot_l      = hparams.n_rot(il);
+        const float   freq_base_l  = model.get_rope_freq_base (cparams, il);
+        const float   freq_scale_l = model.get_rope_freq_scale(cparams, il);
+
+        ggml_tensor * rope_factors = hparams.is_swa(il) ? nullptr : model.layers[il].rope_freqs;
+
+        auto [q, k, v] = build_qkv(model.layers[il], cur_in, n_embd_head, n_head_il, n_head_kv, il);
+
+        q = build_norm(q, model.layers[il].attn_q_norm, NULL, LLM_NORM_RMS, il);
+        cb(q, "model.layers.{}.self_attn.q_layernorm", il);
+        k = build_norm(k, model.layers[il].attn_k_norm, NULL, LLM_NORM_RMS, il);
+        cb(k, "model.layers.{}.self_attn.k_layernorm", il);
+
+        q = ggml_rope_ext(ctx0, q, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
+                          freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+        k = ggml_rope_ext(ctx0, k, inp_pos, rope_factors, n_rot_l, rope_type, n_ctx_orig,
+                          freq_base_l, freq_scale_l, ext_factor, attn_factor, beta_fast, beta_slow);
+
+        ggml_tensor * attn_out = build_attn(inp_attn,
+                NULL, NULL, NULL,
+                q, k, v, nullptr, nullptr, nullptr, 1.0f / sqrtf(float(n_embd_head)), il);
+        cb(attn_out, "attn_out", il);
+
+        ggml_tensor * gate = build_lora_mm(model.layers[il].wqkv_gate, cur_in);
+        gate = ggml_sigmoid(ctx0, gate);
+        cb(gate, "model.layers.{}.self_attn.gate", il);
+
+        ggml_tensor * attn_3d = ggml_reshape_3d(ctx0, attn_out, n_embd_head, n_head_il, n_tokens);
+        ggml_tensor * gate_3d = ggml_reshape_3d(ctx0, gate,     1,           n_head_il, n_tokens);
+        attn_3d = ggml_mul(ctx0, attn_3d, gate_3d);
+        attn_out = ggml_reshape_2d(ctx0, attn_3d, n_embd_head * n_head_il, n_tokens);
+
+        ggml_tensor * out = build_lora_mm(model.layers[il].wo, attn_out, model.layers[il].wo_s);
+        cb(out, "model.layers.{}.self_attn.out_proj", il);
+
+        return out;
+    };
+
+    ggml_tensor * cur = build_inp_embd(model.tok_embd);
+    cb(cur, "model.embed_tokens", -1);
+
+    ggml_build_forward_expand(gf, cur);
+
+    // decision: no memory, the whole prompt is read in one batch
+    auto * inp_hybrid = decision ? nullptr : build_inp_mem_hybrid_iswa();
+    auto * inp_attn   = decision ? build_attn_inp_no_cache() : nullptr;
+
+    ggml_tensor * inp_pos     = build_inp_pos();
+    ggml_tensor * inp_out_ids = build_inp_out_ids();
+
+    for (int il = 0; il < n_layer; ++il) {
+        auto * prev_cur = cur;
+
+        cur = build_norm(cur, model.layers[il].attn_norm, NULL, LLM_NORM_RMS, il);
+        cb(cur, "model.layers.{}.operator_norm", il);
+
+        if (hparams.is_recr(il)) {
+            if constexpr (decision) {
+                cur = build_lfm_shortconv(model, cur, il);
+            } else {
+                const auto * mctx_recr = static_cast<const llama_memory_hybrid_iswa_context *>(mctx)->get_recr();
+                cur = build_lfm_shortconv(model, mctx_recr, inp_hybrid->get_recr(), cur, il);
+            }
+        } else if constexpr (decision) {
+            cur = build_attn_block(cur, inp_pos, inp_attn, il);
+        } else {
+            cur = build_attn_block(cur, inp_pos, inp_hybrid->get_attn(), il);
+        }
+
+        if (model.layers[il].attn_post_norm) {
+            cur = build_norm(cur, model.layers[il].attn_post_norm, NULL, LLM_NORM_RMS, il);
+            cb(cur, "model.layers.{}.post_operator_norm", il);
+        }
+
+        if (il == n_layer - 1 && inp_out_ids) {
+            cur      = ggml_get_rows(ctx0, cur, inp_out_ids);
+            prev_cur = ggml_get_rows(ctx0, prev_cur, inp_out_ids);
+        }
+
+        cur = ggml_add(ctx0, prev_cur, cur);
+
+        auto * ffn_norm_out = build_norm(cur, model.layers[il].ffn_norm, NULL, LLM_NORM_RMS, il);
+        cb(ffn_norm_out, "model.layers.{}.ffn_norm", il);
+
+        ggml_tensor * ffn_out = build_lfm_dense_ffn(model, ffn_norm_out, il);
+
+        if (model.layers[il].ffn_post_norm) {
+            ffn_out = build_norm(ffn_out, model.layers[il].ffn_post_norm, NULL, LLM_NORM_RMS, il);
+            cb(ffn_out, "model.layers.{}.post_ffn_norm", il);
+        }
+
+        cur = ggml_add(ctx0, cur, ffn_out);
+
+        cur = build_cvec(cur, il);
+        cb(cur, "l_out", il);
+    }
+
+    cur = build_norm(cur, model.output_norm, NULL, LLM_NORM_RMS, -1);
+    cb(cur, "result_norm", -1);
+    res->t_embd = cur;
+
+    cur = build_lora_mm(model.output, cur, model.output_s);
+    cb(cur, "result_output", -1);
+
+    res->t_logits = cur;
+}
+
+template struct llama_model_lfm3::graph<false>;
+template struct llama_model_lfm3::graph<true>;

src/models/models.h

@@ -111,6 +111,11 @@ struct llm_build_lfm_base : public llm_graph_context {
                                       ggml_tensor * cur,
                                       int il);
 
+    // no recurrent state (decision graphs): causal over the ubatch from a zero state
+    ggml_tensor * build_lfm_shortconv(const llama_model & model,
+                                      ggml_tensor * cur,
+                                      int il);
+
     ggml_tensor * build_lfm_dense_ffn(const llama_model & model,
                                       ggml_tensor * cur,
                                       int il);
@@ -2433,6 +2438,21 @@ struct llama_model_lfm2moe : public llama_model_base {
 };
 
 
+struct llama_model_lfm3 : public llama_model_base {
+    llama_model_lfm3(const struct llama_model_params & params) : llama_model_base(params) {}
+    void load_arch_hparams(llama_model_loader & ml) override;
+    void load_arch_tensors(llama_model_loader & ml) override;
+
+    // decision: d1, the prompt read in one batch, no memory
+    template <bool decision>
+    struct graph : public llm_build_lfm_base {
+        graph(const llama_model & model, const llm_graph_params & params);
+    };
+
+    std::unique_ptr<llm_graph_context> build_arch_graph(const llm_graph_params & params) const override;
+};
+
+
 struct llama_model_smallthinker : public llama_model_base {
     llama_model_smallthinker(const struct llama_model_params & params) : llama_model_base(params) {}
     void load_arch_hparams(llama_model_loader & ml) override;

tests/fusion/MTL.csv

@@ -155,6 +155,7 @@ laguna              ,0   ,any     ,RMS_NORM+MUL                ,      9
 lfm2                ,0   ,any     ,RMS_NORM+MUL                ,      7
 lfm2moe             ,1   ,any     ,MUL+ADD                     ,      2
 lfm2moe             ,1   ,any     ,RMS_NORM+MUL                ,      7
+lfm3                ,0   ,any     ,RMS_NORM+MUL                ,     17
 llada               ,0   ,any     ,RMS_NORM+MUL                ,      5
 llada-moe           ,1   ,any     ,MUL+ADD                     ,      4
 llada-moe           ,1   ,any     ,RMS_NORM+MUL                ,      9

tests/test-chat.cpp

@@ -986,8 +986,8 @@ const common_chat_msg message_user_parts{
     /* .content = */ "",
     /* .content_parts = */
     {
-     { "text", "Hey" },
-     { "text", "there" },
+     { "text", "Hey", "" },
+     { "text", "there", "" },
      },
     /* .tool_calls = */
     {                 },
@@ -3302,9 +3302,9 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
                 common_chat_msg message_parts;
                 message_parts.role          = "user";
                 message_parts.content_parts = {
-                    { /* .type = */ "text", /* .text = */ "First part.\n\n" },
-                    { /* .type = */ "media_marker", /* .text = */ "<__media__>" },
-                    { /* .type = */ "text", /* .text = */ "Second part.\n\n" },
+                    { /* .type = */ "text", /* .text = */ "First part.\n\n", /* .name = */ "" },
+                    { /* .type = */ "media_marker", /* .text = */ "<__media__>", /* .name = */ "" },
+                    { /* .type = */ "text", /* .text = */ "Second part.\n\n", /* .name = */ "" },
                 };
 
                 common_chat_templates_inputs inputs;
@@ -5589,6 +5589,85 @@ static void test_template_output_peg_parsers(bool detailed_debug) {
         test_lfm2_parser(tmpl, detailed_debug);
     }
 
+    {
+        auto tst = peg_tester("models/templates/LFM3.jinja", detailed_debug);
+
+        tst.test("Hello, world!\nWhat's up?")
+            .enable_thinking(false)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .expect(message_assist)
+            .run();
+
+        tst.test("I'm\nthinking</think>Hello, world!\nWhat's up?")
+            .enable_thinking(true)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .expect(message_assist_thoughts)
+            .run();
+
+        tst.test(
+               "<tool_call>\n"
+               "<function=special_function>\n"
+               "<parameter=arg1>\n1\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .enable_thinking(false)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .tools({ special_function_tool })
+            .expect(message_assist_call)
+            .run();
+
+        tst.test(
+               "I'm\nthinking</think>"
+               "<tool_call>\n"
+               "<function=special_function>\n"
+               "<parameter=arg1>\n1\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .enable_thinking(true)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .tools({ special_function_tool })
+            .expect(message_assist_call_thoughts)
+            .run();
+
+        // a deferred tool is left out of <tools>, a tool_reference renders it, and its schema types the call
+        const auto deferred_tools = common_chat_tools_parse_oaicompat(json::parse(R"([
+            {"type": "function", "function": {"name": "tool_search", "description": "Search for tools",
+             "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}},
+            {"type": "function", "function": {"name": "buy", "description": "Buy an item", "defer_loading": true,
+             "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "qty": {"type": "integer"}}}}}
+        ])"));
+        {
+            auto tmpls = read_templates("models/templates/LFM3.jinja");
+            common_chat_templates_inputs inputs;
+            inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(R"([
+                {"role": "user", "content": "Buy the blue one"},
+                {"role": "assistant", "tool_calls": [{"id": "c0", "type": "function",
+                 "function": {"name": "tool_search", "arguments": "{\"query\": \"buy\"}"}}]},
+                {"role": "tool", "tool_call_id": "c0", "name": "tool_search", "content": [{"type": "tool_reference", "name": "buy"}]}
+            ])"));
+            inputs.tools                 = deferred_tools;
+            inputs.add_generation_prompt = true;
+            const auto prompt = common_chat_templates_apply(tmpls.get(), inputs).prompt.str();
+            assert_contains(prompt, "<tool_response>\n<additional_tools>\n{\"name\": \"buy\"");
+            assert_equals(true, prompt.find("\"name\": \"buy\"") > prompt.find("</tools>"));
+            assert_not_contains(prompt, "defer_loading");
+            assert_not_contains(prompt, "tool_reference");
+        }
+
+        tst.test(
+               "<tool_call>\n"
+               "<function=buy>\n"
+               "<parameter=id>\nblue\n</parameter>\n"
+               "<parameter=qty>\n2\n</parameter>\n"
+               "</function>\n"
+               "</tool_call>")
+            .enable_thinking(false)
+            .reasoning_format(COMMON_REASONING_FORMAT_AUTO)
+            .tools(deferred_tools)
+            .expect_tool_calls({ { "buy", R"({"id": "blue", "qty": 2})", {} } })
+            .run();
+    }
+
     // Thinking cases only apply to LFM2.5-8B-A1B, the one LFM2 template that emits <think>
     {
         auto tst = peg_tester("models/templates/LFM2.5-8B-A1B.jinja", detailed_debug);
@@ -7659,7 +7738,7 @@ static void test_template_generation_prompt() {
         preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "true";
         auto preserve_reasoning_params = common_chat_templates_apply(tmpls.get(), preserve_reasoning_inputs);
         assert_contains(preserve_reasoning_params.prompt.str(), "<｜Assistant｜><think>Previous reasoning</think>Previous answer");
-        assert_equals(true, common_chat_templates_get_caps(tmpls.get()).at("supports_preserve_reasoning"));
+        assert_equals(true, common_chat_templates_get_caps(tmpls.get()).at("supports_preserve_reasoning").get<bool>());
 
         auto no_preserve_reasoning_inputs = default_history_inputs;
         no_preserve_reasoning_inputs.chat_template_kwargs["preserve_reasoning"] = "false";
@@ -8232,7 +8311,7 @@ static void test_reasoning_effort_caps() {
 
     auto assert_supports_effort = [](const std::string & path, bool expected) {
         auto tmpls = read_templates(path);
-        assert_equals(expected, common_chat_templates_get_caps(tmpls.get()).at("supports_reasoning_effort"));
+        assert_equals(expected, common_chat_templates_get_caps(tmpls.get()).at("supports_reasoning_effort").get<bool>());
     };
 
     assert_supports_effort("models/templates/deepseek-ai-DeepSeek-V4.jinja", true);
@@ -8243,6 +8322,24 @@ static void test_reasoning_effort_caps() {
     assert_supports_effort("models/templates/Cohere2MoE.jinja", true);
     assert_supports_effort("models/templates/meta-llama-Llama-3.1-8B-Instruct.jinja", false);
     assert_supports_effort("models/templates/Qwen-Qwen3-0.6B.jinja", false);
+
+    auto assert_effort_levels = [](const std::string & path, const std::vector<std::string> & expected) {
+        auto tmpls = read_templates(path);
+        assert_equals(json(expected).dump(), common_chat_templates_get_caps(tmpls.get()).at("reasoning_effort_levels").dump());
+    };
+
+    const std::vector<std::string> all_levels = { "minimal", "low", "medium", "high", "xhigh", "max" };
+
+    assert_effort_levels("models/templates/deepseek-ai-DeepSeek-V4.jinja", { "max" });
+    assert_effort_levels("models/templates/deepseek-ai-DeepSeek-V4-Flash-0731.jinja", { "high", "max" });
+    assert_effort_levels("models/templates/IFM-K2-Horizon.jinja", { "low", "medium", "high" });
+    assert_effort_levels("models/templates/tencent-Hy3.jinja", { "low", "high" });
+    assert_effort_levels("models/templates/muse-spark.jinja", { "minimal", "low", "medium", "high" });
+    assert_effort_levels("models/templates/upstage-Solar-Open-100B.jinja", { "low" });
+    assert_effort_levels("models/templates/muse-glimmer.jinja", all_levels);
+    assert_effort_levels("models/templates/openai-gpt-oss-120b.jinja", all_levels);
+    assert_effort_levels("models/templates/Cohere2MoE.jinja", {});
+    assert_effort_levels("models/templates/Qwen-Qwen3-0.6B.jinja", {});
 }
 
 static void test_msg_diffs_compute() {

tests/test-jinja.cpp

@@ -2550,6 +2550,23 @@ static void test_caps(testing & t) {
         t.assert_true("does not support string content", !caps.supports_string_content);
         t.assert_true("supports typed content", caps.supports_typed_content);
     });
+
+    t.test("string and typed content, behind a truthiness check", [](testing & t) {
+        // an empty content array is falsy, the parts are only read when there are some (LFM3 case)
+        auto caps = get_caps(
+            "{% for message in messages %}"
+            "{% if message['content'] %}"
+            "{% if message['content'] is string %}"
+            "{{ message['content'] }}"
+            "{% else %}"
+            "{% for part in message['content'] %}{{ part['text'] }}{% endfor %}"
+            "{% endif %}"
+            "{% endif %}"
+            "{% endfor %}"
+        );
+        t.assert_true("supports string content", caps.supports_string_content);
+        t.assert_true("supports typed content", caps.supports_typed_content);
+    });
 }
 
 static void test_string_parts(testing & t) {

tests/test-llama-archs.cpp

@@ -251,6 +251,8 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         n_head = 1;
         n_ff   = 96;
         n_layer = 22; // hparams.n_layer_kv_from_start = 20 is hardcoded
+    } else if (arch == LLM_ARCH_LFM3) {
+        n_layer = 3;
     } else if (arch == LLM_ARCH_DEEPSEEK4) {
         // head size 64 so that GPU flash attention kernels support the model
         n_embd  = 512;
@@ -336,8 +338,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
     ms.add_kv(LLM_KV_FULL_ATTENTION_INTERVAL, uint32_t(2));
 
     if (arch == LLM_ARCH_PLAMO2 || arch == LLM_ARCH_JAMBA || arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE ||
-            arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_KIMI_LINEAR ||
-            arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT) {
+            arch == LLM_ARCH_GRANITE_HYBRID || arch == LLM_ARCH_LFM2 || arch == LLM_ARCH_LFM2MOE || arch == LLM_ARCH_LFM3 ||
+            arch == LLM_ARCH_KIMI_LINEAR || arch == LLM_ARCH_BAILINGMOE3 || arch == LLM_ARCH_KIMI_K3 || arch == LLM_ARCH_GLM5_NEXT
+            ) {
         GGML_ASSERT(n_layer >= 2);
         std::vector<uint32_t> n_head_per_layer;
         n_head_per_layer.reserve(n_layer);
@@ -439,6 +442,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) {
         if (arch == LLM_ARCH_FLINT) {
             ms.add_kv(LLM_KV_ATTENTION_ROPE_PATTERN, pattern);
         }
+    } else if (arch == LLM_ARCH_LFM3) {
+        ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, std::vector<uint32_t>({0, 0, 1}));
+        ms.add_kv(LLM_KV_ROPE_FREQ_BASE_SWA,               10000.0f);
     } else {
         ms.add_kv(LLM_KV_ATTENTION_SLIDING_WINDOW_PATTERN, uint32_t(2));
     }

tests/test-llama-context.cpp

@@ -409,6 +409,10 @@ int main(int argc, char ** argv) {
 
     const int32_t n_threads = std::max<int32_t>(1, n_hw_2 / n_pool);
 
+    // every context of the sweep runs on its share of the cores
+    params.cpuparams.n_threads       = n_threads;
+    params.cpuparams_batch.n_threads = n_threads;
+
     // a column is as wide as its case name, but never narrower than the widest verdict
     const size_t verdict_width = std::max({ test_status_width(test_status::PASS),
                                             test_status_width(test_status::FAIL),

tools/mtmd/CMakeLists.txt

@@ -58,6 +58,9 @@ add_library(mtmd
             models/sensenova-u1.cpp
             models/minimax-m3.cpp
             models/qwen3vl.cpp
+            models/lfm3vl.cpp
+            models/lfm3a.cpp
+            models/lfm3a-gen.cpp
             models/ling3vl.cpp
             models/mimovl.cpp
             models/qwen3a.cpp

tools/mtmd/clip-impl.h

@@ -106,6 +106,11 @@
 #define KEY_GEN_AUDIO_PROJ_TYPE    "clip.gen.audio.projector_type" // for models with mixed modalities
 // name of the weight variant, for settings that are not in the checkpoint
 #define KEY_GEN_AUDIO_VARIANT      "clip.gen.audio.model_variant"
+// sampling defaults of the generated audio, like general.sampling.* for text
+#define KEY_GEN_AUDIO_TEMP           "clip.gen.audio.sampling.temp"
+#define KEY_GEN_AUDIO_TOP_K          "clip.gen.audio.sampling.top_k"
+#define KEY_GEN_AUDIO_PENALTY_REPEAT "clip.gen.audio.sampling.penalty_repeat"
+#define KEY_GEN_AUDIO_PENALTY_LAST_N "clip.gen.audio.sampling.penalty_last_n"
 #define KEY_AUDIO_SUBSMPL_FACTOR   "clip.audio.subsampling_factor"
 
 //
@@ -173,6 +178,12 @@
 #define TN_DEEPSTACK_FC1   "v.deepstack.%d.fc1.%s"      // qwen3vl deepstack
 #define TN_DEEPSTACK_FC2   "v.deepstack.%d.fc2.%s"      // qwen3vl deepstack
 
+// lfm3vl
+#define TN_LFM3VL_EMBEDDING_NORM   "%s.embedding_norm.%s"
+#define TN_LFM3VL_CONV2D_IN_PROJ   "%s.blk.%d.conv2d_in_proj.%s"
+#define TN_LFM3VL_CONV2D_SPATIAL   "%s.blk.%d.conv2d_spatial.%s"
+#define TN_LFM3VL_CONV2D_OUT_PROJ  "%s.blk.%d.conv2d_out_proj.%s"
+
 // mimicpmv
 #define TN_MINICPMV_POS_EMBD_K "resampler.pos_embed_k"
 #define TN_MINICPMV_QUERY      "resampler.query"
@@ -240,6 +251,11 @@
 #define TN_A_GEN_CODE_OUT_EMBD "a.gen.code.out_embd.%s"
 #define TN_A_GEN_CODE_NORM     "a.gen.code.output_norm.%s"
 
+// lfm3a depthformer (MaskGIT), the codec decoder uses the generic layer names with prefix "a.gen.wav.tfm"
+#define TN_A_GEN_CODE_HEAD_NORM "a.gen.code.head_norm.%s"
+#define TN_A_GEN_CODE_MASK_EMBD "a.gen.code.mask_embd"
+#define TN_ATTN_GATE            "%s.blk.%d.attn_gate.%s"
+
 // qwen3tts code2wav (RVQ codes -> raw PCM)
 // pre_transformer layers use the generic TN_ATTN_*/TN_FFN_*/TN_LN_*/TN_LS_* macros, prefix "a.gen.wav.tfm"
 #define TN_A_GEN_WAV_QUANT_FIRST_IN  "a.gen.wav.quant.first.in_proj.%s"
@@ -468,6 +484,9 @@ struct clip_enc_state {
     ggml_backend_buffer_ptr buf;
     std::vector<ggml_tensor *> k; // per layer [n_embd_kv, n_ctx]
     std::vector<ggml_tensor *> v;
+
+    // lfm3a: same order as list_lfm3a_state_slots(), F32 [ne0, ne1], updated in place
+    std::vector<ggml_tensor *> slots;
 };
 
 // per-sequence state of audio generation, see clip_gen_state_init()
@@ -517,6 +536,7 @@ enum projector_type {
     PROJECTOR_TYPE_MERALION,
     PROJECTOR_TYPE_MUSIC_FLAMINGO,
     PROJECTOR_TYPE_LFM2,
+    PROJECTOR_TYPE_LFM3VL,
     PROJECTOR_TYPE_D1OMNI_V,
     PROJECTOR_TYPE_KIMIVL,
     PROJECTOR_TYPE_PADDLEOCR,
@@ -532,6 +552,7 @@ enum projector_type {
     PROJECTOR_TYPE_DEEPSEEK41V,
     PROJECTOR_TYPE_LFM2A,
     PROJECTOR_TYPE_D1OMNI_A,
+    PROJECTOR_TYPE_LFM3A,
     PROJECTOR_TYPE_GLM4V,
     PROJECTOR_TYPE_GLM5V,
     PROJECTOR_TYPE_YOUTUVL,
@@ -555,6 +576,7 @@ enum projector_type {
     PROJECTOR_TYPE_POCKETTTS_SPKENC,
     PROJECTOR_TYPE_POCKETTTS_GEN,
     PROJECTOR_TYPE_SOPRANO,
+    PROJECTOR_TYPE_LFM3A_GEN,
     PROJECTOR_TYPE_MUSE_GLIMMER,
     PROJECTOR_TYPE_SENSENOVA_U1,
     PROJECTOR_TYPE_COHERE2V,
@@ -595,6 +617,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_MERALION,          "meralion"},
     { PROJECTOR_TYPE_MUSIC_FLAMINGO,    "musicflamingo"},
     { PROJECTOR_TYPE_LFM2,              "lfm2"},
+    { PROJECTOR_TYPE_LFM3VL,            "lfm3vl"},
     { PROJECTOR_TYPE_D1OMNI_V,          "d1omni_v"},
     { PROJECTOR_TYPE_KIMIVL,            "kimivl"},
     { PROJECTOR_TYPE_PADDLEOCR,         "paddleocr"},
@@ -610,6 +633,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_DEEPSEEK41V,       "deepseek41v"},
     { PROJECTOR_TYPE_LFM2A,             "lfm2a"},
     { PROJECTOR_TYPE_D1OMNI_A,          "d1omni_a"},
+    { PROJECTOR_TYPE_LFM3A,             "lfm3a"},
     { PROJECTOR_TYPE_GLM4V,             "glm4v"},
     { PROJECTOR_TYPE_GLM5V,             "glm5v"},
     { PROJECTOR_TYPE_YOUTUVL,           "youtuvl"},
@@ -633,6 +657,7 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_POCKETTTS_SPKENC,  "pockettts_spkenc"},
     { PROJECTOR_TYPE_POCKETTTS_GEN,     "pockettts_gen"},
     { PROJECTOR_TYPE_SOPRANO,           "soprano"},
+    { PROJECTOR_TYPE_LFM3A_GEN,         "lfm3a_gen"},
     { PROJECTOR_TYPE_MUSE_GLIMMER,      "muse-glimmer"},
     { PROJECTOR_TYPE_SENSENOVA_U1,      "sensenova_u1"},
     { PROJECTOR_TYPE_COHERE2V,          "cohere2v"},

tools/mtmd/clip-model.h

@@ -175,6 +175,12 @@ struct clip_hparams {
     // name of the weight variant, some pipelines tune themselves on it
     std::string gen_model_variant;
 
+    // sampling defaults of the generated audio: greedy, no top-k, no repetition penalty
+    float   gen_temp           = 0.0f;
+    int32_t gen_top_k          = 0;
+    float   gen_penalty_repeat = 1.0f;
+    int32_t gen_penalty_last_n = 0;
+
     // pocket-tts
     static constexpr int32_t pockettts_max_spk_seconds = 30;
     int32_t seanet_n_stage    = 0;
@@ -195,6 +201,7 @@ struct clip_hparams {
     int32_t wav_dac_n_block      = 0;
     int32_t wav_dac_n_res        = 0;
     int32_t wav_tfm_swa          = 0; // pre_transformer's KV cache size, in frames
+    int32_t wav_frame_stack      = 0; // lfm3a codec: decoder positions per frame
 
     // mimo-v2.5: LLM-side connector (input_local_transformer)
     int32_t audio_local_n_layer = 0;
@@ -286,6 +293,9 @@ struct clip_layer {
     ggml_tensor * k_norm = nullptr;
     ggml_tensor * q_norm = nullptr;
 
+    // lfm3: per-head sigmoid gate on the attention output
+    ggml_tensor * attn_gate_w = nullptr;
+
     ggml_tensor * attn_post_norm_w = nullptr;
 
     ggml_tensor * ff_up_w = nullptr;
@@ -325,6 +335,11 @@ struct clip_layer {
     ggml_tensor * deepstack_fc2_w = nullptr;
     ggml_tensor * deepstack_fc2_b = nullptr;
 
+    // lfm3vl conv layers
+    ggml_tensor * conv2d_in_proj_w  = nullptr;
+    ggml_tensor * conv2d_spatial_w  = nullptr; // may be nullptr
+    ggml_tensor * conv2d_out_proj_w = nullptr;
+
     // sam rel_pos
     ggml_tensor * rel_pos_w = nullptr;
     ggml_tensor * rel_pos_h = nullptr;
@@ -561,6 +576,7 @@ struct clip_code2wav {
     // quantizer: RVQ codebook decode
     ggml_tensor * quant_first_in_w  = nullptr; // semantic RVQ, in_proj (1x1 conv, loaded as 2D)
     ggml_tensor * quant_first_out_w = nullptr;
+    ggml_tensor * quant_first_out_b = nullptr;
     ggml_tensor * quant_first_cb_w  = nullptr; // codebook (1 layer)
     ggml_tensor * quant_rest_in_w   = nullptr; // acoustic RVQ
     ggml_tensor * quant_rest_out_w  = nullptr;
@@ -786,6 +802,9 @@ struct clip_model {
     ggml_tensor * yasa_vision_pos_embed = nullptr;
     std::vector<yasa2_stage> yasa_stages;
 
+    // lfm3vl
+    ggml_tensor * lfm3vl_embedding_norm_w = nullptr;
+
     // pixtral, glm4v
     ggml_tensor * token_embd_img_break = nullptr;
 
@@ -844,6 +863,10 @@ struct clip_model {
     ggml_tensor * gen_code_out_embd_w = nullptr; // codebook-0 embedding, fed back into the talker
     ggml_tensor * gen_code_norm_w     = nullptr; // final norm
 
+    // lfm3a depthformer (MaskGIT)
+    ggml_tensor * gen_code_head_norm_w = nullptr; // per-codebook norm before the tied head, merged 2D
+    ggml_tensor * gen_code_mask_embd   = nullptr;
+
     // qwen3tts code2wav: RVQ codes -> raw PCM
     clip_code2wav c2w;
 

tools/mtmd/clip.cpp

@@ -1031,6 +1031,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
             {
                 builder = std::make_unique<clip_graph_sensenova_u1>(ctx, img);
             } break;
+        case PROJECTOR_TYPE_LFM3VL:
+            {
+                builder = std::make_unique<clip_graph_lfm3vl>(ctx, img);
+            } break;
         case PROJECTOR_TYPE_LING3VL:
             {
                 builder = std::make_unique<clip_graph_ling3vl>(ctx, img);
@@ -1144,6 +1148,10 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
             {
                 builder = std::make_unique<clip_graph_conformer>(ctx, img);
             } break;
+        case PROJECTOR_TYPE_LFM3A:
+            {
+                builder = std::make_unique<clip_graph_lfm3a>(ctx, img);
+            } break;
         case PROJECTOR_TYPE_GEMMA4A:
             {
                 builder = std::make_unique<clip_graph_gemma4a>(ctx, img);
@@ -1192,6 +1200,24 @@ static std::unique_ptr<clip_graph> clip_get_graph_builder(clip_ctx * ctx, const
                 const int  n_frames = params && params->feats ? (int) (params->feats->size() / n_latent) : 1;
                 builder = std::make_unique<clip_graph_pockettts_gen>(ctx, img, gen_process, n_step, n_frames);
             } break;
+        case PROJECTOR_TYPE_LFM3A_GEN:
+            {
+                const auto    gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
+                const int64_t n_cb        = ctx->model.gen_code_embd_w->ne[2];
+                bool all_known = false;
+                int  n_frames  = 1;
+                if (params && params->codes) {
+                    GGML_ASSERT(params->codes->size() % (size_t) n_cb == 0);
+                    if (gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
+                        n_frames = (int) (params->codes->size() / n_cb);
+                        GGML_ASSERT(n_frames > 0);
+                    } else {
+                        // -1 marks a masked codebook, none masked means only the feedback embedding is needed
+                        all_known = std::all_of(params->codes->begin(), params->codes->end(), [](int32_t c) { return c >= 0; });
+                    }
+                }
+                builder = std::make_unique<clip_graph_lfm3a_gen>(ctx, img, gen_process, all_known, n_frames);
+            } break;
         case PROJECTOR_TYPE_QWEN3TTS_GEN:
             {
                 const auto  gen_process = params ? params->gen_process : CLIP_GEN_PROCESS_GEN_CODE;
@@ -1876,6 +1902,39 @@ struct clip_model_loader {
                         }
                         hparams.set_warmup_n_tokens(16*16);
                     } break;
+                case PROJECTOR_TYPE_LFM3VL:
+                    {
+                        std::string resize_algo;
+                        get_string(KEY_IMAGE_RESIZE_ALGO, resize_algo);
+                        // Lfm3VlImageProcessor resizes with torchvision
+                        if (resize_algo == "bilinear") {
+                            hparams.set_image_resize_algo(RESIZE_ALGO_BILINEAR, true);
+                        } else if (resize_algo == "bicubic") {
+                            hparams.set_image_resize_algo(RESIZE_ALGO_BICUBIC, true);
+                        } else {
+                            throw std::runtime_error(string_format("%s: unsupported image resize algo: %s\n", __func__, resize_algo.c_str()));
+                        }
+                        // smart_resize stretches to the aligned size, no letterbox
+                        hparams.image_resize_pad = PAD_NONE;
+                        get_u32(KEY_SPATIAL_MERGE_SIZE, hparams.n_merge);
+                        // token counts and the projector graph assume a 2x2 merge
+                        if (hparams.n_merge != 2) {
+                            throw std::runtime_error(string_format("%s: unsupported spatial_merge_size %d for lfm3vl, only 2 is supported\n", __func__, hparams.n_merge));
+                        }
+                        get_u32(string_format(KEY_N_HEAD_KV, "vision"), hparams.n_head_kv);
+                        hparams.rope_theta = 1000000.0f; // vision_rope_theta
+                        // HF processor size; the max is also capped at 4096 tokens as an OOM guard
+                        int min_pixels = 0;
+                        int max_pixels = 0;
+                        get_u32(KEY_IMAGE_MIN_PIXELS, min_pixels);
+                        get_u32(KEY_IMAGE_MAX_PIXELS, max_pixels);
+                        if (min_pixels <= 0 || max_pixels < min_pixels) {
+                            throw std::runtime_error(string_format("%s: invalid image min/max pixels %d/%d\n", __func__, min_pixels, max_pixels));
+                        }
+                        const int patch_area = hparams.patch_size * hparams.patch_size * hparams.n_merge * hparams.n_merge;
+                        hparams.set_limit_image_tokens(min_pixels / patch_area, std::min(4096, max_pixels / patch_area));
+                        hparams.set_warmup_n_tokens(46*46); // avoid OOM on warmup
+                    } break;
                 case PROJECTOR_TYPE_MINIMAX_M3:
                     {
                         hparams.n_merge = 2; // spatial_merge_size
@@ -2099,6 +2158,33 @@ struct clip_model_loader {
                         hparams.audio_window_len  = 1024;
                         hparams.audio_hop_len     = 256;
                     } break;
+                case PROJECTOR_TYPE_LFM3A_GEN:
+                    {
+                        // fixed by LiquidCodecConfig, head counts come from the tensors
+                        hparams.rope_theta = 1000000.0f;
+
+                        hparams.wav_tfm_n_layer    = 8;
+                        hparams.wav_tfm_n_embd     = 512;
+                        hparams.wav_tfm_eps        = 1e-5f;
+                        hparams.wav_tfm_rope_theta = 1000000.0f;
+                        hparams.wav_tfm_swa        = 30; // decoder_window
+                        hparams.wav_frame_stack    = 6;  // stft_frame_stack, STFT frames per codec frame
+                        // the ISTFT runs on the host, see mtmd_gen_audio_process()
+                        hparams.audio_sample_rate  = 24000;
+                        hparams.audio_n_fft        = 1280;
+                        hparams.audio_window_len   = 1280;
+                        hparams.audio_hop_len      = 320;
+
+                        // speech-code sampling, the converter writes the model card values; else the LFM3-Omni card
+                        hparams.gen_temp           = 0.4f;
+                        hparams.gen_top_k          = 0;
+                        hparams.gen_penalty_repeat = 1.2f;
+                        hparams.gen_penalty_last_n = 64;
+                        get_f32(KEY_GEN_AUDIO_TEMP,           hparams.gen_temp,           false);
+                        get_i32(KEY_GEN_AUDIO_TOP_K,          hparams.gen_top_k,          false);
+                        get_f32(KEY_GEN_AUDIO_PENALTY_REPEAT, hparams.gen_penalty_repeat, false);
+                        get_i32(KEY_GEN_AUDIO_PENALTY_LAST_N, hparams.gen_penalty_last_n, false);
+                    } break;
                 case PROJECTOR_TYPE_QWEN3TTS_GEN:
                     {
                         // TODO: hardcoded for now, read from code_predictor_config instead
@@ -2204,6 +2290,20 @@ struct clip_model_loader {
                         hparams.audio_window_len       = 400;
                         hparams.audio_hop_len          = 160;
                     } break;
+                case PROJECTOR_TYPE_LFM3A:
+                    {
+                        get_u32(KEY_A_ATTN_WINDOW_SIZE, hparams.attn_window_size);
+                        if (hparams.attn_window_size <= 0) {
+                            throw std::runtime_error(string_format("%s: invalid audio attention window %d\n", __func__, hparams.attn_window_size));
+                        }
+                        hparams.rope_theta = 1000000.0f; // checked by the converter
+                        // audio preprocessing params, see mtmd_audio_preprocessor_lfm3a
+                        hparams.audio_chunk_len        = 0;
+                        hparams.audio_sample_rate      = 16000;
+                        hparams.audio_n_fft            = 400;
+                        hparams.audio_window_len       = 400;
+                        hparams.audio_hop_len          = 160;
+                    } break;
                 case PROJECTOR_TYPE_EXAONE4_5:
                     {
                         hparams.n_merge = 2;
@@ -2515,6 +2615,9 @@ struct clip_model_loader {
 
         model.position_embeddings = get_tensor(string_format(TN_POS_EMBD, prefix), false);
 
+        // lfm3vl
+        model.lfm3vl_embedding_norm_w = get_tensor(string_format(TN_LFM3VL_EMBEDDING_NORM, prefix, "weight"), false);
+
         const bool has_standard_layers = (
             model.proj_type != PROJECTOR_TYPE_GEMMA3NV &&
             model.proj_type != PROJECTOR_TYPE_QWEN3TTS_SPKENC &&
@@ -2527,10 +2630,13 @@ struct clip_model_loader {
         model.layers.resize(n_layers_to_load);
         for (int il = 0; il < n_layers_to_load; ++il) {
             auto & layer = model.layers[il];
+            // lfm3vl / lfm3a conv layers have no attention tensors
+            const bool o_w_required = model.proj_type != PROJECTOR_TYPE_LFM3VL
+                                   && model.proj_type != PROJECTOR_TYPE_LFM3A;
             layer.k_w    = get_tensor(string_format(TN_ATTN_K,      prefix, il, "weight"), false);
             layer.q_w    = get_tensor(string_format(TN_ATTN_Q,      prefix, il, "weight"), false);
             layer.v_w    = get_tensor(string_format(TN_ATTN_V,      prefix, il, "weight"), false);
-            layer.o_w    = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"));
+            layer.o_w    = get_tensor(string_format(TN_ATTN_OUTPUT, prefix, il, "weight"), o_w_required);
             layer.qkv_w  = get_tensor(string_format(TN_ATTN_QKV,    prefix, il, "weight"), false);
             layer.k_norm = get_tensor(string_format(TN_ATTN_K_NORM, prefix, il, "weight"), false);
             layer.q_norm = get_tensor(string_format(TN_ATTN_Q_NORM, prefix, il, "weight"), false);
@@ -2580,6 +2686,11 @@ struct clip_model_loader {
                 model.n_deepstack_layers++;
             }
 
+            // lfm3vl conv layers
+            layer.conv2d_in_proj_w  = get_tensor(string_format(TN_LFM3VL_CONV2D_IN_PROJ,  prefix, il, "weight"), false);
+            layer.conv2d_spatial_w  = get_tensor(string_format(TN_LFM3VL_CONV2D_SPATIAL,  prefix, il, "weight"), false);
+            layer.conv2d_out_proj_w = get_tensor(string_format(TN_LFM3VL_CONV2D_OUT_PROJ, prefix, il, "weight"), false);
+
             // some models already exported with legacy (incorrect) naming which is quite messy, let's fix it here
             // note: Qwen model converted from the old surgery script has n_ff = 0, so we cannot use n_ff to check!
             bool is_ffn_swapped = (
@@ -2764,6 +2875,24 @@ struct clip_model_loader {
                     model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
                     model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
                 } break;
+            case PROJECTOR_TYPE_LFM3VL:
+                {
+                    model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);
+                    model.mm_input_norm_b = get_tensor(TN_MM_INP_NORM_B);
+                    model.mm_0_w = get_tensor(string_format(TN_LLAVA_PROJ, 0, "weight"));
+                    model.mm_0_b = get_tensor(string_format(TN_LLAVA_PROJ, 0, "bias"));
+                    model.mm_1_w = get_tensor(string_format(TN_LLAVA_PROJ, 2, "weight"));
+                    model.mm_1_b = get_tensor(string_format(TN_LLAVA_PROJ, 2, "bias"));
+                    // each layer must be either a conv block or a complete attention block
+                    for (size_t il = 0; il < model.layers.size(); ++il) {
+                        const auto & layer = model.layers[il];
+                        const bool is_conv = layer.conv2d_in_proj_w != nullptr;
+                        const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w;
+                        if (is_conv == is_attn) {
+                            throw std::runtime_error(string_format("%s: lfm3vl layer %zu is neither a complete conv nor a complete attention block\n", __func__, il));
+                        }
+                    }
+                } break;
             case PROJECTOR_TYPE_LING3VL:
                 {
                     model.mm_input_norm_w = get_tensor(TN_MM_INP_NORM);        // merger.norm
@@ -3373,6 +3502,65 @@ struct clip_model_loader {
                         layer.ls_2_w    = get_tensor(string_format(TN_LS_2,        p, il, "weight"));
                     }
                 } break;
+            case PROJECTOR_TYPE_LFM3A_GEN:
+                {
+                    // depthformer, the generic loop above loaded the a.gen.code.blk.* tensors
+                    for (int il = 0; il < hparams.n_layer; ++il) {
+                        model.layers[il].attn_gate_w = get_tensor(string_format(TN_ATTN_GATE, prefix, il, "weight"));
+                    }
+                    model.gen_code_proj_in_w   = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN,   "weight"));
+                    model.gen_code_proj_in_b   = get_tensor(string_format(TN_A_GEN_CODE_PROJ_IN,   "bias"));
+                    model.gen_code_embd_w      = get_tensor(string_format(TN_A_GEN_CODE_EMBD,      "weight")); // tied head
+                    model.gen_code_out_embd_w  = get_tensor(string_format(TN_A_GEN_CODE_OUT_EMBD,  "weight"));
+                    model.gen_code_norm_w      = get_tensor(string_format(TN_A_GEN_CODE_NORM,      "weight"));
+                    model.gen_code_head_norm_w = get_tensor(string_format(TN_A_GEN_CODE_HEAD_NORM, "weight"));
+                    model.gen_code_mask_embd   = get_tensor(TN_A_GEN_CODE_MASK_EMBD);
+
+                    // codec decoder: FSQ level table, one projection over all codebooks, LFM3 decoder -> polar STFT frames
+                    auto & c2w = model.c2w;
+                    c2w.quant_first_cb_w  = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_CB,  "weight"));
+                    c2w.quant_first_out_w = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "weight"));
+                    c2w.quant_first_out_b = get_tensor(string_format(TN_A_GEN_WAV_QUANT_FIRST_OUT, "bias"));
+                    c2w.tfm_in_proj_w     = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ,  "weight"));
+                    c2w.tfm_in_proj_b     = get_tensor(string_format(TN_A_GEN_WAV_TFM_IN_PROJ,  "bias"));
+                    c2w.tfm_out_proj_w    = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "weight"));
+                    c2w.tfm_out_proj_b    = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_PROJ, "bias"));
+                    c2w.tfm_output_norm_w = get_tensor(string_format(TN_A_GEN_WAV_TFM_OUT_NORM, "weight"));
+
+                    // each layer is a short conv block or a gated attention block, both with sandwich norms
+                    c2w.tfm_layers.resize(hparams.wav_tfm_n_layer);
+                    for (int il = 0; il < hparams.wav_tfm_n_layer; il++) {
+                        auto & layer = c2w.tfm_layers[il];
+                        const char * p = "a.gen.wav.tfm";
+                        layer.ln_1_w           = get_tensor(string_format(TN_LN_1,           p, il, "weight"));
+                        layer.attn_post_norm_w = get_tensor(string_format(TN_ATTN_POST_NORM, p, il, "weight"));
+                        layer.ln_2_w           = get_tensor(string_format(TN_LN_2,           p, il, "weight"));
+                        layer.ff_post_norm_w   = get_tensor(string_format(TN_FFN_POST_NORM,  p, il, "weight"));
+                        layer.ff_gate_w        = get_tensor(string_format(TN_FFN_GATE,       p, il, "weight"));
+                        layer.ff_up_w          = get_tensor(string_format(TN_FFN_UP,         p, il, "weight"));
+                        layer.ff_down_w        = get_tensor(string_format(TN_FFN_DOWN,       p, il, "weight"));
+                        layer.conv_pw1_w       = get_tensor(string_format(TN_CONV_PW1,       p, il, "weight"), false);
+                        layer.conv_dw_w        = get_tensor(string_format(TN_CONV_DW,        p, il, "weight"), false);
+                        layer.conv_pw2_w       = get_tensor(string_format(TN_CONV_PW2,       p, il, "weight"), false);
+                        layer.q_w              = get_tensor(string_format(TN_ATTN_Q,         p, il, "weight"), false);
+                        layer.k_w              = get_tensor(string_format(TN_ATTN_K,         p, il, "weight"), false);
+                        layer.v_w              = get_tensor(string_format(TN_ATTN_V,         p, il, "weight"), false);
+                        layer.o_w              = get_tensor(string_format(TN_ATTN_OUTPUT,    p, il, "weight"), false);
+                        layer.q_norm           = get_tensor(string_format(TN_ATTN_Q_NORM,    p, il, "weight"), false);
+                        layer.k_norm           = get_tensor(string_format(TN_ATTN_K_NORM,    p, il, "weight"), false);
+                        layer.attn_gate_w      = get_tensor(string_format(TN_ATTN_GATE,      p, il, "weight"), false);
+                        const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w;
+                        const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm && layer.attn_gate_w;
+                        if (is_conv == is_attn) {
+                            throw std::runtime_error(string_format("%s: codec layer %d is neither a conv nor an attention block\n", __func__, il));
+                        }
+                        if (is_attn) {
+                            const int d_head = layer.q_norm->ne[0];
+                            hparams.wav_tfm_n_head    = layer.q_w->ne[1] / d_head;
+                            hparams.wav_tfm_n_head_kv = layer.k_w->ne[1] / d_head;
+                        }
+                    }
+                } break;
             case PROJECTOR_TYPE_QWEN3TTS_GEN:
                 {
                     // code_predictor, proj_in is absent when the talker and the predictor share the hidden size
@@ -3765,6 +3953,39 @@ struct clip_model_loader {
                         layer.conv_pw2_b   = get_tensor(string_format(TN_CONV_PW2,  prefix, il, "bias"));
                     }
                 } break;
+            case PROJECTOR_TYPE_LFM3A:
+                {
+                    for (int i : {0, 2, 3, 5, 6}) {
+                        model.pre_encode_conv_X_w[i] = get_tensor(string_format(TN_CONV1D, i, "weight"));
+                        model.pre_encode_conv_X_b[i] = get_tensor(string_format(TN_CONV1D, i, "bias"));
+                    }
+                    model.pre_encode_out_w = get_tensor(string_format(TN_PRE_ENCODE_OUT, "weight"));
+                    model.pre_encode_out_b = get_tensor(string_format(TN_PRE_ENCODE_OUT, "bias"));
+
+                    model.mm_0_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "weight"));
+                    model.mm_0_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 0, "bias"));
+                    model.mm_1_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "weight"));
+                    model.mm_1_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 1, "bias"));
+                    model.mm_3_w = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "weight"));
+                    model.mm_3_b = get_tensor(string_format(TN_MM_AUDIO_MLP, 3, "bias"));
+
+                    if (!model.post_ln_w) {
+                        throw std::runtime_error(string_format("%s: missing tensor %s\n", __func__, string_format(TN_LN_POST, prefix, "weight").c_str()));
+                    }
+
+                    // each layer is either a short conv block or a complete attention block
+                    for (int il = 0; il < hparams.n_layer; ++il) {
+                        auto & layer = model.layers[il];
+                        layer.conv_pw1_w = get_tensor(string_format(TN_CONV_PW1, prefix, il, "weight"), false);
+                        layer.conv_dw_w  = get_tensor(string_format(TN_CONV_DW,  prefix, il, "weight"), false);
+                        layer.conv_pw2_w = get_tensor(string_format(TN_CONV_PW2, prefix, il, "weight"), false);
+                        const bool is_conv = layer.conv_pw1_w && layer.conv_dw_w && layer.conv_pw2_w;
+                        const bool is_attn = layer.q_w && layer.k_w && layer.v_w && layer.o_w && layer.q_norm && layer.k_norm;
+                        if (is_conv == is_attn) {
+                            throw std::runtime_error(string_format("%s: lfm3a layer %d is neither a complete conv nor a complete attention block\n", __func__, il));
+                        }
+                    }
+                } break;
             case PROJECTOR_TYPE_PARAKEET:
                 {
 
@@ -4384,6 +4605,10 @@ struct clip_init_result clip_init(const char * fname, struct clip_context_params
         if (loader.has_gen_audio) {
             ctx_gen_audio = new clip_ctx(ctx_params);
             loader.load_hparams(ctx_gen_audio->model, CLIP_MODALITY_GEN_AUDIO);
+            // TODO: CUDA has no flash attention kernel for the depthformer head dim 32, enable it once it does
+            if (ctx_gen_audio->model.proj_type == PROJECTOR_TYPE_LFM3A_GEN && ctx_gen_audio->flash_attn_type == CLIP_FLASH_ATTN_TYPE_AUTO) {
+                ctx_gen_audio->flash_attn_type = CLIP_FLASH_ATTN_TYPE_DISABLED;
+            }
             loader.load_tensors(*ctx_gen_audio);
             // TODO: fix warmup
             ctx_gen_audio->buf_compute_meta.resize(ctx_gen_audio->max_nodes * ggml_tensor_overhead() + ggml_graph_overhead());
@@ -4452,6 +4677,7 @@ int clip_n_output_tokens_x(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_HUNYUANVL:
         case PROJECTOR_TYPE_YOUTUVL:
         case PROJECTOR_TYPE_MUSE_GLIMMER:
+        case PROJECTOR_TYPE_LFM3VL:
             return (img->nx() / params.patch_size) / 2;
         case PROJECTOR_TYPE_STEP3VL:
         case PROJECTOR_TYPE_MINICPMV4_6:
@@ -4484,6 +4710,7 @@ int clip_n_output_tokens_y(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_HUNYUANVL:
         case PROJECTOR_TYPE_YOUTUVL:
         case PROJECTOR_TYPE_MUSE_GLIMMER:
+        case PROJECTOR_TYPE_LFM3VL:
             return (img->ny() / params.patch_size) / 2;
         case PROJECTOR_TYPE_STEP3VL:
         case PROJECTOR_TYPE_MINICPMV4_6:
@@ -4571,6 +4798,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
         case PROJECTOR_TYPE_GLM5V:
         case PROJECTOR_TYPE_YOUTUVL:
         case PROJECTOR_TYPE_MUSE_GLIMMER:
+        case PROJECTOR_TYPE_LFM3VL:
             {
                 // dynamic size (2 conv, so double patch size)
                 int x_patch = img->nx() / (params.patch_size * 2);
@@ -4746,6 +4974,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
         } break;
         case PROJECTOR_TYPE_LFM2A:
         case PROJECTOR_TYPE_D1OMNI_A:
+        case PROJECTOR_TYPE_LFM3A:
             {
                 n_patches = ((((img->nx() + 1) / 2) + 1) / 2 + 1) / 2;
             } break;
@@ -4788,6 +5017,7 @@ int clip_n_output_tokens(const clip_ctx * ctx, const clip_image_f32 * img) {
                 n_patches = 1;
             } break;
         case PROJECTOR_TYPE_QWEN3TTS_GEN:
+        case PROJECTOR_TYPE_LFM3A_GEN:
             {
                 // one hidden-state vector fed back to the talker per call
                 n_patches = 1;
@@ -4848,11 +5078,12 @@ bool clip_image_batch_encode(clip_ctx * ctx, int n_threads, const clip_image_f32
     return clip_encode(ctx, &params);
 }
 
-// persisted state slots of the gen-audio decoder, per pipeline
+// persisted state slots of the gen-audio decoders and the streaming audio encoders
 static std::vector<c2w_state_slot> list_gen_state_slots(const clip_hparams & hparams, const clip_model & model) {
     switch (model.proj_type) {
         case PROJECTOR_TYPE_QWEN3TTS_GEN:  return list_c2w_state_slots(hparams, model);
         case PROJECTOR_TYPE_POCKETTTS_GEN: return list_pockettts_state_slots(hparams, model);
+        case PROJECTOR_TYPE_LFM3A_GEN:     return list_lfm3a_gen_state_slots(hparams, model);
         default:                           return {};
     }
 }
@@ -4912,6 +5143,30 @@ void clip_gen_state_reset(clip_gen_state * state) {
 
 clip_enc_state * clip_enc_state_init(clip_ctx * ctx, int32_t n_frames_max) {
     const auto & hparams = ctx->model.hparams;
+    if (ctx->proj_type() == PROJECTOR_TYPE_LFM3A) {
+        // downsampler context, short-conv taps and attention window, see list_lfm3a_state_slots()
+        const auto slots = list_lfm3a_state_slots(hparams, ctx->model);
+        auto st = std::make_unique<clip_enc_state>();
+        ggml_init_params params = {
+            /*.mem_size   =*/ ggml_tensor_overhead() * slots.size(),
+            /*.mem_buffer =*/ nullptr,
+            /*.no_alloc   =*/ true,
+        };
+        st->ctx_data.reset(ggml_init(params));
+        for (const auto & slot : slots) {
+            ggml_tensor * t = ggml_new_tensor_2d(st->ctx_data.get(), GGML_TYPE_F32, slot.ne0, slot.ne1);
+            ggml_format_name(t, "enc_state_%s", slot.name.c_str());
+            st->slots.push_back(t);
+        }
+        st->buf.reset(ggml_backend_alloc_ctx_tensors(st->ctx_data.get(), ctx->backend));
+        if (!st->buf) {
+            LOG_ERR("%s: failed to allocate the encoder state\n", __func__);
+            return nullptr;
+        }
+        // a zero state is the cold start
+        ggml_backend_buffer_clear(st->buf.get(), 0);
+        return st.release();
+    }
     if (ctx->proj_type() != PROJECTOR_TYPE_VOXTRAL_RT || hparams.attn_window_size <= 0) {
         LOG_ERR("%s: the encoder does not support streaming input\n", __func__);
         return nullptr;
@@ -4983,6 +5238,11 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
     // build the inference graph
     ggml_backend_sched_reset(ctx->sched.get());
     ggml_cgraph * gf = clip_get_graph_builder(ctx, imgs, params)->build();
+
+    // the last node is the embedding tensor, code2wav has no out_embd
+    // read it before scheduling: backends may reorder the nodes of graphs with several outputs (Metal)
+    ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
+
     if (!ggml_backend_sched_alloc_graph(ctx->sched.get(), gf)) {
         LOG_ERR("%s: failed to allocate compute graph\n", __func__);
         return false;
@@ -5051,6 +5311,28 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
         }
     };
 
+    // lfm3a encoder and codec: causal sliding window over [cached window | this call]
+    auto set_lfm3a_window_inputs = [&](int base, int n_pos, int window) {
+        std::vector<int32_t> positions(n_pos);
+        for (int i = 0; i < n_pos; i++) {
+            positions[i] = base + i;
+        }
+        set_input_i32("positions", positions);
+
+        // key j holds position base - prefix + j
+        const int prefix = window - 1;
+        const int n_kv   = prefix + n_pos;
+        std::vector<float> mask((size_t) n_kv * n_pos, -INFINITY);
+        for (int q = 0; q < n_pos; q++) {
+            for (int j = std::max(0, q + prefix - window + 1); j <= q + prefix; j++) {
+                if (base - prefix + j >= 0) {
+                    mask[(size_t) q * n_kv + j] = 0.0f;
+                }
+            }
+        }
+        set_input_attn_mask(mask);
+    };
+
     // rope positions and attention mask of the mimi transformers (pocket-tts).
     // the mask is causal with a sliding window, see _build_attention_mask() in the reference
     auto set_pockettts_tfm_inputs = [&]() {
@@ -5122,8 +5404,9 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
         }
         set_input_f32("inp_raw", inp_raw);
 
-    } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV) {
-        // audio input. GEN_WAV is not here: it takes codes or feats, set in the switch below
+    } else if (params->gen_process != CLIP_GEN_PROCESS_GEN_WAV && ggml_graph_get_tensor(gf, "inp_raw") != nullptr) {
+        // audio input. GEN_WAV is not here: it takes codes or feats, set in the switch below.
+        // a GEN_CODE graph may have no hidden-state input (lfm3a_gen feedback embedding)
         GGML_ASSERT(imgs.entries.size() == 1);
 
         const auto & mel_inp = imgs.entries[0];
@@ -5377,6 +5660,24 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
                 set_input_i32("merger_ds_idx_2", m_ds_2);
                 set_input_i32("merger_ds_idx_3", m_ds_3);
             } break;
+        case PROJECTOR_TYPE_LFM3VL:
+            {
+                // patches in raster-scan order (no interleave before encoder layers)
+                const int pw = image_size_width  / patch_size;
+                const int ph = image_size_height / patch_size;
+                std::vector<int> positions(n_pos * 4);
+                int ptr = 0;
+                for (int y = 0; y < ph; y++) {
+                    for (int x = 0; x < pw; x++) {
+                        positions[                  ptr] = y;
+                        positions[    num_patches + ptr] = x;
+                        positions[2 * num_patches + ptr] = y;
+                        positions[3 * num_patches + ptr] = x;
+                        ptr++;
+                    }
+                }
+                set_input_i32("positions", positions);
+            } break;
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN3VL:
         case PROJECTOR_TYPE_LING3VL:
@@ -5917,6 +6218,53 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
             {
                 // do nothing
             } break;
+        case PROJECTOR_TYPE_LFM3A_GEN:
+            {
+                const int64_t n_cb    = model.gen_code_embd_w->ne[2];
+                const int64_t n_vocab = model.gen_code_embd_w->ne[1];
+                if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
+                    GGML_ASSERT(params->codes != nullptr);
+                    // codes index the FSQ table via ggml_get_rows, the end-of-audio code has no waveform
+                    const int64_t n_codes = model.c2w.quant_first_cb_w->ne[1];
+                    for (int32_t c : *params->codes) {
+                        if (c < 0 || c >= n_codes) {
+                            LOG_ERR("%s: code out of range (%d, codebook size %lld)\n", __func__, c, (long long) n_codes);
+                            return false;
+                        }
+                    }
+                    std::vector<int32_t> codes = *params->codes;
+                    set_input_i32("inp_codes", codes);
+                    set_gen_state_in();
+                    // the "pos" slot leads the state
+                    float base = 0.0f;
+                    ggml_backend_tensor_get(params->gen_state->slots[0], &base, 0, sizeof(float));
+                    set_lfm3a_window_inputs((int) base, (int) ggml_nelements(get_inp_tensor("positions")), hparams.wav_tfm_swa);
+                } else {
+                    // rows of the merged per-codebook tables; a masked codebook (-1) takes row 0, the mask drops it
+                    std::vector<int32_t> codes(n_cb, -1);
+                    if (params->codes) {
+                        GGML_ASSERT(params->codes->size() == (size_t) n_cb);
+                        codes = *params->codes;
+                    }
+                    std::vector<float> mask(n_cb, 0.0f);
+                    for (int64_t i = 0; i < n_cb; i++) {
+                        if (codes[i] >= n_vocab) {
+                            LOG_ERR("%s: code out of range (%d, vocab %lld)\n", __func__, codes[i], (long long) n_vocab);
+                            return false;
+                        }
+                        if (codes[i] < 0) {
+                            codes[i] = 0;
+                        } else {
+                            mask[i] = 1.0f;
+                        }
+                        codes[i] += (int32_t) (i * n_vocab);
+                    }
+                    set_input_i32("inp_codes", codes);
+                    if (ggml_graph_get_tensor(gf, "inp_mask") != nullptr) {
+                        set_input_f32("inp_mask", mask);
+                    }
+                }
+            } break;
         case PROJECTOR_TYPE_QWEN3TTS_GEN:
             {
                 if (params->gen_process == CLIP_GEN_PROCESS_GEN_WAV) {
@@ -6185,6 +6533,31 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
                 }
                 set_input_f32("pos_emb", pos_emb);
             } break;
+        case PROJECTOR_TYPE_LFM3A:
+            {
+                GGML_ASSERT(imgs.entries.size() == 1);
+                const int n_pos = clip_n_output_tokens(ctx, &imgs.entries.front());
+
+                // the "pos" slot leads the state; without a stream the state is zero, the clip is encoded from its start
+                int base = 0;
+                if (params->enc_state != nullptr) {
+                    const auto & slots = params->enc_state->slots;
+                    float base_f = 0.0f;
+                    ggml_backend_tensor_get(slots[0], &base_f, 0, sizeof(float));
+                    base = (int) base_f;
+                    size_t i = 0;
+                    for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+                        ggml_backend_tensor_copy(slots[i++], get_inp_tensor(("state_in_" + slot.name).c_str()));
+                    }
+                } else {
+                    for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+                        ggml_tensor * t = get_inp_tensor(("state_in_" + slot.name).c_str());
+                        ggml_backend_tensor_memset(t, 0, 0, ggml_nbytes(t));
+                    }
+                }
+
+                set_lfm3a_window_inputs(base, n_pos, hparams.attn_window_size);
+            } break;
         case PROJECTOR_TYPE_PARAKEET:
             {
                 GGML_ASSERT(imgs.entries.size() == 1);
@@ -6404,9 +6777,6 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
         return false;
     }
 
-    // the last node is the embedding tensor, code2wav has no out_embd
-    ggml_tensor * embeddings = params->out_embd ? ggml_graph_node(gf, -1) : nullptr;
-
     if (embeddings != nullptr) {
         // sanity check (assuming that all images in batch have the same number of tokens, so we only check the first one)
         const int n_tokens_out = embeddings->ne[1];
@@ -6486,7 +6856,15 @@ bool clip_encode(struct clip_ctx * ctx, struct clip_encode_params * params) {
             out_audio.resize((size_t) n_frames * hop);
         }
     }
-    if (params->enc_state != nullptr) {
+    if (params->enc_state != nullptr && model.proj_type == PROJECTOR_TYPE_LFM3A) {
+        const auto & slots = params->enc_state->slots;
+        size_t i = 0;
+        for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+            ggml_tensor * t = ggml_graph_get_tensor(gf, ("state_out_" + slot.name).c_str());
+            GGML_ASSERT(t != nullptr);
+            copy_gen_state_out(t, slots[i++]);
+        }
+    } else if (params->enc_state != nullptr) {
         clip_enc_state * st = params->enc_state;
         ggml_tensor * tail = ggml_graph_get_tensor(gf, "out_stem_tail");
         GGML_ASSERT(tail != nullptr);
@@ -6603,6 +6981,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
             return ctx->model.mm_2_w->ne[1];
         case PROJECTOR_TYPE_QWEN2VL:
         case PROJECTOR_TYPE_QWEN25VL:
+        case PROJECTOR_TYPE_LFM3VL:
         case PROJECTOR_TYPE_EXAONE4_5:
         case PROJECTOR_TYPE_JANUS_PRO:
         case PROJECTOR_TYPE_YOUTUVL:
@@ -6667,6 +7046,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
         case PROJECTOR_TYPE_LFM2A:
             return ctx->model.position_embeddings->ne[0];
         case PROJECTOR_TYPE_D1OMNI_A:
+        case PROJECTOR_TYPE_LFM3A:
             return ctx->model.mm_3_w->ne[1];
         case PROJECTOR_TYPE_GRANITE_SPEECH:
             return ctx->model.qf_proj_blocks[0].qf_proj_linear_w->ne[1];
@@ -6680,6 +7060,7 @@ int clip_n_mmproj_embd(const struct clip_ctx * ctx) {
         case PROJECTOR_TYPE_QWEN3TTS_SPKENC:
             return ctx->model.mm_fc_w->ne[2];
         case PROJECTOR_TYPE_QWEN3TTS_GEN:
+        case PROJECTOR_TYPE_LFM3A_GEN:
             return ctx->model.gen_code_out_embd_w->ne[0];
         case PROJECTOR_TYPE_POCKETTTS_SPKENC:
             return ctx->model.spk_proj_w->ne[1];

tools/mtmd/models/lfm3a-gen.cpp

@@ -0,0 +1,257 @@
+#include "models.h"
+
+#include <cmath>
+
+// LFM3-Audio generation
+//
+// GEN_CODE: one MaskGIT pass of the depthformer, hidden state + known codes (-1 = masked) -> logits per codebook
+//           once all codes are known, only the feedback embedding is built
+// GEN_WAV : a window of frames -> polar STFT frames of the liquid-codec decoder, the ISTFT runs on the host
+
+// state carried between GEN_WAV calls: rope offset, short-conv taps and attention window of the codec decoder
+std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model) {
+    std::vector<c2w_state_slot> slots;
+    if (model.c2w.tfm_layers.empty()) {
+        return slots;
+    }
+    const int64_t prefix = hparams.wav_tfm_swa - 1;
+    slots.push_back({"pos", 1, 1});
+    for (size_t il = 0; il < model.c2w.tfm_layers.size(); il++) {
+        const auto & layer = model.c2w.tfm_layers[il];
+        if (layer.conv_dw_w) {
+            slots.push_back({"conv_" + std::to_string(il), layer.conv_dw_w->ne[0] - 1, hparams.wav_tfm_n_embd});
+        } else {
+            slots.push_back({"k_" + std::to_string(il), layer.k_w->ne[1], prefix});
+            slots.push_back({"v_" + std::to_string(il), layer.v_w->ne[1], prefix});
+        }
+    }
+    return slots;
+}
+
+// pre-norm LFM block, with the LFM3 sandwich norms when the layer has them
+ggml_tensor * clip_graph_lfm3a_gen::build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il,
+                                                    const std::function<ggml_tensor *(ggml_tensor *)> & op) {
+    ggml_tensor * cur = build_norm(inpL, layer.ln_1_w, nullptr, NORM_TYPE_RMS, eps, il);
+    cur = op(cur);
+    if (layer.attn_post_norm_w) {
+        cur = build_norm(cur, layer.attn_post_norm_w, nullptr, NORM_TYPE_RMS, eps, il);
+    }
+    inpL = ggml_add(ctx0, cur, inpL);
+
+    cur = build_norm(inpL, layer.ln_2_w, nullptr, NORM_TYPE_RMS, eps, il);
+    cur = build_ffn(cur, layer.ff_up_w, nullptr, layer.ff_gate_w, nullptr, layer.ff_down_w, nullptr, FFN_SILU, il);
+    if (layer.ff_post_norm_w) {
+        cur = build_norm(cur, layer.ff_post_norm_w, nullptr, NORM_TYPE_RMS, eps, il);
+    }
+    return ggml_add(ctx0, inpL, cur);
+}
+
+// GQA with q/k RMSNorm, NEOX rope, an optional cached K/V prefix and the LFM3 per-head sigmoid gate
+ggml_tensor * clip_graph_lfm3a_gen::build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv,
+                                                   float eps, float rope_theta, int il, ggml_tensor * inp_pos,
+                                                   ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache,
+                                                   ggml_tensor ** k_all, ggml_tensor ** v_all) {
+    const int64_t n_pos   = cur->ne[1];
+    const int64_t head_dim = layer.q_norm->ne[0];
+
+    ggml_tensor * Qcur = ggml_reshape_3d(ctx0, build_mm(layer.q_w, cur), head_dim, n_head,    n_pos);
+    ggml_tensor * Kcur = ggml_reshape_3d(ctx0, build_mm(layer.k_w, cur), head_dim, n_head_kv, n_pos);
+    ggml_tensor * Vcur = build_mm(layer.v_w, cur);
+
+    Qcur = build_norm(Qcur, layer.q_norm, nullptr, NORM_TYPE_RMS, eps, il);
+    Kcur = build_norm(Kcur, layer.k_norm, nullptr, NORM_TYPE_RMS, eps, il);
+    Qcur = ggml_rope_ext(ctx0, Qcur, inp_pos, nullptr, head_dim, GGML_ROPE_TYPE_NEOX, 0, rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+    Kcur = ggml_rope_ext(ctx0, Kcur, inp_pos, nullptr, head_dim, GGML_ROPE_TYPE_NEOX, 0, rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+
+    ggml_tensor * k = ggml_reshape_2d(ctx0, Kcur, head_dim * n_head_kv, n_pos);
+    ggml_tensor * v = Vcur;
+    if (k_cache) {
+        k = ggml_concat(ctx0, k_cache, k, 1);
+        v = ggml_concat(ctx0, v_cache, v, 1);
+    }
+    if (k_all) {
+        *k_all = k;
+        *v_all = v;
+    }
+    const int64_t n_kv = k->ne[1];
+
+    ggml_tensor * attn = build_attn(layer.attn_gate_w ? nullptr : layer.o_w, nullptr,
+        Qcur,
+        ggml_reshape_3d(ctx0, k, head_dim, n_head_kv, n_kv),
+        ggml_reshape_3d(ctx0, v, head_dim, n_head_kv, n_kv),
+        kq_mask, 1.0f / sqrtf((float) head_dim), il);
+    if (layer.attn_gate_w) {
+        ggml_tensor * gate = ggml_sigmoid(ctx0, build_mm(layer.attn_gate_w, cur));
+        attn = ggml_mul(ctx0, ggml_reshape_3d(ctx0, attn, head_dim, n_head, n_pos), ggml_reshape_3d(ctx0, gate, 1, n_head, n_pos));
+        attn = build_mm(layer.o_w, ggml_reshape_2d(ctx0, attn, head_dim * n_head, n_pos));
+    }
+    return attn;
+}
+
+// short conv: in_proj -> b * x -> causal depthwise conv over [conv_state | x] -> c * conv -> out_proj
+ggml_tensor * clip_graph_lfm3a_gen::build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il,
+                                                   ggml_tensor * conv_state, ggml_tensor ** conv_state_out) {
+    const int64_t n_embd_c = cur->ne[0];
+    const int64_t n_pos    = cur->ne[1];
+
+    ggml_tensor * bcx = build_mm(layer.conv_pw1_w, cur);
+    ggml_tensor * b = ggml_view_2d(ctx0, bcx, n_embd_c, n_pos, bcx->nb[1], 0);
+    ggml_tensor * c = ggml_view_2d(ctx0, bcx, n_embd_c, n_pos, bcx->nb[1], ggml_row_size(bcx->type, n_embd_c));
+    ggml_tensor * x = ggml_view_2d(ctx0, bcx, n_embd_c, n_pos, bcx->nb[1], ggml_row_size(bcx->type, 2 * n_embd_c));
+
+    ggml_tensor * bx = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mul(ctx0, b, x)));
+    bx = ggml_concat(ctx0, conv_state, bx, 0);
+    const int64_t d_conv = conv_state->ne[0];
+    *conv_state_out = ggml_cont(ctx0, ggml_view_2d(ctx0, bx, d_conv, n_embd_c, bx->nb[1], (size_t) n_pos * bx->nb[0]));
+
+    cur = ggml_ssm_conv(ctx0, ggml_reshape_3d(ctx0, bx, bx->ne[0], n_embd_c, 1), layer.conv_dw_w);
+    cur = ggml_mul(ctx0, c, ggml_reshape_2d(ctx0, cur, n_embd_c, n_pos));
+    cb(cur, "conv", il);
+    return build_mm(layer.conv_pw2_w, cur);
+}
+
+ggml_tensor * clip_graph_lfm3a_gen::build_gen_code() {
+    ggml_tensor * embd_w = model.gen_code_embd_w; // [n_embd, n_vocab, n_codebooks], tied to the output head
+    const int64_t n_cb    = embd_w->ne[2];
+    const int64_t n_vocab = embd_w->ne[1];
+
+    // rows of the merged per-codebook tables: code + codebook * n_vocab
+    ggml_tensor * inp_codes = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_cb);
+    ggml_set_name(inp_codes, "inp_codes");
+    ggml_set_input(inp_codes);
+
+    // feedback embedding for the backbone
+    ggml_tensor * fb = ggml_get_rows(ctx0, model.gen_code_out_embd_w, inp_codes); // [n_mmproj_embd, n_cb]
+    fb = ggml_sum_rows(ctx0, ggml_cont(ctx0, ggml_transpose(ctx0, fb)));
+    fb = ggml_reshape_2d(ctx0, fb, n_mmproj_embd, 1);
+    cb(fb, "gen_embd", -1);
+    if (all_known) {
+        return fb;
+    }
+
+    ggml_tensor * h_state = ggml_new_tensor_1d(ctx0, GGML_TYPE_F32, n_mmproj_embd);
+    ggml_set_name(h_state, "inp_raw");
+    ggml_set_input(h_state);
+
+    // 1 for a revealed codebook, 0 for a masked one
+    ggml_tensor * inp_mask = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, 1, n_cb);
+    ggml_set_name(inp_mask, "inp_mask");
+    ggml_set_input(inp_mask);
+
+    // one position per codebook: projected hidden state + token embedding or the mask embedding
+    ggml_tensor * cur = ggml_add(ctx0, build_mm(model.gen_code_proj_in_w, h_state), model.gen_code_proj_in_b);
+    cur = ggml_reshape_2d(ctx0, cur, n_embd, n_cb);
+
+    ggml_tensor * tok = ggml_get_rows(ctx0, ggml_reshape_2d(ctx0, embd_w, n_embd, n_vocab * n_cb), inp_codes);
+    tok = ggml_mul(ctx0, tok, inp_mask);
+    ggml_tensor * msk = ggml_mul(ctx0, ggml_repeat(ctx0, model.gen_code_mask_embd, cur), ggml_scale_bias(ctx0, inp_mask, -1.0f, 1.0f));
+    cur = ggml_add(ctx0, cur, ggml_add(ctx0, tok, msk));
+    cb(cur, "depth_inp", -1);
+
+    // bidirectional: no mask, no state
+    ggml_tensor * inp_pos = ggml_cast(ctx0, ggml_arange(ctx0, 0.0f, (float) n_cb, 1.0f), GGML_TYPE_I32);
+    for (int il = 0; il < n_layer; il++) {
+        const auto & layer = model.layers[il];
+        cur = build_lfm_layer(cur, layer, eps, il, [&](ggml_tensor * x) {
+            return build_lfm_attn(x, layer, hparams.n_head, hparams.n_head_kv, eps, hparams.rope_theta, il, inp_pos,
+                                  nullptr, nullptr, nullptr, nullptr, nullptr);
+        });
+    }
+
+    cur = build_norm(cur, model.gen_code_norm_w, nullptr, NORM_TYPE_RMS, eps, -1);
+    cur = build_norm(cur, model.gen_code_head_norm_w, nullptr, NORM_TYPE_RMS, eps, -1); // per codebook
+    ggml_tensor * logits = ggml_mul_mat(ctx0, embd_w, ggml_reshape_3d(ctx0, cur, n_embd, 1, n_cb)); // [n_vocab, 1, n_cb]
+    logits = ggml_reshape_2d(ctx0, logits, n_vocab, n_cb);
+    ggml_set_name(logits, "out_feats");
+    ggml_set_output(logits);
+    ggml_build_forward_expand(gf, logits);
+
+    return fb;
+}
+
+ggml_tensor * clip_graph_lfm3a_gen::build_gen_wav() {
+    const auto & c2w = model.c2w;
+    const int64_t n_cb   = model.gen_code_embd_w->ne[2];
+    const int64_t n_fs   = hparams.wav_frame_stack;
+    const int64_t n_pos  = n_fs * n_frames;
+    const int64_t prefix = hparams.wav_tfm_swa - 1;
+
+    // frame-major codes
+    ggml_tensor * inp_codes = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_cb * n_frames);
+    ggml_set_name(inp_codes, "inp_codes");
+    ggml_set_input(inp_codes);
+
+    std::map<std::string, ggml_tensor *> state_in;
+    std::vector<std::pair<std::string, ggml_tensor *>> state_out;
+    for (const auto & slot : list_lfm3a_gen_state_slots(hparams, model)) {
+        ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+        ggml_set_name(t, ("state_in_" + slot.name).c_str());
+        ggml_set_input(t);
+        state_in[slot.name] = t;
+    }
+
+    // FSQ decode: every codebook shares the level table, one projection over the n_cb code vectors of a frame
+    ggml_tensor * q = ggml_get_rows(ctx0, c2w.quant_first_cb_w, inp_codes); // [n_lvl, n_cb * n_frames]
+    q = ggml_reshape_2d(ctx0, q, q->ne[0] * n_cb, n_frames);
+    ggml_tensor * cur = ggml_add(ctx0, build_mm(c2w.quant_first_out_w, q), c2w.quant_first_out_b);
+    cur = ggml_add(ctx0, build_mm(c2w.tfm_in_proj_w, cur), c2w.tfm_in_proj_b); // [n_embd_dec, n_frames]
+    const int64_t n_embd_dec = cur->ne[0];
+
+    // nearest-exact upsampling: every frame becomes n_fs decoder positions
+    cur = ggml_repeat_4d(ctx0, ggml_reshape_3d(ctx0, cur, n_embd_dec, 1, n_frames), n_embd_dec, n_fs, n_frames, 1);
+    cur = ggml_reshape_2d(ctx0, cur, n_embd_dec, n_pos);
+
+    // positions continue across calls from the "pos" slot, see clip_encode()
+    ggml_tensor * inp_pos = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
+    ggml_set_name(inp_pos, "positions");
+    ggml_set_input(inp_pos);
+    state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_pos)});
+
+    // sliding-window causal mask over [cached window | this chunk]
+    ggml_tensor * kq_mask = build_inp_attn_mask(prefix + n_pos, n_pos);
+
+    const float eps_dec = hparams.wav_tfm_eps;
+    for (size_t il = 0; il < c2w.tfm_layers.size(); il++) {
+        const auto & layer = c2w.tfm_layers[il];
+        cur = build_lfm_layer(cur, layer, eps_dec, (int) il, [&](ggml_tensor * x) {
+            if (layer.conv_dw_w) {
+                const std::string name = "conv_" + std::to_string(il);
+                ggml_tensor * st = nullptr;
+                ggml_tensor * out = build_lfm_conv(x, layer, (int) il, state_in.at(name), &st);
+                state_out.push_back({name, st});
+                return out;
+            }
+            // this chunk's tail of the window is kept for the next call
+            const std::string k_name = "k_" + std::to_string(il);
+            const std::string v_name = "v_" + std::to_string(il);
+            ggml_tensor * k = nullptr;
+            ggml_tensor * v = nullptr;
+            ggml_tensor * out = build_lfm_attn(x, layer, hparams.wav_tfm_n_head, hparams.wav_tfm_n_head_kv, eps_dec,
+                                               hparams.wav_tfm_rope_theta, (int) il, inp_pos, kq_mask,
+                                               state_in.at(k_name), state_in.at(v_name), &k, &v);
+            state_out.push_back({k_name, ggml_cont(ctx0, ggml_view_2d(ctx0, k, k->ne[0], prefix, k->nb[1], (size_t) n_pos * k->nb[1]))});
+            state_out.push_back({v_name, ggml_cont(ctx0, ggml_view_2d(ctx0, v, v->ne[0], prefix, v->nb[1], (size_t) n_pos * v->nb[1]))});
+            return out;
+        });
+    }
+
+    cur = build_norm(cur, c2w.tfm_output_norm_w, nullptr, NORM_TYPE_RMS, eps_dec, -1);
+    cur = ggml_add(ctx0, build_mm(c2w.tfm_out_proj_w, cur), c2w.tfm_out_proj_b); // [2 * n_bins, n_pos]
+    cb(cur, "polar_out", -1);
+
+    for (const auto & s : state_out) {
+        ggml_set_name(s.second, ("state_out_" + s.first).c_str());
+        ggml_set_output(s.second);
+        ggml_build_forward_expand(gf, s.second);
+    }
+
+    ggml_set_name(cur, "out_feats");
+    ggml_set_output(cur);
+    return cur;
+}
+
+ggml_cgraph * clip_graph_lfm3a_gen::build() {
+    ggml_tensor * out = gen_process == CLIP_GEN_PROCESS_GEN_WAV ? build_gen_wav() : build_gen_code();
+    ggml_build_forward_expand(gf, out);
+    return gf;
+}

tools/mtmd/models/lfm3a.cpp

@@ -0,0 +1,189 @@
+#include "models.h"
+
+// streaming encoder state: base position, conv left context, short-conv taps and attention window
+// a zero state gives the same result as encoding the whole clip
+std::vector<c2w_state_slot> list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model) {
+    std::vector<c2w_state_slot> slots;
+
+    const int64_t n_mel  = hparams.n_mel_bins;
+    const int64_t n_ch   = model.pre_encode_conv_X_w[0]->ne[3];
+    const int64_t prefix = hparams.attn_window_size - 1;
+
+    slots.push_back({"pos", 1, 1});
+    slots.push_back({"ds0", 1, n_mel});
+    slots.push_back({"ds1", 1, (n_mel / 2) * n_ch});
+    slots.push_back({"ds2", 1, (n_mel / 4) * n_ch});
+    for (size_t il = 0; il < model.layers.size(); il++) {
+        const auto & layer = model.layers[il];
+        if (layer.conv_dw_w) {
+            slots.push_back({"conv_" + std::to_string(il), layer.conv_dw_w->ne[0] - 1, hparams.n_embd});
+        } else {
+            slots.push_back({"k_" + std::to_string(il), layer.k_w->ne[1], prefix});
+            slots.push_back({"v_" + std::to_string(il), layer.v_w->ne[1], prefix});
+        }
+    }
+
+    return slots;
+}
+
+// LFM3-Audio encoder: conv downsampler (8x in time) -> causal LFM2 stack -> MLP projector
+ggml_cgraph * clip_graph_lfm3a::build() {
+    const norm_type norm_t = NORM_TYPE_RMS;
+    const float rope_theta = hparams.rope_theta;
+    const int64_t prefix   = hparams.attn_window_size - 1;
+
+    // the graph always streams, a zero state encodes a clip from its start
+    std::map<std::string, ggml_tensor *> state_in;
+    std::vector<std::pair<std::string, ggml_tensor *>> state_out;
+    for (const auto & slot : list_lfm3a_state_slots(hparams, model)) {
+        ggml_tensor * t = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, slot.ne0, slot.ne1);
+        ggml_set_name(t, ("state_in_" + slot.name).c_str());
+        ggml_set_input(t);
+        state_in[slot.name] = t;
+    }
+
+    // stride-2 conv over [time, freq, ch]; the left column comes from the state and the right edge is zero-padded
+    auto conv_down = [&](ggml_tensor * x, const std::string & name, ggml_tensor * w, ggml_tensor * b, bool depthwise) {
+        ggml_tensor * left = ggml_reshape_4d(ctx0, state_in.at(name), 1, x->ne[1], x->ne[2], 1);
+        x = ggml_concat(ctx0, left, x, 0);
+        ggml_tensor * last = ggml_view_4d(ctx0, x, 1, x->ne[1], x->ne[2], 1, x->nb[1], x->nb[2], x->nb[3], (x->ne[0] - 1) * x->nb[0]);
+        state_out.push_back({name, ggml_reshape_2d(ctx0, ggml_cont(ctx0, last), 1, x->ne[1] * x->ne[2])});
+        x = ggml_pad_ext(ctx0, x, 0, 1, 0, 0, 0, 0, 0, 0);
+        x = depthwise ? ggml_conv_2d_dw_direct(ctx0, w, x, 2, 2, 0, 1, 1, 1)
+                      : ggml_conv_2d(ctx0, w, x, 2, 2, 0, 1, 1, 1);
+        return ggml_add(ctx0, x, b);
+    };
+
+    // inp: [n_frames, n_mel, 1]
+    ggml_tensor * cur = build_inp_raw(1);
+
+    // downsampler, same layout as the lfm2a pre-encoder but with SiLU
+    cur = conv_down(cur, "ds0", model.pre_encode_conv_X_w[0], model.pre_encode_conv_X_b[0], false);
+    cur = ggml_silu_inplace(ctx0, cur);
+
+    cur = conv_down(cur, "ds1", model.pre_encode_conv_X_w[2], model.pre_encode_conv_X_b[2], true);
+    cur = ggml_conv_2d_direct(ctx0, model.pre_encode_conv_X_w[3], cur, 1, 1, 0, 0, 1, 1);
+    cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[3]);
+    cur = ggml_silu_inplace(ctx0, cur);
+
+    cur = conv_down(cur, "ds2", model.pre_encode_conv_X_w[5], model.pre_encode_conv_X_b[5], true);
+    cur = ggml_conv_2d_direct(ctx0, model.pre_encode_conv_X_w[6], cur, 1, 1, 0, 0, 1, 1);
+    cur = ggml_add(ctx0, cur, model.pre_encode_conv_X_b[6]);
+    cur = ggml_silu_inplace(ctx0, cur);
+    cb(cur, "downsample.conv", -1);
+
+    // [n_pos, n_mel/8, n_ch] -> [n_ch * n_mel/8, n_pos]
+    const int n_pos = cur->ne[0];
+    cur = ggml_reshape_2d(ctx0, cur, n_pos, cur->ne[1] * cur->ne[2]);
+    cur = ggml_cont(ctx0, ggml_transpose(ctx0, cur));
+    cur = build_mm(model.pre_encode_out_w, cur);
+    cur = ggml_add(ctx0, cur, model.pre_encode_out_b);
+    cb(cur, "downsample.out", -1);
+
+    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, n_pos);
+    ggml_set_name(positions, "positions");
+    ggml_set_input(positions);
+    state_out.push_back({"pos", ggml_scale_bias(ctx0, state_in.at("pos"), 1.0f, (float) n_pos)});
+
+    // causal mask with a sliding window, shared by all attention layers
+    ggml_tensor * kq_mask = build_inp_attn_mask(prefix + n_pos, n_pos);
+
+    ggml_tensor * inpL = cur;
+
+    for (int il = 0; il < n_layer; il++) {
+        const auto & layer = model.layers[il];
+
+        cur = build_norm(inpL, layer.ln_1_w, nullptr, norm_t, eps, il);
+        cb(cur, "operator_norm", il);
+
+        if (layer.conv_dw_w) {
+            // short conv: in_proj -> b * x -> causal depthwise conv -> c * conv -> out_proj
+            ggml_tensor * bcx = build_mm(layer.conv_pw1_w, cur);
+            cb(bcx, "conv_in_proj", il);
+
+            ggml_tensor * b = ggml_view_2d(ctx0, bcx, n_embd, n_pos, bcx->nb[1], 0);
+            ggml_tensor * c = ggml_view_2d(ctx0, bcx, n_embd, n_pos, bcx->nb[1], ggml_row_size(bcx->type, n_embd));
+            ggml_tensor * x = ggml_view_2d(ctx0, bcx, n_embd, n_pos, bcx->nb[1], ggml_row_size(bcx->type, 2 * n_embd));
+
+            ggml_tensor * bx = ggml_cont(ctx0, ggml_transpose(ctx0, ggml_mul(ctx0, b, x)));
+            const int d_conv = layer.conv_dw_w->ne[0] - 1;
+            const std::string name = "conv_" + std::to_string(il);
+            bx = ggml_concat(ctx0, state_in.at(name), bx, 0);
+            state_out.push_back({name, ggml_cont(ctx0, ggml_view_2d(ctx0, bx, d_conv, n_embd, bx->nb[1], (size_t) n_pos * bx->nb[0]))});
+            bx = ggml_reshape_3d(ctx0, bx, bx->ne[0], n_embd, 1);
+
+            cur = ggml_ssm_conv(ctx0, bx, layer.conv_dw_w);
+            cur = ggml_reshape_2d(ctx0, cur, n_embd, n_pos);
+            cb(cur, "conv_conv", il);
+
+            cur = ggml_mul(ctx0, c, cur);
+            cur = build_mm(layer.conv_pw2_w, cur);
+            cb(cur, "conv_out_proj", il);
+        } else {
+            ggml_tensor * Qcur = build_mm(layer.q_w, cur);
+            ggml_tensor * Kcur = build_mm(layer.k_w, cur);
+            ggml_tensor * Vcur = build_mm(layer.v_w, cur);
+
+            Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head,    n_pos);
+            Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head_kv, n_pos);
+            Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head_kv, n_pos);
+
+            Qcur = build_norm(Qcur, layer.q_norm, nullptr, norm_t, eps, il);
+            Kcur = build_norm(Kcur, layer.k_norm, nullptr, norm_t, eps, il);
+
+            Qcur = ggml_rope_ext(ctx0, Qcur, positions, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0,
+                                 rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+            Kcur = ggml_rope_ext(ctx0, Kcur, positions, nullptr, d_head, GGML_ROPE_TYPE_NEOX, 0,
+                                 rope_theta, 1.0f, 0.0f, 1.0f, 0.0f, 0.0f);
+
+            // the cached window goes in front, this chunk's tail is kept for the next call
+            const std::string k_name = "k_" + std::to_string(il);
+            const std::string v_name = "v_" + std::to_string(il);
+            const int64_t n_embd_kv = d_head * n_head_kv;
+            ggml_tensor * K = ggml_concat(ctx0, state_in.at(k_name), ggml_reshape_2d(ctx0, Kcur, n_embd_kv, n_pos), 1);
+            ggml_tensor * V = ggml_concat(ctx0, state_in.at(v_name), ggml_reshape_2d(ctx0, Vcur, n_embd_kv, n_pos), 1);
+            state_out.push_back({k_name, ggml_cont(ctx0, ggml_view_2d(ctx0, K, n_embd_kv, prefix, K->nb[1], (size_t) n_pos * K->nb[1]))});
+            state_out.push_back({v_name, ggml_cont(ctx0, ggml_view_2d(ctx0, V, n_embd_kv, prefix, V->nb[1], (size_t) n_pos * V->nb[1]))});
+            Kcur = ggml_reshape_3d(ctx0, K, d_head, n_head_kv, prefix + n_pos);
+            Vcur = ggml_reshape_3d(ctx0, V, d_head, n_head_kv, prefix + n_pos);
+
+            cur = build_attn(layer.o_w, nullptr, Qcur, Kcur, Vcur, kq_mask, kq_scale, il);
+            cb(cur, "attn_out", il);
+        }
+
+        cur = ggml_add(ctx0, cur, inpL);
+        inpL = cur;
+
+        cur = build_norm(cur, layer.ln_2_w, nullptr, norm_t, eps, il);
+        cb(cur, "ffn_norm", il);
+
+        cur = build_ffn(cur,
+            layer.ff_up_w, nullptr,
+            layer.ff_gate_w, nullptr,
+            layer.ff_down_w, nullptr,
+            FFN_SILU, il);
+        cb(cur, "ffn_out", il);
+
+        cur = ggml_add(ctx0, inpL, cur);
+        inpL = cur;
+    }
+
+    cur = build_norm(inpL, model.post_ln_w, nullptr, norm_t, eps, -1);
+    cb(cur, "embedding_norm", -1);
+
+    // projector: LayerNorm -> Linear -> GELU -> Linear
+    cur = build_norm(cur, model.mm_0_w, model.mm_0_b, NORM_TYPE_NORMAL, 1e-5, -1);
+    cur = build_ffn(cur, model.mm_1_w, model.mm_1_b, nullptr, nullptr, model.mm_3_w, model.mm_3_b, FFN_GELU_ERF, -1);
+    cb(cur, "projected", -1);
+
+    // the state outputs go first so that the embeddings stay the last node of the graph
+    for (const auto & s : state_out) {
+        ggml_set_name(s.second, ("state_out_" + s.first).c_str());
+        ggml_set_output(s.second);
+        ggml_build_forward_expand(gf, s.second);
+    }
+
+    ggml_build_forward_expand(gf, cur);
+
+    return gf;
+}

tools/mtmd/models/lfm3vl.cpp

@@ -0,0 +1,182 @@
+#include "models.h"
+
+ggml_cgraph * clip_graph_lfm3vl::build() {
+    GGML_ASSERT(model.patch_embeddings_0 != nullptr);
+    GGML_ASSERT(model.lfm3vl_embedding_norm_w != nullptr);
+    GGML_ASSERT(model.class_embedding == nullptr);
+    GGML_ASSERT(n_batch == 1);
+
+    const int batch_size       = 1;
+    const int n_pos            = n_patches;
+    const int num_position_ids = n_pos * 4; // m-rope requires 4 dim per position
+
+    const norm_type norm_t = NORM_TYPE_RMS;
+
+    // 2D rope: pairs (j, j+n_dims); first d_head/4 pairs rotate by y, next d_head/4 by x
+    int mrope_sections[4] = {d_head/4, d_head/4, 0, 0};
+
+    ggml_tensor * inp_raw = build_inp_raw();
+
+    // one patchifier kernel per temporal slice: a still image fills both slices, so both kernels apply to it
+    ggml_tensor * inp = ggml_conv_2d(ctx0, model.patch_embeddings_0, inp_raw, patch_size, patch_size, 0, 0, 1, 1);
+    if (model.patch_embeddings_1) {
+        inp = ggml_add(ctx0, inp, ggml_conv_2d(ctx0, model.patch_embeddings_1, inp_raw, patch_size, patch_size, 0, 0, 1, 1));
+    }
+    inp = ggml_reshape_2d(ctx0, inp, n_pos, n_embd);
+    inp = ggml_cont(ctx0, ggml_transpose(ctx0, inp));
+    cb(inp, "patchifier_out", -1);
+
+    // the 2x2 interleave waits for the projector, the conv layers need raster-scan order
+    inp = build_norm(inp, model.lfm3vl_embedding_norm_w, nullptr, norm_t, eps, -1);
+    cb(inp, "embedding_norm", -1);
+
+    ggml_tensor * inpL = inp;
+
+    // position IDs for 2D RoPE (same format as Qwen3VL)
+    ggml_tensor * positions = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, num_position_ids);
+    ggml_set_name(positions, "positions");
+    ggml_set_input(positions);
+
+    const float rope_theta = hparams.rope_theta;
+
+    for (int il = 0; il < n_layer; il++) {
+        auto & layer = model.layers[il];
+
+        const bool is_conv_layer = (layer.conv2d_in_proj_w != nullptr);
+
+        ggml_tensor * cur = inpL;
+
+        cur = build_norm(cur, layer.ln_1_w, layer.ln_1_b, norm_t, eps, il);
+        cb(cur, "ln1", il);
+
+        if (is_conv_layer) {
+            ggml_tensor * proj = build_mm(layer.conv2d_in_proj_w, cur);
+            cb(proj, "conv_in_proj", il);
+
+            ggml_tensor * first  = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+                proj->nb[1], 0);
+            ggml_tensor * middle = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+                proj->nb[1], ggml_row_size(proj->type, n_embd));
+            ggml_tensor * third  = ggml_view_2d(ctx0, proj, n_embd, n_pos,
+                proj->nb[1], ggml_row_size(proj->type, 2 * n_embd));
+
+            ggml_tensor * gated = ggml_mul(ctx0, first, third);
+            cb(gated, "conv_gated", il);
+
+            ggml_tensor * conv_out;
+            if (layer.conv2d_spatial_w) {
+                ggml_tensor * gated_3d = ggml_reshape_4d(ctx0, gated,
+                    n_embd, n_patches_x, n_patches_y, batch_size);
+                ggml_tensor * gated_2d = ggml_cont(ctx0,
+                    ggml_permute(ctx0, gated_3d, 2, 0, 1, 3));
+                cb(gated_2d, "conv_gated_2d", il);
+
+                conv_out = ggml_conv_2d_dw_direct(ctx0, layer.conv2d_spatial_w, gated_2d,
+                    1, 1, 1, 1, 1, 1);
+                cb(conv_out, "conv_spatial", il);
+
+                conv_out = ggml_cont(ctx0,
+                    ggml_permute(ctx0, conv_out, 1, 2, 0, 3));
+                conv_out = ggml_reshape_2d(ctx0, conv_out, n_embd, n_pos);
+            } else {
+                conv_out = gated;
+            }
+
+            cur = ggml_mul(ctx0, middle, conv_out);
+            cb(cur, "conv_mul", il);
+
+            cur = build_mm(layer.conv2d_out_proj_w, cur);
+            cb(cur, "conv_out_proj", il);
+        } else {
+            ggml_tensor * Qcur = build_mm(layer.q_w, cur);
+            ggml_tensor * Kcur = build_mm(layer.k_w, cur);
+            ggml_tensor * Vcur = build_mm(layer.v_w, cur);
+
+            cb(Qcur, "Qcur", il);
+            cb(Kcur, "Kcur", il);
+            cb(Vcur, "Vcur", il);
+
+            Qcur = ggml_reshape_3d(ctx0, Qcur, d_head, n_head,    n_pos);
+            Kcur = ggml_reshape_3d(ctx0, Kcur, d_head, n_head_kv, n_pos);
+            Vcur = ggml_reshape_3d(ctx0, Vcur, d_head, n_head_kv, n_pos);
+
+            if (layer.q_norm) {
+                Qcur = build_norm(Qcur, layer.q_norm, nullptr, norm_t, eps, il);
+            }
+            if (layer.k_norm) {
+                Kcur = build_norm(Kcur, layer.k_norm, nullptr, norm_t, eps, il);
+            }
+
+            Qcur = ggml_rope_multi(
+                ctx0, Qcur, positions, nullptr,
+                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
+                32768, rope_theta, 1, 0, 1, 32, 1);
+            Kcur = ggml_rope_multi(
+                ctx0, Kcur, positions, nullptr,
+                d_head/2, mrope_sections, GGML_ROPE_TYPE_VISION,
+                32768, rope_theta, 1, 0, 1, 32, 1);
+
+            cb(Qcur, "Qcur_rope", il);
+            cb(Kcur, "Kcur_rope", il);
+
+            cur = build_attn(layer.o_w, layer.o_b,
+                Qcur, Kcur, Vcur, nullptr, kq_scale, il);
+            cb(cur, "attn_out", il);
+        }
+
+        cur = ggml_add(ctx0, cur, inpL);
+        inpL = cur;
+        cb(cur, "ffn_inp", il);
+
+        cur = build_norm(cur, layer.ln_2_w, layer.ln_2_b, norm_t, eps, il);
+        cb(cur, "ffn_inp_normed", il);
+
+        cur = build_ffn(cur,
+            layer.ff_up_w, layer.ff_up_b,
+            layer.ff_gate_w, layer.ff_gate_b,
+            layer.ff_down_w, layer.ff_down_b,
+            hparams.ffn_op, il);
+        cb(cur, "ffn_out", il);
+
+        cur = ggml_add(ctx0, inpL, cur);
+        cb(cur, "layer_out", il);
+
+        inpL = cur;
+    }
+
+    // no post-layernorm for LFM3-VL
+
+    ggml_tensor * embeddings = inpL;
+
+    // projector input LayerNorm, eps is projector_layer_norm_eps (not the encoder eps)
+    embeddings = build_norm(embeddings, model.mm_input_norm_w, model.mm_input_norm_b, NORM_TYPE_NORMAL, 1e-6f, -1);
+    cb(embeddings, "proj_input_norm", -1);
+
+    // 2x2 patch interleave (same as Qwen3VL) so the reshape below merges spatial 2x2 blocks
+    {
+        embeddings = ggml_cont_4d(
+            ctx0, embeddings,
+            n_embd * 2, n_patches_x / 2, n_patches_y, batch_size);
+        embeddings = ggml_reshape_4d(
+            ctx0, embeddings,
+            n_embd * 2, n_patches_x / 2, 2, batch_size * (n_patches_y / 2));
+        embeddings = ggml_permute(ctx0, embeddings, 0, 2, 1, 3);
+        embeddings = ggml_cont_3d(
+            ctx0, embeddings,
+            n_embd, n_patches_x * n_patches_y, batch_size);
+    }
+    cb(embeddings, "proj_interleaved", -1);
+
+    embeddings = ggml_reshape_3d(ctx0, embeddings, n_embd * 4, n_pos / 4, batch_size);
+
+    // projector MLP: linear_1 -> GELU (erf, as HF "gelu") -> linear_2
+    embeddings = build_ffn(embeddings,
+        model.mm_0_w, model.mm_0_b,
+        nullptr, nullptr,
+        model.mm_1_w, model.mm_1_b,
+        ffn_op_type::FFN_GELU_ERF, -1);
+
+    ggml_build_forward_expand(gf, embeddings);
+
+    return gf;
+}

tools/mtmd/models/models.h

@@ -2,6 +2,7 @@
 
 #include "../clip-graph.h"
 
+#include <functional>
 #include <map>
 #include <string>
 #include <utility>
@@ -54,6 +55,11 @@ struct clip_graph_sensenova_u1 : clip_graph {
     ggml_cgraph * build() override;
 };
 
+struct clip_graph_lfm3vl : clip_graph {
+    clip_graph_lfm3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+    ggml_cgraph * build() override;
+};
+
 struct clip_graph_ling3vl : clip_graph_qwen3vl {
     clip_graph_ling3vl(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph_qwen3vl(ctx, img) {}
     ggml_cgraph * build() override;
@@ -184,6 +190,11 @@ struct clip_graph_conformer : clip_graph {
     ggml_cgraph * build() override;
 };
 
+struct clip_graph_lfm3a : clip_graph {
+    clip_graph_lfm3a(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
+    ggml_cgraph * build() override;
+};
+
 struct clip_graph_granite_speech : clip_graph {
     clip_graph_granite_speech(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
     ggml_cgraph * build() override;
@@ -428,6 +439,39 @@ struct clip_graph_soprano : clip_graph {
     int n_frames;
 };
 
+//
+// LFM3-Audio generation:
+// GEN_CODE = one MaskGIT pass of the depthformer, or only the feedback embedding once all codes are known
+// GEN_WAV  = liquid-codec decoder, a window of frames -> polar STFT frames
+//
+struct clip_graph_lfm3a_gen : clip_graph {
+    clip_graph_lfm3a_gen(clip_ctx * ctx, const clip_image_f32 & img, clip_gen_process_type gen_process, bool all_known, int n_frames)
+        : clip_graph(ctx, img), gen_process(gen_process), all_known(all_known), n_frames(n_frames) {}
+    ggml_cgraph * build() override;
+
+    clip_gen_process_type gen_process;
+    bool all_known; // GEN_CODE only
+    int  n_frames;  // GEN_WAV only
+
+    ggml_tensor * build_gen_code();
+    ggml_tensor * build_gen_wav();
+
+    ggml_tensor * build_lfm_layer(ggml_tensor * inpL, const clip_layer & layer, float eps, int il,
+                                  const std::function<ggml_tensor *(ggml_tensor *)> & op);
+    ggml_tensor * build_lfm_attn(ggml_tensor * cur, const clip_layer & layer, int n_head, int n_head_kv,
+                                 float eps, float rope_theta, int il, ggml_tensor * inp_pos,
+                                 ggml_tensor * kq_mask, ggml_tensor * k_cache, ggml_tensor * v_cache,
+                                 ggml_tensor ** k_all, ggml_tensor ** v_all);
+    ggml_tensor * build_lfm_conv(ggml_tensor * cur, const clip_layer & layer, int il,
+                                 ggml_tensor * conv_state, ggml_tensor ** conv_state_out);
+};
+
+// state slots of the streaming codec decoder (lfm3a_gen GEN_WAV)
+std::vector<c2w_state_slot> list_lfm3a_gen_state_slots(const clip_hparams & hparams, const clip_model & model);
+
+// state slots of the streaming LFM3-Audio encoder (lfm3a)
+std::vector<c2w_state_slot> list_lfm3a_state_slots(const clip_hparams & hparams, const clip_model & model);
+
 struct clip_graph_kimik25 : clip_graph {
     clip_graph_kimik25(clip_ctx * ctx, const clip_image_f32 & img) : clip_graph(ctx, img) {}
     ggml_cgraph * build() override;

tools/mtmd/mtmd-audio.cpp

@@ -1055,6 +1055,61 @@ bool mtmd_audio_preprocessor_d1omni::preprocess(const float *                 sa
     return true;
 }
 
+//
+// mtmd_audio_preprocessor_lfm3a
+//
+
+void mtmd_audio_preprocessor_lfm3a::initialize() {
+    cache.fill_sin_cos_table(hparams.audio_n_fft);
+    cache.fill_hann_window(hparams.audio_window_len, true);
+    cache.fill_mel_filterbank_matrix(hparams.n_mel_bins, hparams.audio_n_fft, hparams.audio_sample_rate);
+}
+
+bool mtmd_audio_preprocessor_lfm3a::preprocess(const float *                 samples,
+                                               size_t                        n_samples,
+                                               std::vector<mtmd_audio_mel> & output) const {
+    if (n_samples == 0) {
+        return false;
+    }
+
+    const size_t win = hparams.audio_window_len;
+    const size_t hop = hparams.audio_hop_len;
+
+    // pad on the right so that no sample is dropped by the last frame
+    size_t n_padded = win;
+    if (n_samples > win) {
+        n_padded = n_samples + (hop - (n_samples - win) % hop) % hop;
+    }
+    std::vector<float> padded(samples, samples + n_samples);
+    padded.resize(n_padded, 0.0f);
+
+    filter_params params;
+    params.n_mel            = hparams.n_mel_bins;
+    params.n_fft_bins       = 1 + (hparams.audio_n_fft / 2);
+    params.hann_window_size = hparams.audio_window_len;
+    params.hop_length       = hparams.audio_hop_len;
+    params.sample_rate      = hparams.audio_sample_rate;
+    params.no_padding       = true;
+    params.mel_floor        = 1e-12f; // log10 floor of -12
+
+    GGML_ASSERT(!cache.sin_vals.empty());
+    GGML_ASSERT(!cache.cos_vals.empty());
+    GGML_ASSERT(!cache.filters.data.empty());
+
+    mtmd_audio_mel out;
+    bool ok = log_mel_spectrogram(padded.data(), (int) padded.size(), 4, params, cache, out);
+    if (!ok) {
+        return false;
+    }
+
+    for (auto & v : out.data) {
+        v = (v + 4.0f) / 4.0f;
+    }
+
+    output.push_back(std::move(out));
+    return true;
+}
+
 //
 // mtmd_audio_preprocessor_granite_speech
 //

tools/mtmd/mtmd-audio.h

@@ -95,6 +95,15 @@ struct mtmd_audio_preprocessor_d1omni : mtmd_audio_preprocessor_conformer {
     bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
 };
 
+struct mtmd_audio_preprocessor_lfm3a : mtmd_audio_preprocessor {
+    mtmd_audio_preprocessor_lfm3a(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
+    void initialize() override;
+    bool preprocess(const float * samples, size_t n_samples, std::vector<mtmd_audio_mel> & output) const override;
+
+  private:
+    mtmd_audio_cache cache;
+};
+
 struct mtmd_audio_preprocessor_granite_speech : mtmd_audio_preprocessor {
     mtmd_audio_preprocessor_granite_speech(const clip_ctx * ctx) : mtmd_audio_preprocessor(ctx) {}
     void initialize() override;

tools/mtmd/mtmd-cli.cpp

@@ -120,7 +120,8 @@ struct mtmd_cli_context {
         init_vision_context(params);
 
         mtmd_cap = mtmd_helper_get_capabilities(model, ctx_vision.get());
-        if (!mtmd_cap.chat) {
+        // a realtime transcription model has no chat, it streams the audio below
+        if (!mtmd_cap.chat && !mtmd_cap.audio_stream) {
             LOG_ERR("Model does not support chat mode\n");
             LOG_ERR("Hint: for TTS models, please use llama-tts\n");
             exit(1);
@@ -567,7 +568,10 @@ int main(int argc, char ** argv) {
 
     if (g_is_interrupted) return 130;
 
-    if (ctx.mtmd_cap.audio_stream) {
+    // realtime transcription models (no chat) transcribe while they listen, the others answer after the audio
+    const bool transcribes = ctx.mtmd_cap.audio_stream && !ctx.mtmd_cap.chat;
+
+    if (transcribes) {
         if (params.image.size() != 1) {
             LOG_ERR("This model transcribes audio in realtime, pass exactly one --audio file\n");
             return 1;

tools/mtmd/mtmd-helper-gen.cpp

@@ -8,7 +8,9 @@
 #include <cctype>
 #include <cmath>
 #include <cstring>
+#include <functional>
 #include <memory>
+#include <random>
 #include <string>
 #include <unordered_map>
 #include <vector>
@@ -103,6 +105,11 @@ public:
     virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
     // forces any buffered codes through code2wav now, regardless of window_frames
     virtual int32_t flush() = 0;
+    // continuing a sequence (inp->n_past > 0) with the model's own text/audio switching
+    virtual bool supports_continue() const { return false; }
+    virtual bool next_is_text() const { return false; }
+
+    std::vector<llama_logit_bias> logit_bias; // see mtmd_helper_gen_audio_get_logit_bias()
 
 protected:
     // one state per generation, so a fixed seed always gives the same output
@@ -1261,6 +1268,455 @@ private:
     bool sent    = false;
 };
 
+// LFM3-Audio TTS, see generate() in the reference: each step samples a frame of codes by MaskGIT
+// and feeds its embedding back, the end-of-audio code stops
+class lfm3a_gen_audio_pipeline : public mtmd_gen_audio_pipeline {
+public:
+    using mtmd_gen_audio_pipeline::mtmd_gen_audio_pipeline;
+
+    void reset() override {
+        seq_id = 0;
+        pos = 0;
+        continuing = false;
+        mode = MODE_AUDIO;
+        slots_left = 0;
+        speaking = false;
+        text_done = false;
+        prompt_tokens.clear();
+        prompt_pos = 0;
+        hist.clear();
+        gen_state.reset();
+        pending_tail = false;
+        pcm_sent = 0;
+        wav_header_sent = false;
+        audio_pcm.clear();
+        h_state_buf.clear();
+        out_buf.clear();
+    }
+
+    int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+        reset();
+        seq_id = inp->seq_id;
+
+        if (inp->speaker_ref) {
+            LOG_WRN("mtmd_helper_gen_audio: the model has no voice conditioning, the speaker file is ignored\n");
+        }
+
+        // the prompt is rendered by the caller: the "tts" chat template of the model, or the next turn of a conversation
+        continuing = inp->n_past > 0;
+        pos        = continuing ? inp->n_past : 0;
+        const std::string full(inp->prompt, inp->prompt_len);
+        prompt_tokens.resize(full.size() + 16);
+        const int n_ids = llama_tokenize(vocab, full.c_str(), (int32_t) full.size(), prompt_tokens.data(),
+                                         (int32_t) prompt_tokens.size(), false, true);
+        if (n_ids <= 0) {
+            LOG_ERR("mtmd_helper_gen_audio: tokenization failed\n");
+            return 1;
+        }
+        prompt_tokens.resize((size_t) n_ids);
+
+        if (!find_tokens()) {
+            return 1;
+        }
+        // a continued response opens with the token of the asked mode, a text-only one cannot switch to audio
+        if (continuing) {
+            switch (inp->out_mode) {
+                case MTMD_HELPER_GEN_AUDIO_OUT_AUDIO: prompt_tokens.push_back(tok_audio_start); break;
+                case MTMD_HELPER_GEN_AUDIO_OUT_MIXED: prompt_tokens.push_back(tok_mixed_start); break;
+                case MTMD_HELPER_GEN_AUDIO_OUT_TEXT:
+                    logit_bias = { { tok_audio_start, -INFINITY }, { tok_mixed_start, -INFINITY } };
+                    break;
+                default: break;
+            }
+        }
+        // the last prompt token picks the mode, see generate() of the reference
+        const llama_token last = prompt_tokens.back();
+        mode = last == tok_audio_start ? MODE_AUDIO : last == tok_mixed_start ? MODE_MIXED : MODE_TEXT;
+        slots_left = n_text_block;
+        if (!continuing && mode != MODE_AUDIO) {
+            LOG_ERR("mtmd_helper_gen_audio: the prompt must come from the \"tts\" chat template of the model\n");
+            return 1;
+        }
+
+        // the sampling defaults of the model, the caller's top-k/top-p are not used
+        const mtmd_gen_inp def = mtmd_gen_inp_default(mctx);
+        temp        = def.temp;
+        top_k       = def.top_k;
+        rep_penalty = def.penalty_repeat;
+        rep_window  = def.penalty_last_n;
+
+        out_type = inp->out_type;
+        stream   = inp->stream;
+        rng.seed(inp->seed == UINT32_MAX ? std::random_device{}() : inp->seed);
+        return init_gen_state(inp->seed) ? 0 : 1;
+    }
+
+    int32_t step_prompt(int32_t n_batch) override {
+        GGML_ASSERT(n_batch > 0);
+        const int n_prompt = (int) prompt_tokens.size();
+        if (prompt_pos >= n_prompt) {
+            return 0;
+        }
+        const int n = std::min(n_batch, n_prompt - prompt_pos);
+        llama_batch batch = llama_batch_init(n, 0, 1);
+        for (int i = 0; i < n; i++) {
+            batch.token   [i]    = prompt_tokens[(size_t) (prompt_pos + i)];
+            batch.pos     [i]    = pos + i;
+            batch.n_seq_id[i]    = 1;
+            batch.seq_id  [i][0] = seq_id;
+            batch.logits  [i]    = prompt_pos + i == n_prompt - 1;
+        }
+        batch.n_tokens = n;
+        const int ret = llama_decode(lctx, batch);
+        llama_batch_free(batch);
+        if (ret != 0) {
+            LOG_ERR("mtmd_helper_gen_audio: prompt decode failed\n");
+            return -1;
+        }
+        pos        += n;
+        prompt_pos += n;
+        return n_prompt - prompt_pos;
+    }
+
+    bool supports_continue() const override {
+        return true;
+    }
+
+    bool next_is_text() const override {
+        return mode == MODE_TEXT || (mode == MODE_MIXED && !speaking);
+    }
+
+    // one generate() step: a text token from the caller or an audio frame from the hidden state
+    int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) override {
+        const bool interleaving = mode == MODE_MIXED;
+        if (next_is_text()) {
+            // <|im_end|> ends the response in any text slot, <|text_end|> outside an interleaved block too
+            if (sampled == tok_im_end || (!interleaving && sampled == tok_text_end)) {
+                *out_stop    = true;
+                *h_state_out = nullptr;
+                return 0;
+            }
+            // the switching tokens are fed back like any text token
+            if (!interleaving) {
+                if (sampled == tok_audio_start) {
+                    mode = MODE_AUDIO;
+                } else if (sampled == tok_mixed_start) {
+                    mode       = MODE_MIXED;
+                    slots_left = n_text_block;
+                    speaking   = false;
+                    text_done  = false;
+                }
+            } else if (sampled == tok_text_end) {
+                text_done = true;
+                speaking  = true;
+            }
+            llama_batch batch = llama_batch_get_one(&sampled, 1);
+            if (!decode(batch)) {
+                return 1;
+            }
+        } else {
+            std::vector<int32_t> codes;
+            const float * fb = nullptr;
+            bool is_eos = false;
+            if (!sample_frame(h_state_in, codes, &fb, &is_eos)) {
+                return 1;
+            }
+            if (is_eos) {
+                if (!continuing) {
+                    // TTS: the end of audio ends the generation
+                    *out_stop    = true;
+                    *h_state_out = nullptr;
+                    return 0;
+                }
+                // the segment ends, the codec tail goes out and the response continues in text
+                if (flush() != 0) {
+                    return 1;
+                }
+                mode = MODE_TEXT;
+            } else {
+                hist.insert(hist.end(), codes.begin(), codes.end());
+                if (!decode_wav(codes.data(), codes.size())) {
+                    return 1;
+                }
+            }
+            decode_embd_batch batch_embd(const_cast<float *>(fb), 1, 1, n_embd);
+            batch_embd.set_position_normal(pos, seq_id);
+            batch_embd.logits[0] = 1;
+            if (llama_process(lctx, LLAMA_PROCESS_TYPE_DECODE, batch_embd.render(lctx, 0, 1)) != 0) {
+                LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+                return 1;
+            }
+            pos++;
+        }
+
+        // a slot emitted while interleaving counts down the block, the switching token itself does not
+        if (interleaving && !text_done && --slots_left == 0) {
+            speaking   = !speaking;
+            slots_left = speaking ? n_audio_block : n_text_block;
+        }
+
+        const float * he = llama_get_embeddings_ith(lctx, -1);
+        h_state_buf.assign(he, he + n_embd);
+        *h_state_out = h_state_buf.data();
+        return 0;
+    }
+
+    int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
+        *out_sample_rate = info.sample_rate;
+
+        if (!stream) {
+            if (flush() != 0) {
+                return 1;
+            }
+            if (out_n_samples) {
+                *out_n_samples = (int64_t) audio_pcm.size();
+            }
+            if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+                *out_data     = (const char *) audio_pcm.data();
+                *out_data_len = audio_pcm.size() * sizeof(float);
+                return 0;
+            }
+            out_buf.clear();
+            if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
+                LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+                return 1;
+            }
+            *out_data     = out_buf.data();
+            *out_data_len = out_buf.size();
+            return 0;
+        }
+
+        // streaming: only the audio produced since the previous call, every frame is decoded as it is sampled
+        const size_t n_new = audio_pcm.size() - pcm_sent;
+        if (out_n_samples) {
+            *out_n_samples = (int64_t) n_new;
+        }
+        if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+            *out_data     = (const char *) (audio_pcm.data() + pcm_sent);
+            *out_data_len = n_new * sizeof(float);
+            pcm_sent = audio_pcm.size();
+            return 0;
+        }
+        out_buf.clear();
+        if (!wav_header_sent) {
+            write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
+            wav_header_sent = true;
+        }
+        append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+        pcm_sent = audio_pcm.size();
+        *out_data     = out_buf.data();
+        *out_data_len = out_buf.size();
+        return 0;
+    }
+
+    // the ISTFT holds back the overlap of the last frame
+    int32_t flush() override {
+        if (pending_tail) {
+            if (!decode_wav(nullptr, 0)) {
+                return 1;
+            }
+            pending_tail = false;
+        }
+        return 0;
+    }
+
+private:
+    // checked by the converter
+    static constexpr int   n_cb        = 12;
+    static constexpr int   schedule[]  = { 4, 4, 4 }; // codebooks revealed per depthformer pass
+
+    // a random order of the codebooks, revealed by the schedule; fb is the feedback embedding of the codes.
+    // at the end of audio every code is the end code, its feedback embedding is only built when continuing
+    bool sample_frame(const float * h_state, std::vector<int32_t> & codes, const float ** fb, bool * is_eos) {
+        std::vector<int> order(n_cb);
+        for (int i = 0; i < n_cb; i++) {
+            order[(size_t) i] = i;
+        }
+        std::shuffle(order.begin(), order.end(), rng);
+
+        codes.assign(n_cb, -1);
+        mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+        inp.type    = MTMD_GEN_PROCESS_TYPE_GEN_CODE;
+        inp.embd    = const_cast<float *>(h_state);
+        inp.codes   = codes.data();
+        inp.n_codes = codes.size();
+        mtmd_gen_out out{};
+
+        int k = 0;
+        for (int n_pick : schedule) {
+            if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0 || out.n_feats == 0 || out.n_feats % n_cb != 0) {
+                LOG_ERR("mtmd_helper_gen_audio: depthformer pass failed\n");
+                return false;
+            }
+            const size_t n_vocab = out.n_feats / n_cb;
+            logits.assign(out.feats, out.feats + out.n_feats);
+            for (int j = 0; j < n_pick; j++, k++) {
+                const int i = order[(size_t) k];
+                float * l = logits.data() + (size_t) i * n_vocab;
+                apply_rep_penalty(l, n_vocab, i);
+                codes[(size_t) i] = sample(l, n_vocab);
+            }
+        }
+
+        // the last code of every codebook ends the audio
+        const int32_t eos = (int32_t) (logits.size() / n_cb) - 1;
+        *is_eos = std::find(codes.begin(), codes.end(), eos) != codes.end();
+        if (*is_eos) {
+            codes.assign(n_cb, eos);
+            if (!continuing) {
+                *fb = nullptr;
+                return true;
+            }
+        }
+
+        // all codes known: only the feedback embedding
+        if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+            LOG_ERR("mtmd_helper_gen_audio: codebook embedding failed\n");
+            return false;
+        }
+        fb_buf.assign(out.embd, out.embd + n_embd);
+        *fb = fb_buf.data();
+        return true;
+    }
+
+    // CTRL penalty over the codes codebook cb took in the last rep_window frames
+    void apply_rep_penalty(float * l, size_t n_vocab, int cb) const {
+        if (rep_penalty <= 1.0f || rep_window <= 0) {
+            return;
+        }
+        const size_t n_hist = hist.size() / n_cb;
+        const size_t n_win  = std::min(n_hist, (size_t) rep_window);
+        std::vector<bool> seen(n_vocab, false);
+        for (size_t f = n_hist - n_win; f < n_hist; f++) {
+            const int32_t c = hist[f * n_cb + (size_t) cb];
+            if (c >= 0 && (size_t) c < n_vocab && !seen[(size_t) c]) {
+                seen[(size_t) c] = true;
+                l[c] = l[c] > 0.0f ? l[c] / rep_penalty : l[c] * rep_penalty;
+            }
+        }
+    }
+
+    // TODO: reuse the samplers of common (top-k, temperature, penalties) instead of this one
+    int32_t sample(const float * l, size_t n_vocab) {
+        if (temp <= 0.0f) {
+            return (int32_t) (std::max_element(l, l + n_vocab) - l);
+        }
+        // top-k keeps the codes scoring at least the k-th best, ties included
+        float lmin = -INFINITY;
+        if (top_k > 0 && (size_t) top_k < n_vocab) {
+            std::vector<float> sorted(l, l + n_vocab);
+            std::nth_element(sorted.begin(), sorted.begin() + (top_k - 1), sorted.end(), std::greater<float>());
+            lmin = sorted[(size_t) top_k - 1];
+        }
+        const float lmax = *std::max_element(l, l + n_vocab);
+        std::vector<double> prob(n_vocab);
+        double sum = 0.0;
+        for (size_t v = 0; v < n_vocab; v++) {
+            prob[v] = l[v] < lmin ? 0.0 : std::exp((double) (l[v] - lmax) / temp);
+            sum += prob[v];
+        }
+        double r = std::uniform_real_distribution<double>(0.0, 1.0)(rng) * sum;
+        for (size_t v = 0; v < n_vocab; v++) {
+            r -= prob[v];
+            if (r <= 0.0) {
+                return (int32_t) v;
+            }
+        }
+        return (int32_t) n_vocab - 1;
+    }
+
+    // n_codes == 0 flushes the ISTFT tail
+    bool decode_wav(const int32_t * codes, size_t n_codes) {
+        mtmd_gen_inp inp = mtmd_gen_inp_default(mctx);
+        inp.type       = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
+        inp.codes      = const_cast<int32_t *>(codes);
+        inp.n_codes    = n_codes;
+        mtmd_gen_out out{};
+        if (mtmd_gen_audio_process(mctx, gen_state.get(), &inp, &out) != 0) {
+            LOG_ERR("mtmd_helper_gen_audio: codec decode failed\n");
+            return false;
+        }
+        audio_pcm.insert(audio_pcm.end(), out.audio, out.audio + out.n_samples);
+        pending_tail = n_codes > 0;
+        return true;
+    }
+
+    bool find_tokens() {
+        tok_audio_start = find_special_token(vocab, "<|audio_start|>");
+        tok_mixed_start = find_special_token(vocab, "<|mixed_start|>");
+        tok_text_end    = find_special_token(vocab, "<|text_end|>");
+        tok_im_end      = find_special_token(vocab, "<|im_end|>");
+        if (tok_audio_start == LLAMA_TOKEN_NULL || tok_mixed_start == LLAMA_TOKEN_NULL ||
+            tok_text_end == LLAMA_TOKEN_NULL || tok_im_end == LLAMA_TOKEN_NULL) {
+            LOG_ERR("mtmd_helper_gen_audio: missing LFM3-Audio control tokens in vocab\n");
+            return false;
+        }
+        return true;
+    }
+
+    bool decode(llama_batch batch) {
+        std::vector<llama_pos>      positions(batch.n_tokens);
+        std::vector<int32_t>        n_seq(batch.n_tokens, 1);
+        std::vector<llama_seq_id>   seq(batch.n_tokens, seq_id);
+        std::vector<llama_seq_id *> seq_ptr(batch.n_tokens);
+        std::vector<int8_t>         logits(batch.n_tokens, 1);
+        for (int i = 0; i < batch.n_tokens; i++) {
+            positions[i] = pos + i;
+            seq_ptr[i]   = &seq[i];
+        }
+        batch.pos      = positions.data();
+        batch.n_seq_id = n_seq.data();
+        batch.seq_id   = seq_ptr.data();
+        batch.logits   = logits.data();
+        if (llama_decode(lctx, batch) != 0) {
+            LOG_ERR("mtmd_helper_gen_audio: decode failed\n");
+            return false;
+        }
+        pos += batch.n_tokens;
+        return true;
+    }
+
+    // generate() modes: text, audio until end of audio, interleaved until <|text_end|>
+    enum gen_mode { MODE_TEXT, MODE_AUDIO, MODE_MIXED };
+    // checked by the converter
+    static constexpr int n_text_block  = 6;  // interleaved_n_text
+    static constexpr int n_audio_block = 12; // interleaved_n_audio
+
+    llama_token tok_audio_start = LLAMA_TOKEN_NULL;
+    llama_token tok_mixed_start = LLAMA_TOKEN_NULL;
+    llama_token tok_text_end    = LLAMA_TOKEN_NULL;
+    llama_token tok_im_end      = LLAMA_TOKEN_NULL;
+
+    bool     continuing = false; // the prompt continues a sequence, the response follows the model's switching
+    gen_mode mode       = MODE_AUDIO;
+    int      slots_left = 0;
+    bool     speaking   = false; // interleaving: the current block is audio
+    bool     text_done  = false; // interleaving: <|text_end|> was emitted, audio until the end-of-audio frame
+
+    llama_seq_id seq_id = 0;
+    int pos = 0;
+    std::vector<llama_token> prompt_tokens;
+    int prompt_pos = 0;
+
+    float   temp        = 0.0f;
+    int32_t top_k       = 0;
+    float   rep_penalty = 1.0f;
+    int32_t rep_window  = 0; // frames
+    std::mt19937 rng;
+    std::vector<float>   logits;
+    std::vector<float>   fb_buf;
+    std::vector<int32_t> hist; // frame-major codes of the frames so far
+
+    bool pending_tail = false; // the ISTFT holds back the overlap of the last frame
+    std::vector<float>   audio_pcm;
+    std::vector<float>   h_state_buf;
+    mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+    std::vector<char> out_buf;
+    bool   stream          = false;
+    size_t pcm_sent        = 0;     // samples already returned by get_output()
+    bool   wav_header_sent = false;
+};
+
 static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
     switch (mtmd_gen_audio_get_info(mctx).type) {
         case MTMD_GEN_AUDIO_TYPE_QWEN3TTS:
@@ -1269,6 +1725,8 @@ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lc
             return std::unique_ptr<mtmd_gen_audio_pipeline>(new pockettts_gen_audio_pipeline(lctx, mctx));
         case MTMD_GEN_AUDIO_TYPE_SOPRANO:
             return std::unique_ptr<mtmd_gen_audio_pipeline>(new soprano_gen_audio_pipeline(lctx, mctx));
+        case MTMD_GEN_AUDIO_TYPE_LFM3A:
+            return std::unique_ptr<mtmd_gen_audio_pipeline>(new lfm3a_gen_audio_pipeline(lctx, mctx));
         default:
             return nullptr;
     }
@@ -1308,6 +1766,11 @@ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_
         LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
         return 1;
     }
+    if (inp->n_past > 0 && !ctx->pipeline->supports_continue()) {
+        LOG_ERR("mtmd_helper_gen_audio: the pipeline cannot continue a sequence\n");
+        return 1;
+    }
+    ctx->pipeline->logit_bias.clear();
     return ctx->pipeline->set_input(inp);
 }
 
@@ -1318,6 +1781,34 @@ int32_t mtmd_helper_gen_audio_step_prompt(mtmd_helper_gen_audio * ctx, int32_t n
     return ctx->pipeline->step_prompt(n_batch);
 }
 
+bool mtmd_helper_gen_audio_next_is_text(mtmd_helper_gen_audio * ctx) {
+    return ctx->pipeline && ctx->pipeline->next_is_text();
+}
+
+const llama_logit_bias * mtmd_helper_gen_audio_get_logit_bias(mtmd_helper_gen_audio * ctx, size_t * n_bias) {
+    *n_bias = ctx->pipeline ? ctx->pipeline->logit_bias.size() : 0;
+    return *n_bias ? ctx->pipeline->logit_bias.data() : nullptr;
+}
+
+int32_t mtmd_helper_audio_stream_get_logit_bias(const mtmd_context * mctx, const llama_model * model,
+                                                llama_logit_bias * out, int32_t n_max) {
+    std::vector<llama_logit_bias> bias;
+    if (mtmd_gen_audio_get_info(mctx).type == MTMD_GEN_AUDIO_TYPE_LFM3A) {
+        // LFM3-Audio live ASR: only <|text_end|> ends the text of a chunk, the stream adds the final <|mixed_end|>
+        const llama_vocab * vocab = llama_model_get_vocab(model);
+        for (const char * piece : { "<|mixed_end|>", "<|mixed_start|>", "<|audio_start|>", "<|im_start|>", "<|im_end|>", "<|endoftext|>" }) {
+            const llama_token tok = find_special_token(vocab, piece);
+            if (tok != LLAMA_TOKEN_NULL) {
+                bias.push_back({ tok, -INFINITY });
+            }
+        }
+    }
+    for (int32_t i = 0; i < n_max && i < (int32_t) bias.size(); i++) {
+        out[i] = bias[i];
+    }
+    return (int32_t) bias.size();
+}
+
 int32_t mtmd_helper_gen_audio_step_gen(mtmd_helper_gen_audio * ctx, llama_token sampled,
                                        const float * h_state_in, const float ** h_state_out,
                                        bool * out_stop) {

tools/mtmd/mtmd-helper.cpp

@@ -11,6 +11,7 @@
 #include "mtmd-helper.h"
 #include "mtmd-helper-common.h"
 #include "llama.h"
+#include "../src/llama-ext.h"
 
 #include "hash/hash.h"
 
@@ -1043,8 +1044,10 @@ mtmd_helper_cap mtmd_helper_get_capabilities(const llama_model * model, const mt
     cap.audio_gen_spk_ref = gen_audio_type == MTMD_GEN_AUDIO_TYPE_QWEN3TTS
                          || gen_audio_type == MTMD_GEN_AUDIO_TYPE_POCKETTTS;
 
-    // tts-only model cannot be used for chat (no chat template)
-    cap.chat = !cap.audio_gen || llama_model_chat_template(model, nullptr) != nullptr;
+    // a tts model without a chat template only outputs audio, it cannot be used for chat
+    // neither can a realtime transcription model (e.g. Voxtral Realtime): its text follows the audio by a token delay
+    cap.text_gen = !cap.audio_gen || llama_model_chat_template(model, nullptr) != nullptr;
+    cap.chat     = cap.text_gen && llama_model_n_tokens_delay(model) == 0;
 
     return cap;
 }

tools/mtmd/mtmd-helper.h

@@ -184,7 +184,7 @@ MTMD_API int32_t mtmd_helper_video_read_next(mtmd_helper_video * ctx,
             char ** out_text);
 
 struct mtmd_helper_cap {
-    bool chat; // false for tts-only model (no chat template)
+    bool chat; // false for tts-only models (no chat template) and realtime transcription models (token delay)
 
     // input capabilities
     bool vision;
@@ -195,7 +195,7 @@ struct mtmd_helper_cap {
     bool audio_stream; // realtime transcription, see mtmd_audio_stream_init()
 
     // output capabilities
-    bool text_gen; // always true for now
+    bool text_gen; // false for an audio-only model (tts-only, no chat template)
     bool audio_gen;
     bool audio_gen_spk_ref; // accepts a speaker reference audio for voice cloning
 };
@@ -223,15 +223,28 @@ enum mtmd_helper_gen_audio_outtype {
     MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM, // raw PCM
     MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono
 };
+// what a response that continues a sequence (n_past > 0) outputs
+enum mtmd_helper_gen_audio_out_mode {
+    MTMD_HELPER_GEN_AUDIO_OUT_AUTO,  // the model switches between text and audio itself
+    MTMD_HELPER_GEN_AUDIO_OUT_TEXT,
+    MTMD_HELPER_GEN_AUDIO_OUT_AUDIO,
+    MTMD_HELPER_GEN_AUDIO_OUT_MIXED, // interleaved text and audio
+};
 struct mtmd_helper_gen_audio_inp {
     bool stream; // if true, output() must be called after each step_gen()
     llama_seq_id seq_id;
 
+    // the text to speak; for a model with a "tts" chat template (e.g. LFM3-Audio), the prompt rendered from it
     const char * prompt;
     size_t       prompt_len;
 
+    // > 0: continue a sequence of n_past positions with raw text (LFM3-Audio only)
+    int32_t n_past;
+    enum mtmd_helper_gen_audio_out_mode out_mode; // n_past > 0 only (default: auto)
+
     mtmd_bitmap * speaker_ref; // optional, can be NULL
     const char * lang; // optional, can be NULL
+    const char * voice; // optional, can be NULL, model-dependent
 
     int32_t  top_k;
     float    top_p;
@@ -271,6 +284,18 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
                         const float ** h_state_out,
                         bool * out_stop);
 
+// true if the next step_gen() takes a text token: the caller samples it from the logits and passes it as sampled
+// false if the next step generates an audio frame from h_state_in
+MTMD_API bool mtmd_helper_gen_audio_next_is_text(mtmd_helper_gen_audio * ctx);
+
+// bias for the text tokens the caller samples (e.g. no switch to audio in a text-only response), n_bias is set to its size
+MTMD_API const llama_logit_bias * mtmd_helper_gen_audio_get_logit_bias(mtmd_helper_gen_audio * ctx, size_t * n_bias);
+
+// bias for the text the caller samples while it transcribes an audio stream opened with a prompt (see mtmd_audio_stream_params)
+// writes up to n_max entries to out, returns the size of the bias
+MTMD_API int32_t mtmd_helper_audio_stream_get_logit_bias(const mtmd_context * mctx, const struct llama_model * model,
+                                                         llama_logit_bias * out, int32_t n_max);
+
 // out_data valid until next get_output() or reset() call
 // out_n_samples (optional, can be NULL) receives the number of generated PCM samples
 // if inp->stream is true: returns only audio produced since the previous call, and
@@ -293,6 +318,7 @@ MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
 #ifdef __cplusplus
 #include <set>
 #include <memory>
+#include <vector>
 
 namespace mtmd_helper {
 
@@ -318,6 +344,7 @@ struct gen_audio {
         mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
         std::string prompt_str;
         std::string lang_str;
+        std::string voice_str;
         mtmd::bitmap_ptr speaker_ref_ptr;
 
         inp()                             = default;
@@ -328,6 +355,7 @@ struct gen_audio {
 
         void set_prompt     (std::string p)        { prompt_str = std::move(p); }
         void set_lang       (std::string l)        { lang_str   = std::move(l); }
+        void set_voice      (std::string v)        { voice_str  = std::move(v); }
         void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
 
         // pointers are only valid as long as *this is alive
@@ -335,6 +363,7 @@ struct gen_audio {
             data.prompt      = prompt_str.c_str();
             data.prompt_len  = prompt_str.size();
             data.lang        = lang_str.empty() ? nullptr : lang_str.c_str();
+            data.voice       = voice_str.empty() ? nullptr : voice_str.c_str();
             data.speaker_ref = speaker_ref_ptr.get();
             return &data;
         }
@@ -359,6 +388,14 @@ struct gen_audio {
     int32_t step_gen(llama_token sampled, const float * h_state, const float ** h_state_out, bool * out_stop = nullptr) {
         return mtmd_helper_gen_audio_step_gen(ctx.get(), sampled, h_state, h_state_out, out_stop);
     }
+    bool next_is_text() {
+        return mtmd_helper_gen_audio_next_is_text(ctx.get());
+    }
+    std::vector<llama_logit_bias> logit_bias() {
+        size_t n = 0;
+        const llama_logit_bias * bias = mtmd_helper_gen_audio_get_logit_bias(ctx.get(), &n);
+        return std::vector<llama_logit_bias>(bias, bias + n);
+    }
     int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) {
         return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples);
     }
@@ -367,6 +404,12 @@ struct gen_audio {
     }
 };
 
+inline std::vector<llama_logit_bias> audio_stream_logit_bias(const mtmd_context * mctx, const llama_model * model) {
+    std::vector<llama_logit_bias> bias(mtmd_helper_audio_stream_get_logit_bias(mctx, model, nullptr, 0));
+    mtmd_helper_audio_stream_get_logit_bias(mctx, model, bias.data(), (int32_t) bias.size());
+    return bias;
+}
+
 } // namespace mtmd_helper
 #endif
 

tools/mtmd/mtmd-image.cpp

@@ -809,8 +809,7 @@ clip_image_size mtmd_image_preprocessor_dyn_size::get_target_size(const clip_hpa
 
 mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_image_u8 & img) const {
     clip_image_u8 resized_image;
-    const clip_image_size target_size = get_target_size(hparams, img.get_size());
-    img_tool::resize(img, resized_image, target_size,
+    img_tool::resize(img, resized_image, target_size(img.get_size()),
                         hparams.image_resize_algo,
                         hparams.image_resize_pad,
                         hparams.image_pad_color);
@@ -819,6 +818,34 @@ mtmd_image_preproc_out mtmd_image_preprocessor_dyn_size::preprocess(const clip_i
     return output;
 }
 
+//
+// mtmd_image_preprocessor_lfm3vl
+//
+
+// rounds before the pixel checks (an edge may round to 0) and computes in double, as transformers does
+clip_image_size mtmd_image_preprocessor_lfm3vl::smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels) {
+    const double h = height;
+    const double w = width;
+    int h_bar = static_cast<int>(std::nearbyint(h / factor)) * factor;
+    int w_bar = static_cast<int>(std::nearbyint(w / factor)) * factor;
+    if (static_cast<int64_t>(h_bar) * w_bar > max_pixels) {
+        const double beta = std::sqrt(h * w / max_pixels);
+        h_bar = static_cast<int>(std::floor(h / beta / factor)) * factor;
+        w_bar = static_cast<int>(std::floor(w / beta / factor)) * factor;
+    } else if (static_cast<int64_t>(h_bar) * w_bar < min_pixels) {
+        const double beta = std::sqrt(min_pixels / (h * w));
+        h_bar = static_cast<int>(std::ceil(h * beta / factor)) * factor;
+        w_bar = static_cast<int>(std::ceil(w * beta / factor)) * factor;
+    }
+
+    return {std::max(factor, w_bar), std::max(factor, h_bar)};
+}
+
+clip_image_size mtmd_image_preprocessor_lfm3vl::target_size(const clip_image_size & original_size) const {
+    return smart_resize(original_size.width, original_size.height, hparams.patch_size * hparams.n_merge,
+                        hparams.image_min_pixels, hparams.image_max_pixels);
+}
+
 //
 // mtmd_image_preprocessor_glm5v
 //

tools/mtmd/mtmd-image.h

@@ -135,6 +135,15 @@ struct mtmd_image_preprocessor_dyn_size : mtmd_image_preprocessor {
     mtmd_image_preprocessor_dyn_size(const clip_ctx * ctx) : mtmd_image_preprocessor(ctx) {}
     static clip_image_size get_target_size(const clip_hparams & hparams, const clip_image_size & original_size);
     mtmd_image_preproc_out preprocess(const clip_image_u8 & img) const override;
+    virtual clip_image_size target_size(const clip_image_size & original_size) const { return get_target_size(hparams, original_size); }
+};
+
+// LFM3-VL, same as dyn_size but sized exactly as the transformers Qwen2-VL smart_resize
+struct mtmd_image_preprocessor_lfm3vl : mtmd_image_preprocessor_dyn_size {
+    mtmd_image_preprocessor_lfm3vl(const clip_ctx * ctx) : mtmd_image_preprocessor_dyn_size(ctx) {}
+    clip_image_size target_size(const clip_image_size & original_size) const override;
+
+    static clip_image_size smart_resize(int width, int height, int factor, int64_t min_pixels, int64_t max_pixels);
 };
 
 // GLM 5.3 flash, similar to dyn_size, but each edge is aligned up to patch_size*n_merge, and max budget is met by a search over the height with the width scaled proportionally

tools/mtmd/mtmd.cpp

@@ -772,6 +772,13 @@ struct mtmd_context {
                     img_end = "<|image_end|>";
                     image_preproc = std::make_unique<mtmd_image_preprocessor_muse_glimmer>(ctx_v);
                 } break;
+            case PROJECTOR_TYPE_LFM3VL:
+                {
+                    // <|vision_start|> ... (image embeddings) ... <|vision_end|>
+                    img_beg = "<|vision_start|>";
+                    img_end = "<|vision_end|>";
+                    image_preproc = std::make_unique<mtmd_image_preprocessor_lfm3vl>(ctx_v);
+                } break;
             case PROJECTOR_TYPE_YOUTUVL:
                 {
                     // <|vision_start|> ... (image embeddings) ... <|vision_end|>
@@ -1073,6 +1080,15 @@ struct mtmd_context {
                 {
                     audio_preproc = std::make_unique<mtmd_audio_preprocessor_d1omni>(ctx_a);
                 } break;
+            case PROJECTOR_TYPE_LFM3A:
+                {
+                    // LFM3-Omni wraps the audio in the image span, LFM3-Audio (no vision tower) places it bare
+                    if (ctx_v) {
+                        aud_beg = img_beg;
+                        aud_end = img_end;
+                    }
+                    audio_preproc = std::make_unique<mtmd_audio_preprocessor_lfm3a>(ctx_a);
+                } break;
             case PROJECTOR_TYPE_GRANITE_SPEECH:
                 {
                     audio_preproc = std::make_unique<mtmd_audio_preprocessor_granite_speech>(ctx_a);
@@ -2094,6 +2110,10 @@ mtmd_gen_audio_info mtmd_gen_audio_get_info(const mtmd_context * ctx) {
             info.type = MTMD_GEN_AUDIO_TYPE_SOPRANO;
             info.sample_rate = 32000;
             break;
+        case PROJECTOR_TYPE_LFM3A_GEN:
+            info.type = MTMD_GEN_AUDIO_TYPE_LFM3A;
+            info.sample_rate = 24000;
+            break;
         default:
             info.type = MTMD_GEN_AUDIO_TYPE_NONE;
             break;
@@ -2124,6 +2144,16 @@ mtmd_gen_inp mtmd_gen_inp_default(const mtmd_context * ctx) {
         case PROJECTOR_TYPE_SOPRANO:
             inp.type = MTMD_GEN_PROCESS_TYPE_GEN_WAV;
             break;
+        case PROJECTOR_TYPE_LFM3A_GEN:
+            {
+                // MaskGIT sampling runs in the helper, with the settings of the checkpoint
+                const auto * hparams = clip_get_hparams(ctx->ctx_gen_a);
+                inp.top_k          = hparams->gen_top_k;
+                inp.top_p          = 1.0f;
+                inp.temp           = hparams->gen_temp;
+                inp.penalty_repeat = hparams->gen_penalty_repeat;
+                inp.penalty_last_n = hparams->gen_penalty_last_n;
+            } break;
         default:
             break;
     }
@@ -2139,6 +2169,9 @@ struct mtmd_gen_state {
     std::vector<float>   out_embd;  // next-step hidden state fed back to backbone (GEN_CODE)
     std::vector<float>   out_audio; // decoded PCM samples for the current frame (GEN_WAV)
 
+    // lfm3a: the codec graph yields STFT frames, the ISTFT runs on the host
+    std::unique_ptr<mtmd_audio_streaming_istft> istft;
+
     ~mtmd_gen_state() {
         clip_gen_state_free(clip_state);
     }
@@ -2161,6 +2194,10 @@ mtmd_gen_state * mtmd_gen_state_init(mtmd_context * ctx, mtmd_gen_state_params p
     if (!st->clip_state) {
         return nullptr;
     }
+    if (clip_get_projector_type(ctx->ctx_gen_a) == PROJECTOR_TYPE_LFM3A_GEN) {
+        const auto * hparams = clip_get_hparams(ctx->ctx_gen_a);
+        st->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
+    }
     return st.release();
 }
 
@@ -2170,6 +2207,72 @@ void mtmd_gen_state_free(mtmd_gen_state * state) {
 
 void mtmd_gen_state_reset(mtmd_gen_state * state) {
     clip_gen_state_reset(state->clip_state);
+    if (state->istft) {
+        const auto * hparams = clip_get_hparams(state->ctx_clip);
+        state->istft = std::make_unique<mtmd_audio_streaming_istft>(hparams->audio_n_fft, hparams->audio_hop_len);
+    }
+}
+
+// lfm3a GEN_WAV: ISTFT of the codec output, n_codes == 0 flushes and resets the state
+static int32_t mtmd_gen_wav_lfm3a(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
+    clip_ctx * ctx_clip = ctx->ctx_gen_a;
+    const auto & hparams = *clip_get_hparams(ctx_clip);
+
+    const int n_bins = hparams.audio_n_fft / 2 + 1;
+    const int n_trim = (hparams.audio_n_fft - hparams.audio_hop_len) / 2;
+
+    auto & audio = state->out_audio;
+    audio.clear();
+
+    if (!inp->codes || inp->n_codes == 0) {
+        // the tail beyond the last frame is padding, like the centered ISTFT of the reference
+        audio = state->istft->flush();
+        audio.resize(audio.size() - std::min(audio.size(), (size_t) n_trim));
+        mtmd_gen_state_reset(state);
+    } else {
+        std::vector<int32_t> in_codes(inp->codes, inp->codes + inp->n_codes);
+        std::vector<float>   feats;
+
+        // no hidden-state input, the batch entry is an unused placeholder
+        clip_image_f32 dummy;
+        dummy.set_size({1, 1}, false, true);
+        dummy.cpy_buf(std::vector<float>(1, 0.0f));
+
+        clip_image_f32_batch batch;
+        batch.is_audio = true;
+        batch.entries.push_back(std::move(dummy));
+
+        clip_encode_params params;
+        params.imgs        = &batch;
+        params.n_threads   = ctx->n_threads;
+        params.gen_process = CLIP_GEN_PROCESS_GEN_WAV;
+        params.codes       = &in_codes;
+        params.out_feats   = &feats;
+        params.gen_state   = state->clip_state;
+        if (!clip_encode(ctx_clip, &params)) {
+            LOG_ERR("%s: clip_encode failed (codec)\n", __func__);
+            return 1;
+        }
+
+        // [2 * n_bins, n_stft_frames]: log magnitude, then phase
+        GGML_ASSERT(feats.size() % (2 * (size_t) n_bins) == 0);
+        const size_t n_frames = feats.size() / (2 * (size_t) n_bins);
+        std::vector<float> spec(2 * (size_t) n_bins);
+        for (size_t f = 0; f < n_frames; f++) {
+            const float * fr = feats.data() + f * 2 * n_bins;
+            for (int j = 0; j < n_bins; j++) {
+                const float mag = std::exp(fr[j]);
+                spec[2 * j + 0] = mag * std::cos(fr[n_bins + j]);
+                spec[2 * j + 1] = mag * std::sin(fr[n_bins + j]);
+            }
+            const std::vector<float> pcm = state->istft->process_frame(spec.data());
+            audio.insert(audio.end(), pcm.begin(), pcm.end());
+        }
+    }
+
+    out->audio     = audio.data();
+    out->n_samples = audio.size();
+    return 0;
 }
 
 static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state * state, const mtmd_gen_inp * inp, mtmd_gen_out * out) {
@@ -2193,12 +2296,22 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
         }
     }
 
+    if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_WAV && clip_get_projector_type(ctx_clip) == PROJECTOR_TYPE_LFM3A_GEN) {
+        return mtmd_gen_wav_lfm3a(ctx, state, inp, out);
+    }
+
     if (inp->type == MTMD_GEN_PROCESS_TYPE_GEN_CODE) {
         const size_t n_embd = (size_t) clip_n_mmproj_embd(ctx_clip);
 
+        // the hidden state may be omitted when the graph does not use it
         clip_image_f32 hidden_state;
         hidden_state.set_size({(int) n_embd, 1}, false, true);
-        hidden_state.cpy_buf(std::vector<float>(inp->embd, inp->embd + n_embd));
+        hidden_state.cpy_buf(inp->embd ? std::vector<float>(inp->embd, inp->embd + n_embd) : std::vector<float>(n_embd, 0.0f));
+
+        std::vector<int32_t> in_codes;
+        if (inp->codes) {
+            in_codes.assign(inp->codes, inp->codes + inp->n_codes);
+        }
 
         clip_image_f32_batch batch;
         batch.is_audio = true;
@@ -2220,6 +2333,7 @@ static int32_t mtmd_gen_audio_process_impl(mtmd_context * ctx, mtmd_gen_state *
         params.top_k         = inp->top_k;
         params.top_p         = inp->top_p;
         params.temp          = inp->temp;
+        params.codes         = inp->codes ? &in_codes : nullptr;
         params.out_is_eos    = &is_eos;
         params.gen_state     = state->clip_state;
 
@@ -3088,26 +3202,18 @@ mtmd_memory_usage mtmd_get_memory_usage(const char * mmproj_fname,
 // realtime audio input
 //
 
+// the model-independent part of a stream, the model-specific part is in the subclasses below
 struct mtmd_audio_stream {
     mtmd_context * ctx;
     clip_ctx * ctx_clip;
-    clip_enc_state * enc_state = nullptr;
 
     // model constants
     int n_mel;
-    int n_fft;
     int hop;
     int n_embd;          // decoder embedding size
     int frames_per_tok;  // mel frames per decoder row
     int samples_per_tok;
-    int n_frames_max;    // encoder frames per clip_encode() call
     int n_step_mel;      // mel frames per encoder call
-    int n_right_pad;     // silence tokens added by flush()
-    float log_mel_max;   // global_log_mel_max in params.json
-
-    std::vector<llama_token> prefix; // text tokens paired with the first rows
-
-    mtmd_audio_cache cache;
 
     // samples not yet turned into mel frames, buf[0] is at absolute index buf_pos
     std::vector<float> buf;
@@ -3116,103 +3222,143 @@ struct mtmd_audio_stream {
     int64_t n_frames  = 0; // mel frames computed so far
 
     std::vector<float> mel;     // pending mel frames, frame-major
-    std::vector<float> mel_ctx; // last 2 mel frames, the left context of the conv stem
     std::vector<float> embd;    // pending audio embeddings
     size_t embd_off = 0;        // rows of embd already consumed
     std::vector<float> out;     // rows of the last step
-    std::vector<float> tok_embd;
 
     int64_t n_rows = 0;         // rows produced so far
-    int64_t n_rows_flushed = 0; // rows produced once the audio up to the last flush is consumed
+    int64_t n_rows_done = 0;    // audio rows whose text has been sampled, see mtmd_audio_stream_get_n_rows()
+    int64_t n_rows_flushed = 0; // audio rows up to the end of the last flush
     bool pending_flush = false;
     bool ended = false;
+    bool seq_start = false;     // the rows of the last step start a new sequence
 
-    ~mtmd_audio_stream() {
-        clip_enc_state_free(enc_state);
+
+    virtual ~mtmd_audio_stream() = default;
+
+    // model constants and initial state, false on failure
+    virtual bool    init(const mtmd_audio_stream_params & params) = 0;
+    virtual int32_t flush(bool end) = 0;
+    virtual int32_t step(llama_token last_token) = 0;
+
+    void add_silence(int64_t n) {
+        buf.insert(buf.end(), n, 0.0f);
+        n_samples += n;
     }
 };
 
+// Voxtral Realtime: a row is an audio embedding plus the embedding of the previous token
+struct mtmd_audio_stream_voxtral_rt : mtmd_audio_stream {
+    clip_enc_state * enc_state = nullptr;
+
+    int n_fft;
+    int n_frames_max;    // encoder frames per clip_encode() call
+    int n_right_pad;     // silence tokens added by flush()
+    float log_mel_max;   // global_log_mel_max in params.json
+
+    std::vector<llama_token> prefix; // text tokens paired with the first rows
+
+    mtmd_audio_cache cache;
+
+    std::vector<float> mel_ctx; // last 2 mel frames, the left context of the conv stem
+    std::vector<float> tok_embd;
+
+    ~mtmd_audio_stream_voxtral_rt() override {
+        clip_enc_state_free(enc_state);
+    }
+
+    bool    init(const mtmd_audio_stream_params & params) override;
+    int32_t flush(bool end) override;
+    int32_t step(llama_token last_token) override;
+};
+
+// LFM3-Audio: the rows are plain audio embeddings of a chat turn, the model answers after the audio
+// live ASR (with a prompt), trained layout: <prompt> [A x 12] text<|text_end|> ... [A x r] <|mixed_end|> text<|text_end|>
+// training samples are max 30 s: restart the sequence at a silent chunk after 15 s, always after 30 s; the encoder keeps its state
+struct mtmd_audio_stream_lfm3a : mtmd_audio_stream {
+    int win; // samples per mel frame
+    clip_enc_state * enc_state = nullptr;
+
+    // live ASR
+    static constexpr int64_t chunk_rows       = 12;
+    static constexpr int32_t max_chunk_tokens = 48; // text tokens per chunk, then the chunk is closed
+    static constexpr int32_t max_final_tokens = 96;
+
+    std::vector<llama_token> prompt; // empty: the rows of a chat turn
+    llama_token tok_text_end  = LLAMA_TOKEN_NULL;
+    llama_token tok_mixed_end = LLAMA_TOKEN_NULL;
+    int64_t soft_seq_rows = 0;
+    int64_t max_seq_rows  = 0;
+
+    bool    started        = false;
+    bool    done           = false; // the text of the final chunk is complete
+    bool    in_chunk       = false; // the text of the last chunk is being sampled
+    bool    final_chunk    = false;
+    bool    chunk_has_text = false;
+    int32_t n_chunk_text   = 0;
+    int64_t n_chunk_rows   = 0;
+    int64_t n_seq_rows     = 0;     // audio rows since the sequence started
+    std::vector<llama_token> closing; // the tokens that close the last chunk, they go in before the next one
+    std::vector<float> tok_embd;
+    // the audio markers of the model (LFM3-Omni), as rows around the audio
+    std::vector<float> beg_rows;
+    std::vector<float> end_rows;
+    bool beg_done = false;
+    bool end_done = false;
+
+    ~mtmd_audio_stream_lfm3a() override {
+        clip_enc_state_free(enc_state);
+    }
+
+    bool    init(const mtmd_audio_stream_params & params) override;
+    int32_t flush(bool end) override;
+    int32_t step(llama_token last_token) override;
+
+    bool    add_tokens(const llama_token * tokens, size_t n);
+    int32_t step_live(llama_token last_token);
+};
+
 bool mtmd_support_audio_stream(const mtmd_context * ctx) {
-    return ctx->ctx_a && ctx->text_model && clip_get_projector_type(ctx->ctx_a) == PROJECTOR_TYPE_VOXTRAL_RT;
+    if (!ctx->ctx_a || !ctx->text_model) {
+        return false;
+    }
+    const projector_type proj = clip_get_projector_type(ctx->ctx_a);
+    return proj == PROJECTOR_TYPE_VOXTRAL_RT || proj == PROJECTOR_TYPE_LFM3A;
 }
 
 mtmd_audio_stream_params mtmd_audio_stream_params_default(void) {
     mtmd_audio_stream_params params;
-    params.step_ms = 320;
+    params.step_ms  = 320;
+    params.prompt   = nullptr;
+    params.n_prompt = 0;
     return params;
 }
 
-static void mtmd_audio_stream_add_silence(mtmd_audio_stream * s, int64_t n) {
-    s->buf.insert(s->buf.end(), n, 0.0f);
-    s->n_samples += n;
-}
-
 mtmd_audio_stream * mtmd_audio_stream_init(mtmd_context * ctx, mtmd_audio_stream_params params) {
     if (!mtmd_support_audio_stream(ctx)) {
         LOG_ERR("%s: the model does not support streaming audio input\n", __func__);
         return nullptr;
     }
     const clip_hparams * hparams = clip_get_hparams(ctx->ctx_a);
-    const llama_vocab * vocab = ctx->vocab;
 
-    auto s = std::make_unique<mtmd_audio_stream>();
+    std::unique_ptr<mtmd_audio_stream> s;
+    switch (clip_get_projector_type(ctx->ctx_a)) {
+        case PROJECTOR_TYPE_VOXTRAL_RT: s = std::make_unique<mtmd_audio_stream_voxtral_rt>(); break;
+        case PROJECTOR_TYPE_LFM3A:      s = std::make_unique<mtmd_audio_stream_lfm3a>();      break;
+        default:                        GGML_ABORT("unsupported projector");
+    }
     s->ctx      = ctx;
     s->ctx_clip = ctx->ctx_a;
     s->n_mel    = hparams->n_mel_bins;
-    s->n_fft    = hparams->audio_n_fft;
     s->hop      = hparams->audio_hop_len;
     s->n_embd   = clip_n_mmproj_embd(ctx->ctx_a);
     if (s->n_embd != ctx->n_embd_text) {
         LOG_ERR("%s: mmproj embedding size %d does not match the text model (%d)\n", __func__, s->n_embd, ctx->n_embd_text);
         return nullptr;
     }
-
-    // one decoder row per 80 ms of audio: the conv stem has stride 2, then frames are stacked
-    s->frames_per_tok  = 2 * hparams->proj_stack_factor;
-    s->samples_per_tok = s->frames_per_tok * s->hop;
-    s->n_frames_max    = (hparams->warmup_audio_size - 2) / 2;
-    s->log_mel_max     = 1.5f;
-
-    // the encoder works in whole decoder rows
-    int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / s->hop);
-    n_step = std::max(n_step, s->frames_per_tok);
-    n_step = CLIP_ALIGN(n_step, s->frames_per_tok);
-    s->n_step_mel = std::min(n_step, 2 * s->n_frames_max);
-
-    // prefix: BOS then one pad token per silence token and per delay token
-    // the output lags behind the audio by n_delay rows, the text model is conditioned on the same value
-    const int n_delay    = llama_model_n_tokens_delay(ctx->text_model);
-    const int n_left_pad = 32;
-    s->n_right_pad = n_delay + 1 + 10;
-    {
-        const std::string pad_str = "[STREAMING_PAD]";
-        std::vector<llama_token> pad(8);
-        const int n_pad = llama_tokenize(vocab, pad_str.c_str(), pad_str.size(), pad.data(), pad.size(), false, true);
-        if (n_pad != 1) {
-            LOG_ERR("%s: cannot find the %s token\n", __func__, pad_str.c_str());
-            return nullptr;
-        }
-        s->prefix.push_back(llama_vocab_bos(vocab));
-        s->prefix.insert(s->prefix.end(), n_left_pad + n_delay, pad[0]);
-    }
-
-    s->cache.fill_sin_cos_table(s->n_fft);
-    s->cache.fill_hann_window(hparams->audio_window_len, true);
-    s->cache.fill_mel_filterbank_matrix(s->n_mel, s->n_fft, hparams->audio_sample_rate);
-
-    s->enc_state = clip_enc_state_init(s->ctx_clip, s->n_frames_max);
-    if (!s->enc_state) {
+    if (!s->init(params)) {
         return nullptr;
     }
-
-    // the first mel frame is centered on the first sample, so its window starts before the audio
-    s->buf.assign(s->n_fft / 2, 0.0f);
-    s->buf_pos = -s->n_fft / 2;
-    s->mel_ctx.assign(2 * s->n_mel, 0.0f);
-
-    // silence before the audio
-    mtmd_audio_stream_add_silence(s.get(), (int64_t) n_left_pad * s->samples_per_tok);
-
     return s.release();
 }
 
@@ -3233,18 +3379,99 @@ int32_t mtmd_audio_stream_flush(mtmd_audio_stream * s, bool end) {
     if (s->ended) {
         return 1;
     }
+    return s->flush(end);
+}
+
+int32_t mtmd_audio_stream_step(mtmd_audio_stream * s, llama_token last_token) {
+    return s->step(last_token);
+}
+
+const float * mtmd_audio_stream_get_output_embd(const mtmd_audio_stream * s) {
+    return s->out.data();
+}
+
+bool mtmd_audio_stream_is_seq_start(const mtmd_audio_stream * s) {
+    return s->seq_start;
+}
+
+
+//
+// Voxtral Realtime
+//
+
+bool mtmd_audio_stream_voxtral_rt::init(const mtmd_audio_stream_params & params) {
+    const clip_hparams * hparams = clip_get_hparams(ctx_clip);
+    const llama_vocab * vocab = ctx->vocab;
+
+    n_fft = hparams->audio_n_fft;
+
+    // one decoder row per 80 ms of audio: the conv stem has stride 2, then frames are stacked
+    frames_per_tok  = 2 * hparams->proj_stack_factor;
+    samples_per_tok = frames_per_tok * hop;
+    n_frames_max    = (hparams->warmup_audio_size - 2) / 2;
+    log_mel_max     = 1.5f;
+
+    // the encoder works in whole decoder rows
+    int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop);
+    n_step = std::max(n_step, frames_per_tok);
+    n_step = CLIP_ALIGN(n_step, frames_per_tok);
+    n_step_mel = std::min(n_step, 2 * n_frames_max);
+
+    if (params.n_prompt > 0) {
+        LOG_ERR("%s: Voxtral Realtime takes no prompt\n", __func__);
+        return false;
+    }
+
+    // prefix: BOS then one pad token per silence token and per delay token
+    // the output lags behind the audio by n_delay rows, the text model is conditioned on the same value
+    const int n_delay    = llama_model_n_tokens_delay(ctx->text_model);
+    const int n_left_pad = 32;
+    n_right_pad = n_delay + 1 + 10;
+    {
+        const std::string pad_str = "[STREAMING_PAD]";
+        std::vector<llama_token> pad(8);
+        const int n_pad = llama_tokenize(vocab, pad_str.c_str(), pad_str.size(), pad.data(), pad.size(), false, true);
+        if (n_pad != 1) {
+            LOG_ERR("%s: cannot find the %s token\n", __func__, pad_str.c_str());
+            return false;
+        }
+        prefix.push_back(llama_vocab_bos(vocab));
+        prefix.insert(prefix.end(), n_left_pad + n_delay, pad[0]);
+    }
+
+    cache.fill_sin_cos_table(n_fft);
+    cache.fill_hann_window(hparams->audio_window_len, true);
+    cache.fill_mel_filterbank_matrix(n_mel, n_fft, hparams->audio_sample_rate);
+
+    enc_state = clip_enc_state_init(ctx_clip, n_frames_max);
+    if (!enc_state) {
+        return false;
+    }
+
+    // the first mel frame is centered on the first sample, so its window starts before the audio
+    buf.assign(n_fft / 2, 0.0f);
+    buf_pos = -n_fft / 2;
+    mel_ctx.assign(2 * n_mel, 0.0f);
+
+    // silence before the audio
+    add_silence((int64_t) n_left_pad * samples_per_tok);
+
+    return true;
+}
+
+int32_t mtmd_audio_stream_voxtral_rt::flush(bool end) {
     // align to a whole row, then add the silence the model expects after the audio
-    const int64_t n_align = (s->samples_per_tok - s->n_samples % s->samples_per_tok) % s->samples_per_tok;
-    mtmd_audio_stream_add_silence(s, n_align + (int64_t) s->n_right_pad * s->samples_per_tok);
+    const int64_t n_align = (samples_per_tok - n_samples % samples_per_tok) % samples_per_tok;
+    add_silence(n_align + (int64_t) n_right_pad * samples_per_tok);
     // one row per samples_per_tok, the left padding included
-    s->n_rows_flushed = s->n_samples / s->samples_per_tok;
-    s->pending_flush = true;
-    s->ended = end;
+    n_rows_flushed = n_samples / samples_per_tok;
+    pending_flush = true;
+    ended = end;
     return 0;
 }
 
 // mel frame k is centered on sample k * hop, with force the samples past the end are taken as silence
-static void mtmd_audio_stream_compute_mel(mtmd_audio_stream * s, bool force) {
+static void mtmd_audio_stream_compute_mel(mtmd_audio_stream_voxtral_rt * s, bool force) {
     const int half = s->n_fft / 2;
     std::vector<float> window(s->n_fft);
     std::vector<float> frame(s->n_mel);
@@ -3274,7 +3501,7 @@ static void mtmd_audio_stream_compute_mel(mtmd_audio_stream * s, bool force) {
     }
 }
 
-static bool mtmd_audio_stream_encode(mtmd_audio_stream * s, bool force) {
+static bool mtmd_audio_stream_encode(mtmd_audio_stream_voxtral_rt * s, bool force) {
     while (true) {
         const int n_avail = (int) (s->mel.size() / s->n_mel);
         int n = n_avail - n_avail % s->frames_per_tok;
@@ -3326,7 +3553,7 @@ static bool mtmd_audio_stream_encode(mtmd_audio_stream * s, bool force) {
 }
 
 // out rows = audio embeddings + token embeddings
-static bool mtmd_audio_stream_add_rows(mtmd_audio_stream * s, const llama_token * tokens, size_t n) {
+static bool mtmd_audio_stream_add_rows(mtmd_audio_stream_voxtral_rt * s, const llama_token * tokens, size_t n) {
     s->tok_embd.resize(n * s->n_embd);
     if (llama_model_get_tok_embd_rows(s->ctx->text_model, tokens, n, s->tok_embd.data()) == 0) {
         LOG_ERR("%s: invalid token\n", __func__);
@@ -3341,55 +3568,330 @@ static bool mtmd_audio_stream_add_rows(mtmd_audio_stream * s, const llama_token
     return true;
 }
 
-int32_t mtmd_audio_stream_step(mtmd_audio_stream * s, llama_token last_token) {
-    const bool force = s->pending_flush;
-    s->pending_flush = false;
+int32_t mtmd_audio_stream_voxtral_rt::step(llama_token last_token) {
+    const bool force = pending_flush;
+    pending_flush = false;
 
-    mtmd_audio_stream_compute_mel(s, force);
-    if (!mtmd_audio_stream_encode(s, force)) {
+    // last_token was sampled from the last row, the text of every row so far is out
+    n_rows_done = n_rows;
+
+    mtmd_audio_stream_compute_mel(this, force);
+    if (!mtmd_audio_stream_encode(this, force)) {
         return -2;
     }
 
     // drop the consumed rows once in a while
-    if (s->embd_off >= 1024) {
-        s->embd.erase(s->embd.begin(), s->embd.begin() + s->embd_off * s->n_embd);
-        s->embd_off = 0;
+    if (embd_off >= 1024) {
+        embd.erase(embd.begin(), embd.begin() + embd_off * n_embd);
+        embd_off = 0;
     }
 
-    const int64_t n_avail = (int64_t) (s->embd.size() / s->n_embd) - s->embd_off;
-    s->out.clear();
+    const int64_t n_avail = (int64_t) (embd.size() / n_embd) - embd_off;
+    out.clear();
 
-    if (s->n_rows == 0) {
+    if (n_rows == 0) {
         // the prefix rows go out together, the caller samples from the last one
-        const int64_t n_prefix = (int64_t) s->prefix.size();
+        const int64_t n_prefix = (int64_t) prefix.size();
         if (n_avail < n_prefix) {
-            return s->ended ? -1 : 0;
+            return ended ? -1 : 0;
         }
-        if (!mtmd_audio_stream_add_rows(s, s->prefix.data(), s->prefix.size())) {
+        if (!mtmd_audio_stream_add_rows(this, prefix.data(), prefix.size())) {
             return -2;
         }
         return (int32_t) n_prefix;
     }
 
     if (n_avail == 0) {
-        return s->ended ? -1 : 0;
+        return ended ? -1 : 0;
     }
     if (last_token == LLAMA_TOKEN_NULL) {
         LOG_ERR("%s: the token sampled from the previous row is required\n", __func__);
         return -2;
     }
-    if (!mtmd_audio_stream_add_rows(s, &last_token, 1)) {
+    if (!mtmd_audio_stream_add_rows(this, &last_token, 1)) {
         return -2;
     }
     return 1;
 }
 
-const float * mtmd_audio_stream_get_output_embd(const mtmd_audio_stream * s) {
-    return s->out.data();
+//
+// LFM3-Audio
+//
+
+bool mtmd_audio_stream_lfm3a::init(const mtmd_audio_stream_params & params) {
+    const clip_hparams * hparams = clip_get_hparams(ctx_clip);
+
+    // one row per 8 mel frames (three stride-2 convs), mel frames are not centered
+    win             = hparams->audio_window_len;
+    frames_per_tok  = 8;
+    samples_per_tok = frames_per_tok * hop;
+
+    const int n_step = (int) ((int64_t) params.step_ms * hparams->audio_sample_rate / 1000 / hop);
+    n_step_mel = CLIP_ALIGN(std::max(n_step, frames_per_tok), frames_per_tok);
+
+    if (params.n_prompt > 0) {
+        // live ASR, a chunk per encoder call
+        n_step_mel = (int) chunk_rows * frames_per_tok;
+        prompt.assign(params.prompt, params.prompt + params.n_prompt);
+
+        const llama_vocab * vocab = llama_model_get_vocab(ctx->text_model);
+        auto find_token = [&](const char * piece) {
+            llama_token tok[2];
+            const int n = llama_tokenize(vocab, piece, (int32_t) strlen(piece), tok, 2, false, true);
+            return n == 1 ? tok[0] : LLAMA_TOKEN_NULL;
+        };
+        tok_text_end  = find_token("<|text_end|>");
+        tok_mixed_end = find_token("<|mixed_end|>");
+        if (tok_text_end == LLAMA_TOKEN_NULL || tok_mixed_end == LLAMA_TOKEN_NULL) {
+            LOG_ERR("%s: the vocab lacks the tokens of LFM3-Audio live ASR\n", __func__);
+            return false;
+        }
+        soft_seq_rows = std::lround(15.0 * hparams->audio_sample_rate / samples_per_tok);
+        max_seq_rows  = std::lround(30.0 * hparams->audio_sample_rate / samples_per_tok);
+    }
+
+    enc_state = clip_enc_state_init(ctx_clip, n_step_mel);
+    if (!enc_state) {
+        return false;
+    }
+
+    // the markers the tokenizer puts around an audio clip go around the stream as well
+    const llama_vocab * vocab = llama_model_get_vocab(ctx->text_model);
+    auto marker_rows = [&](const std::string & text, std::vector<float> & rows) {
+        if (text.empty()) {
+            return true;
+        }
+        std::vector<llama_token> tokens(text.size() + 4);
+        const int32_t n = llama_tokenize(vocab, text.c_str(), (int32_t) text.size(), tokens.data(), (int32_t) tokens.size(), false, true);
+        if (n <= 0) {
+            return false;
+        }
+        rows.resize((size_t) n * n_embd);
+        return llama_model_get_tok_embd_rows(ctx->text_model, tokens.data(), n, rows.data()) != 0;
+    };
+    return marker_rows(ctx->aud_beg, beg_rows) && marker_rows(ctx->aud_end, end_rows);
+}
+
+// end: pad the tail like mtmd_audio_preprocessor_lfm3a, so the rows match encoding the clip at once
+// otherwise only whole rows are encoded, a padded partial row would change the encoder state
+// live ASR: pad the last chunk with silence, so its text comes out
+int32_t mtmd_audio_stream_lfm3a::flush(bool end) {
+    if (end) {
+        const int64_t n_padded = n_samples <= win ? win : n_samples + (hop - (n_samples - win) % hop) % hop;
+        add_silence(n_padded - n_samples);
+    } else if (!prompt.empty() && n_samples > 0) {
+        const int64_t frames_per_chunk = chunk_rows * frames_per_tok;
+        const int64_t n_frames_audio   = n_samples <= win ? 1 : (n_samples - win + hop - 1) / hop + 1;
+        const int64_t n_frames_padded  = (n_frames_audio + frames_per_chunk - 1) / frames_per_chunk * frames_per_chunk;
+        add_silence(win + (n_frames_padded - 1) * hop - n_samples);
+    }
+    n_rows_flushed = (n_samples >= win ? (n_samples - win) / hop + 1 : 0) / frames_per_tok;
+    pending_flush = true;
+    ended = end;
+    return 0;
+}
+
+// mel frame k covers samples [k * hop, k * hop + win), computed with the offline preprocessor
+static bool mtmd_audio_stream_compute_mel(mtmd_audio_stream_lfm3a * s) {
+    const int64_t n_total = s->n_samples >= s->win ? (s->n_samples - s->win) / s->hop + 1 : 0;
+    const int64_t n_new   = n_total - s->n_frames;
+    if (n_new <= 0) {
+        return true;
+    }
+    const int64_t start = s->n_frames * s->hop - s->buf_pos;
+    const size_t  len   = (size_t) (s->win + (n_new - 1) * s->hop);
+    std::vector<mtmd_audio_mel> mel_chunks;
+    if (!s->ctx->audio_preproc->preprocess(s->buf.data() + start, len, mel_chunks) || mel_chunks.size() != 1) {
+        LOG_ERR("%s: unable to preprocess audio\n", __func__);
+        return false;
+    }
+    const auto & mel = mel_chunks[0];
+    GGML_ASSERT(mel.n_len == n_new && mel.n_mel == s->n_mel);
+    for (int64_t t = 0; t < n_new; t++) {
+        for (int m = 0; m < s->n_mel; m++) {
+            s->mel.push_back(mel.data[(size_t) m * mel.n_len + t]);
+        }
+    }
+    s->n_frames = n_total;
+
+    const int64_t keep_from = s->n_frames * s->hop;
+    if (keep_from > s->buf_pos) {
+        s->buf.erase(s->buf.begin(), s->buf.begin() + (keep_from - s->buf_pos));
+        s->buf_pos = keep_from;
+    }
+    return true;
+}
+
+// whole rows only, except for the last call of an ended stream; the encoder state carries the rest
+static bool mtmd_audio_stream_encode(mtmd_audio_stream_lfm3a * s, bool flush) {
+    const int n_avail = (int) (s->mel.size() / s->n_mel);
+    int n = n_avail - n_avail % s->frames_per_tok;
+    if (flush && s->ended) {
+        n = n_avail;
+    } else if (!flush) {
+        n -= n % s->n_step_mel;
+    }
+    if (n <= 0) {
+        return true;
+    }
+
+    std::vector<float> data((size_t) s->n_mel * n);
+    for (int m = 0; m < s->n_mel; m++) {
+        for (int t = 0; t < n; t++) {
+            data[(size_t) m * n + t] = s->mel[(size_t) t * s->n_mel + m];
+        }
+    }
+    clip_image_f32 img;
+    img.set_size({n, s->n_mel}, false, /* is_audio */ true);
+    img.cpy_buf(data);
+    const int n_rows = clip_n_output_tokens(s->ctx_clip, &img);
+
+    clip_image_f32_batch batch;
+    batch.is_audio = true;
+    batch.entries.push_back(std::move(img));
+
+    std::vector<float> res((size_t) s->n_embd * n_rows);
+    clip_encode_params params;
+    params.n_threads = s->ctx->n_threads;
+    params.imgs      = &batch;
+    params.out_embd  = &res;
+    params.enc_state = s->enc_state;
+    if (!clip_encode(s->ctx_clip, &params)) {
+        LOG_ERR("%s: failed to encode audio\n", __func__);
+        return false;
+    }
+    s->embd.insert(s->embd.end(), res.begin(), res.end());
+    s->mel.erase(s->mel.begin(), s->mel.begin() + (size_t) n * s->n_mel);
+    return true;
+}
+
+int32_t mtmd_audio_stream_lfm3a::step(llama_token last_token) {
+    const bool flush = pending_flush;
+    pending_flush = false;
+
+    if (!mtmd_audio_stream_compute_mel(this) || !mtmd_audio_stream_encode(this, flush)) {
+        return -2;
+    }
+    if (!prompt.empty()) {
+        return step_live(last_token);
+    }
+
+    // rows carry no text, all of them go out at once and last_token is not used
+    const int64_t n_avail = (int64_t) (embd.size() / n_embd) - embd_off;
+    out.clear();
+    if (n_avail > 0 || ended) {
+        if (!beg_done) {
+            beg_done = true;
+            out = beg_rows;
+        }
+        out.insert(out.end(), embd.begin() + embd_off * n_embd, embd.end());
+        embd.clear();
+        embd_off = 0;
+        if (ended && !end_done) {
+            end_done = true;
+            out.insert(out.end(), end_rows.begin(), end_rows.end());
+        }
+    }
+    const int64_t n_out = (int64_t) (out.size() / n_embd);
+    if (n_out == 0) {
+        return ended ? -1 : 0;
+    }
+    n_rows      += n_out;
+    n_rows_done += n_avail; // the markers are not audio rows
+    return (int32_t) n_out;
+}
+
+bool mtmd_audio_stream_lfm3a::add_tokens(const llama_token * tokens, size_t n) {
+    if (n == 0) {
+        return true;
+    }
+    tok_embd.resize(n * n_embd);
+    if (llama_model_get_tok_embd_rows(ctx->text_model, tokens, n, tok_embd.data()) == 0) {
+        LOG_ERR("%s: invalid token\n", __func__);
+        return false;
+    }
+    out.insert(out.end(), tok_embd.begin(), tok_embd.end());
+    return true;
+}
+
+// last_token continues the text of the chunk, or <|text_end|> closes it and the next 12 rows go out (the rest at the end)
+int32_t mtmd_audio_stream_lfm3a::step_live(llama_token last_token) {
+    out.clear();
+    seq_start = false;
+
+    // drop the consumed rows once in a while
+    if (embd_off >= 1024) {
+        embd.erase(embd.begin(), embd.begin() + embd_off * n_embd);
+        embd_off = 0;
+    }
+
+    if (in_chunk) {
+        if (last_token == LLAMA_TOKEN_NULL) {
+            LOG_ERR("%s: the token sampled from the previous row is required\n", __func__);
+            return -2;
+        }
+        if (last_token != tok_text_end) {
+            chunk_has_text = true;
+            if (n_chunk_text + 1 < (final_chunk ? max_final_tokens : max_chunk_tokens)) {
+                n_chunk_text++;
+                return add_tokens(&last_token, 1) ? 1 : -2;
+            }
+            // close a runaway chunk like the model should have
+            closing.push_back(last_token);
+        }
+        // the text of the chunk is complete, its closing tokens go in before the next chunk
+        in_chunk     = false;
+        n_rows_done += n_chunk_rows;
+        if (final_chunk) {
+            done = true;
+        }
+        closing.push_back(tok_text_end);
+    }
+    if (done) {
+        return -1;
+    }
+
+    const int64_t n_avail = (int64_t) (embd.size() / n_embd) - embd_off;
+    const bool    drained = ended && mel.empty();
+    const bool    final   = drained && n_avail <= chunk_rows;
+    if (!final && n_avail < chunk_rows) {
+        return 0;
+    }
+    if (final && n_avail == 0 && !started) {
+        done = true; // no audio at all
+        return -1;
+    }
+
+    if (!started || n_seq_rows >= max_seq_rows || (n_seq_rows >= soft_seq_rows && !chunk_has_text)) {
+        closing    = prompt; // the closing tokens go away with the sequence
+        seq_start  = true;
+        n_seq_rows = 0;
+    }
+    const int64_t n_feed = final ? n_avail : chunk_rows;
+    if (!add_tokens(closing.data(), closing.size())) {
+        return -2;
+    }
+    closing.clear();
+    const float * rows = embd.data() + embd_off * n_embd;
+    out.insert(out.end(), rows, rows + (size_t) n_feed * n_embd);
+    embd_off += n_feed;
+    if (final && !add_tokens(&tok_mixed_end, 1)) {
+        return -2;
+    }
+
+    started        = true;
+    in_chunk       = true;
+    final_chunk    = final;
+    chunk_has_text = false;
+    n_chunk_text   = 0;
+    n_chunk_rows   = n_feed;
+    n_seq_rows    += n_feed;
+    n_rows        += n_feed;
+    return (int32_t) (out.size() / n_embd);
 }
 
 int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * s) {
-    return s->n_rows;
+    return s->n_rows_done;
 }
 
 int64_t mtmd_audio_stream_get_n_rows_flushed(const mtmd_audio_stream * s) {

tools/mtmd/mtmd.h

@@ -383,6 +383,11 @@ typedef struct mtmd_audio_stream mtmd_audio_stream;
 
 struct mtmd_audio_stream_params {
     int32_t step_ms; // audio encoded per encoder call, rounded up to the model step (default: 320)
+
+    // tokens that open the stream, they go out with the first rows (default: none)
+    // with the prompt of its "asr" chat template, LFM3-Audio transcribes live
+    const llama_token * prompt;
+    size_t              n_prompt;
 };
 MTMD_API struct mtmd_audio_stream_params mtmd_audio_stream_params_default(void);
 
@@ -412,11 +417,14 @@ MTMD_API int32_t mtmd_audio_stream_step(mtmd_audio_stream * stream, llama_token
 // rows of the last step, the reading size (in bytes) is n_rows * llama_model_n_embd_inp(model) * sizeof(float)
 MTMD_API const float * mtmd_audio_stream_get_output_embd(const mtmd_audio_stream * stream);
 
-// rows produced so far, the prefix rows included
+// true if the rows of the last step start a new sequence: the caller drops its sequence before decoding them
+MTMD_API bool mtmd_audio_stream_is_seq_start(const mtmd_audio_stream * stream);
+
+// audio rows whose text has been sampled (prefix rows included), step() updates it from last_token
 MTMD_API int64_t mtmd_audio_stream_get_n_rows(const mtmd_audio_stream * stream);
 
-// rows the stream produces up to the end of the last flush
-// once get_n_rows() reaches it, every row of the flushed audio has been produced
+// audio rows up to the end of the last flush
+// once get_n_rows() reaches it, the text of the flushed audio has been sampled
 MTMD_API int64_t mtmd_audio_stream_get_n_rows_flushed(const mtmd_audio_stream * stream);
 
 /////////////////////////////////////////
@@ -428,6 +436,7 @@ enum mtmd_gen_audio_type {
     MTMD_GEN_AUDIO_TYPE_QWEN3TTS,
     MTMD_GEN_AUDIO_TYPE_POCKETTTS,
     MTMD_GEN_AUDIO_TYPE_SOPRANO,
+    MTMD_GEN_AUDIO_TYPE_LFM3A,
 };
 
 struct mtmd_gen_audio_info {
@@ -444,6 +453,7 @@ enum mtmd_gen_process_type {
     MTMD_GEN_PROCESS_TYPE_GEN_WAV,  // convert semantic to PCM audio
                                     // for qwen3tts, this is code2wav
                                     // for pocket-tts, this is mimi decoder
+                                    // for lfm3a, this is the codec decoder + ISTFT, n_codes == 0 flushes the tail
 };
 
 struct mtmd_gen_inp {
@@ -455,9 +465,12 @@ struct mtmd_gen_inp {
     int32_t top_k;
     float   top_p;
     float   temp; // sampling temperature, or noise scale for flow-matching decoders
+    float   penalty_repeat; // repetition penalty of the sampled codes, <= 1 is off
+    int32_t penalty_last_n; // frames the repetition penalty looks back on
 
     // for MTMD_GEN_PROCESS_TYPE_GEN_WAV
     // pass either codes (discrete) or feats (continuous), depending on the pipeline
+    // lfm3a GEN_CODE also takes codes: the codes revealed so far, -1 for a masked codebook
     int32_t * codes;
     size_t    n_codes;
     const float * feats;

tools/server/CMakeLists.txt

@@ -17,6 +17,8 @@ add_library(${TARGET} STATIC
     server-decision.h
     server-context.cpp
     server-context.h
+    server-realtime.cpp
+    server-realtime.h
     server-stream.cpp
     server-stream.h
     server-tools.cpp

tools/server/README.md

@@ -857,6 +857,8 @@ Returns raw audio bytes (`audio/wav` by default) rather than JSON. For more info
 
 `lang`: Language code for the utterance (model-dependent, e.g. `en`, `zh`). Optional.
 
+`voice`: Voice for the utterance (model-dependent, e.g. `female` or `male` for LFM3-Audio). Optional.
+
 `speaker_ref_b64`: Base64-encoded reference audio to clone the speaker's voice. Optional. Alternatively, upload the reference audio as a `speaker_ref` file field via `multipart/form-data` (see example below) - if both are provided, the uploaded file takes precedence.
 
 `top_k`, `top_p`: Sampling params for the acoustic code predictor. Optional, model-dependent defaults apply.
@@ -1017,7 +1019,7 @@ By default, it is read-only. To make POST request to change global properties, y
 - `total_slots` - the total number of slots for process requests (defined by `--parallel` option)
 - `model_path` - the path to model file (same with `-m` argument)
 - `chat_template` - the model's original Jinja2 prompt template
-- `chat_template_caps` - capabilities of the chat template (see `common/jinja/caps.h` for more info)
+- `chat_template_caps` - capabilities of the chat template (see `common/jinja/caps.h` for more info), `reasoning_effort_levels` lists the `reasoning_effort` values the template renders distinctly, from lowest to highest
 - `modalities` - the list of supported modalities
 - `is_sleeping` - sleeping status, see [Sleeping on idle](#sleeping-on-idle)
 
@@ -1585,6 +1587,10 @@ Returns a JSON object with a boolean `success` field, and an optional `message`
 
 Transcribes streamed audio, for models with a streaming audio encoder. Compatible with OAI `/v1/realtime` schema.
 
+Models that write the text while they listen use this endpoint: Voxtral Realtime, and live ASR models, which carry the prompt of their transcription in a chat template named `asr` (e.g. the LFM3-Audio live ASR checkpoints, the text of every 0.96 s of audio follows it). A live ASR model restarts its context at a pause after 15 s and always after 30 s, the transcript then goes on in a new line. A commit completes the audio with silence, so that its text comes out.
+
+Live ASR models can also hold a conversation, so they transcribe live only in sessions opened with `/v1/realtime?intent=transcription`, as OpenAI transcription sessions are; other sessions are a [realtime conversation](#realtime-conversation).
+
 Audio must be PCM16, mono, at the model sample rate. The sample rate is returned in `session.created` as `audio.input.format.rate`. Each `input_audio_buffer.append` carries a base64-encoded chunk of PCM16 samples.
 
 *Client events:*
@@ -1606,6 +1612,38 @@ Closing the websocket also ends the input.
 
 Note that words held back by the model's delay at the time of a commit are attributed to the next item.
 
+#### Realtime conversation
+
+Models that answer after the audio, like LFM3-Audio and LFM3-Omni, hold a conversation on the same endpoint (with an `asr` template, unless the session is opened with `?intent=transcription`). The audio is prefilled as it arrives, so the response starts right after the last chunk. Audio input is PCM16 mono at the model sample rate (`audio.input.format.rate` of `session.created`), audio output is PCM16 mono at `audio.output.format.rate`.
+
+`session.type` selects the session, it can be changed with `session.update` before the first audio or message:
+
+- `realtime` (default): a conversation, every response stays in context for the next turns
+- `transcription`: each committed audio buffer is transcribed on its own, `audio.input.transcription.language` picks the language
+
+*Client events:*
+
+- `session.update`: `type`, `instructions` (the system prompt, before the first audio or message), `output_modalities`, `max_output_tokens` (an integer or `"inf"`, the default)
+- `input_audio_buffer.append`: `audio` holds base64 PCM16 samples
+- `input_audio_buffer.commit`: ends the audio of the user turn, in a transcription session the transcript follows
+- `conversation.item.create`: a user message with `input_text` content
+- `response.create`: ends the user turn and generates the response, `response.output_modalities` and `response.max_output_tokens` override the session
+- `response.cancel`: stops the response being generated
+
+LFM3-Omni is not trained to reason, run it with `--reasoning off` so that its turns open with an empty think block.
+
+`output_modalities` picks the mode of the responses: left out, the model switches between text and audio on its own; `["text"]` keeps the response in text, `["audio"]` makes it speech and `["text", "audio"]` interleaves text with the speech.
+
+*Server events:*
+
+- `session.created` and `session.updated`: carry the `session` object
+- `input_audio_buffer.committed`, `conversation.item.added`, `conversation.item.done`: the user item
+- `response.created`, `response.output_item.added`, `response.content_part.added`, then `response.output_text.delta` and `response.output_audio.delta` (base64 PCM16) as they are generated, then the `.done` events and `response.done`
+- `conversation.item.input_audio_transcription.delta` and `.completed`: in a transcription session
+- `error`: `error.message` describes the problem
+
+See [examples/realtime_client.py](../../examples/realtime_client.py) for a client with the `openai` library.
+
 ### POST `/v1/responses`: OpenAI-compatible Responses API
 
 *Options:*

tools/server/server-chat.cpp

@@ -464,7 +464,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                         // Single-pass: build both text and content_parts, decide format at the end
                         std::string result_text;
                         json content_parts = json::array();
-                        bool has_images = false;
+                        bool keep_parts = false; // images or tool references
 
                         for (const auto & c : result_content) {
                             std::string c_type = json_value(c, "type", std::string());
@@ -476,7 +476,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                                     {"text", text}
                                 });
                             } else if (c_type == "image") {
-                                has_images = true;
+                                keep_parts = true;
                                 json source = json_value(c, "source", json::object());
                                 std::string source_type = json_value(source, "type", std::string());
                                 if (source_type == "base64") {
@@ -493,10 +493,17 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                                         {"image_url", {{"url", json_value(source, "url", std::string())}}}
                                     });
                                 }
+                            } else if (c_type == "tool_reference") {
+                                // tool search result naming a deferred tool
+                                keep_parts = true;
+                                content_parts.push_back({
+                                    {"type", "tool_reference"},
+                                    {"name", json_value(c, "tool_name", std::string())}
+                                });
                             }
                         }
 
-                        if (!has_images) {
+                        if (!keep_parts) {
                             // Text-only: collapse to a plain string for maximum compatibility
                             tool_results.push_back({
                                 {"role", "tool"},
@@ -504,7 +511,7 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                                 {"content", result_text}
                             });
                         } else {
-                            // Mixed or image-only: use array content parts (OpenAI multimodal tool format)
+                            // images or tool references: keep array content parts
                             tool_results.push_back({
                                 {"role", "tool"},
                                 {"tool_call_id", tool_use_id},
@@ -559,6 +566,9 @@ json server_chat_convert_anthropic_to_oai(const json & body) {
                         {"parameters", tool.contains("input_schema") ? tool.at("input_schema") : json::object()}
                     }}
                 });
+                if (tool.contains("defer_loading")) {
+                    oai_tools.back()["defer_loading"] = tool.at("defer_loading");
+                }
             }
             oai_body["tools"] = oai_tools;
         }
@@ -677,11 +687,11 @@ json convert_transcriptions_to_chatcmpl(
     if (response_format != "json") {
         throw std::invalid_argument("Only 'json' response_format is supported for transcription");
     }
-    const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls);
+    const common_chat_prompt_preset preset = common_chat_get_asr_prompt(tmpls, language);
     if (prompt.empty()) {
         prompt = preset.user;
     }
-    if (!language.empty()) {
+    if (!language.empty() && !preset.has_language) {
         prompt += string_format(" (language: %s)", language.c_str());
     }
     prompt += get_media_marker();
@@ -711,3 +721,22 @@ json convert_transcriptions_to_chatcmpl(
 
     return chatcmpl_body;
 }
+
+llama_tokens server_chat_get_asr_prompt(const llama_model * model) {
+    const char * src = llama_model_chat_template(model, "asr");
+    if (src == nullptr) {
+        return {};
+    }
+    const llama_vocab * vocab = llama_model_get_vocab(model);
+    const llama_token bos = llama_vocab_bos(vocab);
+    const common_chat_template tmpl(src, bos == LLAMA_TOKEN_NULL ? "" : common_token_to_piece(vocab, bos, true), "");
+    jinja::context ctx(tmpl.source());
+    jinja::global_from_json(ctx, json {
+        {"bos_token",             tmpl.bos_token()},
+        {"messages",              json::array()},
+        {"add_generation_prompt", true},
+    }, false);
+    jinja::runtime runtime(ctx);
+    const std::string prompt = jinja::runtime::gather_string_parts(runtime.execute(tmpl.prog))->as_string().str();
+    return common_tokenize(vocab, prompt, false, true);
+}

tools/server/server-chat.h

@@ -21,4 +21,8 @@ json convert_transcriptions_to_chatcmpl(
     const std::map<std::string, uploaded_file> & in_files,
     std::vector<raw_buffer> & out_files);
 
+// prompt of the "asr" chat template, live ASR models (e.g. LFM3-Audio live ASR) transcribe with it
+// empty if the model has no such template, throws on a template error
+llama_tokens server_chat_get_asr_prompt(const llama_model * model);
+
 json server_chat_msg_diff_to_json_oaicompat(const common_chat_msg_diff & diff);

tools/server/server-common.cpp

@@ -220,6 +220,8 @@ std::vector<std::string> server_model_output_modalities(common_decision_type dec
         case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
         case COMMON_DECISION_TYPE_H2O_LIGHTNING:
         case COMMON_DECISION_TYPE_MINICPM_OCR_JEV:
+        case COMMON_DECISION_TYPE_LFM3_D1:
+        case COMMON_DECISION_TYPE_LFM3_D1_OMNI:
             out = {"decisions"};
             break;
         case COMMON_DECISION_TYPE_GRANITE_SWITCH:
@@ -1316,7 +1318,7 @@ static void oaicompat_content_load_media(json & content, const server_chat_param
             p.erase("input_video");
             p.erase("video_url");
 
-        } else if (type != "text") {
+        } else if (type != "text" && type != "tool_reference") {
             throw std::invalid_argument("unsupported content[].type");
         }
     }
@@ -1442,7 +1444,7 @@ json oaicompat_chat_params_parse(
     inputs.json_schema            = json_schema.is_null() ? "" : json_schema.dump();
     inputs.grammar                = grammar;
     inputs.use_jinja              = opt.use_jinja;
-    inputs.parallel_tool_calls    = json_value(body, "parallel_tool_calls", caps["supports_parallel_tool_calls"]);
+    inputs.parallel_tool_calls    = json_value(body, "parallel_tool_calls", caps.at("supports_parallel_tool_calls").get<bool>());
     inputs.add_generation_prompt  = json_value(body, "add_generation_prompt", true);
     inputs.continue_final_message = body.contains("continue_final_message") ?
         common_chat_continuation_parse(body.at("continue_final_message")) :

tools/server/server-context.cpp

@@ -8,6 +8,7 @@
 #include "server-http.h"
 #include "server-task.h"
 #include "server-queue.h"
+#include "server-realtime.h"
 #include "server-schema.h"
 #include "server-stream.h"
 
@@ -402,6 +403,9 @@ struct server_slot {
     bool audio_wait = false; // the stream needs more audio before it can produce rows
     std::deque<int64_t> audio_commit_rows; // end row of each non-final commit not yet completed
 
+    // realtime conversation, see SERVER_TASK_TYPE_REALTIME
+    std::unique_ptr<server_realtime_slot> rt;
+
     server_slot_stats stats;
 
     // accepted tokens per draft position
@@ -460,6 +464,7 @@ struct server_slot {
         audio_wait = false;
         audio_commit_rows.clear();
         inp_embd.clear();
+        rt.reset();
     }
 
     void init_sampler() const {
@@ -529,6 +534,12 @@ struct server_slot {
             return false;
         }
 
+        // lfm3 d1 has no memory, its prompt is read as one flat ubatch and the short convs would mix the sequences
+        // TODO: batch them along the sequence dimension, this needs ubatches of sequences with unequal lengths
+        if (task->decision.no_batching || other_slot.task->decision.no_batching) {
+            return false;
+        }
+
         return task->type == other_slot.task->type
             && inp_embd.size() == other_slot.inp_embd.size()
             && are_lora_equal(lora, other_slot.lora);
@@ -654,7 +665,7 @@ struct server_slot {
 
             // do not keep context of the child slots - the parent's context is enough
             // nor of the stateless slots - no task ever reuses it
-            if (task->is_child() || task->is_stateless() || audio_stream) {
+            if (task->is_child() || task->is_stateless() || audio_stream || task->type == SERVER_TASK_TYPE_REALTIME) {
                 prompt_clear();
             }
 
@@ -1017,6 +1028,9 @@ public:
     mtmd_helper_init_opt init_opt = mtmd_helper_init_opt_default();
     const llama_vocab * vocab = nullptr;
 
+    // prompt of the "asr" chat template, for live ASR
+    llama_tokens asr_prompt;
+
     bool has_cap_tts() const {
         return mtmd_cap.audio_gen;
     }
@@ -1699,6 +1713,13 @@ private:
                 return false;
             }
 
+            try {
+                asr_prompt = server_chat_get_asr_prompt(model_tgt);
+            } catch (const std::exception & e) {
+                SRV_ERR("%s: \"asr\" chat template error: %s\n", __func__, e.what());
+                return false;
+            }
+
             // IMPORTANT: chat_params is reused across sleeping / resuming states,
             //            never store llama_context/llama_model pointers in chat_params,
             //            as they may be invalidated after sleeping
@@ -1721,7 +1742,7 @@ private:
             {
                 auto caps = common_chat_templates_get_caps(chat_params.tmpls.get());
                 auto it = params_base.default_template_kwargs.find("preserve_reasoning");
-                bool supported = caps.at("supports_preserve_reasoning");
+                bool supported = caps.at("supports_preserve_reasoning").get<bool>();
                 bool specified = params_base.preserve_reasoning_specified;
                 // note: the kwarg is enabled by default if not specified explicitly, so check the value
                 bool enabled = it != params_base.default_template_kwargs.end() && it->second == "true";
@@ -2014,6 +2035,23 @@ private:
             }
         }
 
+        // realtime transcription: the rows come from the audio stream, the helper may bias some tokens
+        if (task.type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
+            auto sparams = mtmd_audio_stream_params_default();
+            sparams.prompt   = asr_prompt.data();
+            sparams.n_prompt = asr_prompt.size();
+            slot.audio_stream.reset(mtmd_audio_stream_init(mctx, sparams));
+            if (!slot.audio_stream) {
+                send_error(task, "failed to initialize the audio stream", ERROR_TYPE_SERVER);
+                return false;
+            }
+            if (!asr_prompt.empty()) {
+                for (const auto & b : mtmd_helper::audio_stream_logit_bias(mctx, model_tgt)) {
+                    task.params.sampling.logit_bias.push_back(b);
+                }
+            }
+        }
+
         // initialize samplers
         if (task.need_sampling()) {
             try {
@@ -2069,12 +2107,6 @@ private:
 
         // no prompt, the rows come from the audio stream, which audio controls reach from now on
         if (slot.task->type == SERVER_TASK_TYPE_TRANSCRIBE_RT) {
-            slot.audio_stream.reset(mtmd_audio_stream_init(mctx, mtmd_audio_stream_params_default()));
-            if (!slot.audio_stream) {
-                send_error(slot, "failed to initialize the audio stream", ERROR_TYPE_SERVER);
-                slot.release();
-                return false;
-            }
             slot.prompt_clear();
             slot.init_sampler();
             slot.sampled = LLAMA_TOKEN_NULL;
@@ -2084,6 +2116,17 @@ private:
             send_partial_response(slot, {}, false, true);
         }
 
+        // no prompt either, the session appends text and audio and asks for responses with its controls
+        if (slot.task->type == SERVER_TASK_TYPE_REALTIME) {
+            slot.rt = std::make_unique<server_realtime_slot>(ctx_tgt, mctx, slot.id, slot.n_ctx, slot.task->params.sampling,
+                                                             queue_results, slot.task->id, slot.task->index);
+            slot.prompt_clear();
+            slot.stats.update_prompt_start();
+            slot.state      = SLOT_STATE_GENERATING;
+            slot.audio_wait = true;
+            send_partial_response(slot, {}, false, true);
+        }
+
         // reset server kill-switch counter
         n_empty_consecutive = 0;
 
@@ -2966,6 +3009,7 @@ private:
             case SERVER_TASK_TYPE_COMPLETION:
             case SERVER_TASK_TYPE_INFILL:
             case SERVER_TASK_TYPE_TRANSCRIBE_RT:
+            case SERVER_TASK_TYPE_REALTIME:
             case SERVER_TASK_TYPE_EMBEDDING:
             case SERVER_TASK_TYPE_RERANK:
             case SERVER_TASK_TYPE_TTS:
@@ -3078,6 +3122,20 @@ private:
                         // act on the live slot mid generation, never defer
                         common_sampler_reasoning_budget_force(slot->smpl.get());
                         res->success = true;
+                    } else if (slot->rt) {
+                        // a cancel acts now, everything else runs after what is already queued
+                        if (task.params.control_action == SERVER_CONTROL_ACTION_RESPONSE_CANCEL) {
+                            slot->rt->cancel();
+                        } else {
+                            slot->rt->push({
+                                task.params.control_action,
+                                std::move(task.params.control_text),
+                                std::move(task.params.control_audio),
+                                task.params.control_response,
+                            });
+                        }
+                        slot->audio_wait = false;
+                        res->success = true;
                     } else if (task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_APPEND
                             || task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_COMMIT
                             || task.params.control_action == SERVER_CONTROL_ACTION_AUDIO_END) {
@@ -3424,7 +3482,9 @@ private:
             return true; // rows from a previous step, not decoded yet
         }
 
-        // every produced row is decoded and its delta queued, the commits ending there are complete
+        const int32_t n = mtmd_audio_stream_step(slot.audio_stream.get(), slot.sampled);
+
+        // complete the commits whose rows have their text sampled, before the new rows: their text goes to the next item
         const int64_t n_rows = mtmd_audio_stream_get_n_rows(slot.audio_stream.get());
         while (!slot.audio_commit_rows.empty() && slot.audio_commit_rows.front() <= n_rows) {
             slot.audio_commit_rows.pop_front();
@@ -3440,8 +3500,19 @@ private:
             queue_results.send(std::move(res));
         }
 
-        const int32_t n = mtmd_audio_stream_step(slot.audio_stream.get(), slot.sampled);
         if (n > 0) {
+            // new sequence (e.g. live ASR restart): drop the old one, the text goes on in a new line
+            if (mtmd_audio_stream_is_seq_start(slot.audio_stream.get())) {
+                slot.prompt_clear();
+                if (slot.n_sent_text > 0) {
+                    completion_token_output sep;
+                    sep.tok          = LLAMA_TOKEN_NULL;
+                    sep.text_to_send = "\n";
+                    slot.generated.append(common_chat_input(sep.text_to_send));
+                    slot.n_sent_text += sep.text_to_send.size();
+                    send_partial_response(slot, sep, false);
+                }
+            }
             const float * rows = mtmd_audio_stream_get_output_embd(slot.audio_stream.get());
             slot.inp_embd.assign(rows, rows + (size_t) n * batch.n_embd);
             slot.audio_wait = false;
@@ -3527,6 +3598,14 @@ private:
             abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));
         }
 
+        // so do the realtime conversations
+        try {
+            process_realtime_slots();
+        } catch (const std::exception & e) {
+            SRV_ERR("process_realtime_slots() failed: %s\n", e.what());
+            abort_all_slots("process_realtime_slots() failed: " + std::string(e.what()));
+        }
+
         GGML_ASSERT(batch.slot_batched || batch.size() == 0);
 
         if (batch.slot_batched) {
@@ -3589,6 +3668,21 @@ private:
         }
     }
 
+    // realtime conversations (SERVER_TASK_TYPE_REALTIME) drive their own decode loop, see server_realtime_slot
+    void process_realtime_slots() {
+        iterate(slots, [&](server_slot & slot) {
+            if (!slot.is_processing() || !slot.rt) {
+                return;
+            }
+            if (!slot.rt->update()) {
+                send_error(slot, slot.rt->error, slot.rt->error_type);
+                slot.release();
+                return;
+            }
+            slot.audio_wait = slot.rt->is_idle();
+        });
+    }
+
     void process_tts_slots() {
         iterate(slots, [&](server_slot & slot) {
             if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
@@ -3668,8 +3762,8 @@ private:
         // apply context-shift if needed
         // TODO: simplify and improve
         iterate(slots, [&](server_slot & slot) {
-            if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {
-                // TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch
+            if (slot.task && (slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME)) {
+                // TTS and realtime conversation slots drive their own decode loop, never enter the shared batch
                 return;
             }
             if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
@@ -3744,7 +3838,7 @@ private:
 
         // determine which slots are generating and drafting
         iterate(slots, [&](server_slot & slot) {
-            if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
+            if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME) {
                 return;
             }
 
@@ -3890,7 +3984,7 @@ private:
                     return; // batch is full, skip remaining slots
                 }
 
-                if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
+                if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS || slot.task->type == SERVER_TASK_TYPE_REALTIME) {
                     return;
                 }
 
@@ -3954,7 +4048,8 @@ private:
                         }
 
                         // TODO: support memory-less logits computation
-                        if (slot.task->need_logits() && !llama_get_memory(ctx_tgt)) {
+                        // a decision reads its logits from the one batch of its prompt
+                        if (slot.task->need_logits() && !llama_get_memory(ctx_tgt) && slot.task->type != SERVER_TASK_TYPE_DECISION) {
                             send_error(slot, "the current context does not support logits computation. skipping", ERROR_TYPE_SERVER);
                             slot.release();
                             return;
@@ -6016,7 +6111,7 @@ void server_routes::init_routes() {
     // realtime transcription over websocket, modeled after the OpenAI realtime API
     // audio is PCM16 mono at the model sample rate, sent as base64 in input_audio_buffer.append events
     // input_audio_buffer.commit with "final": true ends the input, closing the socket also ends it
-    this->ws_realtime = [this](const server_http_req &, server_http_context::ws_conn & conn) {
+    this->ws_realtime = [this](const server_http_req & req, server_http_context::ws_conn & conn) {
         auto send_event = [&conn](json ev) {
             ev["event_id"] = "event_" + random_string();
             conn.send(safe_json_to_str(ev));
@@ -6033,6 +6128,14 @@ void server_routes::init_routes() {
             });
         };
 
+        // models that answer after the audio hold a conversation, see server-realtime.h
+        // live transcription: Voxtral Realtime (no chat), or an "asr" model in a session opened with ?intent=transcription
+        const bool live_asr = !ctx_server.asr_prompt.empty() && req.get_param("intent") == "transcription";
+        if (ctx_server.mtmd_cap.audio_stream && ctx_server.mtmd_cap.chat && !live_asr) {
+            server_realtime_conversation({queue_tasks, queue_results, *meta, params.sampling, ctx_server.mctx, ctx_server.mtmd_cap, ctx_server.vocab}, conn);
+            return;
+        }
+
         if (!meta->has_audio_stream) {
             send_error_event("The current model does not support realtime transcription.", "invalid_request_error");
             conn.close();
@@ -6367,8 +6470,13 @@ void server_routes::init_routes() {
         server_task task(SERVER_TASK_TYPE_TTS);
         server_schema::eval_tts_schema(params, task, body);
         const bool stream = task.params.stream;
-        task.tts_inp.set_prompt(prompt);
-        task.tts_inp.set_lang(json_value(body, "lang", std::string()));
+        // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
+        const std::string lang       = json_value(body, "lang", std::string());
+        const std::string voice      = json_value(body, "voice", std::string());
+        const std::string tts_prompt = common_chat_get_tts_prompt(meta->chat_params.tmpls.get(), prompt, lang, voice);
+        task.tts_inp.set_prompt(tts_prompt.empty() ? prompt : tts_prompt);
+        task.tts_inp.set_lang(lang);
+        task.tts_inp.set_voice(voice);
         task.tts_inp.data.stream    = stream;
         task.tts_inp.data.out_type  = response_format == "pcm"
             ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM

tools/server/server-context.h

@@ -33,7 +33,7 @@ struct server_context_meta {
 
     // chat params
     server_chat_params & chat_params;
-    std::map<std::string, bool> chat_template_caps;
+    json chat_template_caps;
 
     // tokens
     std::string bos_token_str;

tools/server/server-decision.cpp

@@ -162,9 +162,13 @@ void server_decision_context::init(const llama_model * model) {
         noul_criteria_order = true;
         strict_payload      = true;
         canonical_legend    = true;
-    } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1) {
+    } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1 || model_type == COMMON_DECISION_TYPE_LFM3_D1 || model_type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
         n_options_max   = 255;
         noul_true_first = true;
+        if (model_type != COMMON_DECISION_TYPE_LFM2_D1) {
+            // the texts of the special tokens are broken in the input
+            init_quoted_spellings(LLAMA_TOKEN_ATTR_CONTROL, 2);
+        }
     } else if (model_type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
         token_marker = llama_vocab_mask(vocab);
         if (token_marker == LLAMA_TOKEN_NULL) {
@@ -186,22 +190,7 @@ void server_decision_context::init(const llama_model * model) {
             throw std::runtime_error("decision model has no labels");
         }
         // the texts of the control and user-defined tokens are quoted in the input, so that they stay text
-        for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
-            if (llama_vocab_get_attr(vocab, id) & (LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED)) {
-                const std::string text = llama_vocab_get_text(vocab, id);
-                if (!text.empty()) {
-                    quoted_spellings.push_back(text);
-                }
-            }
-        }
-        std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) {
-            return a.size() > b.size();
-        });
-        for (const auto & spelling : quoted_spellings) {
-            if (quoted_first.find(spelling[0]) == std::string::npos) {
-                quoted_first += spelling[0];
-            }
-        }
+        init_quoted_spellings(LLAMA_TOKEN_ATTR_CONTROL | LLAMA_TOKEN_ATTR_USER_DEFINED, 1);
         n_options_max   = labels.size();
         noul_true_first = true;
     } else if (model_type == COMMON_DECISION_TYPE_MINICPM_OCR_JEV) {
@@ -221,17 +210,42 @@ void server_decision_context::init(const llama_model * model) {
     SRV_INF("decision model type: %s\n", type_name.c_str());
 }
 
+void server_decision_context::init_quoted_spellings(uint32_t attrs, size_t min_size) {
+    for (llama_token id = 0; id < llama_vocab_n_tokens(vocab); id++) {
+        if (llama_vocab_get_attr(vocab, id) & attrs) {
+            const std::string text = llama_vocab_get_text(vocab, id);
+            if (text.size() >= min_size) {
+                quoted_spellings.push_back(text);
+            }
+        }
+    }
+    std::stable_sort(quoted_spellings.begin(), quoted_spellings.end(), [](const std::string & a, const std::string & b) {
+        return a.size() > b.size();
+    });
+    for (const auto & spelling : quoted_spellings) {
+        if (quoted_first.find(spelling[0]) == std::string::npos) {
+            quoted_first += spelling[0];
+        }
+    }
+}
+
 //
 // request parsing
 //
 
+// lfm3-d1-omni: a spoken question, its instructions are an input_audio part
+static bool decision_is_spoken(const json & instructions) {
+    return instructions.is_object() && json_value(instructions, "type", std::string()) == "input_audio";
+}
+
 static bool decision_is_payload(const json & val) {
     return val.is_string() || val.is_object() || val.is_array();
 }
 
 std::vector<server_decision_question> server_decision_context::parse_questions(const json & body) const {
-    // lfm2-d1, lfm2-d1-omni and vllm-d3 accept a null state, for example to ask about images only
-    if (!body.contains("state") || (body.at("state").is_null() && type != COMMON_DECISION_TYPE_LFM2_D1 && type != COMMON_DECISION_TYPE_LFM2_D1_OMNI && type != COMMON_DECISION_TYPE_VLLM_D3)) {
+    // lfm2-d1, lfm2-d1-omni, lfm3-d1 and vllm-d3 accept a null state, for example to ask about images only
+    const bool null_state = type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM2_D1_OMNI || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI;
+    if (!body.contains("state") || (body.at("state").is_null() && !null_state)) {
         throw std::invalid_argument("\"state\" must be provided");
     }
     if (strict_payload && !decision_is_payload(body.at("state"))) {
@@ -264,6 +278,9 @@ std::vector<server_decision_question> server_decision_context::parse_questions(c
         if (strict_payload && (!decision_is_payload(instructions) || instructions == "")) {
             throw err("\"instructions\" must be non-empty text, an object or an array");
         }
+        if (type == COMMON_DECISION_TYPE_LFM3_D1 && decision_is_spoken(instructions)) {
+            throw err("spoken instructions need a model with audio input");
+        }
 
         server_decision_question question;
         question.id           = id;
@@ -581,31 +598,65 @@ static std::string decision_kev_text(const json & val) {
     return server_decision_escape_special_tokens(decision_kev_render(val));
 }
 
-// lfm2-d1-omni: special tokens written in the input must not be parsed as such, in keys too (d1-omni prompt.py: escape)
-static json decision_d1omni_escape(const json & val, int depth = 0) {
+// lfm2-d1-omni, lfm3-d1: special tokens written in the input must not be parsed as such, in keys too
+template <typename F>
+static json decision_escape(const json & val, const F & escape, int depth = 0) {
     if (depth >= RECURSIVE_LIMIT) {
         throw std::invalid_argument("JSON is nested more than " + std::to_string(RECURSIVE_LIMIT) + " levels deep");
     }
     if (val.is_string()) {
-        return server_decision_escape_special_tokens(val.get<std::string>());
+        return escape(val.get<std::string>());
     }
     if (val.is_array()) {
         json out = json::array();
         for (const auto & item : val) {
-            out.push_back(decision_d1omni_escape(item, depth + 1));
+            out.push_back(decision_escape(item, escape, depth + 1));
         }
         return out;
     }
     if (val.is_object()) {
         json out = json::object();
         for (const auto & [key, item] : val.items()) {
-            out[server_decision_escape_special_tokens(key)] = decision_d1omni_escape(item, depth + 1);
+            out[escape(key)] = decision_escape(item, escape, depth + 1);
         }
         return out;
     }
     return val;
 }
 
+std::string server_decision_context::d1_break(const std::string & text) const {
+    static const std::string zwsp = "\xE2\x80\x8B";
+    std::string cur = text;
+    // again for a match that overlapped the one broken before it
+    for (bool changed = true; changed;) {
+        changed = false;
+        std::string out;
+        for (size_t i = 0; i < cur.size();) {
+            const std::string * hit = nullptr;
+            if (quoted_first.find(cur[i]) != std::string::npos) {
+                for (const auto & spelling : quoted_spellings) {
+                    if (cur.compare(i, spelling.size(), spelling) == 0) {
+                        hit = &spelling;
+                        break;
+                    }
+                }
+            }
+            if (hit == nullptr) {
+                out += cur[i++];
+                continue;
+            }
+            // after the first character, its UTF-8 bytes
+            const unsigned char c = (*hit)[0];
+            const size_t n = c < 0x80 ? 1 : c < 0xE0 ? 2 : c < 0xF0 ? 3 : 4;
+            out += hit->substr(0, n) + zwsp + hit->substr(n);
+            i += hit->size();
+            changed = true;
+        }
+        cur = std::move(out);
+    }
+    return cur;
+}
+
 //
 // h2o-lightning: the texts are rendered the way the reference shim renders them with Python (json.dumps, str.lstrip)
 //
@@ -955,6 +1006,9 @@ void server_decision_context::d1_labels(const server_decision_question & questio
             codes.push_back(question.options[i].key);
         } else if (n_options <= 26) {
             codes.push_back(std::string(1, 'A' + i));
+        } else if (type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
+            // A..Z, then AA, AB..
+            codes.push_back(i < 26 ? std::string(1, 'A' + i) : std::string{(char) ('A' + (i - 26) / 26), (char) ('A' + (i - 26) % 26)});
         } else {
             codes.push_back(string_format("%02zu", i));
         }
@@ -1012,7 +1066,7 @@ json server_decision_context::render_options(const server_decision_question & qu
 
     std::vector<std::string>  d1_texts;
     std::vector<llama_tokens> d1_groups;
-    if (type == COMMON_DECISION_TYPE_LFM2_D1) {
+    if (type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
         d1_labels(question, d1_texts, d1_groups);
     }
 
@@ -1092,8 +1146,17 @@ std::string server_decision_context::render(
         inp = decision_replace_text(inp, text_marker, " ");
     }
 
+    // lfm3-d1-omni: a spoken question is heard where its text would be, after the media of the state
+    const bool is_spoken = decision_is_spoken(question.instructions);
+    if (is_spoken) {
+        inp["instructions"] = "";
+    }
+    if (is_lfm3_d1()) {
+        inp = decision_escape(inp, [this](const std::string & text) { return d1_break(text); });
+    }
+
     if (type == COMMON_DECISION_TYPE_LFM2_D1_OMNI) {
-        inp = decision_replace_text(decision_d1omni_escape(inp), D1OMNI_MARKER, "<<d1omni ");
+        inp = decision_replace_text(decision_escape(inp, server_decision_escape_special_tokens), D1OMNI_MARKER, "<<d1omni ");
         inp["audio"]         = is_audio;
         inp["sep"]           = D1OMNI_SEP;
         inp["mark_state"]    = D1OMNI_MARK_STATE;
@@ -1103,13 +1166,16 @@ std::string server_decision_context::render(
 
     // the template puts one media marker per image
     json images = json::array();
-    if (n_images > 0) {
+    if (n_images > 0 || is_spoken) {
         inp = decision_replace_text(inp, get_media_marker(), " ");
         for (size_t i = 0; i < n_images; i++) {
             images.push_back(get_media_marker());
         }
     }
     inp["images"] = images;
+    if (is_spoken) {
+        inp["instructions"] = get_media_marker();
+    }
 
     jinja::context ctx(tmpl->source());
     jinja::global_from_json(ctx, inp, false);
@@ -1145,12 +1211,18 @@ void server_decision_context::fill_task(
         return;
     }
 
+    std::vector<raw_buffer> media = files;
+    if (decision_is_spoken(question.instructions)) {
+        const json audio = json_value(question.instructions, "input_audio", json::object());
+        decision_load_audio(audio.contains("data") ? audio.at("data") : json(), media);
+    }
+
     if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_GRANITE_SWITCH || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 ||
         type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
         // lev reads the ratings of a noul question at its first labels, not at the digits
         task.decision.labels.assign(labels.begin(), labels.begin() + n_outputs(question));
     }
-    if (type == COMMON_DECISION_TYPE_LFM2_D1) {
+    if (type == COMMON_DECISION_TYPE_LFM2_D1 || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
         std::vector<std::string>  texts;
         std::vector<llama_tokens> groups;
         d1_labels(question, texts, groups);
@@ -1158,11 +1230,13 @@ void server_decision_context::fill_task(
             task.decision.labels.insert(task.decision.labels.end(), group.begin(), group.end());
             task.decision.label_groups.push_back(group.size());
         }
+        // the short convs of lfm3 d1 have no memory to separate the sequences of a batch
+        task.decision.no_batching = is_lfm3_d1();
     }
     if (type == COMMON_DECISION_TYPE_OPENJEV || type == COMMON_DECISION_TYPE_LEV || type == COMMON_DECISION_TYPE_NIMBLE || type == COMMON_DECISION_TYPE_PPLX_DECIDER || type == COMMON_DECISION_TYPE_VLLM_D3 || type == COMMON_DECISION_TYPE_LFM2_D1 ||
-        type == COMMON_DECISION_TYPE_H2O_LIGHTNING) {
-        if (!files.empty()) {
-            task.tokens = process_mtmd_prompt(mctx, prompt, files, init_opt);
+        type == COMMON_DECISION_TYPE_H2O_LIGHTNING || type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI) {
+        if (!media.empty()) {
+            task.tokens = process_mtmd_prompt(mctx, prompt, media, init_opt);
             return;
         }
     }
@@ -1556,6 +1630,10 @@ double server_decision_context::get_decision_confidence(server_decision_question
     if (type == COMMON_DECISION_TYPE_VLLM_DECISION2 || type == COMMON_DECISION_TYPE_VLLM_D3) {
         return decision_confidence_entropy(probs);
     }
+    // lfm3-d1: the probability of the answer (d1 api.py)
+    if (is_lfm3_d1()) {
+        return *std::max_element(probs.begin(), probs.end());
+    }
     switch (question_type) {
         case SERVER_DECISION_QUESTION_CHOICE: return decision_confidence_choice(probs);
         case SERVER_DECISION_QUESTION_SCORE:  return decision_confidence_score(probs);
@@ -1644,6 +1722,16 @@ json server_decision_context::format_answer(const server_decision_question & que
             const json & description = question.options[i].description;
             legend[question.options[i].key] = canonical_legend && !description.is_string() ? json(decision_sort_keys(description).dump()) : description;
         }
+        // lfm3-d1: the median level, the first whose cumulative probability reaches one half (d1 api.py)
+        if (is_lfm3_d1()) {
+            double mass = 0.0;
+            size_t level = n - 1;
+            for (size_t i = 0; i < n && mass < 0.5; i++) {
+                mass += probs[i];
+                level = i;
+            }
+            expected = (double) level;
+        }
         answer["score"]         = expected;
         answer["legend"]        = legend;
         answer["probabilities"] = probabilities;

tools/server/server-decision.h

@@ -59,6 +59,10 @@ struct server_decision_context {
         return type == COMMON_DECISION_TYPE_CLEF;
     }
 
+    bool is_lfm3_d1() const {
+        return type == COMMON_DECISION_TYPE_LFM3_D1 || type == COMMON_DECISION_TYPE_LFM3_D1_OMNI;
+    }
+
     // true if the prompt of the model has a place for images
     bool can_use_images() const {
         switch (type) {
@@ -70,6 +74,8 @@ struct server_decision_context {
             case COMMON_DECISION_TYPE_LFM2_D1_OMNI:
             case COMMON_DECISION_TYPE_H2O_LIGHTNING:
             case COMMON_DECISION_TYPE_MINICPM_OCR_JEV:
+            case COMMON_DECISION_TYPE_LFM3_D1:
+            case COMMON_DECISION_TYPE_LFM3_D1_OMNI:
                 return true;
             default:
                 return false;
@@ -164,6 +170,10 @@ private:
     json render_h2o(const json & state, const server_decision_question & question) const;
     std::string h2o_cut_state(const std::string & text) const;
     std::string h2o_quote(const std::string & text) const;
+    // the texts of the tokens with these attributes, quoted_spellings and quoted_first
+    void init_quoted_spellings(uint32_t attrs, size_t min_size);
+    // LFM3_D1: a zero-width space after the first character of each special token text (d1 prompt.py: plain)
+    std::string d1_break(const std::string & text) const;
     void fill_task_laya(llama_tokens & tokens, const server_decision_question & question, server_task & task) const;
     void fill_task_d1omni(
             const json & state,

tools/server/server-realtime.cpp

@@ -0,0 +1,949 @@
+#include "server-realtime.h"
+#include "server-common.h"
+#include "server-context.h"
+#include "server-task.h"
+
+#include "base64.hpp"
+#include "chat.h"
+#include "common.h"
+#include "mtmd.h"
+#include "mtmd-helper.h"
+
+#include <algorithm>
+#include <atomic>
+#include <climits>
+#include <cmath>
+#include <cstring>
+#include <mutex>
+#include <thread>
+
+//
+// a realtime conversation, modeled after the OpenAI realtime API (GA)
+//
+// one slot holds the conversation for the whole session: text and audio go into it as they arrive, so a response
+// starts right after the last audio. session.type "realtime" holds a conversation, "transcription" transcribes each
+// committed audio buffer on its own.
+//
+
+namespace {
+
+// output_modalities of a response, unset: the model switches between text and audio itself
+bool parse_output_modalities(const json & j, mtmd_helper_gen_audio_out_mode & mode, std::string & err) {
+    if (j.is_null()) {
+        mode = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
+        return true;
+    }
+    if (!j.is_array() || j.empty()) {
+        err = "output_modalities must be a non-empty array";
+        return false;
+    }
+    bool text  = false;
+    bool audio = false;
+    for (const auto & m : j) {
+        if (m == "text") {
+            text = true;
+        } else if (m == "audio") {
+            audio = true;
+        } else {
+            err = "output_modalities: unknown modality " + m.dump();
+            return false;
+        }
+    }
+    if (text && audio) {
+        mode = MTMD_HELPER_GEN_AUDIO_OUT_MIXED;
+    } else {
+        mode = audio ? MTMD_HELPER_GEN_AUDIO_OUT_AUDIO : MTMD_HELPER_GEN_AUDIO_OUT_TEXT;
+    }
+    return true;
+}
+
+bool parse_max_output_tokens(const json & j, int32_t & n) {
+    if (j == "inf") {
+        n = -1;
+        return true;
+    }
+    if (!j.is_number_integer() || j.get<int64_t>() < 1) {
+        return false;
+    }
+    n = (int32_t) std::min<int64_t>(j.get<int64_t>(), INT32_MAX);
+    return true;
+}
+
+json output_modalities_json(mtmd_helper_gen_audio_out_mode mode) {
+    switch (mode) {
+        case MTMD_HELPER_GEN_AUDIO_OUT_TEXT:  return json::array({"text"});
+        case MTMD_HELPER_GEN_AUDIO_OUT_AUDIO: return json::array({"audio"});
+        case MTMD_HELPER_GEN_AUDIO_OUT_MIXED: return json::array({"text", "audio"});
+        default:                 return nullptr;
+    }
+}
+
+struct realtime_conversation {
+    const server_realtime_deps &  deps;
+    server_http_context::ws_conn & conn;
+
+    std::string id = "sess_" + random_string();
+
+    // session config
+    std::string                    type = "realtime";
+    std::string                    instructions;
+    std::string                    language; // transcription
+    mtmd_helper_gen_audio_out_mode out_mode          = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
+    int32_t                        max_output_tokens = -1; // "inf"
+    int                            input_rate        = 16000;
+    int                            output_rate       = 24000;
+
+    // the text around a turn, from the chat template
+    std::string first_header; // system prompt + first user header
+    std::string user_header;
+    std::string user_footer;
+    std::string assistant_header;
+    std::string assistant_footer;
+
+
+    // conversation state, owned by the reader thread
+    bool        started   = false; // something went into the slot, the session config is fixed
+    bool        turn_open = false; // a user turn is being filled
+    bool        has_audio = false; // the open turn has audio not committed yet
+    int         n_turns   = 0;
+    std::string next_header;       // text that goes in before the next user turn
+
+    // the item or response the results belong to, shared with the result loop
+    std::mutex  mutex;
+    std::string last_item_id;
+    std::string transcribe_item_id; // transcription: the item being transcribed
+    std::string transcript;
+    std::string response_id;
+    std::string response_item_id;
+    std::string response_text;
+    int         idx_text  = -1;
+    int         idx_audio = -1;
+    json        parts     = json::array();
+
+    std::atomic<bool> closed{false};
+
+    realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn) : deps(deps), conn(conn) {}
+
+    //
+    // events
+    //
+
+    void send_event(json ev) {
+        ev["event_id"] = "event_" + random_string();
+        conn.send(safe_json_to_str(ev));
+    }
+
+    void send_error(const std::string & message, const std::string & type_err = "invalid_request_error") {
+        send_event({
+            {"type",  "error"},
+            {"error", {{"type", type_err}, {"code", nullptr}, {"message", message}, {"param", nullptr}}},
+        });
+    }
+
+    json session_json() const {
+        json audio_in = {{"format", {{"type", "audio/pcm"}, {"rate", input_rate}}}, {"turn_detection", nullptr}};
+        if (type == "transcription") {
+            audio_in["transcription"] = {{"model", deps.meta.model_name}, {"language", language.empty() ? json(nullptr) : json(language)}};
+            return {
+                {"id",     id},
+                {"object", "realtime.transcription_session"},
+                {"type",   "transcription"},
+                {"audio",  {{"input", audio_in}}},
+            };
+        }
+        json audio = {{"input", audio_in}};
+        if (deps.cap.audio_gen) {
+            audio["output"] = {{"format", {{"type", "audio/pcm"}, {"rate", output_rate}}}};
+        }
+        return {
+            {"id",                id},
+            {"object",            "realtime.session"},
+            {"type",              "realtime"},
+            {"model",             deps.meta.model_name},
+            {"instructions",      instructions},
+            {"output_modalities", output_modalities_json(out_mode)},
+            {"max_output_tokens", max_output_tokens < 0 ? json("inf") : json(max_output_tokens)},
+            {"audio",             audio},
+        };
+    }
+
+    //
+    // slot controls, run on the slot in the order they are posted
+    //
+
+    bool control(server_control_action action, std::string text = {}, std::vector<float> audio = {},
+                 server_control_response response = {}) {
+        server_response_reader ctl(deps.queue_tasks, deps.queue_results, 1);
+        server_task task(SERVER_TASK_TYPE_CONTROL);
+        task.id                       = ctl.get_new_id();
+        task.params.control_action    = action;
+        task.params.control_cmpl_id   = id;
+        task.params.control_text      = std::move(text);
+        task.params.control_audio     = std::move(audio);
+        task.params.control_response  = response;
+        ctl.post_task(std::move(task));
+        auto result = ctl.next([this]() { return closed.load() || !conn.is_open(); });
+        auto * res = result ? dynamic_cast<server_task_result_control *>(result.get()) : nullptr;
+        return res && res->success;
+    }
+
+    //
+    // chat template
+    //
+
+    std::string render(const std::vector<common_chat_msg> & messages, bool add_generation_prompt = false) const {
+        const auto & chat = deps.meta.chat_params;
+        common_chat_templates_inputs inputs;
+        inputs.messages              = messages;
+        inputs.add_generation_prompt = add_generation_prompt;
+        inputs.use_jinja             = chat.use_jinja;
+        inputs.enable_thinking       = false;
+        inputs.chat_template_kwargs  = chat.chat_template_kwargs;
+        return common_chat_templates_apply(chat.tmpls.get(), inputs).prompt.str();
+    }
+
+    // the text around a user and an assistant turn, from placeholder messages
+    bool derive_turn_text() {
+        const std::string user  = "@@USER@@";
+        const std::string reply = "@@REPLY@@";
+        auto msg = [](const char * role, const std::string & content) {
+            common_chat_msg m;
+            m.role    = role;
+            m.content = content;
+            return m;
+        };
+        std::vector<common_chat_msg> sys;
+        if (!instructions.empty()) {
+            sys.push_back(msg("system", instructions));
+        }
+        auto with = [&](std::vector<common_chat_msg> rest, bool add_generation_prompt = false) {
+            std::vector<common_chat_msg> all = sys;
+            all.insert(all.end(), rest.begin(), rest.end());
+            return render(all, add_generation_prompt);
+        };
+        const std::string a = with({msg("user", user)});
+        const std::string g = with({msg("user", user)}, true);
+        const std::string c = with({msg("user", user), msg("assistant", reply)});
+        const std::string d = with({msg("user", user), msg("assistant", reply), msg("user", user)});
+
+        const size_t pa = a.find(user);
+        const size_t pc = c.find(reply);
+        const size_t pd = d.find(user, c.size());
+        if (pa == std::string::npos || pc == std::string::npos || pd == std::string::npos ||
+            c.compare(0, a.size(), a) != 0 || d.compare(0, c.size(), c) != 0 || g.compare(0, a.size(), a) != 0 || pc < a.size()) {
+            return false;
+        }
+        // the response opens with the generation prompt, a past turn may render differently (e.g. its think block)
+        first_header     = a.substr(0, pa);
+        user_footer      = a.substr(pa + user.size());
+        assistant_header = g.substr(a.size());
+        assistant_footer = c.substr(pc + reply.size());
+        user_header      = d.substr(c.size(), pd - c.size());
+        return true;
+    }
+
+    //
+    // conversation
+    //
+
+    // the header of a user turn goes in before its first audio or text
+    bool ensure_turn() {
+        if (turn_open) {
+            return true;
+        }
+        const std::string header = n_turns == 0 ? first_header : next_header + user_header;
+        if (!control(SERVER_CONTROL_ACTION_TEXT_APPEND, header)) {
+            return false;
+        }
+        next_header.clear();
+        started   = true;
+        turn_open = true;
+        return true;
+    }
+
+    void on_append(const json & ev) {
+        std::string pcm;
+        try {
+            pcm = base64::decode(json_value(ev, "audio", std::string()));
+        } catch (const std::exception &) {
+            send_error("invalid base64 audio");
+            return;
+        }
+        std::vector<float> samples(pcm.size() / 2);
+        for (size_t i = 0; i < samples.size(); i++) {
+            int16_t v;
+            memcpy(&v, pcm.data() + 2 * i, sizeof(v));
+            samples[i] = v / 32768.0f;
+        }
+        if (samples.empty() || !ensure_turn()) {
+            return;
+        }
+        if (control(SERVER_CONTROL_ACTION_AUDIO_APPEND, {}, std::move(samples))) {
+            has_audio = true;
+        }
+    }
+
+    // ends the audio of the open turn, it becomes a user item
+    bool commit() {
+        if (!has_audio) {
+            send_error("input_audio_buffer.commit: the buffer is empty", "input_audio_buffer_commit_empty");
+            return false;
+        }
+        if (!control(SERVER_CONTROL_ACTION_AUDIO_END)) {
+            return false;
+        }
+        has_audio = false;
+
+        std::lock_guard<std::mutex> lock(mutex);
+        const json prev = last_item_id.empty() ? json(nullptr) : json(last_item_id);
+        last_item_id = "item_" + random_string();
+        send_event({{"type", "input_audio_buffer.committed"}, {"previous_item_id", prev}, {"item_id", last_item_id}});
+        const json item = {{"id", last_item_id}, {"object", "realtime.item"}, {"type", "message"}, {"role", "user"},
+                           {"status", "completed"}, {"content", json::array({{{"type", "input_audio"}}})}};
+        send_event({{"type", "conversation.item.added"}, {"previous_item_id", prev}, {"item", item}});
+        send_event({{"type", "conversation.item.done"},  {"previous_item_id", prev}, {"item", item}});
+        return true;
+    }
+
+    void on_commit() {
+        if (!commit() || type != "transcription") {
+            return;
+        }
+        // transcription: the committed audio is transcribed on its own, then the slot is cleared for the next one
+        {
+            std::lock_guard<std::mutex> lock(mutex);
+            transcribe_item_id = last_item_id;
+            transcript.clear();
+        }
+        server_control_response response;
+        response.n_predict = max_output_tokens;
+        response.out_mode  = MTMD_HELPER_GEN_AUDIO_OUT_TEXT;
+        if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, user_footer + assistant_header, {}, response)) {
+            return;
+        }
+        control(SERVER_CONTROL_ACTION_CLEAR);
+        turn_open = false;
+        n_turns   = 0;
+    }
+
+    void on_item_create(const json & ev) {
+        if (type == "transcription") {
+            send_error("conversation.item.create: not supported in a transcription session");
+            return;
+        }
+        const json item = ev.value("item", json::object());
+        if (json_value(item, "type", std::string()) != "message" || json_value(item, "role", std::string()) != "user") {
+            send_error("conversation.item.create: only user messages are supported");
+            return;
+        }
+        std::string text;
+        for (const auto & part : item.value("content", json::array())) {
+            const std::string t = json_value(part, "type", std::string());
+            if (t != "input_text") {
+                send_error("conversation.item.create: unsupported content type: " + t);
+                return;
+            }
+            text += json_value(part, "text", std::string());
+        }
+        if (!ensure_turn() || !control(SERVER_CONTROL_ACTION_TEXT_APPEND, text)) {
+            return;
+        }
+        std::lock_guard<std::mutex> lock(mutex);
+        const json prev = last_item_id.empty() ? json(nullptr) : json(last_item_id);
+        last_item_id = item.contains("id") && item["id"].is_string() ? item["id"].get<std::string>() : "item_" + random_string();
+        const json out = {{"id", last_item_id}, {"object", "realtime.item"}, {"type", "message"}, {"role", "user"},
+                          {"status", "completed"}, {"content", json::array({{{"type", "input_text"}, {"text", text}}})}};
+        send_event({{"type", "conversation.item.added"}, {"previous_item_id", prev}, {"item", out}});
+        send_event({{"type", "conversation.item.done"},  {"previous_item_id", prev}, {"item", out}});
+    }
+
+    void on_response_create(const json & ev) {
+        if (type == "transcription") {
+            send_error("response.create: not supported in a transcription session");
+            return;
+        }
+        if (!turn_open) {
+            send_error("response.create: commit audio or create a message first", "conversation_empty");
+            return;
+        }
+        mtmd_helper_gen_audio_out_mode mode = out_mode;
+        const json resp = ev.value("response", json::object());
+        if (resp.contains("output_modalities")) {
+            std::string err;
+            if (!parse_output_modalities(resp["output_modalities"], mode, err)) {
+                send_error("response.create: " + err);
+                return;
+            }
+        }
+        if (!deps.cap.audio_gen && mode != MTMD_HELPER_GEN_AUDIO_OUT_TEXT && mode != MTMD_HELPER_GEN_AUDIO_OUT_AUTO) {
+            send_error("response.create: the model cannot generate audio");
+            return;
+        }
+        if (!deps.cap.text_gen && mode != MTMD_HELPER_GEN_AUDIO_OUT_AUDIO && mode != MTMD_HELPER_GEN_AUDIO_OUT_AUTO) {
+            send_error("response.create: the model cannot generate text");
+            return;
+        }
+        // audio not committed yet belongs to this turn
+        if (has_audio && !commit()) {
+            return;
+        }
+
+        // the gen-audio pipeline opens the response in the asked mode
+        const std::string prompt = user_footer + assistant_header;
+        server_control_response response;
+        response.n_predict = max_output_tokens;
+        if (resp.contains("max_output_tokens") && !parse_max_output_tokens(resp["max_output_tokens"], response.n_predict)) {
+            send_error("response.create: max_output_tokens must be an integer or \"inf\"");
+            return;
+        }
+        response.out_mode = mode;
+
+        {
+            std::lock_guard<std::mutex> lock(mutex);
+            response_id      = "resp_" + random_string();
+            response_item_id = "item_" + random_string();
+            response_text.clear();
+            parts     = json::array();
+            idx_text  = -1;
+            idx_audio = -1;
+            send_event({{"type", "response.created"}, {"response", {{"id", response_id}, {"object", "realtime.response"},
+                        {"status", "in_progress"}, {"output_modalities", output_modalities_json(mode)}, {"output", json::array()}}}});
+            send_event({{"type", "response.output_item.added"}, {"response_id", response_id}, {"output_index", 0},
+                        {"item", {{"id", response_item_id}, {"object", "realtime.item"}, {"type", "message"}, {"role", "assistant"},
+                                  {"status", "in_progress"}, {"content", json::array()}}}});
+        }
+        if (!control(SERVER_CONTROL_ACTION_RESPONSE_CREATE, prompt, {}, response)) {
+            return;
+        }
+        // the response ends the turn, its tokens are in the slot; the footer goes in before the next turn
+        next_header = assistant_footer;
+        turn_open   = false;
+        n_turns++;
+    }
+
+    void on_session_update(const json & ev) {
+        const json session = ev.value("session", json::object());
+        if (session.contains("type") && session["type"].is_string() && session["type"] != type) {
+            const std::string t = session["type"];
+            if (t != "realtime" && t != "transcription") {
+                send_error("session.update: unknown session type " + t);
+                return;
+            }
+            if (started) {
+                send_error("session.update: the session type cannot change after the first audio or message");
+                return;
+            }
+            type = t;
+        }
+        if (session.contains("instructions") && session["instructions"].is_string()) {
+            if (started) {
+                send_error("session.update: instructions cannot change after the first audio or message");
+            } else {
+                instructions = session["instructions"];
+            }
+        }
+        if (session.contains("output_modalities")) {
+            std::string err;
+            if (!parse_output_modalities(session["output_modalities"], out_mode, err)) {
+                send_error("session.update: " + err);
+                return;
+            }
+        }
+        if (session.contains("max_output_tokens") && !parse_max_output_tokens(session["max_output_tokens"], max_output_tokens)) {
+            send_error("session.update: max_output_tokens must be an integer or \"inf\"");
+            return;
+        }
+        const json audio = session.value("audio", json::object());
+        const json in    = audio.value("input", json::object());
+        if (in.contains("format") && in["format"].is_object() && json_value(in["format"], "rate", input_rate) != input_rate) {
+            send_error("session.update: the input audio must be PCM16 at " + std::to_string(input_rate) + " Hz");
+        }
+        if (in.contains("transcription") && in["transcription"].is_object()) {
+            language = json_value(in["transcription"], "language", language);
+        }
+        if (!started && !configure()) {
+            send_error("the chat template does not fit the realtime turn format", "server_error");
+            return;
+        }
+        send_event({{"type", "session.updated"}, {"session", session_json()}});
+    }
+
+    // turn text of the session type, before anything went into the slot
+    bool configure() {
+        const std::string saved = instructions;
+        if (type == "transcription") {
+            const auto preset = common_chat_get_asr_prompt(deps.meta.chat_params.tmpls.get(), language);
+            instructions = preset.system;
+        }
+        const bool ok = derive_turn_text();
+        instructions = saved;
+        return ok;
+    }
+
+    void handle(const std::string & msg) {
+        json ev;
+        try {
+            ev = json::parse(msg);
+        } catch (const std::exception &) {
+            send_error("invalid JSON");
+            return;
+        }
+        const std::string t = json_value(ev, "type", std::string());
+        if (t == "session.update" || t == "transcription_session.update") {
+            on_session_update(ev);
+        } else if (t == "input_audio_buffer.append") {
+            on_append(ev);
+        } else if (t == "input_audio_buffer.commit") {
+            on_commit();
+        } else if (t == "input_audio_buffer.clear") {
+            // the audio already went into the conversation
+            send_event({{"type", "input_audio_buffer.cleared"}});
+        } else if (t == "conversation.item.create") {
+            on_item_create(ev);
+        } else if (t == "response.create") {
+            on_response_create(ev);
+        } else if (t == "response.cancel") {
+            control(SERVER_CONTROL_ACTION_RESPONSE_CANCEL);
+        } else {
+            send_error("unknown event type: " + t);
+        }
+    }
+
+    //
+    // results of the slot
+    //
+
+    static std::string encode_pcm16(const std::vector<float> & samples) {
+        std::string bytes(samples.size() * sizeof(int16_t), '\0');
+        for (size_t i = 0; i < samples.size(); i++) {
+            const int16_t v = (int16_t) std::lround(std::max(-1.0f, std::min(1.0f, samples[i])) * 32767.0f);
+            memcpy(&bytes[2 * i], &v, sizeof(v));
+        }
+        return base64::encode(bytes);
+    }
+
+    // a part of the response is opened on its first delta, in order of appearance
+    int open_part(const char * part_type) {
+        json part = std::string(part_type) == "output_text" ? json{{"type", part_type}, {"text", ""}} : json{{"type", part_type}, {"transcript", ""}};
+        parts.push_back(part);
+        send_event({{"type", "response.content_part.added"}, {"response_id", response_id}, {"item_id", response_item_id},
+                    {"output_index", 0}, {"content_index", parts.size() - 1}, {"part", part}});
+        return (int) parts.size() - 1;
+    }
+
+    void on_result(const server_task_result_realtime & res) {
+        std::lock_guard<std::mutex> lock(mutex);
+        if (!transcribe_item_id.empty()) {
+            if (!res.text.empty()) {
+                transcript += res.text;
+                send_event({{"type", "conversation.item.input_audio_transcription.delta"}, {"item_id", transcribe_item_id},
+                            {"content_index", 0}, {"delta", res.text}});
+            }
+            if (res.response_done) {
+                send_event({{"type", "conversation.item.input_audio_transcription.completed"}, {"item_id", transcribe_item_id},
+                            {"content_index", 0}, {"transcript", transcript}});
+                transcribe_item_id.clear();
+            }
+            return;
+        }
+
+        const json base = {{"response_id", response_id}, {"item_id", response_item_id}, {"output_index", 0}};
+        if (!res.text.empty()) {
+            if (idx_text < 0) {
+                idx_text = open_part("output_text");
+            }
+            response_text += res.text;
+            json ev = base;
+            ev["type"]          = "response.output_text.delta";
+            ev["content_index"] = idx_text;
+            ev["delta"]         = res.text;
+            send_event(ev);
+        }
+        if (!res.audio.empty()) {
+            if (idx_audio < 0) {
+                idx_audio = open_part("output_audio");
+            }
+            json ev = base;
+            ev["type"]          = "response.output_audio.delta";
+            ev["content_index"] = idx_audio;
+            ev["delta"]         = encode_pcm16(res.audio);
+            send_event(ev);
+        }
+        if (!res.response_done) {
+            return;
+        }
+
+        const std::string status = res.cancelled ? "cancelled" : "completed";
+        for (size_t i = 0; i < parts.size(); i++) {
+            json ev = base;
+            ev["content_index"] = i;
+            if ((int) i == idx_text) {
+                parts[i]["text"] = response_text;
+                ev["type"]       = "response.output_text.done";
+                ev["text"]       = response_text;
+            } else {
+                ev["type"] = "response.output_audio.done";
+            }
+            send_event(ev);
+            ev.erase("text");
+            ev["type"] = "response.content_part.done";
+            ev["part"] = parts[i];
+            send_event(ev);
+        }
+        const json item = {{"id", response_item_id}, {"object", "realtime.item"}, {"type", "message"}, {"role", "assistant"},
+                           {"status", status}, {"content", parts}};
+        send_event({{"type", "response.output_item.done"}, {"response_id", response_id}, {"output_index", 0}, {"item", item}});
+        send_event({{"type", "response.done"}, {"response", {{"id", response_id}, {"object", "realtime.response"}, {"status", status},
+                    {"output", json::array({item})}, {"usage", {{"output_tokens", res.n_decoded}}}}}});
+        last_item_id = response_item_id;
+    }
+
+    //
+    // session
+    //
+
+    void run() {
+        input_rate = mtmd_get_audio_sample_rate(deps.mctx);
+        if (deps.cap.audio_gen) {
+            output_rate = mtmd_gen_audio_get_info(deps.mctx).sample_rate;
+        }
+        if (!configure()) {
+            send_error("the chat template does not fit the realtime turn format", "server_error");
+            conn.close();
+            return;
+        }
+
+        server_response_reader rd(deps.queue_tasks, deps.queue_results, 1);
+        {
+            server_task task(SERVER_TASK_TYPE_REALTIME);
+            task.id                       = rd.get_new_id();
+            task.params.sampling          = deps.sampling;
+            task.params.stream            = true;
+            task.params.oaicompat_cmpl_id = id;
+            task.params.oaicompat_model   = deps.meta.model_name;
+            rd.post_task(std::move(task));
+        }
+        // the slot answers once it holds the session
+        {
+            auto result = rd.next([this]() { return !conn.is_open(); });
+            if (!result || result->is_error()) {
+                if (result) {
+                    send_error(json_value(result->to_json(), "message", std::string("server error")), "server_error");
+                }
+                conn.close();
+                return;
+            }
+        }
+        send_event({{"type", "session.created"}, {"session", session_json()}});
+
+        // client events are handled on this thread, results are sent from the loop below
+        std::thread reader([this]() {
+            std::string msg;
+            while (conn.read(msg) != 0) {
+                handle(msg);
+            }
+            closed = true;
+            conn.close();
+        });
+
+        while (true) {
+            auto result = rd.next([this]() { return !conn.is_open(); });
+            if (!result) {
+                break;
+            }
+            if (result->is_error()) {
+                send_error(json_value(result->to_json(), "message", std::string("server error")), "server_error");
+                break;
+            }
+            if (auto * res = dynamic_cast<server_task_result_realtime *>(result.get())) {
+                on_result(*res);
+            }
+        }
+
+        closed = true;
+        conn.close();
+        reader.join();
+    }
+};
+
+} // namespace
+
+void server_realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn) {
+    realtime_conversation session(deps, conn);
+    session.run();
+}
+
+//
+// server_realtime_slot
+//
+
+server_realtime_slot::server_realtime_slot(llama_context * ctx, mtmd_context * mctx, llama_seq_id seq_id, int32_t n_ctx,
+                                           const common_params_sampling & sampling, server_response & queue_results, int id_task, int index)
+    : ctx(ctx), model(llama_get_model(ctx)), vocab(llama_model_get_vocab(llama_get_model(ctx))), mctx(mctx), seq_id(seq_id),
+      n_ctx(n_ctx), sampling(sampling), queue_results(queue_results), id_task(id_task), index(index) {
+    if (mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE) {
+        gen.init(ctx, mctx);
+    }
+}
+
+void server_realtime_slot::push(server_realtime_op && op) {
+    ops.push_back(std::move(op));
+}
+
+void server_realtime_slot::cancel() {
+    cancelled = responding;
+}
+
+bool server_realtime_slot::is_idle() const {
+    return ops.empty() && !responding;
+}
+
+bool server_realtime_slot::fail(const std::string & msg, enum error_type type) {
+    error      = msg;
+    error_type = type;
+    return false;
+}
+
+void server_realtime_slot::send_result(std::string text, std::vector<float> audio, bool response_done) {
+    auto res = std::make_unique<server_task_result_realtime>();
+    res->id            = id_task;
+    res->index         = index;
+    res->text          = std::move(text);
+    res->audio         = std::move(audio);
+    res->response_done = response_done;
+    res->cancelled     = response_done && cancelled;
+    res->n_decoded     = n_decoded;
+    queue_results.send(std::move(res));
+}
+
+// appends tokens or embedding rows to the conversation, logits of the last one if asked
+bool server_realtime_slot::decode(const llama_token * tokens, const float * rows, int32_t n, bool logits_last) {
+    const int32_t n_embd  = llama_model_n_embd_inp(model);
+    const int32_t n_batch = llama_n_batch(ctx);
+    if (n_past + n >= n_ctx) {
+        return fail("the conversation does not fit in the context", ERROR_TYPE_EXCEED_CONTEXT_SIZE);
+    }
+    for (int32_t i0 = 0; i0 < n; i0 += n_batch) {
+        const int32_t n_cur = std::min(n_batch, n - i0);
+        llama_batch b = llama_batch_init(n_cur, rows ? n_embd : 0, 1);
+        for (int32_t i = 0; i < n_cur; i++) {
+            if (rows) {
+                memcpy(b.embd + (size_t) i * n_embd, rows + (size_t) (i0 + i) * n_embd, n_embd * sizeof(float));
+            } else {
+                b.token[i] = tokens[i0 + i];
+            }
+            b.pos[i]       = n_past + i0 + i;
+            b.n_seq_id[i]  = 1;
+            b.seq_id[i][0] = seq_id;
+            b.logits[i]    = logits_last && i0 + i == n - 1;
+        }
+        b.n_tokens = n_cur;
+        const int ret = llama_decode(ctx, b);
+        llama_batch_free(b);
+        if (ret != 0) {
+            return fail("failed to decode the conversation");
+        }
+    }
+    n_past += n;
+    return true;
+}
+
+// decodes the rows the audio stream has ready
+bool server_realtime_slot::audio_rows() {
+    while (true) {
+        const int32_t n = mtmd_audio_stream_step(stream.get(), LLAMA_TOKEN_NULL);
+        if (n == -2) {
+            return fail("audio stream error");
+        }
+        if (n <= 0) {
+            return true;
+        }
+        if (!decode(nullptr, mtmd_audio_stream_get_output_embd(stream.get()), n, false)) {
+            return false;
+        }
+    }
+}
+
+// samples the next text token of a response from the logits of the slot's own last decode; several slots
+// respond in turns on the same context, so this runs before any other slot decodes
+void server_realtime_slot::sample_next() {
+    sampled = LLAMA_TOKEN_NULL;
+    if (gen.valid() && !gen.next_is_text()) {
+        return;
+    }
+    sampled = common_sampler_sample(smpl.get(), ctx, -1);
+    common_sampler_accept(smpl.get(), sampled, true);
+}
+
+bool server_realtime_slot::response_start(const server_realtime_op & op) {
+    auto sparams = sampling;
+    n_predict = op.response.n_predict;
+    n_decoded = 0;
+    cancelled = false;
+
+    if (gen.valid()) {
+        // the gen-audio pipeline continues the conversation and follows the model's text/audio switching
+        mtmd_helper::gen_audio::inp inp;
+        inp.set_prompt(op.text);
+        inp.data.seq_id   = seq_id;
+        inp.data.n_past   = n_past;
+        inp.data.seed     = sampling.seed;
+        inp.data.stream   = true;
+        inp.data.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM;
+        inp.data.out_mode = op.response.out_mode;
+        if (gen.set_input(inp.get()) != 0) {
+            return fail("failed to start the response");
+        }
+        // the pipeline may bias some text tokens, e.g. the switch to audio of a text-only response
+        for (const auto & b : gen.logit_bias()) {
+            sparams.logit_bias.push_back(b);
+        }
+        int32_t ret;
+        while ((ret = gen.step_prompt(llama_n_batch(ctx))) > 0) {
+        }
+        if (ret < 0) {
+            return fail("failed to start the response");
+        }
+        const float * h = llama_get_embeddings_ith(ctx, -1);
+        h_state.assign(h, h + llama_model_n_embd(model));
+    } else {
+        const llama_tokens toks = common_tokenize(vocab, op.text, false, true);
+        if (toks.empty()) {
+            return fail("failed to start the response");
+        }
+        if (!decode(toks.data(), nullptr, (int32_t) toks.size(), true)) {
+            return false;
+        }
+    }
+    smpl.reset(common_sampler_init(model, sparams));
+    // the logits of the last decode are this slot's only until another slot decodes
+    sample_next();
+    responding = true;
+    return true;
+}
+
+// one text token or audio frame of the response being generated
+bool server_realtime_slot::response_step() {
+    bool        stop = false;
+    std::string text;
+    std::vector<float> audio;
+
+    if (gen.valid()) {
+        const float * h_next = nullptr;
+        if (gen.step_gen(sampled, h_state.data(), &h_next, &stop) != 0) {
+            return fail("response generation failed");
+        }
+        if (sampled != LLAMA_TOKEN_NULL && !stop && !llama_vocab_is_control(vocab, sampled)) {
+            text = common_token_to_piece(vocab, sampled, false);
+        }
+        if (h_next) {
+            h_state.assign(h_next, h_next + llama_model_n_embd(model));
+        }
+    } else {
+        stop = llama_vocab_is_eog(vocab, sampled);
+        if (!stop) {
+            if (!llama_vocab_is_control(vocab, sampled)) {
+                text = common_token_to_piece(vocab, sampled, false);
+            }
+            if (!decode(&sampled, nullptr, 1, true)) {
+                return false;
+            }
+        }
+    }
+    n_decoded++;
+
+    // the pipeline decodes on its own, the response ends before the context is full
+    const bool ctx_full = gen.valid() ?
+        llama_memory_seq_pos_max(llama_get_memory(ctx), seq_id) + 2 >= n_ctx :
+        n_past + 1 >= n_ctx;
+    const bool done = stop || cancelled || ctx_full || (n_predict > 0 && n_decoded >= n_predict);
+    if (!done) {
+        sample_next();
+    }
+    if (gen.valid()) {
+        if (done) {
+            gen.flush();
+        }
+        int32_t sample_rate = 0;
+        const char * data   = nullptr;
+        size_t data_len     = 0;
+        if (gen.get_output(&sample_rate, &data, &data_len) == 0 && data_len > 0) {
+            audio.assign((const float *) data, (const float *) data + data_len / sizeof(float));
+        }
+    }
+    if (!text.empty() || !audio.empty()) {
+        send_result(std::move(text), std::move(audio));
+    }
+    if (done) {
+        if (gen.valid()) {
+            // the pipeline decoded the response, the sequence ends where it stopped
+            n_past = llama_memory_seq_pos_max(llama_get_memory(ctx), seq_id) + 1;
+        }
+        send_result({}, {}, true);
+        responding = false;
+        cancelled  = false;
+    }
+    return true;
+}
+
+bool server_realtime_slot::update() {
+    llama_set_embeddings(ctx, true);
+
+    if (responding) {
+        return response_step();
+    }
+
+    while (!ops.empty() && !responding) {
+        server_realtime_op op = std::move(ops.front());
+        ops.pop_front();
+        switch (op.action) {
+            case SERVER_CONTROL_ACTION_TEXT_APPEND:
+                {
+                    const llama_tokens toks = common_tokenize(vocab, op.text, false, true);
+                    if (!toks.empty() && !decode(toks.data(), nullptr, (int32_t) toks.size(), false)) {
+                        return false;
+                    }
+                } break;
+            case SERVER_CONTROL_ACTION_AUDIO_APPEND:
+                {
+                    if (!stream) {
+                        stream.reset(mtmd_audio_stream_init(mctx, mtmd_audio_stream_params_default()));
+                        if (!stream) {
+                            return fail("failed to initialize the audio stream");
+                        }
+                    }
+                    mtmd_audio_stream_push(stream.get(), op.audio.data(), op.audio.size());
+                    if (!audio_rows()) {
+                        return false;
+                    }
+                } break;
+            case SERVER_CONTROL_ACTION_AUDIO_COMMIT:
+            case SERVER_CONTROL_ACTION_AUDIO_END:
+                {
+                    // the end of the audio of a turn
+                    if (stream) {
+                        mtmd_audio_stream_flush(stream.get(), true);
+                        if (!audio_rows()) {
+                            return false;
+                        }
+                        stream.reset();
+                    }
+                } break;
+            case SERVER_CONTROL_ACTION_RESPONSE_CREATE:
+                {
+                    if (!response_start(op)) {
+                        return false;
+                    }
+                } break;
+            case SERVER_CONTROL_ACTION_CLEAR:
+                {
+                    llama_memory_seq_rm(llama_get_memory(ctx), seq_id, -1, -1);
+                    stream.reset();
+                    n_past = 0;
+                } break;
+            default:
+                break;
+        }
+    }
+    return true;
+}

tools/server/server-realtime.h

@@ -0,0 +1,94 @@
+#pragma once
+
+#include "server-http.h"
+#include "server-queue.h"
+#include "server-task.h"
+
+#include "llama.h"
+#include "mtmd.h"
+#include "mtmd-helper.h"
+#include "sampling.h"
+
+#include <deque>
+#include <string>
+#include <vector>
+
+struct server_context_meta;
+struct common_params_sampling;
+struct mtmd_context;
+struct mtmd_helper_cap;
+struct llama_vocab;
+
+struct server_realtime_deps {
+    server_queue &                 queue_tasks;
+    server_response &              queue_results;
+    const server_context_meta &    meta;
+    const common_params_sampling & sampling;
+    mtmd_context *                 mctx;
+    const mtmd_helper_cap &        cap;
+    const llama_vocab *            vocab;
+};
+
+// a realtime conversation over websocket, for models that answer after the audio (e.g. LFM3-Audio):
+// the audio is prefilled as it arrives, responses follow output_modalities or the model's own switching
+void server_realtime_conversation(const server_realtime_deps & deps, server_http_context::ws_conn & conn);
+
+// one control of a realtime conversation
+struct server_realtime_op {
+    server_control_action   action;
+    std::string             text;
+    std::vector<float>      audio;
+    server_control_response response;
+};
+
+// the conversation of a slot that holds a realtime session (SERVER_TASK_TYPE_REALTIME), it runs on the server thread
+// the controls run in order; the sequence is decoded by its own calls, not in the shared batch
+struct server_realtime_slot {
+    server_realtime_slot(llama_context * ctx, mtmd_context * mctx, llama_seq_id seq_id, int32_t n_ctx,
+                         const common_params_sampling & sampling, server_response & queue_results, int id_task, int index);
+
+    void push(server_realtime_op && op);
+    void cancel(); // stops the response being generated
+
+    // runs the queued controls, or one step of the response; false on error, see error and error_type
+    bool update();
+
+    // nothing to run until the next control
+    bool is_idle() const;
+
+    std::string error;
+    enum error_type error_type = ERROR_TYPE_SERVER;
+
+private:
+    llama_context *        ctx;
+    const llama_model *    model;
+    const llama_vocab *    vocab;
+    mtmd_context *         mctx;
+    llama_seq_id           seq_id;
+    int32_t                n_ctx;
+    common_params_sampling sampling;
+    server_response &      queue_results;
+    int                    id_task;
+    int                    index;
+
+    std::deque<server_realtime_op> ops;
+    llama_pos   n_past     = 0;     // positions of the conversation in the sequence
+    bool        responding = false;
+    bool        cancelled  = false;
+    int32_t     n_predict  = 0;
+    int32_t     n_decoded  = 0;
+    llama_token sampled    = LLAMA_TOKEN_NULL; // next text token of the response, sampled right after the slot's own decode
+
+    mtmd::audio_stream_ptr stream;   // the audio of the open turn
+    mtmd_helper::gen_audio gen;      // set if the model generates audio
+    std::vector<float>     h_state;  // hidden state that the gen-audio pipeline steps from
+    common_sampler_ptr     smpl;
+
+    bool fail(const std::string & msg, enum error_type type = ERROR_TYPE_SERVER);
+    void send_result(std::string text, std::vector<float> audio, bool response_done = false);
+    bool decode(const llama_token * tokens, const float * rows, int32_t n, bool logits_last);
+    bool audio_rows();
+    void sample_next();
+    bool response_start(const server_realtime_op & op);
+    bool response_step();
+};

tools/server/server-task.cpp

@@ -1511,6 +1511,16 @@ json server_task_result_rerank::to_json() {
 //
 // server_task_result_tts
 //
+json server_task_result_realtime::to_json() {
+    return json {
+        {"text",          text},
+        {"n_samples",     audio.size()},
+        {"response_done", response_done},
+        {"cancelled",     cancelled},
+        {"n_decoded",     n_decoded},
+    };
+}
+
 json server_task_result_tts::to_json() {
     return json {
         {"sample_rate", sample_rate},

tools/server/server-task.h

@@ -20,6 +20,7 @@ enum server_task_type {
     SERVER_TASK_TYPE_DECISION,
     SERVER_TASK_TYPE_INFILL,
     SERVER_TASK_TYPE_TRANSCRIBE_RT,
+    SERVER_TASK_TYPE_REALTIME, // a conversation over /v1/realtime, see server-realtime.cpp
     SERVER_TASK_TYPE_CANCEL,
     SERVER_TASK_TYPE_CONTROL,
     SERVER_TASK_TYPE_NEXT_RESPONSE,
@@ -57,6 +58,17 @@ enum server_control_action {
     SERVER_CONTROL_ACTION_AUDIO_APPEND, // push control_audio into the slot's audio stream
     SERVER_CONTROL_ACTION_AUDIO_COMMIT, // flush the audio stream, more audio can follow
     SERVER_CONTROL_ACTION_AUDIO_END,    // flush the audio stream and end it
+    // realtime conversations (SERVER_TASK_TYPE_REALTIME), applied in order with the audio actions
+    SERVER_CONTROL_ACTION_TEXT_APPEND,     // append control_text to the conversation
+    SERVER_CONTROL_ACTION_RESPONSE_CREATE, // generate after control_text, see control_response
+    SERVER_CONTROL_ACTION_RESPONSE_CANCEL, // stop the response being generated
+    SERVER_CONTROL_ACTION_CLEAR,           // drop the conversation
+};
+
+// a response of a realtime conversation
+struct server_control_response {
+    mtmd_helper_gen_audio_out_mode out_mode  = MTMD_HELPER_GEN_AUDIO_OUT_AUTO;
+    int32_t                        n_predict = -1; // text tokens and audio frames, -1: until the context is full
 };
 
 struct task_params {
@@ -100,6 +112,8 @@ struct task_params {
     server_control_action control_action = SERVER_CONTROL_ACTION_NONE;
     std::string           control_cmpl_id;
     std::vector<float>    control_audio; // PCM F32 samples for SERVER_CONTROL_ACTION_AUDIO_APPEND
+    std::string           control_text;  // raw text, special tokens are parsed
+    server_control_response control_response;
 
     // reported in generation_settings, parsing itself is owned by the chat session
     common_chat_format      chat_format      = COMMON_CHAT_FORMAT_CONTENT_ONLY;
@@ -196,6 +210,8 @@ struct server_task {
         int32_t                  column = 0;
         // if set, embeddings is [q | k], and the output is instead the scaled dot product of q[pointer] and k[marker]
         int32_t                  pointer = -1;
+        // if set, the prompt must be the only sequence of its batch
+        bool                     no_batching = false;
 
         // if set, the outputs are the pooled embeddings of the sequence, one score per class
         bool                     pooled = false;
@@ -536,6 +552,21 @@ struct server_task_result_embd : server_task_result {
     json to_json_oaicompat();
 };
 
+// one event of a realtime conversation: text and audio of the response being generated, or its end
+struct server_task_result_realtime : server_task_result {
+    std::string        text;
+    std::vector<float> audio;               // PCM F32 at the sample rate of mtmd_gen_audio_get_info()
+    bool               response_done = false;
+    bool               cancelled     = false;
+    int32_t            n_decoded     = 0;
+
+    virtual bool is_stop() override {
+        return false; // the session ends when the client leaves, the task is cancelled then
+    }
+
+    virtual json to_json() override;
+};
+
 struct server_task_result_tts : server_task_result {
     std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type)
     int32_t     sample_rate = 0;

tools/server/tests/unit/test_compat_anthropic.py

@@ -498,6 +498,54 @@ def test_anthropic_tool_result_error():
     assert res.body["type"] == "message"
 
 
+def test_anthropic_tool_reference():
+    """Anthropic defer_loading and tool_reference convert to their OpenAI forms (behaviour: test_tool_call.py)"""
+    server.jinja = True
+    server.chat_template_file = '../../../models/templates/LFM3.jinja'
+    server.start()
+
+    search = {"name": "tool_search", "description": "Search for available tools by keyword",
+              "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}
+    purchase = {"name": "markItemPurchased", "description": "Mark an item as purchased",
+                "parameters": {"type": "object", "properties": {"id": {"type": "string"}}}}
+
+    anthropic = server.make_request("POST", "/v1/messages/count_tokens", data={
+        "model": "test",
+        "tools": [
+            {"name": t["name"], "description": t["description"], "input_schema": t["parameters"], **extra}
+            for t, extra in ((search, {}), (purchase, {"defer_loading": True}))
+        ],
+        "messages": [
+            {"role": "user", "content": "Mark item abc as purchased"},
+            {"role": "assistant", "content": [
+                {"type": "tool_use", "id": "call_1", "name": "tool_search", "input": {"query": "purchase"}},
+            ]},
+            {"role": "user", "content": [
+                {"type": "tool_result", "tool_use_id": "call_1",
+                 "content": [{"type": "tool_reference", "tool_name": "markItemPurchased"}]},
+            ]},
+        ],
+    })
+    oai = server.make_request("POST", "/chat/completions/input_tokens", data={
+        "tools": [
+            {"type": "function", "function": search},
+            {"type": "function", "function": {**purchase, "defer_loading": True}},
+        ],
+        "messages": [
+            {"role": "user", "content": "Mark item abc as purchased"},
+            {"role": "assistant", "content": None, "tool_calls": [
+                {"id": "call_1", "type": "function",
+                 "function": {"name": "tool_search", "arguments": "{\"query\": \"purchase\"}"}},
+            ]},
+            {"role": "tool", "tool_call_id": "call_1",
+             "content": [{"type": "tool_reference", "name": "markItemPurchased"}]},
+        ],
+    })
+    assert anthropic.status_code == 200, anthropic.body
+    assert oai.status_code == 200, oai.body
+    assert anthropic.body["input_tokens"] == oai.body["input_tokens"]
+
+
 def test_anthropic_tool_streaming():
     """Test streaming with tool use"""
     server.jinja = True

tools/server/tests/unit/test_tool_call.py

@@ -642,3 +642,54 @@ def do_test_hello_world(server: ServerProcess, **kwargs):
     code = actual_arguments["code"]
     assert isinstance(code, str), f"Expected code to be a string, got {type(code)}: {json.dumps(code)}"
     assert re.match(r'''print\(("[Hh]ello,? [Ww]orld!?"|'[Hh]ello,? [Ww]orld!?')\)''', re.sub(r'#.*\n?', '', code)), f'Expected hello world, got {code}'
+
+
+SEARCH_TOOL = {
+    "type": "function",
+    "function": {
+        "name": "tool_search",
+        "description": "Search for available tools by keyword",
+        "parameters": {"type": "object", "properties": {"query": {"type": "string"}}},
+    },
+}
+PURCHASE_FUNCTION = {
+    "name": "markItemPurchased",
+    "description": "Mark an item as purchased",
+    "parameters": {"type": "object", "properties": {"id": {"type": "string"}, "qty": {"type": "integer"}}},
+}
+
+PURCHASE_TOOL = {"type": "function", "function": PURCHASE_FUNCTION}
+
+
+def count_deferred_tokens(tools, tool_content):
+    res = server.make_request("POST", "/chat/completions/input_tokens", data={
+        "tools": tools,
+        "messages": [
+            {"role": "user", "content": "Mark item abc as purchased"},
+            {"role": "assistant", "content": None, "tool_calls": [
+                {"id": "call_1", "type": "function",
+                 "function": {"name": "tool_search", "arguments": "{\"query\": \"purchase\"}"}},
+            ]},
+            {"role": "tool", "tool_call_id": "call_1", "content": tool_content},
+        ],
+    })
+    assert res.status_code == 200, res.body
+    return res.body["input_tokens"]
+
+
+def test_deferred_tool_reference():
+    """A defer_loading tool is withheld from the system prompt, a tool_reference reveals its definition"""
+    server.jinja = True
+    server.chat_template_file = '../../../models/templates/LFM3.jinja'
+    server.start()
+
+    deferred = {"type": "function", "function": {**PURCHASE_FUNCTION, "defer_loading": True}}
+    tools = [SEARCH_TOOL, deferred]
+
+    # withheld from the system prompt
+    assert count_deferred_tokens(tools, "none") == count_deferred_tokens([SEARCH_TOOL], "none")
+
+    # the reference renders like the definition sent inline
+    by_reference = count_deferred_tokens(tools, [{"type": "tool_reference", "name": "markItemPurchased"}])
+    inline = count_deferred_tokens(tools, "<additional_tools>\n" + json.dumps(PURCHASE_FUNCTION) + "\n</additional_tools>")
+    assert by_reference == inline

tools/tts/README.md

@@ -84,3 +84,29 @@ Both [Soprano-80M](https://huggingface.co/ekwek/Soprano-80M) and [Soprano-1.1-80
 python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --outtype f32 --outfile soprano.gguf
 python convert_hf_to_gguf.py path/to/Soprano-1.1-80M --mmproj --outtype f16 --outfile mmproj-soprano.gguf
 ```
+
+## LFM3-Audio
+
+Available params:
+- `--tts-lang` can be `ar`, `da`, `de`, `el`, `en`, `en-gb`, `es`, `fi`, `fr`, `he`, `it`, `ja`, `ko`, `nl`, `pl`, `pt`, `sv`, `tr`, `zh` (default: `en`), other values are used as the language name
+- `--tts-voice` can be `female` (default) or `male`
+- Note: `--tts-speaker-file` is not used, the model has no voice cloning
+- Note: `--top-k` and `--top-p` are not used, audio frames are sampled with the settings of the model card (LFM3-Audio: temperature 1.0, top-k 128, repetition penalty 1.1 over 4096 frames; LFM3-Omni: temperature 0.4, repetition penalty 1.2 over 64 frames), the `audio_*` keys of `generation_config.json` override them at conversion
+- Each frame is 80 ms, generation stops at the end-of-audio frame
+
+Example usage:
+
+```sh
+llama-tts -m lfm3-audio.gguf \
+    -mm mmproj-lfm3-audio.gguf \
+    -p "Hello world" \
+    --tts-voice male \
+    --output out.wav
+```
+
+The mmproj holds the audio encoder (for ASR), the depthformer and the codec decoder, the latter two are exported when the checkpoint has an `audio_tokenizer` directory:
+
+```sh
+python convert_hf_to_gguf.py path/to/lfm3-audio --outfile lfm3-audio.gguf
+python convert_hf_to_gguf.py path/to/lfm3-audio --mmproj --outfile mmproj-lfm3-audio.gguf
+```

tools/tts/tts.cpp

@@ -1,4 +1,5 @@
 #include "arg.h"
+#include "chat.h"
 #include "common.h"
 #include "sampling.h"
 #include "log.h"
@@ -36,7 +37,7 @@ static void print_usage(int, char ** argv) {
     LOG("\nexample usage:\n");
     LOG("\n    %s -m backbone.gguf -mm mmproj.gguf -p \"text to speak\" -o output.wav", argv[0]);
     LOG("\n    %s -hf user/model -p \"text to speak\" -o output.wav\n", argv[0]);
-    LOG("\nnote: --tts-lang and --tts-speaker-file may not be supported in all models");
+    LOG("\nnote: --tts-lang, --tts-voice and --tts-speaker-file may not be supported in all models");
     LOG("\n      use -n to limit the output length");
     LOG("\n      see tts/README.md for per-model usage notes");
     LOG("\n\n");
@@ -111,14 +112,22 @@ int main(int argc, char ** argv) {
         speaker_bitmap.reset(wrapper.bitmap);
     }
 
+    // a model with a "tts" chat template (e.g. LFM3-Audio) speaks the prompt rendered from it
+    auto tmpls = common_chat_templates_init(model, "");
+    std::string prompt = common_chat_get_tts_prompt(tmpls.get(), params.prompt, params.tts_lang, params.tts_voice);
+    if (prompt.empty()) {
+        prompt = params.prompt;
+    }
+
     mtmd_helper::gen_audio gen;
     gen.init(lctx, mctx.get());
     mtmd_helper_gen_audio_inp inp = mtmd_helper_gen_audio_inp_default();
     inp.seq_id      = 0;
-    inp.prompt      = params.prompt.c_str();
-    inp.prompt_len  = params.prompt.size();
+    inp.prompt      = prompt.c_str();
+    inp.prompt_len  = prompt.size();
     inp.speaker_ref = speaker_bitmap.get();
     inp.lang        = params.tts_lang.c_str();
+    inp.voice       = params.tts_voice.c_str();
     inp.top_k       = params.sampling.top_k;
     inp.top_p       = params.sampling.top_p;
     inp.seed        = params.sampling.seed;